trace: test for main API, fix bugs discovered by them

fixes #320
refs #318

.circleci/config.yml

@@ -6,8 +6,11 @@ workflows:
       - build-1.11
       - build-1.12
       - build-1.13
+      - build-1.14
       - build-latest
+      - test-build-in-docker
 jobs:
+
   # build-latest serves as the template
   # we use YAML anchors & aliases to exchange the docker image (and hence Go version used for the build)
   build-latest: &build-latest
@@ -108,3 +111,35 @@ jobs:
     <<: *build-latest
     docker:
     - image: circleci/golang:1.13
+
+  build-1.14:
+    <<: *build-latest
+    docker:
+    - image: circleci/golang:1.14
+
+  # this job tries to mimic the build-in-docker instructions
+  # given in docs/installation.rst
+  #
+  # However, CircleCI doesn't support volume mounts, so we have to copy
+  # the source into the build-container by modifying the Dockerfile
+  test-build-in-docker:
+    description: Check that build-in-docker works
+    docker:
+      - image: circleci/golang:latest
+    environment:
+    working_directory: /go/src/github.com/zrepl/zrepl
+    steps:
+    - checkout
+    - setup_remote_docker
+    - run:
+        name: (hacky) circleci doesn't allow volume mounts, so copy src to container
+        command: echo "ADD . /src" >> build.Dockerfile
+    - run:
+        name: (hacky) commit modified Dockerfile to avoid failing git clean check in Makefile
+        command: git -c user.name='circleci' -c user.email='circleci@localhost' commit -m 'CIRCLECI modified Dockerfile with zrepl src' --author 'autoauthor <circleci@localhost>' -- build.Dockerfile
+    - run:
+        name: build the build image (build deps)
+        command: docker build -t zrepl_build -f build.Dockerfile .
+    - run:
+        name: try compiling
+        command: docker run -it zrepl_build make release

Makefile

@@ -17,19 +17,27 @@ endif
 GO := go
 GOOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOOS"')
 GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"')
+GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"')
 GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"')
 GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
 GO_ENV_VARS := GO111MODULE=on
 GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
 GO_MOD_READONLY := -mod=readonly
-GO_BUILDFLAGS := $(GO_MOD_READONLY)
+GO_EXTRA_BUILDFLAGS :=
+GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
 GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
 GOLANGCI_LINT := golangci-lint
+ifneq ($(GOARM),)
+	ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM)
+else
+	ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)
+endif
 
 .PHONY: printvars
 printvars:
 	@echo GOOS=$(GOOS)
 	@echo GOARCH=$(GOARCH)
+	@echo GOARM=$(GOARM)
 
 
 ##################### PRODUCING A RELEASE #############
@@ -58,6 +66,7 @@ wrapup-and-checksum:
 		-acf $(NOARCH_TARBALL) \
 		$(ARTIFACTDIR)/docs/html \
 		$(ARTIFACTDIR)/bash_completion \
+		$(ARTIFACTDIR)/_zrepl.zsh_completion \
 		$(ARTIFACTDIR)/go_env.txt \
 		dist \
 		config/samples
@@ -99,6 +108,7 @@ bins-all:
 	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=386
 	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=amd64
 	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm     GOARM=7
 	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=386
 	$(MAKE) $(BINS_ALL_TARGETS) GOOS=darwin    GOARCH=amd64
 	$(MAKE) $(BINS_ALL_TARGETS) GOOS=solaris   GOARCH=amd64
@@ -118,10 +128,10 @@ vet:
 	$(GO_ENV_VARS) $(GO) vet $(GO_BUILDFLAGS) ./...
 
 zrepl-bin:
-	$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(GOOS)-$(GOARCH)"
+	$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(ZREPL_TARGET_TUPLE)"
 
 platformtest-bin:
-		$(GO_BUILD) -o "$(ARTIFACTDIR)/platformtest-$(GOOS)-$(GOARCH)" ./platformtest/harness
+	$(GO_BUILD) -o "$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" ./platformtest/harness
 
 ##################### DEV TARGETS #####################
 # not part of the build, must do that manually
@@ -136,11 +146,22 @@ format:
 
 ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
 ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
+ZREPL_PLATFORMTEST_MOUNTPOINT := /tmp/zreplplatformtest.pool
+ZREPL_PLATFORMTEST_ZFS_LOG := /tmp/zreplplatformtest.zfs.log
+# ZREPL_PLATFORMTEST_STOP_AND_KEEP := -failure.stop-and-keep-pool
+ZREPL_PLATFORMTEST_ARGS := 
 platformtest: # do not track dependency on platformtest-bin to allow build of platformtest outside of test VM
-	"$(ARTIFACTDIR)/platformtest-$(GOOS)-$(GOARCH)" -poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" -imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)"
+	rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
+	platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
+	"$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" \
+		-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
+		-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
+		-mountpoint "$(ZREPL_PLATFORMTEST_MOUNTPOINT)" \
+		$(ZREPL_PLATFORMTEST_STOP_AND_KEEP) \
+		$(ZREPL_PLATFORMTEST_ARGS)
 
 ##################### NOARCH #####################
-.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
+.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
 
 
 $(ARTIFACTDIR):
@@ -148,12 +169,16 @@ $(ARTIFACTDIR):
 $(ARTIFACTDIR)/docs: $(ARTIFACTDIR)
 	mkdir -p "$@"
 
-noarch: $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_env.txt docs
+noarch: $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs
 	# pass
 
 $(ARTIFACTDIR)/bash_completion:
 	$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
-	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) bashcomp "$@"
+	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion bash "$@"
+
+$(ARTIFACTDIR)/_zrepl.zsh_completion:
+	$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
+	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion zsh "$@"
 
 $(ARTIFACTDIR)/go_env.txt:
 	$(GO_ENV_VARS) $(GO) env > $@

README.md

@@ -1,8 +1,9 @@
 [![GitHub license](https://img.shields.io/github/license/zrepl/zrepl.svg)](https://github.com/zrepl/zrepl/blob/master/LICENSE)
 [![Language: Go](https://img.shields.io/badge/language-Go-6ad7e5.svg)](https://golang.org/)
 [![User Docs](https://img.shields.io/badge/docs-web-blue.svg)](https://zrepl.github.io)
-[![Donate via PayPal](https://img.shields.io/badge/donate-paypal-yellow.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
+[![Donate via Patreon](https://img.shields.io/endpoint.svg?url=https%3A%2F%2Fshieldsio-patreon.herokuapp.com%2Fzrepl%2Fpledges&style=flat&color=yellow)](https://www.patreon.com/zrepl)
 [![Donate via Liberapay](https://img.shields.io/liberapay/receives/zrepl.svg?logo=liberapay)](https://liberapay.com/zrepl/donate)
+[![Donate via PayPal](https://img.shields.io/badge/donate-paypal-yellow.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
 [![Twitter](https://img.shields.io/twitter/url/https/github.com/zrepl/zrepl.svg?style=social)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl)
 
 # zrepl
@@ -23,6 +24,7 @@ zrepl is a one-stop ZFS backup & replication solution.
    If so, think of an expressive configuration example.
 2. Think of at least one use case that generalizes from your concrete application.
 3. Open an issue on GitHub with example conf & use case attached.
+4. **Optional**: [Post a bounty](https://www.bountysource.com/teams/zrepl) on the issue, or [contact Christian Schwarz](https://cschwarz.com) for contract work.
 
 The above does not apply if you already implemented everything.
 Check out the *Coding Workflow* section below for details.
@@ -46,11 +48,8 @@ zrepl is written in [Go](https://golang.org) and uses [Go modules](https://githu
 The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
 
 To get started, run `./lazy.sh devsetup` to easily install build dependencies and read `docs/installation.rst -> Compiling from Source`.
-
+`lazy.sh` uses `python3-pip` to fetch the build dependencies for the docs - you might want to use a [venv](https://docs.python.org/3/library/venv.html).
-### Overall Architecture
+If you just want to install the Go dependencies, run `./lazy.sh godep`.
-
-The application architecture is documented as part of the user docs in the *Implementation* section (`docs/content/impl`).
-Make sure to develop an understanding how zrepl is typically used by studying the user docs first.
 
 ### Project Structure
 
@@ -62,14 +61,15 @@ Make sure to develop an understanding how zrepl is typically used by studying th
 │   └── samples
 ├── daemon                  # the implementation of `zrepl daemon` subcommand
 │   ├── filters
+│   ├── hooks               # snapshot hooks
 │   ├── job                 # job implementations
 │   ├── logging             # logging outlets + formatters
 │   ├── nethelpers
 │   ├── prometheus
 │   ├── pruner              # pruner implementation
 │   ├── snapper             # snapshotter implementation
-├── docs                    # sphinx-based documentation
 ├── dist                    # supplemental material for users & package maintainers
+├── docs                    # sphinx-based documentation
 │   ├── **/*.rst            # documentation in reStructuredText
 │   ├── sphinxconf
 │   │   └── conf.py         # sphinx config (see commit 445a280 why its not in docs/)
@@ -78,6 +78,7 @@ Make sure to develop an understanding how zrepl is typically used by studying th
 │   └── public_git          # checkout of zrepl.github.io managed by above shell script
 ├── endpoint                # implementation of replication endpoints (=> package replication)
 ├── logger                  # our own logger package
+├── platformtest            # test suite for our zfs abstractions (error classification, etc)
 ├── pruning                 # pruning rules (the logic, not the actual execution)
 │   └── retentiongrid
 ├── replication
@@ -92,14 +93,13 @@ Make sure to develop an understanding how zrepl is typically used by studying th
 │   ├── transportmux        # TCP connecter and listener used to split control & data traffic
 │   └── versionhandshake    # replication protocol version handshake perfomed on newly established connections
 ├── tlsconf                 # abstraction for Go TLS server + client config
-├── transport               # transports implementation
+├── transport               # transport implementations
 │   ├── fromconfig
 │   ├── local
 │   ├── ssh
 │   ├── tcp
 │   └── tls
 ├── util
-├── vendor                  # managed by dep
 ├── version                 # abstraction for versions (filled during build by Makefile)
 └── zfs                     # zfs(8) wrappers
 ```
@@ -134,3 +134,15 @@ There will not be a big refactoring (an attempt was made, but it's destroying to
 
 However, new contributions & patches should fix naming without further notice in the commit message.
 
+### RPC debugging
+
+Optionally, there are various RPC-related environment variables, that if set to something != `""` will produce additional debug output on stderr:
+
+https://github.com/zrepl/zrepl/blob/master/rpc/rpc_debug.go#L11
+
+https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/dataconn_debug.go#L11
+
+https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/stream/stream_debug.go#L11
+
+https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/heartbeatconn/heartbeatconn_debug.go#L11
+

build.Dockerfile

@@ -2,7 +2,8 @@ FROM golang:latest
 
 RUN apt-get update && apt-get install -y \
     python3-pip \
-    unzip
+    unzip \
+    gawk
 
 ADD build.installprotoc.bash ./
 RUN bash build.installprotoc.bash
@@ -22,7 +23,7 @@ RUN mkdir -p /.cache && chmod -R 0777 /.cache
 WORKDIR /src
 
 # Install build tools (e.g. protobuf generator, stringer) into $GOPATH/bin
-ADD go.mod go.sum ./
+ADD build/ /tmp/build
 RUN /tmp/lazy.sh godep
 
 RUN chmod -R 0777 /go

build/tools.go

@@ -3,9 +3,9 @@
 package main
 
 import (
-	_ "golang.org/x/tools/cmd/stringer"
-	_ "github.com/golang/protobuf/protoc-gen-go"
 	_ "github.com/alvaroloes/enumer"
-	_ "golang.org/x/tools/cmd/goimports"
+	_ "github.com/golang/protobuf/protoc-gen-go"
 	_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
+	_ "golang.org/x/tools/cmd/goimports"
+	_ "golang.org/x/tools/cmd/stringer"
 )

cli/cli.go

@@ -1,11 +1,13 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 	"os"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"
 )
@@ -19,29 +21,55 @@ var rootCmd = &cobra.Command{
 	Short: "One-stop ZFS replication solution",
 }
 
-var bashcompCmd = &cobra.Command{
+func init() {
-	Use:   "bashcomp path/to/out/file",
+	rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
-	Short: "generate bash completions",
+}
-	Run: func(cmd *cobra.Command, args []string) {
+
-		if len(args) != 1 {
+var genCompletionCmd = &cobra.Command{
-			fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
+	Use:   "gencompletion",
-			err := cmd.Usage()
+	Short: "generate shell auto-completions",
-			if err != nil {
+}
-				panic(err)
+
-			}
+type completionCmdInfo struct {
-			os.Exit(1)
+	genFunc func(outpath string) error
-		}
+	help    string
-		if err := rootCmd.GenBashCompletionFile(args[0]); err != nil {
+}
-			fmt.Fprintf(os.Stderr, "error generating bash completion: %s", err)
+
-			os.Exit(1)
+var completionCmdMap = map[string]completionCmdInfo{
-		}
+	"zsh": {
+		rootCmd.GenZshCompletionFile,
+		"  save to file `_zrepl` in your zsh's $fpath",
+	},
+	"bash": {
+		rootCmd.GenBashCompletionFile,
+		"  save to a path and source that path in your .bashrc",
 	},
-	Hidden: true,
 }
 
 func init() {
-	rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
+	for sh, info := range completionCmdMap {
-	rootCmd.AddCommand(bashcompCmd)
+		sh, info := sh, info
+		genCompletionCmd.AddCommand(&cobra.Command{
+			Use:     fmt.Sprintf("%s path/to/out/file", sh),
+			Short:   fmt.Sprintf("generate %s completions", sh),
+			Example: info.help,
+			Run: func(cmd *cobra.Command, args []string) {
+				if len(args) != 1 {
+					fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
+					err := cmd.Usage()
+					if err != nil {
+						panic(err)
+					}
+					os.Exit(1)
+				}
+				if err := info.genFunc(args[0]); err != nil {
+					fmt.Fprintf(os.Stderr, "error generating %s completion: %s", sh, err)
+					os.Exit(1)
+				}
+			},
+		})
+	}
+	rootCmd.AddCommand(genCompletionCmd)
 }
 
 type Subcommand struct {
@@ -49,7 +77,7 @@ type Subcommand struct {
 	Short            string
 	Example          string
 	NoRequireConfig  bool
-	Run              func(subcommand *Subcommand, args []string) error
+	Run              func(ctx context.Context, subcommand *Subcommand, args []string) error
 	SetupFlags       func(f *pflag.FlagSet)
 	SetupSubcommands func() []*Subcommand
 
@@ -70,7 +98,11 @@ func (s *Subcommand) Config() *config.Config {
 
 func (s *Subcommand) run(cmd *cobra.Command, args []string) {
 	s.tryParseConfig()
-	err := s.Run(s, args)
+	ctx := context.Background()
+	endTask := trace.WithTaskFromStackUpdateCtx(&ctx)
+	defer endTask()
+	err := s.Run(ctx, s, args)
+	endTask()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%s\n", err)
 		os.Exit(1)

client/configcheck.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -29,7 +30,7 @@ var ConfigcheckCmd = &cli.Subcommand{
 		f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]")
 		f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]")
 	},
-	Run: func(subcommand *cli.Subcommand, args []string) error {
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
 		formatMap := map[string]func(interface{}){
 			"": func(i interface{}) {},
 			"pretty": func(i interface{}) {

client/migrate.go

@@ -4,9 +4,13 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/fatih/color"
+	"github.com/kr/pretty"
 	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
 
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/zfs"
 
 	"github.com/zrepl/zrepl/cli"
@@ -31,20 +35,26 @@ var migrations = []*cli.Subcommand{
 			f.BoolVar(&migratePlaceholder0_1Args.dryRun, "dry-run", false, "dry run")
 		},
 	},
+	&cli.Subcommand{
+		Use: "replication-cursor:v1-v2",
+		Run: doMigrateReplicationCursor,
+		SetupFlags: func(f *pflag.FlagSet) {
+			f.BoolVar(&migrateReplicationCursorArgs.dryRun, "dry-run", false, "dry run")
+		},
+	},
 }
 
 var migratePlaceholder0_1Args struct {
 	dryRun bool
 }
 
-func doMigratePlaceholder0_1(sc *cli.Subcommand, args []string) error {
+func doMigratePlaceholder0_1(ctx context.Context, sc *cli.Subcommand, args []string) error {
 	if len(args) != 0 {
 		return fmt.Errorf("migration does not take arguments, got %v", args)
 	}
 
 	cfg := sc.Config()
 
-	ctx := context.Background()
 	allFSS, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
 	if err != nil {
 		return errors.Wrap(err, "cannot list filesystems")
@@ -88,7 +98,7 @@ func doMigratePlaceholder0_1(sc *cli.Subcommand, args []string) error {
 		}
 		for _, fs := range wi.fss {
 			fmt.Printf("\t%q ... ", fs.ToString())
-			r, err := zfs.ZFSMigrateHashBasedPlaceholderToCurrent(fs, migratePlaceholder0_1Args.dryRun)
+			r, err := zfs.ZFSMigrateHashBasedPlaceholderToCurrent(ctx, fs, migratePlaceholder0_1Args.dryRun)
 			if err != nil {
 				fmt.Printf("error: %s\n", err)
 			} else if !r.NeedsModification {
@@ -102,3 +112,157 @@ func doMigratePlaceholder0_1(sc *cli.Subcommand, args []string) error {
 
 	return nil
 }
+
+var migrateReplicationCursorArgs struct {
+	dryRun bool
+}
+
+var bold = color.New(color.Bold)
+var succ = color.New(color.FgGreen)
+var fail = color.New(color.FgRed)
+
+var migrateReplicationCursorSkipSentinel = fmt.Errorf("skipping this filesystem")
+
+func doMigrateReplicationCursor(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	if len(args) != 0 {
+		return fmt.Errorf("migration does not take arguments, got %v", args)
+	}
+
+	cfg := sc.Config()
+	jobs, err := job.JobsFromConfig(cfg)
+	if err != nil {
+		fmt.Printf("cannot parse config:\n%s\n\n", err)
+		fmt.Printf("NOTE: this migration was released together with a change in job name requirements.\n")
+		return fmt.Errorf("exiting migration after error")
+	}
+
+	v1cursorJobs := make([]job.Job, 0, len(cfg.Jobs))
+	for i, j := range cfg.Jobs {
+		if jobs[i].Name() != j.Name() {
+			panic("implementation error")
+		}
+		switch j.Ret.(type) {
+		case *config.PushJob:
+			v1cursorJobs = append(v1cursorJobs, jobs[i])
+		case *config.SourceJob:
+			v1cursorJobs = append(v1cursorJobs, jobs[i])
+		default:
+			fmt.Printf("ignoring job %q (%d/%d, type %T), not supposed to create v1 replication cursors\n", j.Name(), i, len(cfg.Jobs), j.Ret)
+			continue
+		}
+	}
+
+	// scan all filesystems for v1 replication cursors
+
+	fss, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
+	if err != nil {
+		return errors.Wrap(err, "list filesystems")
+	}
+
+	var hadError bool
+	for _, fs := range fss {
+
+		bold.Printf("INSPECT FILESYSTEM %q\n", fs.ToString())
+
+		err := doMigrateReplicationCursorFS(ctx, v1cursorJobs, fs)
+		if err == migrateReplicationCursorSkipSentinel {
+			bold.Printf("FILESYSTEM SKIPPED\n")
+		} else if err != nil {
+			hadError = true
+			fail.Printf("MIGRATION FAILED: %s\n", err)
+		} else {
+			succ.Printf("FILESYSTEM %q COMPLETE\n", fs.ToString())
+		}
+	}
+
+	if hadError {
+		fail.Printf("\n\none or more filesystems could not be migrated, please inspect output and or re-run migration")
+		return errors.Errorf("")
+	}
+	return nil
+}
+
+func doMigrateReplicationCursorFS(ctx context.Context, v1CursorJobs []job.Job, fs *zfs.DatasetPath) error {
+
+	var owningJob job.Job = nil
+	for _, job := range v1CursorJobs {
+		conf := job.SenderConfig()
+		if conf == nil {
+			continue
+		}
+		pass, err := conf.FSF.Filter(fs)
+		if err != nil {
+			return errors.Wrapf(err, "filesystem filter error in job %q for fs %q", job.Name(), fs.ToString())
+		}
+		if !pass {
+			continue
+		}
+		if owningJob != nil {
+			return errors.Errorf("jobs %q and %q both match %q\ncannot attribute replication cursor to either one", owningJob.Name(), job.Name(), fs)
+		}
+		owningJob = job
+	}
+	if owningJob == nil {
+		fmt.Printf("no job's Filesystems filter matches\n")
+		return migrateReplicationCursorSkipSentinel
+	}
+	fmt.Printf("identified owning job %q\n", owningJob.Name())
+
+	bookmarks, err := zfs.ZFSListFilesystemVersions(ctx, fs, zfs.ListFilesystemVersionsOptions{
+		Types: zfs.Bookmarks,
+	})
+	if err != nil {
+		return errors.Wrapf(err, "list filesystem versions of %q", fs.ToString())
+	}
+
+	var oldCursor *zfs.FilesystemVersion
+	for i, fsv := range bookmarks {
+		_, _, err := endpoint.ParseReplicationCursorBookmarkName(fsv.ToAbsPath(fs))
+		if err != endpoint.ErrV1ReplicationCursor {
+			continue
+		}
+
+		if oldCursor != nil {
+			fmt.Printf("unexpected v1 replication cursor candidate: %q", fsv.ToAbsPath(fs))
+			return errors.Wrap(err, "multiple filesystem versions identified as v1 replication cursors")
+		}
+
+		oldCursor = &bookmarks[i]
+
+	}
+
+	if oldCursor == nil {
+		bold.Printf("no v1 replication cursor found for filesystem %q\n", fs.ToString())
+		return migrateReplicationCursorSkipSentinel
+	}
+
+	fmt.Printf("found v1 replication cursor:\n%s\n", pretty.Sprint(oldCursor))
+
+	mostRecentNew, err := endpoint.GetMostRecentReplicationCursorOfJob(ctx, fs.ToString(), owningJob.SenderConfig().JobID)
+	if err != nil {
+		return errors.Wrapf(err, "get most recent v2 replication cursor")
+	}
+
+	if mostRecentNew == nil {
+		return errors.Errorf("no v2 replication cursor found for job %q on filesystem %q", owningJob.SenderConfig().JobID, fs.ToString())
+	}
+
+	fmt.Printf("most recent v2 replication cursor:\n%#v", oldCursor)
+
+	if !(mostRecentNew.CreateTXG >= oldCursor.CreateTXG) {
+		return errors.Errorf("v1 replication cursor createtxg is higher than v2 cursor's, skipping this filesystem")
+	}
+
+	fmt.Printf("determined that v2 cursor is bookmark of same or newer version than v1 cursor\n")
+	fmt.Printf("destroying v1 cursor %q\n", oldCursor.ToAbsPath(fs))
+
+	if migrateReplicationCursorArgs.dryRun {
+		succ.Printf("DRY RUN\n")
+	} else {
+		if err := zfs.ZFSDestroyFilesystemVersion(ctx, fs, oldCursor); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

client/pprof.go

@@ -1,67 +1,10 @@
 package client
 
-import (
+import "github.com/zrepl/zrepl/cli"
-	"errors"
-	"log"
-	"os"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon"
-)
-
-var pprofArgs struct {
-	daemon.PprofServerControlMsg
-}
 
 var PprofCmd = &cli.Subcommand{
-	Use:   "pprof off | [on TCP_LISTEN_ADDRESS]",
+	Use: "pprof",
-	Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
+	SetupSubcommands: func() []*cli.Subcommand {
-	Run: func(subcommand *cli.Subcommand, args []string) error {
+		return []*cli.Subcommand{PprofListenCmd, pprofActivityTraceCmd}
-		if len(args) < 1 {
-			goto enargs
-		}
-		switch args[0] {
-		case "on":
-			pprofArgs.Run = true
-			if len(args) != 2 {
-				return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
-			}
-			pprofArgs.HttpListenAddress = args[1]
-		case "off":
-			if len(args) != 1 {
-				goto enargs
-			}
-			pprofArgs.Run = false
-		}
-
-		RunPProf(subcommand.Config())
-		return nil
-	enargs:
-		return errors.New("invalid number of positional arguments")
-
 	},
 }
-
-func RunPProf(conf *config.Config) {
-	log := log.New(os.Stderr, "", 0)
-
-	die := func() {
-		log.Printf("exiting after error")
-		os.Exit(1)
-	}
-
-	log.Printf("connecting to zrepl daemon")
-
-	httpc, err := controlHttpClient(conf.Global.Control.SockPath)
-	if err != nil {
-		log.Printf("error creating http client: %s", err)
-		die()
-	}
-	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofArgs.PprofServerControlMsg, struct{}{})
-	if err != nil {
-		log.Printf("error sending control message: %s", err)
-		die()
-	}
-	log.Printf("finished")
-}

client/pprof_activity_trace.go

@@ -0,0 +1,44 @@
+package client
+
+import (
+	"context"
+	"io"
+	"log"
+	"os"
+
+	"golang.org/x/net/websocket"
+
+	"github.com/zrepl/zrepl/cli"
+)
+
+var pprofActivityTraceCmd = &cli.Subcommand{
+	Use:   "activity-trace ZREPL_PPROF_HOST:ZREPL_PPROF_PORT",
+	Short: "attach to zrepl daemon with activated pprof listener and dump an activity-trace to stdout",
+	Run:   runPProfActivityTrace,
+}
+
+func runPProfActivityTrace(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+	log := log.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Printf("exiting after error")
+		os.Exit(1)
+	}
+
+	if len(args) != 1 {
+		log.Printf("exactly one positional argument is required")
+		die()
+	}
+
+	url := "ws://" + args[0] + "/debug/zrepl/activity-trace" // FIXME dont' repeat that
+
+	log.Printf("attaching to activity trace stream %s", url)
+	ws, err := websocket.Dial(url, "", url)
+	if err != nil {
+		log.Printf("error: %s", err)
+		die()
+	}
+
+	_, err = io.Copy(os.Stdout, ws)
+	return err
+}

client/pprof_listen.go

@@ -0,0 +1,68 @@
+package client
+
+import (
+	"context"
+	"errors"
+	"log"
+	"os"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+)
+
+var pprofListenCmd struct {
+	daemon.PprofServerControlMsg
+}
+
+var PprofListenCmd = &cli.Subcommand{
+	Use:   "listen off | [on TCP_LISTEN_ADDRESS]",
+	Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		if len(args) < 1 {
+			goto enargs
+		}
+		switch args[0] {
+		case "on":
+			pprofListenCmd.Run = true
+			if len(args) != 2 {
+				return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
+			}
+			pprofListenCmd.HttpListenAddress = args[1]
+		case "off":
+			if len(args) != 1 {
+				goto enargs
+			}
+			pprofListenCmd.Run = false
+		}
+
+		RunPProf(subcommand.Config())
+		return nil
+	enargs:
+		return errors.New("invalid number of positional arguments")
+
+	},
+}
+
+func RunPProf(conf *config.Config) {
+	log := log.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Printf("exiting after error")
+		os.Exit(1)
+	}
+
+	log.Printf("connecting to zrepl daemon")
+
+	httpc, err := controlHttpClient(conf.Global.Control.SockPath)
+	if err != nil {
+		log.Printf("error creating http client: %s", err)
+		die()
+	}
+	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofListenCmd.PprofServerControlMsg, struct{}{})
+	if err != nil {
+		log.Printf("error sending control message: %s", err)
+		die()
+	}
+	log.Printf("finished")
+}

client/signal.go

@@ -1,6 +1,8 @@
 package client
 
 import (
+	"context"
+
 	"github.com/pkg/errors"
 
 	"github.com/zrepl/zrepl/cli"
@@ -11,7 +13,7 @@ import (
 var SignalCmd = &cli.Subcommand{
 	Use:   "signal [wakeup|reset] JOB",
 	Short: "wake up a job from wait state or abort its current invocation",
-	Run: func(subcommand *cli.Subcommand, args []string) error {
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
 		return runSignalCmd(subcommand.Config(), args)
 	},
 }

client/status.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"math"
@@ -11,7 +12,7 @@ import (
 	"sync"
 	"time"
 
-	// tcell is the termbox-compatbile library for abstracting away escape sequences, etc.
+	// tcell is the termbox-compatible library for abstracting away escape sequences, etc.
 	// as of tcell#252, the number of default distributed terminals is relatively limited
 	// additional terminal definitions can be included via side-effect import
 	// See https://github.com/gdamore/tcell/blob/master/terminfo/base/base.go
@@ -81,7 +82,7 @@ type tui struct {
 	indent int
 
 	lock   sync.Mutex //For report and error
-	report map[string]job.Status
+	report map[string]*job.Status
 	err    error
 
 	jobFilter string
@@ -180,7 +181,7 @@ var StatusCmd = &cli.Subcommand{
 	Run: runStatus,
 }
 
-func runStatus(s *cli.Subcommand, args []string) error {
+func runStatus(ctx context.Context, s *cli.Subcommand, args []string) error {
 	httpc, err := controlHttpClient(s.Config().Global.Control.SockPath)
 	if err != nil {
 		return err
@@ -219,7 +220,7 @@ func runStatus(s *cli.Subcommand, args []string) error {
 	defer termbox.Close()
 
 	update := func() {
-		m := make(map[string]job.Status)
+		var m daemon.Status
 
 		err2 := jsonRequestResponse(httpc, daemon.ControlJobEndpointStatus,
 			struct{}{},
@@ -228,7 +229,7 @@ func runStatus(s *cli.Subcommand, args []string) error {
 
 		t.lock.Lock()
 		t.err = err2
-		t.report = m
+		t.report = m.Jobs
 		t.lock.Unlock()
 		t.draw()
 	}
@@ -264,7 +265,7 @@ loop:
 
 }
 
-func (t *tui) getReplicationProgresHistory(jobName string) *bytesProgressHistory {
+func (t *tui) getReplicationProgressHistory(jobName string) *bytesProgressHistory {
 	p, ok := t.replicationProgress[jobName]
 	if !ok {
 		p = &bytesProgressHistory{}
@@ -329,7 +330,7 @@ func (t *tui) draw() {
 				t.printf("Replication:")
 				t.newline()
 				t.addIndent(1)
-				t.renderReplicationReport(activeStatus.Replication, t.getReplicationProgresHistory(k))
+				t.renderReplicationReport(activeStatus.Replication, t.getReplicationProgressHistory(k))
 				t.addIndent(-1)
 
 				t.printf("Pruning Sender:")
@@ -461,12 +462,23 @@ func (t *tui) renderReplicationReport(rep *report.Report, history *bytesProgress
 	if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
 		// Draw global progress bar
 		// Progress: [---------------]
-		expected, replicated := latest.BytesSum()
+		expected, replicated, containsInvalidSizeEstimates := latest.BytesSum()
 		rate, changeCount := history.Update(replicated)
+		eta := time.Duration(0)
+		if rate > 0 {
+			eta = time.Duration((expected-replicated)/rate) * time.Second
+		}
 		t.write("Progress: ")
 		t.drawBar(50, replicated, expected, changeCount)
 		t.write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinary(replicated), ByteCountBinary(expected), ByteCountBinary(rate)))
+		if eta != 0 {
+			t.write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
+		}
 		t.newline()
+		if containsInvalidSizeEstimates {
+			t.write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
+			t.newline()
+		}
 
 		var maxFSLen int
 		for _, fs := range latest.Filesystems {
@@ -686,7 +698,7 @@ func rightPad(str string, length int, pad string) string {
 
 var arrowPositions = `>\|/`
 
-// changeCount = 0 indicates stall / no progresss
+// changeCount = 0 indicates stall / no progress
 func (t *tui) drawBar(length int, bytes, totalBytes int64, changeCount int) {
 	var completedLength int
 	if totalBytes > 0 {
@@ -707,11 +719,20 @@ func (t *tui) drawBar(length int, bytes, totalBytes int64, changeCount int) {
 
 func (t *tui) printFilesystemStatus(rep *report.FilesystemReport, active bool, maxFS int) {
 
-	expected, replicated := rep.BytesSum()
+	expected, replicated, containsInvalidSizeEstimates := rep.BytesSum()
-	status := fmt.Sprintf("%s (step %d/%d, %s/%s)",
+	sizeEstimationImpreciseNotice := ""
+	if containsInvalidSizeEstimates {
+		sizeEstimationImpreciseNotice = " (some steps lack size estimation)"
+	}
+	if rep.CurrentStep < len(rep.Steps) && rep.Steps[rep.CurrentStep].Info.BytesExpected == 0 {
+		sizeEstimationImpreciseNotice = " (step lacks size estimation)"
+	}
+
+	status := fmt.Sprintf("%s (step %d/%d, %s/%s)%s",
 		strings.ToUpper(string(rep.State)),
 		rep.CurrentStep, len(rep.Steps),
 		ByteCountBinary(replicated), ByteCountBinary(expected),
+		sizeEstimationImpreciseNotice,
 	)
 
 	activeIndicator := " "
@@ -731,8 +752,17 @@ func (t *tui) printFilesystemStatus(rep *report.FilesystemReport, active bool, m
 			if nextStep.IsIncremental() {
 				next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
 			} else {
-				next = fmt.Sprintf("next: %s (full)", nextStep.Info.To)
+				next = fmt.Sprintf("next: full send %s", nextStep.Info.To)
 			}
+			attribs := []string{}
+
+			if nextStep.Info.Resumed {
+				attribs = append(attribs, "resumed")
+			}
+
+			attribs = append(attribs, fmt.Sprintf("encrypted=%s", nextStep.Info.Encrypted))
+
+			next += fmt.Sprintf(" (%s)", strings.Join(attribs, ", "))
 		} else {
 			next = "" // individual FSes may still be in planning state
 		}
@@ -755,3 +785,27 @@ func ByteCountBinary(b int64) string {
 	}
 	return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
 }
+
+func humanizeDuration(duration time.Duration) string {
+	days := int64(duration.Hours() / 24)
+	hours := int64(math.Mod(duration.Hours(), 24))
+	minutes := int64(math.Mod(duration.Minutes(), 60))
+	seconds := int64(math.Mod(duration.Seconds(), 60))
+
+	var parts []string
+
+	force := false
+	chunks := []int64{days, hours, minutes, seconds}
+	for i, chunk := range chunks {
+		if force || chunk > 0 {
+			padding := 0
+			if force {
+				padding = 2
+			}
+			parts = append(parts, fmt.Sprintf("%*d%c", padding, chunk, "dhms"[i]))
+			force = true
+		}
+	}
+
+	return strings.Join(parts, " ")
+}

client/stdinserver.go

@@ -17,7 +17,7 @@ import (
 var StdinserverCmd = &cli.Subcommand{
 	Use:   "stdinserver CLIENT_IDENTITY",
 	Short: "stdinserver transport mode (started from authorized_keys file as forced command)",
-	Run: func(subcommand *cli.Subcommand, args []string) error {
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
 		return runStdinserver(subcommand.Config(), args)
 	},
 }

client/testcmd.go

@@ -1,7 +1,10 @@
 package client
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
+	"os"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
@@ -15,7 +18,7 @@ import (
 var TestCmd = &cli.Subcommand{
 	Use: "test",
 	SetupSubcommands: func() []*cli.Subcommand {
-		return []*cli.Subcommand{testFilter, testPlaceholder}
+		return []*cli.Subcommand{testFilter, testPlaceholder, testDecodeResumeToken}
 	},
 }
 
@@ -36,7 +39,7 @@ var testFilter = &cli.Subcommand{
 	Run: runTestFilterCmd,
 }
 
-func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
+func runTestFilterCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
 
 	if testFilterArgs.job == "" {
 		return fmt.Errorf("must specify --job flag")
@@ -57,6 +60,8 @@ func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
 		confFilter = j.Filesystems
 	case *config.PushJob:
 		confFilter = j.Filesystems
+	case *config.SnapJob:
+		confFilter = j.Filesystems
 	default:
 		return fmt.Errorf("job type %T does not have filesystems filter", j)
 	}
@@ -70,7 +75,7 @@ func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
 	if testFilterArgs.input != "" {
 		fsnames = []string{testFilterArgs.input}
 	} else {
-		out, err := zfs.ZFSList([]string{"name"})
+		out, err := zfs.ZFSList(ctx, []string{"name"})
 		if err != nil {
 			return fmt.Errorf("could not list ZFS filesystems: %s", err)
 		}
@@ -131,13 +136,13 @@ var testPlaceholder = &cli.Subcommand{
 	Run: runTestPlaceholder,
 }
 
-func runTestPlaceholder(subcommand *cli.Subcommand, args []string) error {
+func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
 
 	var checkDPs []*zfs.DatasetPath
 
 	// all actions first
 	if testPlaceholderArgs.all {
-		out, err := zfs.ZFSList([]string{"name"})
+		out, err := zfs.ZFSList(ctx, []string{"name"})
 		if err != nil {
 			return errors.Wrap(err, "could not list ZFS filesystems")
 		}
@@ -161,7 +166,7 @@ func runTestPlaceholder(subcommand *cli.Subcommand, args []string) error {
 
 	fmt.Printf("IS_PLACEHOLDER\tDATASET\tzrepl:placeholder\n")
 	for _, dp := range checkDPs {
-		ph, err := zfs.ZFSGetFilesystemPlaceholderState(dp)
+		ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, dp)
 		if err != nil {
 			return errors.Wrap(err, "cannot get placeholder state")
 		}
@@ -176,3 +181,32 @@ func runTestPlaceholder(subcommand *cli.Subcommand, args []string) error {
 	}
 	return nil
 }
+
+var testDecodeResumeTokenArgs struct {
+	token string
+}
+
+var testDecodeResumeToken = &cli.Subcommand{
+	Use:   "decoderesumetoken --token TOKEN",
+	Short: "decode resume token",
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&testDecodeResumeTokenArgs.token, "token", "", "the resume token obtained from the receive_resume_token property")
+	},
+	Run: runTestDecodeResumeTokenCmd,
+}
+
+func runTestDecodeResumeTokenCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+	if testDecodeResumeTokenArgs.token == "" {
+		return fmt.Errorf("token argument must be specified")
+	}
+	token, err := zfs.ParseResumeToken(ctx, testDecodeResumeTokenArgs.token)
+	if err != nil {
+		return err
+	}
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", " ")
+	if err := enc.Encode(&token); err != nil {
+		panic(err)
+	}
+	return nil
+}

client/version.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"context"
 	"fmt"
 	"os"
 
@@ -25,7 +26,7 @@ var VersionCmd = &cli.Subcommand{
 	SetupFlags: func(f *pflag.FlagSet) {
 		f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)")
 	},
-	Run: func(subcommand *cli.Subcommand, args []string) error {
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
 		versionArgs.Config = subcommand.Config()
 		versionArgs.ConfigErr = subcommand.ConfigParsingError()
 		return runVersionCmd()

client/zfsabstractions.go

@@ -0,0 +1,147 @@
+package client
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var (
+	ZFSAbstractionsCmd = &cli.Subcommand{
+		Use:   "zfs-abstraction",
+		Short: "manage abstractions that zrepl builds on top of ZFS",
+		SetupSubcommands: func() []*cli.Subcommand {
+			return []*cli.Subcommand{
+				zabsCmdList,
+				zabsCmdReleaseAll,
+				zabsCmdReleaseStale,
+				zabsCmdCreate,
+			}
+		},
+	}
+)
+
+// a common set of CLI flags that map to the fields of an
+// endpoint.ListZFSHoldsAndBookmarksQuery
+type zabsFilterFlags struct {
+	Filesystems FilesystemsFilterFlag
+	Job         JobIDFlag
+	Types       AbstractionTypesFlag
+	Concurrency int64
+}
+
+// produce a query from the CLI flags
+func (f zabsFilterFlags) Query() (endpoint.ListZFSHoldsAndBookmarksQuery, error) {
+	q := endpoint.ListZFSHoldsAndBookmarksQuery{
+		FS:          f.Filesystems.FlagValue(),
+		What:        f.Types.FlagValue(),
+		JobID:       f.Job.FlagValue(),
+		Concurrency: f.Concurrency,
+	}
+	return q, q.Validate()
+}
+
+func (f *zabsFilterFlags) registerZabsFilterFlags(s *pflag.FlagSet, verb string) {
+	// Note: the default value is defined in the .FlagValue methods
+	s.Var(&f.Filesystems, "fs", fmt.Sprintf("only %s holds on the specified filesystem [default: all filesystems] [comma-separated list of <dataset-pattern>:<ok|!> pairs]", verb))
+	s.Var(&f.Job, "job", fmt.Sprintf("only %s holds created by the specified job [default: any job]", verb))
+
+	variants := make([]string, 0, len(endpoint.AbstractionTypesAll))
+	for v := range endpoint.AbstractionTypesAll {
+		variants = append(variants, string(v))
+	}
+	variants = sort.StringSlice(variants)
+	variantsJoined := strings.Join(variants, "|")
+	s.Var(&f.Types, "type", fmt.Sprintf("only %s holds of the specified type [default: all] [comma-separated list of %s]", verb, variantsJoined))
+
+	s.Int64VarP(&f.Concurrency, "concurrency", "p", 1, "number of concurrently queried filesystems")
+}
+
+type JobIDFlag struct{ J *endpoint.JobID }
+
+func (f *JobIDFlag) Set(s string) error {
+	if len(s) == 0 {
+		*f = JobIDFlag{J: nil}
+		return nil
+	}
+
+	jobID, err := endpoint.MakeJobID(s)
+	if err != nil {
+		return err
+	}
+	*f = JobIDFlag{J: &jobID}
+	return nil
+}
+func (f JobIDFlag) Type() string               { return "job-ID" }
+func (f JobIDFlag) String() string             { return fmt.Sprint(f.J) }
+func (f JobIDFlag) FlagValue() *endpoint.JobID { return f.J }
+
+type AbstractionTypesFlag map[endpoint.AbstractionType]bool
+
+func (f *AbstractionTypesFlag) Set(s string) error {
+	ats, err := endpoint.AbstractionTypeSetFromStrings(strings.Split(s, ","))
+	if err != nil {
+		return err
+	}
+	*f = AbstractionTypesFlag(ats)
+	return nil
+}
+func (f AbstractionTypesFlag) Type() string { return "abstraction-type" }
+func (f AbstractionTypesFlag) String() string {
+	return endpoint.AbstractionTypeSet(f).String()
+}
+func (f AbstractionTypesFlag) FlagValue() map[endpoint.AbstractionType]bool {
+	if len(f) > 0 {
+		return f
+	}
+	return endpoint.AbstractionTypesAll
+}
+
+type FilesystemsFilterFlag struct {
+	F endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter
+}
+
+func (flag *FilesystemsFilterFlag) Set(s string) error {
+	mappings := strings.Split(s, ",")
+	if len(mappings) == 1 && !strings.Contains(mappings[0], ":") {
+		flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &mappings[0],
+		}
+		return nil
+	}
+
+	f := filters.NewDatasetMapFilter(len(mappings), true)
+	for _, m := range mappings {
+		thisMappingErr := fmt.Errorf("expecting comma-separated list of <dataset-pattern>:<ok|!> pairs, got %q", m)
+		lhsrhs := strings.SplitN(m, ":", 2)
+		if len(lhsrhs) != 2 {
+			return thisMappingErr
+		}
+		err := f.Add(lhsrhs[0], lhsrhs[1])
+		if err != nil {
+			return fmt.Errorf("%s: %s", thisMappingErr, err)
+		}
+	}
+	flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+		Filter: f,
+	}
+	return nil
+}
+func (flag FilesystemsFilterFlag) Type() string { return "filesystem filter spec" }
+func (flag FilesystemsFilterFlag) String() string {
+	return fmt.Sprintf("%v", flag.F)
+}
+func (flag FilesystemsFilterFlag) FlagValue() endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter {
+	var z FilesystemsFilterFlag
+	if flag == z {
+		return endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{Filter: zfs.NoFilter()}
+	}
+	return flag.F
+}

client/zfsabstractions_create.go

@@ -0,0 +1,14 @@
+package client
+
+import "github.com/zrepl/zrepl/cli"
+
+var zabsCmdCreate = &cli.Subcommand{
+	Use:             "create",
+	NoRequireConfig: true,
+	Short:           `create zrepl ZFS abstractions (mostly useful for debugging & development, users should not need to use this command)`,
+	SetupSubcommands: func() []*cli.Subcommand {
+		return []*cli.Subcommand{
+			zabsCmdCreateStepHold,
+		}
+	},
+}

client/zfsabstractions_create_step_hold.go

@@ -0,0 +1,58 @@
+package client
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var zabsCreateStepHoldFlags struct {
+	target string
+	jobid  JobIDFlag
+}
+
+var zabsCmdCreateStepHold = &cli.Subcommand{
+	Use:             "step",
+	Run:             doZabsCreateStep,
+	NoRequireConfig: true,
+	Short:           `create a step hold or bookmark`,
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVarP(&zabsCreateStepHoldFlags.target, "target", "t", "", "snapshot to be held / bookmark to be held")
+		f.VarP(&zabsCreateStepHoldFlags.jobid, "jobid", "j", "jobid for which the hold is installed")
+	},
+}
+
+func doZabsCreateStep(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	if len(args) > 0 {
+		return errors.New("subcommand takes no arguments")
+	}
+
+	f := &zabsCreateStepHoldFlags
+
+	fs, _, _, err := zfs.DecomposeVersionString(f.target)
+	if err != nil {
+		return errors.Wrapf(err, "%q invalid target", f.target)
+	}
+
+	if f.jobid.FlagValue() == nil {
+		return errors.Errorf("jobid must be set")
+	}
+
+	v, err := zfs.ZFSGetFilesystemVersion(ctx, f.target)
+	if err != nil {
+		return errors.Wrapf(err, "get info about target %q", f.target)
+	}
+
+	step, err := endpoint.HoldStep(ctx, fs, v, *f.jobid.FlagValue())
+	if err != nil {
+		return errors.Wrap(err, "create step hold")
+	}
+	fmt.Println(step.String())
+	return nil
+}

client/zfsabstractions_list.go

@@ -0,0 +1,99 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/util/chainlock"
+)
+
+var zabsListFlags struct {
+	Filter zabsFilterFlags
+	Json   bool
+}
+
+var zabsCmdList = &cli.Subcommand{
+	Use:             "list",
+	Short:           `list zrepl ZFS abstractions`,
+	Run:             doZabsList,
+	NoRequireConfig: true,
+	SetupFlags: func(f *pflag.FlagSet) {
+		zabsListFlags.Filter.registerZabsFilterFlags(f, "list")
+		f.BoolVar(&zabsListFlags.Json, "json", false, "emit JSON")
+	},
+}
+
+func doZabsList(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsListFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	abstractions, errors, err := endpoint.ListAbstractionsStreamed(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+
+	var line chainlock.L
+	var wg sync.WaitGroup
+	defer wg.Wait()
+	wg.Add(1)
+
+	// print results
+	go func() {
+		defer wg.Done()
+		enc := json.NewEncoder(os.Stdout)
+		for a := range abstractions {
+			func() {
+				defer line.Lock().Unlock()
+				if zabsListFlags.Json {
+					enc.SetIndent("", "  ")
+					if err := enc.Encode(abstractions); err != nil {
+						panic(err)
+					}
+					fmt.Println()
+				} else {
+					fmt.Println(a)
+				}
+			}()
+		}
+	}()
+
+	// print errors to stderr
+	errorColor := color.New(color.FgRed)
+	var errorsSlice []endpoint.ListAbstractionsError
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for err := range errors {
+			func() {
+				defer line.Lock().Unlock()
+				errorsSlice = append(errorsSlice, err)
+				errorColor.Fprintf(os.Stderr, "%s\n", err)
+			}()
+		}
+	}()
+	wg.Wait()
+	if len(errorsSlice) > 0 {
+		errorColor.Add(color.Bold).Fprintf(os.Stderr, "there were errors in listing the abstractions")
+		return fmt.Errorf("")
+	} else {
+		return nil
+	}
+
+}

client/zfsabstractions_release.go

@@ -0,0 +1,143 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+)
+
+// shared between release-all and release-step
+var zabsReleaseFlags struct {
+	Filter zabsFilterFlags
+	Json   bool
+	DryRun bool
+}
+
+func registerZabsReleaseFlags(s *pflag.FlagSet) {
+	zabsReleaseFlags.Filter.registerZabsFilterFlags(s, "release")
+	s.BoolVar(&zabsReleaseFlags.Json, "json", false, "emit json instead of pretty-printed")
+	s.BoolVar(&zabsReleaseFlags.DryRun, "dry-run", false, "do a dry-run")
+}
+
+var zabsCmdReleaseAll = &cli.Subcommand{
+	Use:             "release-all",
+	Run:             doZabsReleaseAll,
+	NoRequireConfig: true,
+	Short:           `(DANGEROUS) release ALL zrepl ZFS abstractions (mostly useful for uninstalling zrepl completely or for "de-zrepl-ing" a filesystem)`,
+	SetupFlags:      registerZabsReleaseFlags,
+}
+
+var zabsCmdReleaseStale = &cli.Subcommand{
+	Use:             "release-stale",
+	Run:             doZabsReleaseStale,
+	NoRequireConfig: true,
+	Short:           `release stale zrepl ZFS abstractions (useful if zrepl has a bug and does not do it by itself)`,
+	SetupFlags:      registerZabsReleaseFlags,
+}
+
+func doZabsReleaseAll(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsReleaseFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	abstractions, listErrors, err := endpoint.ListAbstractions(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+	if len(listErrors) > 0 {
+		color.New(color.FgRed).Fprintf(os.Stderr, "there were errors in listing the abstractions:\n%s\n", listErrors)
+		// proceed anyways with rest of abstractions
+	}
+
+	return doZabsRelease_Common(ctx, abstractions)
+}
+
+func doZabsReleaseStale(ctx context.Context, sc *cli.Subcommand, args []string) error {
+
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsReleaseFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	stalenessInfo, err := endpoint.ListStale(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+
+	return doZabsRelease_Common(ctx, stalenessInfo.Stale)
+}
+
+func doZabsRelease_Common(ctx context.Context, destroy []endpoint.Abstraction) error {
+
+	if zabsReleaseFlags.DryRun {
+		if zabsReleaseFlags.Json {
+			m, err := json.MarshalIndent(destroy, "", "  ")
+			if err != nil {
+				panic(err)
+			}
+			if _, err := os.Stdout.Write(m); err != nil {
+				panic(err)
+			}
+			fmt.Println()
+		} else {
+			for _, a := range destroy {
+				fmt.Printf("would destroy %s\n", a)
+			}
+		}
+		return nil
+	}
+
+	outcome := endpoint.BatchDestroy(ctx, destroy)
+	hadErr := false
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	colorErr := color.New(color.FgRed)
+	printfSuccess := color.New(color.FgGreen).FprintfFunc()
+	printfSection := color.New(color.Bold).FprintfFunc()
+
+	for res := range outcome {
+		hadErr = hadErr || res.DestroyErr != nil
+		if zabsReleaseFlags.Json {
+			err := enc.Encode(res)
+			if err != nil {
+				colorErr.Fprintf(os.Stderr, "cannot marshal there were errors in destroying the abstractions")
+			}
+		} else {
+			printfSection(os.Stdout, "destroy %s ...", res.Abstraction)
+			if res.DestroyErr != nil {
+				colorErr.Fprintf(os.Stdout, " failed:\n%s\n", res.DestroyErr)
+			} else {
+				printfSuccess(os.Stdout, " OK\n")
+			}
+		}
+	}
+
+	if hadErr {
+		colorErr.Add(color.Bold).Fprintf(os.Stderr, "there were errors in destroying the abstractions")
+		return fmt.Errorf("")
+	} else {
+		return nil
+	}
+}

config/config.go

@@ -75,16 +75,42 @@ type SnapJob struct {
 	Filesystems  FilesystemsFilter `yaml:"filesystems"`
 }
 
+type SendOptions struct {
+	Encrypted bool `yaml:"encrypted"`
+}
+
+var _ yaml.Defaulter = (*SendOptions)(nil)
+
+func (l *SendOptions) SetDefault() {
+	*l = SendOptions{Encrypted: false}
+}
+
+type RecvOptions struct {
+	// Note: we cannot enforce encrypted recv as the ZFS cli doesn't provide a mechanism for it
+	// Encrypted bool `yaml:"may_encrypted"`
+
+	// Future:
+	// Reencrypt bool `yaml:"reencrypt"`
+}
+
+var _ yaml.Defaulter = (*RecvOptions)(nil)
+
+func (l *RecvOptions) SetDefault() {
+	*l = RecvOptions{}
+}
+
 type PushJob struct {
 	ActiveJob    `yaml:",inline"`
 	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
 	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+	Send         *SendOptions      `yaml:"send,fromdefaults,optional"`
 }
 
 type PullJob struct {
 	ActiveJob `yaml:",inline"`
 	RootFS    string                   `yaml:"root_fs"`
 	Interval  PositiveDurationOrManual `yaml:"interval"`
+	Recv      *RecvOptions             `yaml:"recv,fromdefaults,optional"`
 }
 
 type PositiveDurationOrManual struct {
@@ -120,13 +146,15 @@ func (i *PositiveDurationOrManual) UnmarshalYAML(u func(interface{}, bool) error
 
 type SinkJob struct {
 	PassiveJob `yaml:",inline"`
-	RootFS     string `yaml:"root_fs"`
+	RootFS     string       `yaml:"root_fs"`
+	Recv       *RecvOptions `yaml:"recv,optional,fromdefaults"`
 }
 
 type SourceJob struct {
 	PassiveJob   `yaml:",inline"`
 	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
 	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+	Send         *SendOptions      `yaml:"send,optional,fromdefaults"`
 }
 
 type FilesystemsFilter map[string]bool
@@ -245,14 +273,16 @@ type ServeCommon struct {
 }
 
 type TCPServe struct {
-	ServeCommon `yaml:",inline"`
+	ServeCommon    `yaml:",inline"`
-	Listen      string            `yaml:"listen,hostport"`
+	Listen         string            `yaml:"listen,hostport"`
-	Clients     map[string]string `yaml:"clients"`
+	ListenFreeBind bool              `yaml:"listen_freebind,default=false"`
+	Clients        map[string]string `yaml:"clients"`
 }
 
 type TLSServe struct {
 	ServeCommon      `yaml:",inline"`
 	Listen           string        `yaml:"listen,hostport"`
+	ListenFreeBind   bool          `yaml:"listen_freebind,default=false"`
 	Ca               string        `yaml:"ca"`
 	Cert             string        `yaml:"cert"`
 	Key              string        `yaml:"key"`
@@ -331,8 +361,9 @@ type MonitoringEnum struct {
 }
 
 type PrometheusMonitoring struct {
-	Type   string `yaml:"type"`
+	Type           string `yaml:"type"`
-	Listen string `yaml:"listen,hostport"`
+	Listen         string `yaml:"listen,hostport"`
+	ListenFreeBind bool   `yaml:"listen_freebind,default=false"`
 }
 
 type SyslogFacility syslog.Priority
@@ -589,7 +620,7 @@ func ParseConfigBytes(bytes []byte) (*Config, error) {
 
 var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
 
-func parsePostitiveDuration(e string) (d time.Duration, err error) {
+func parsePositiveDuration(e string) (d time.Duration, err error) {
 	comps := durationStringRegex.FindStringSubmatch(e)
 	if len(comps) != 3 {
 		err = fmt.Errorf("does not match regex: %s %#v", e, comps)

config/config_global_test.go

@@ -21,7 +21,7 @@ jobs:
     clients: {
       "10.0.0.1":"foo"
     }
-  root_fs: zoot/foo
+  root_fs: zroot/foo
 `
 	_, err := ParseConfigBytes([]byte(jobdef))
 	require.NoError(t, err)

config/config_jobid_test.go

@@ -0,0 +1 @@
+package config

config/config_send_test.go

@@ -0,0 +1,74 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSendOptions(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  %s
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+	encrypted_false := `
+  send:
+    encrypted: false
+`
+
+	encrypted_true := `
+  send:
+    encrypted: true
+`
+
+	encrypted_unspecified := `
+  send: {}
+`
+
+	send_not_specified := `
+`
+
+	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
+	var c *Config
+
+	t.Run("encrypted_false", func(t *testing.T) {
+		c = testValidConfig(t, fill(encrypted_false))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, false, encrypted)
+	})
+	t.Run("encrypted_true", func(t *testing.T) {
+		c = testValidConfig(t, fill(encrypted_true))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, true, encrypted)
+	})
+
+	t.Run("encrypted_unspecified", func(t *testing.T) {
+		c, err := testConfig(t, fill(encrypted_unspecified))
+		assert.Error(t, err)
+		assert.Nil(t, c)
+	})
+
+	t.Run("send_not_specified", func(t *testing.T) {
+		c, err := testConfig(t, fill(send_not_specified))
+		assert.NoError(t, err)
+		assert.NotNil(t, c)
+	})
+
+}

config/config_test.go

@@ -42,7 +42,7 @@ func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
 
 }
 
-// template must be a template/text template with a single '{{ . }}' as placehodler for val
+// template must be a template/text template with a single '{{ . }}' as placeholder for val
 //nolint[:deadcode,unused]
 func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
 	tmp, err := template.New("master").Parse(tmpl)

config/retentiongrid.go

@@ -64,7 +64,7 @@ func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval,
 		return nil, fmt.Errorf("contains factor <= 0")
 	}
 
-	duration, err := parsePostitiveDuration(comps[2])
+	duration, err := parsePositiveDuration(comps[2])
 	if err != nil {
 		return nil, err
 	}

config/samples/push.yml

@@ -10,6 +10,8 @@ jobs:
       address: "backup-server.foo.bar:8888"
     snapshotting:
       type: manual
+    send:
+      encrypted: false
     pruning:
       keep_sender:
         - type: not_replicated

daemon/control.go

@@ -15,10 +15,12 @@ import (
 
 	"github.com/zrepl/zrepl/daemon/job"
 	"github.com/zrepl/zrepl/daemon/nethelpers"
+	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/util/envconst"
 	"github.com/zrepl/zrepl/version"
 	"github.com/zrepl/zrepl/zfs"
+	"github.com/zrepl/zrepl/zfs/zfscmd"
 )
 
 type controlJob struct {
@@ -44,6 +46,8 @@ func (j *controlJob) Status() *job.Status { return &job.Status{Type: job.TypeInt
 
 func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
 
+func (j *controlJob) SenderConfig() *endpoint.SenderConfig { return nil }
+
 var promControl struct {
 	requestBegin    *prometheus.CounterVec
 	requestFinished *prometheus.HistogramVec
@@ -61,7 +65,7 @@ func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) {
 		Namespace: "zrepl",
 		Subsystem: "control",
 		Name:      "request_finished",
-		Help:      "time it took a request to finih",
+		Help:      "time it took a request to finish",
 		Buckets:   []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3, 10e-3, 100e-3, 200e-3, 400e-3, 800e-3, 1, 10, 20},
 	}, []string{"endpoint"})
 	registerer.MustRegister(promControl.requestBegin)
@@ -114,7 +118,9 @@ func (j *controlJob) Run(ctx context.Context) {
 	mux.Handle(ControlJobEndpointStatus,
 		// don't log requests to status endpoint, too spammy
 		jsonResponder{log, func() (interface{}, error) {
-			s := j.jobs.status()
+			jobs := j.jobs.status()
+			globalZFS := zfscmd.GetReport()
+			s := Status{Jobs: jobs, Global: GlobalStatus{ZFSCmds: globalZFS}}
 			return s, nil
 		}})
 
@@ -247,7 +253,7 @@ func (j jsonRequestResponder) ServeHTTP(w http.ResponseWriter, r *http.Request)
 	var buf bytes.Buffer
 	encodeErr := json.NewEncoder(&buf).Encode(res)
 	if encodeErr != nil {
-		j.log.WithError(producerErr).Error("control handler json marhsal error")
+		j.log.WithError(producerErr).Error("control handler json marshal error")
 		w.WriteHeader(http.StatusInternalServerError)
 		_, err := io.WriteString(w, encodeErr.Error())
 		logIoErr(err)

daemon/daemon.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/job"
@@ -20,11 +21,11 @@ import (
 	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/version"
+	"github.com/zrepl/zrepl/zfs/zfscmd"
 )
 
-func Run(conf *config.Config) error {
+func Run(ctx context.Context, conf *config.Config) error {
-
+	ctx, cancel := context.WithCancel(ctx)
-	ctx, cancel := context.WithCancel(context.Background())
 
 	defer cancel()
 	sigChan := make(chan os.Signal, 1)
@@ -38,6 +39,7 @@ func Run(conf *config.Config) error {
 	if err != nil {
 		return errors.Wrap(err, "cannot build logging from config")
 	}
+	outlets.Add(newPrometheusLogOutlet(), logger.Debug)
 
 	confJobs, err := job.JobsFromConfig(conf)
 	if err != nil {
@@ -47,14 +49,14 @@ func Run(conf *config.Config) error {
 	log := logger.NewLogger(outlets, 1*time.Second)
 	log.Info(version.NewZreplVersionInformation().String())
 
+	ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
+
 	for _, job := range confJobs {
 		if IsInternalJobName(job.Name()) {
 			panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME
 		}
 	}
 
-	ctx = job.WithLogger(ctx, log)
-
 	jobs := newJobs()
 
 	// start control socket
@@ -76,11 +78,15 @@ func Run(conf *config.Config) error {
 			return errors.Errorf("unknown monitoring job #%d (type %T)", i, v)
 		}
 		if err != nil {
-			return errors.Wrapf(err, "cannot build monitorin gjob #%d", i)
+			return errors.Wrapf(err, "cannot build monitoring job #%d", i)
 		}
 		jobs.start(ctx, job, true)
 	}
 
+	// register global (=non job-local) metrics
+	zfscmd.RegisterMetrics(prometheus.DefaultRegisterer)
+	trace.RegisterMetrics(prometheus.DefaultRegisterer)
+
 	log.Info("starting daemon")
 
 	// start regular jobs
@@ -94,6 +100,8 @@ func Run(conf *config.Config) error {
 	case <-ctx.Done():
 		log.WithError(ctx.Err()).Info("context finished")
 	}
+	log.Info("waiting for jobs to finish")
+	<-jobs.wait()
 	log.Info("daemon exiting")
 	return nil
 }
@@ -116,18 +124,24 @@ func newJobs() *jobs {
 	}
 }
 
-const (
-	logJobField string = "job"
-)
-
 func (s *jobs) wait() <-chan struct{} {
 	ch := make(chan struct{})
 	go func() {
 		s.wg.Wait()
+		close(ch)
 	}()
 	return ch
 }
 
+type Status struct {
+	Jobs   map[string]*job.Status
+	Global GlobalStatus
+}
+
+type GlobalStatus struct {
+	ZFSCmds *zfscmd.Report
+}
+
 func (s *jobs) status() map[string]*job.Status {
 	s.m.RLock()
 	defer s.m.RUnlock()
@@ -189,9 +203,8 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
 	s.m.Lock()
 	defer s.m.Unlock()
 
-	jobLog := job.GetLogger(ctx).
+	ctx = logging.WithInjectedField(ctx, logging.JobField, j.Name())
-		WithField(logJobField, j.Name()).
+
-		WithOutlet(newPrometheusLogOutlet(j.Name()), logger.Debug)
 	jobName := j.Name()
 	if !internal && IsInternalJobName(jobName) {
 		panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName))
@@ -206,7 +219,7 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
 	j.RegisterMetrics(prometheus.DefaultRegisterer)
 
 	s.jobs[jobName] = j
-	ctx = job.WithLogger(ctx, jobLog)
+	ctx = zfscmd.WithJobID(ctx, j.Name())
 	ctx, wakeup := wakeup.Context(ctx)
 	ctx, resetFunc := reset.Context(ctx)
 	s.wakeups[jobName] = wakeup
@@ -215,8 +228,8 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
 	s.wg.Add(1)
 	go func() {
 		defer s.wg.Done()
-		jobLog.Info("starting job")
+		job.GetLogger(ctx).Info("starting job")
-		defer jobLog.Info("job exited")
+		defer job.GetLogger(ctx).Info("job exited")
 		j.Run(ctx)
 	}()
 }

daemon/filters/fsmapfilter.go

@@ -161,7 +161,7 @@ func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
 }
 
 // Construct a new filter-only DatasetMapFilter from a mapping
-// The new filter allows excactly those paths that were not forbidden by the mapping.
+// The new filter allows exactly those paths that were not forbidden by the mapping.
 func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
 
 	if m.filterMode {

daemon/filters/fsmapfilter_test.go

@@ -1,11 +1,11 @@
 package filters
 
 import (
+	"testing"
+
 	"github.com/zrepl/zrepl/zfs"
 )
 
-import "testing"
-
 func TestDatasetMapFilter(t *testing.T) {
 
 	type testCase struct {

daemon/filters/fsvfilter.go

@@ -1,41 +0,0 @@
-package filters
-
-import (
-	"strings"
-
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type AnyFSVFilter struct{}
-
-func NewAnyFSVFilter() AnyFSVFilter {
-	return AnyFSVFilter{}
-}
-
-var _ zfs.FilesystemVersionFilter = AnyFSVFilter{}
-
-func (AnyFSVFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) {
-	return true, nil
-}
-
-type PrefixFilter struct {
-	prefix    string
-	fstype    zfs.VersionType
-	fstypeSet bool // optionals anyone?
-}
-
-var _ zfs.FilesystemVersionFilter = &PrefixFilter{}
-
-func NewPrefixFilter(prefix string) *PrefixFilter {
-	return &PrefixFilter{prefix: prefix}
-}
-
-func NewTypedPrefixFilter(prefix string, versionType zfs.VersionType) *PrefixFilter {
-	return &PrefixFilter{prefix, versionType, true}
-}
-
-func (f *PrefixFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) {
-	fstypeMatches := (!f.fstypeSet || t == f.fstype)
-	prefixMatches := strings.HasPrefix(name, f.prefix)
-	return fstypeMatches && prefixMatches, nil
-}

daemon/hooks/hook_logging.go

@@ -6,29 +6,17 @@ import (
 	"context"
 	"sync"
 
+	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/util/envconst"
 )
 
-type contextKey int
-
-const (
-	contextKeyLog contextKey = 0
-)
-
 type Logger = logger.Logger
 
-func WithLogger(ctx context.Context, log Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLog, log)
-}
-
 func GetLogger(ctx context.Context) Logger { return getLogger(ctx) }
 
 func getLogger(ctx context.Context) Logger {
-	if log, ok := ctx.Value(contextKeyLog).(Logger); ok {
+	return logging.GetLogger(ctx, logging.SubsysHooks)
-		return log
-	}
-	return logger.NewNullLogger()
 }
 
 const MAX_HOOK_LOG_SIZE_DEFAULT int = 1 << 20

daemon/hooks/hooks_test.go

@@ -10,9 +10,11 @@ import (
 	"text/template"
 
 	"github.com/stretchr/testify/require"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/hooks"
+	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/zfs"
 )
@@ -69,6 +71,9 @@ func curry(f comparisonAssertionFunc, expected interface{}, right bool) (ret val
 }
 
 func TestHooks(t *testing.T) {
+	ctx, end := trace.WithTaskFromStack(context.Background())
+	defer end()
+
 	testFSName := "testpool/testdataset"
 	testSnapshotName := "testsnap"
 
@@ -418,9 +423,8 @@ jobs:
 
 			cbReached = false
 
-			ctx := context.Background()
 			if testing.Verbose() && !tt.SuppressOutput {
-				ctx = hooks.WithLogger(ctx, log)
+				ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
 			}
 			plan.Run(ctx, false)
 			report := plan.Report()

daemon/job/active.go

@@ -2,17 +2,19 @@ package job
 
 import (
 	"context"
+	"fmt"
 	"sync"
 	"time"
 
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/log"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/filters"
 	"github.com/zrepl/zrepl/daemon/job/reset"
 	"github.com/zrepl/zrepl/daemon/job/wakeup"
-	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/daemon/pruner"
 	"github.com/zrepl/zrepl/daemon/snapper"
 	"github.com/zrepl/zrepl/endpoint"
@@ -28,7 +30,7 @@ import (
 
 type ActiveSide struct {
 	mode      activeMode
-	name      string
+	name      endpoint.JobID
 	connecter transport.Connecter
 
 	prunerFactory *pruner.PrunerFactory
@@ -78,31 +80,33 @@ func (a *ActiveSide) updateTasks(u func(*activeSideTasks)) activeSideTasks {
 }
 
 type activeMode interface {
-	ConnectEndpoints(rpcLoggers rpc.Loggers, connecter transport.Connecter)
+	ConnectEndpoints(ctx context.Context, connecter transport.Connecter)
 	DisconnectEndpoints()
 	SenderReceiver() (logic.Sender, logic.Receiver)
 	Type() Type
+	PlannerPolicy() logic.PlannerPolicy
 	RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{})
 	SnapperReport() *snapper.Report
 	ResetConnectBackoff()
 }
 
 type modePush struct {
-	setupMtx sync.Mutex
+	setupMtx      sync.Mutex
-	sender   *endpoint.Sender
+	sender        *endpoint.Sender
-	receiver *rpc.Client
+	receiver      *rpc.Client
-	fsfilter endpoint.FSFilter
+	senderConfig  *endpoint.SenderConfig
-	snapper  *snapper.PeriodicOrManual
+	plannerPolicy *logic.PlannerPolicy
+	snapper       *snapper.PeriodicOrManual
 }
 
-func (m *modePush) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Connecter) {
+func (m *modePush) ConnectEndpoints(ctx context.Context, connecter transport.Connecter) {
 	m.setupMtx.Lock()
 	defer m.setupMtx.Unlock()
 	if m.receiver != nil || m.sender != nil {
 		panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
 	}
-	m.sender = endpoint.NewSender(m.fsfilter)
+	m.sender = endpoint.NewSender(*m.senderConfig)
-	m.receiver = rpc.NewClient(connecter, loggers)
+	m.receiver = rpc.NewClient(connecter, rpc.GetLoggersOrPanic(ctx))
 }
 
 func (m *modePush) DisconnectEndpoints() {
@@ -121,6 +125,8 @@ func (m *modePush) SenderReceiver() (logic.Sender, logic.Receiver) {
 
 func (m *modePush) Type() Type { return TypePush }
 
+func (m *modePush) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
+
 func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
 	m.snapper.Run(ctx, wakeUpCommon)
 }
@@ -137,13 +143,22 @@ func (m *modePush) ResetConnectBackoff() {
 	}
 }
 
-func modePushFromConfig(g *config.Global, in *config.PushJob) (*modePush, error) {
+func modePushFromConfig(g *config.Global, in *config.PushJob, jobID endpoint.JobID) (*modePush, error) {
 	m := &modePush{}
+
 	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
 	if err != nil {
-		return nil, errors.Wrap(err, "cannnot build filesystem filter")
+		return nil, errors.Wrap(err, "cannot build filesystem filter")
+	}
+
+	m.senderConfig = &endpoint.SenderConfig{
+		FSF:     fsf,
+		Encrypt: &zfs.NilBool{B: in.Send.Encrypted},
+		JobID:   jobID,
+	}
+	m.plannerPolicy = &logic.PlannerPolicy{
+		EncryptedSend: logic.TriFromBool(in.Send.Encrypted),
 	}
-	m.fsfilter = fsf
 
 	if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
 		return nil, errors.Wrap(err, "cannot build snapper")
@@ -153,21 +168,23 @@ func modePushFromConfig(g *config.Global, in *config.PushJob) (*modePush, error)
 }
 
 type modePull struct {
-	setupMtx sync.Mutex
+	setupMtx       sync.Mutex
-	receiver *endpoint.Receiver
+	receiver       *endpoint.Receiver
-	sender   *rpc.Client
+	receiverConfig endpoint.ReceiverConfig
-	rootFS   *zfs.DatasetPath
+	sender         *rpc.Client
-	interval config.PositiveDurationOrManual
+	rootFS         *zfs.DatasetPath
+	plannerPolicy  *logic.PlannerPolicy
+	interval       config.PositiveDurationOrManual
 }
 
-func (m *modePull) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Connecter) {
+func (m *modePull) ConnectEndpoints(ctx context.Context, connecter transport.Connecter) {
 	m.setupMtx.Lock()
 	defer m.setupMtx.Unlock()
 	if m.receiver != nil || m.sender != nil {
 		panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
 	}
-	m.receiver = endpoint.NewReceiver(m.rootFS, false)
+	m.receiver = endpoint.NewReceiver(m.receiverConfig)
-	m.sender = rpc.NewClient(connecter, loggers)
+	m.sender = rpc.NewClient(connecter, rpc.GetLoggersOrPanic(ctx))
 }
 
 func (m *modePull) DisconnectEndpoints() {
@@ -186,6 +203,8 @@ func (m *modePull) SenderReceiver() (logic.Sender, logic.Receiver) {
 
 func (*modePull) Type() Type { return TypePull }
 
+func (m *modePull) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
+
 func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
 	if m.interval.Manual {
 		GetLogger(ctx).Info("manual pull configured, periodic pull disabled")
@@ -223,7 +242,7 @@ func (m *modePull) ResetConnectBackoff() {
 	}
 }
 
-func modePullFromConfig(g *config.Global, in *config.PullJob) (m *modePull, err error) {
+func modePullFromConfig(g *config.Global, in *config.PullJob, jobID endpoint.JobID) (m *modePull, err error) {
 	m = &modePull{}
 	m.interval = in.Interval
 
@@ -235,26 +254,56 @@ func modePullFromConfig(g *config.Global, in *config.PullJob) (m *modePull, err
 		return nil, errors.New("RootFS must not be empty") // duplicates error check of receiver
 	}
 
+	m.plannerPolicy = &logic.PlannerPolicy{
+		EncryptedSend: logic.DontCare,
+	}
+
+	m.receiverConfig = endpoint.ReceiverConfig{
+		JobID:                      jobID,
+		RootWithoutClientComponent: m.rootFS,
+		AppendClientIdentity:       false, // !
+		UpdateLastReceivedHold:     true,
+	}
+	if err := m.receiverConfig.Validate(); err != nil {
+		return nil, errors.Wrap(err, "cannot build receiver config")
+	}
+
 	return m, nil
 }
 
-func activeSide(g *config.Global, in *config.ActiveJob, mode activeMode) (j *ActiveSide, err error) {
+func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}) (j *ActiveSide, err error) {
+
+	j = &ActiveSide{}
+	j.name, err = endpoint.MakeJobID(in.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid job name")
+	}
+
+	switch v := configJob.(type) {
+	case *config.PushJob:
+		j.mode, err = modePushFromConfig(g, v, j.name) // shadow
+	case *config.PullJob:
+		j.mode, err = modePullFromConfig(g, v, j.name) // shadow
+	default:
+		panic(fmt.Sprintf("implementation error: unknown job type %T", v))
+	}
+	if err != nil {
+		return nil, err // no wrapping required
+	}
 
-	j = &ActiveSide{mode: mode}
-	j.name = in.Name
 	j.promRepStateSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace:   "zrepl",
 		Subsystem:   "replication",
 		Name:        "state_time",
 		Help:        "seconds spent during replication",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
 	}, []string{"state"})
 	j.promBytesReplicated = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace:   "zrepl",
 		Subsystem:   "replication",
 		Name:        "bytes_replicated",
 		Help:        "number of bytes replicated from sender to receiver per filesystem",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
 	}, []string{"filesystem"})
 
 	j.connecter, err = fromconfig.ConnecterFromConfig(g, in.Connect)
@@ -267,7 +316,7 @@ func activeSide(g *config.Global, in *config.ActiveJob, mode activeMode) (j *Act
 		Subsystem:   "pruning",
 		Name:        "time",
 		Help:        "seconds spent in pruner",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
 	}, []string{"prune_side"})
 	j.prunerFactory, err = pruner.NewPrunerFactory(in.Pruning, j.promPruneSecs)
 	if err != nil {
@@ -283,7 +332,7 @@ func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) {
 	registerer.MustRegister(j.promBytesReplicated)
 }
 
-func (j *ActiveSide) Name() string { return j.name }
+func (j *ActiveSide) Name() string { return j.name.String() }
 
 type ActiveSideStatus struct {
 	Replication                    *report.Report
@@ -318,16 +367,28 @@ func (j *ActiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
 	return pull.rootFS.Copy(), true
 }
 
+func (j *ActiveSide) SenderConfig() *endpoint.SenderConfig {
+	push, ok := j.mode.(*modePush)
+	if !ok {
+		_ = j.mode.(*modePull) // make sure we didn't introduce a new job type
+		return nil
+	}
+	return push.senderConfig
+}
+
 func (j *ActiveSide) Run(ctx context.Context) {
+	ctx, endTask := trace.WithTaskAndSpan(ctx, "active-side-job", j.Name())
+	defer endTask()
 	log := GetLogger(ctx)
-	ctx = logging.WithSubsystemLoggers(ctx, log)
 
 	defer log.Info("job exiting")
 
 	periodicDone := make(chan struct{})
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
-	go j.mode.RunPeriodic(ctx, periodicDone)
+	periodicCtx, endTask := trace.WithTask(ctx, "periodic")
+	defer endTask()
+	go j.mode.RunPeriodic(periodicCtx, periodicDone)
 
 	invocationCount := 0
 outer:
@@ -343,17 +404,15 @@ outer:
 		case <-periodicDone:
 		}
 		invocationCount++
-		invLog := log.WithField("invocation", invocationCount)
+		invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", invocationCount))
-		j.do(WithLogger(ctx, invLog))
+		j.do(invocationCtx)
+		endSpan()
 	}
 }
 
 func (j *ActiveSide) do(ctx context.Context) {
 
-	log := GetLogger(ctx)
+	j.mode.ConnectEndpoints(ctx, j.connecter)
-	ctx = logging.WithSubsystemLoggers(ctx, log)
-	loggers := rpc.GetLoggersOrPanic(ctx) // filled by WithSubsystemLoggers
-	j.mode.ConnectEndpoints(loggers, j.connecter)
 	defer j.mode.DisconnectEndpoints()
 
 	// allow cancellation of an invocation (this function)
@@ -376,20 +435,22 @@ func (j *ActiveSide) do(ctx context.Context) {
 			return
 		default:
 		}
+		ctx, endSpan := trace.WithSpan(ctx, "replication")
 		ctx, repCancel := context.WithCancel(ctx)
 		var repWait driver.WaitFunc
 		j.updateTasks(func(tasks *activeSideTasks) {
 			// reset it
 			*tasks = activeSideTasks{}
-			tasks.replicationCancel = repCancel
+			tasks.replicationCancel = func() { repCancel(); endSpan() }
 			tasks.replicationReport, repWait = replication.Do(
-				ctx, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver),
+				ctx, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver, j.mode.PlannerPolicy()),
 			)
 			tasks.state = ActiveSideReplicating
 		})
-		log.Info("start replication")
+		GetLogger(ctx).Info("start replication")
 		repWait(true) // wait blocking
 		repCancel()   // always cancel to free up context resources
+		endSpan()
 	}
 
 	{
@@ -398,16 +459,18 @@ func (j *ActiveSide) do(ctx context.Context) {
 			return
 		default:
 		}
+		ctx, endSpan := trace.WithSpan(ctx, "prune_sender")
 		ctx, senderCancel := context.WithCancel(ctx)
 		tasks := j.updateTasks(func(tasks *activeSideTasks) {
 			tasks.prunerSender = j.prunerFactory.BuildSenderPruner(ctx, sender, sender)
-			tasks.prunerSenderCancel = senderCancel
+			tasks.prunerSenderCancel = func() { senderCancel(); endSpan() }
 			tasks.state = ActiveSidePruneSender
 		})
-		log.Info("start pruning sender")
+		GetLogger(ctx).Info("start pruning sender")
 		tasks.prunerSender.Prune()
-		log.Info("finished pruning sender")
+		GetLogger(ctx).Info("finished pruning sender")
 		senderCancel()
+		endSpan()
 	}
 	{
 		select {
@@ -415,16 +478,18 @@ func (j *ActiveSide) do(ctx context.Context) {
 			return
 		default:
 		}
+		ctx, endSpan := trace.WithSpan(ctx, "prune_recever")
 		ctx, receiverCancel := context.WithCancel(ctx)
 		tasks := j.updateTasks(func(tasks *activeSideTasks) {
 			tasks.prunerReceiver = j.prunerFactory.BuildReceiverPruner(ctx, receiver, sender)
-			tasks.prunerReceiverCancel = receiverCancel
+			tasks.prunerReceiverCancel = func() { receiverCancel(); endSpan() }
 			tasks.state = ActiveSidePruneReceiver
 		})
-		log.Info("start pruning receiver")
+		GetLogger(ctx).Info("start pruning receiver")
 		tasks.prunerReceiver.Prune()
-		log.Info("finished pruning receiver")
+		GetLogger(ctx).Info("finished pruning receiver")
 		receiverCancel()
+		endSpan()
 	}
 
 	j.updateTasks(func(tasks *activeSideTasks) {

daemon/job/build_jobs.go

@@ -48,20 +48,12 @@ func buildJob(c *config.Global, in config.JobEnum) (j Job, err error) {
 	// FIXME prettify this
 	switch v := in.Ret.(type) {
 	case *config.SinkJob:
-		m, err := modeSinkFromConfig(c, v)
+		j, err = passiveSideFromConfig(c, &v.PassiveJob, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-		j, err = passiveSideFromConfig(c, &v.PassiveJob, m)
 		if err != nil {
 			return cannotBuildJob(err, v.Name)
 		}
 	case *config.SourceJob:
-		m, err := modeSourceFromConfig(c, v)
+		j, err = passiveSideFromConfig(c, &v.PassiveJob, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-		j, err = passiveSideFromConfig(c, &v.PassiveJob, m)
 		if err != nil {
 			return cannotBuildJob(err, v.Name)
 		}
@@ -71,20 +63,12 @@ func buildJob(c *config.Global, in config.JobEnum) (j Job, err error) {
 			return cannotBuildJob(err, v.Name)
 		}
 	case *config.PushJob:
-		m, err := modePushFromConfig(c, v)
+		j, err = activeSide(c, &v.ActiveJob, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-		j, err = activeSide(c, &v.ActiveJob, m)
 		if err != nil {
 			return cannotBuildJob(err, v.Name)
 		}
 	case *config.PullJob:
-		m, err := modePullFromConfig(c, v)
+		j, err = activeSide(c, &v.ActiveJob, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-		j, err = activeSide(c, &v.ActiveJob, m)
 		if err != nil {
 			return cannotBuildJob(err, v.Name)
 		}
@@ -104,6 +88,11 @@ func validateReceivingSidesDoNotOverlap(receivingRootFSs []string) error {
 	sort.Slice(rfss, func(i, j int) bool {
 		return strings.Compare(rfss[i], rfss[j]) == -1
 	})
+	// add tailing slash because of hierarchy-simulation
+	// rootfs/ is not root of rootfs2/
+	for i := 0; i < len(rfss); i++ {
+		rfss[i] += "/"
+	}
 	// idea:
 	//   no path in rfss must be prefix of another
 	//

daemon/job/build_jobs_test.go

@@ -1,9 +1,13 @@
 package job
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/config"
 )
 
 func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
@@ -18,6 +22,7 @@ func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
 		{false, []string{"a"}},
 		{false, []string{"some/path"}},
 		{false, []string{"zroot/sink1", "zroot/sink2", "zroot/sink3"}},
+		{false, []string{"zroot/foo", "zroot/foobar"}},
 		{true, []string{"zroot/b", "zroot/b"}},
 		{true, []string{"zroot/foo", "zroot/foo/bar", "zroot/baz"}},
 		{false, []string{"a/x", "b/x"}},
@@ -40,3 +45,66 @@ func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
 		}
 	}
 }
+
+func TestJobIDErrorHandling(t *testing.T) {
+	tmpl := `
+jobs:
+- name: %s
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
+
+	type Case struct {
+		jobName string
+		valid   bool
+	}
+	cases := []Case{
+		{"validjobname", true},
+		{"valid with spaces", true},
+		{"invalid\twith\ttabs", false},
+		{"invalid#withdelimiter", false},
+		{"invalid@withdelimiter", false},
+		{"withnewline\\nmiddle", false},
+		{"withnewline\\n", false},
+		{"withslash/", false},
+		{"withslash/inthemiddle", false},
+		{"/", false},
+	}
+
+	for i := range cases {
+		t.Run(cases[i].jobName, func(t *testing.T) {
+			c := cases[i]
+
+			conf, err := config.ParseConfigBytes([]byte(fill(c.jobName)))
+			require.NoError(t, err, "not expecting yaml-config to know about job ids")
+			require.NotNil(t, conf)
+			jobs, err := JobsFromConfig(conf)
+
+			if c.valid {
+				assert.NoError(t, err)
+				require.Len(t, jobs, 1)
+				assert.Equal(t, c.jobName, jobs[0].Name())
+			} else {
+				t.Logf("error: %s", err)
+				assert.Error(t, err)
+				assert.Nil(t, jobs)
+			}
+
+		})
+	}
+
+}

daemon/job/job.go

@@ -7,27 +7,16 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/zfs"
 )
 
 type Logger = logger.Logger
 
-type contextKey int
-
-const (
-	contextKeyLog contextKey = iota
-)
-
 func GetLogger(ctx context.Context) Logger {
-	if l, ok := ctx.Value(contextKeyLog).(Logger); ok {
+	return logging.GetLogger(ctx, logging.SubsysJob)
-		return l
-	}
-	return logger.NewNullLogger()
-}
-
-func WithLogger(ctx context.Context, l Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLog, l)
 }
 
 type Job interface {
@@ -38,6 +27,7 @@ type Job interface {
 	// Jobs that return a subtree of the dataset hierarchy
 	// must return the root of that subtree as rfs and ok = true
 	OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool)
+	SenderConfig() *endpoint.SenderConfig
 }
 
 type Type string

daemon/job/passive.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/filters"
@@ -20,7 +21,7 @@ import (
 
 type PassiveSide struct {
 	mode   passiveMode
-	name   string
+	name   endpoint.JobID
 	listen transport.AuthenticatedListenerFactory
 }
 
@@ -32,43 +33,56 @@ type passiveMode interface {
 }
 
 type modeSink struct {
-	rootDataset *zfs.DatasetPath
+	receiverConfig endpoint.ReceiverConfig
 }
 
 func (m *modeSink) Type() Type { return TypeSink }
 
 func (m *modeSink) Handler() rpc.Handler {
-	return endpoint.NewReceiver(m.rootDataset, true)
+	return endpoint.NewReceiver(m.receiverConfig)
 }
 
 func (m *modeSink) RunPeriodic(_ context.Context)  {}
 func (m *modeSink) SnapperReport() *snapper.Report { return nil }
 
-func modeSinkFromConfig(g *config.Global, in *config.SinkJob) (m *modeSink, err error) {
+func modeSinkFromConfig(g *config.Global, in *config.SinkJob, jobID endpoint.JobID) (m *modeSink, err error) {
 	m = &modeSink{}
-	m.rootDataset, err = zfs.NewDatasetPath(in.RootFS)
+
+	rootDataset, err := zfs.NewDatasetPath(in.RootFS)
 	if err != nil {
 		return nil, errors.New("root dataset is not a valid zfs filesystem path")
 	}
-	if m.rootDataset.Length() <= 0 {
+
-		return nil, errors.New("root dataset must not be empty") // duplicates error check of receiver
+	m.receiverConfig = endpoint.ReceiverConfig{
+		JobID:                      jobID,
+		RootWithoutClientComponent: rootDataset,
+		AppendClientIdentity:       true, // !
+		UpdateLastReceivedHold:     true,
 	}
+	if err := m.receiverConfig.Validate(); err != nil {
+		return nil, errors.Wrap(err, "cannot build receiver config")
+	}
+
 	return m, nil
 }
 
 type modeSource struct {
-	fsfilter zfs.DatasetFilter
+	senderConfig *endpoint.SenderConfig
-	snapper  *snapper.PeriodicOrManual
+	snapper      *snapper.PeriodicOrManual
 }
 
-func modeSourceFromConfig(g *config.Global, in *config.SourceJob) (m *modeSource, err error) {
+func modeSourceFromConfig(g *config.Global, in *config.SourceJob, jobID endpoint.JobID) (m *modeSource, err error) {
 	// FIXME exact dedup of modePush
 	m = &modeSource{}
 	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
 	if err != nil {
-		return nil, errors.Wrap(err, "cannnot build filesystem filter")
+		return nil, errors.Wrap(err, "cannot build filesystem filter")
+	}
+	m.senderConfig = &endpoint.SenderConfig{
+		FSF:     fsf,
+		Encrypt: &zfs.NilBool{B: in.Send.Encrypted},
+		JobID:   jobID,
 	}
-	m.fsfilter = fsf
 
 	if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
 		return nil, errors.Wrap(err, "cannot build snapper")
@@ -80,7 +94,7 @@ func modeSourceFromConfig(g *config.Global, in *config.SourceJob) (m *modeSource
 func (m *modeSource) Type() Type { return TypeSource }
 
 func (m *modeSource) Handler() rpc.Handler {
-	return endpoint.NewSender(m.fsfilter)
+	return endpoint.NewSender(*m.senderConfig)
 }
 
 func (m *modeSource) RunPeriodic(ctx context.Context) {
@@ -91,9 +105,25 @@ func (m *modeSource) SnapperReport() *snapper.Report {
 	return m.snapper.Report()
 }
 
-func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, mode passiveMode) (s *PassiveSide, err error) {
+func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, configJob interface{}) (s *PassiveSide, err error) {
+
+	s = &PassiveSide{}
+
+	s.name, err = endpoint.MakeJobID(in.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid job name")
+	}
+
+	switch v := configJob.(type) {
+	case *config.SinkJob:
+		s.mode, err = modeSinkFromConfig(g, v, s.name) // shadow
+	case *config.SourceJob:
+		s.mode, err = modeSourceFromConfig(g, v, s.name) // shadow
+	}
+	if err != nil {
+		return nil, err // no wrapping necessary
+	}
 
-	s = &PassiveSide{mode: mode, name: in.Name}
 	if s.listen, err = fromconfig.ListenerFactoryFromConfig(g, in.Serve); err != nil {
 		return nil, errors.Wrap(err, "cannot build listener factory")
 	}
@@ -101,7 +131,7 @@ func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, mode passive
 	return s, nil
 }
 
-func (j *PassiveSide) Name() string { return j.name }
+func (j *PassiveSide) Name() string { return j.name.String() }
 
 type PassiveStatus struct {
 	Snapper *snapper.Report
@@ -120,18 +150,29 @@ func (j *PassiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool)
 		_ = j.mode.(*modeSource) // make sure we didn't introduce a new job type
 		return nil, false
 	}
-	return sink.rootDataset.Copy(), true
+	return sink.receiverConfig.RootWithoutClientComponent.Copy(), true
+}
+
+func (j *PassiveSide) SenderConfig() *endpoint.SenderConfig {
+	source, ok := j.mode.(*modeSource)
+	if !ok {
+		_ = j.mode.(*modeSink) // make sure we didn't introduce a new job type
+		return nil
+	}
+	return source.senderConfig
 }
 
 func (*PassiveSide) RegisterMetrics(registerer prometheus.Registerer) {}
 
 func (j *PassiveSide) Run(ctx context.Context) {
-
+	ctx, endTask := trace.WithTaskAndSpan(ctx, "passive-side-job", j.Name())
+	defer endTask()
 	log := GetLogger(ctx)
 	defer log.Info("job exiting")
-	ctx = logging.WithSubsystemLoggers(ctx, log)
 	{
-		ctx, cancel := context.WithCancel(ctx) // shadowing
+		ctx, endTask := trace.WithTask(ctx, "periodic") // shadowing
+		defer endTask()
+		ctx, cancel := context.WithCancel(ctx)
 		defer cancel()
 		go j.mode.RunPeriodic(ctx)
 	}
@@ -141,8 +182,14 @@ func (j *PassiveSide) Run(ctx context.Context) {
 		panic(fmt.Sprintf("implementation error: j.mode.Handler() returned nil: %#v", j))
 	}
 
-	ctxInterceptor := func(handlerCtx context.Context) context.Context {
+	ctxInterceptor := func(handlerCtx context.Context, info rpc.HandlerContextInterceptorData, handler func(ctx context.Context)) {
-		return logging.WithSubsystemLoggers(handlerCtx, log)
+		// the handlerCtx is clean => need to inherit logging and tracing config from job context
+		handlerCtx = logging.WithInherit(handlerCtx, ctx)
+		handlerCtx = trace.WithInherit(handlerCtx, ctx)
+
+		handlerCtx, endTask := trace.WithTaskAndSpan(handlerCtx, "handler", fmt.Sprintf("job=%q client=%q method=%q", j.Name(), info.ClientIdentity(), info.FullMethod()))
+		defer endTask()
+		handler(handlerCtx)
 	}
 
 	rpcLoggers := rpc.GetLoggersOrPanic(ctx) // WithSubsystemLoggers above

daemon/job/snapjob.go

@@ -7,11 +7,11 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/filters"
 	"github.com/zrepl/zrepl/daemon/job/wakeup"
-	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/daemon/pruner"
 	"github.com/zrepl/zrepl/daemon/snapper"
 	"github.com/zrepl/zrepl/endpoint"
@@ -20,7 +20,7 @@ import (
 )
 
 type SnapJob struct {
-	name     string
+	name     endpoint.JobID
 	fsfilter zfs.DatasetFilter
 	snapper  *snapper.PeriodicOrManual
 
@@ -31,7 +31,7 @@ type SnapJob struct {
 	pruner *pruner.Pruner
 }
 
-func (j *SnapJob) Name() string { return j.name }
+func (j *SnapJob) Name() string { return j.name.String() }
 
 func (j *SnapJob) Type() Type { return TypeSnap }
 
@@ -39,20 +39,23 @@ func snapJobFromConfig(g *config.Global, in *config.SnapJob) (j *SnapJob, err er
 	j = &SnapJob{}
 	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
 	if err != nil {
-		return nil, errors.Wrap(err, "cannnot build filesystem filter")
+		return nil, errors.Wrap(err, "cannot build filesystem filter")
 	}
 	j.fsfilter = fsf
 
 	if j.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
 		return nil, errors.Wrap(err, "cannot build snapper")
 	}
-	j.name = in.Name
+	j.name, err = endpoint.MakeJobID(in.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid job name")
+	}
 	j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace:   "zrepl",
 		Subsystem:   "pruning",
 		Name:        "time",
 		Help:        "seconds spent in pruner",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
 	}, []string{"prune_side"})
 	j.prunerFactory, err = pruner.NewLocalPrunerFactory(in.Pruning, j.promPruneSecs)
 	if err != nil {
@@ -84,16 +87,21 @@ func (j *SnapJob) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
 	return nil, false
 }
 
+func (j *SnapJob) SenderConfig() *endpoint.SenderConfig { return nil }
+
 func (j *SnapJob) Run(ctx context.Context) {
+	ctx, endTask := trace.WithTaskAndSpan(ctx, "snap-job", j.Name())
+	defer endTask()
 	log := GetLogger(ctx)
-	ctx = logging.WithSubsystemLoggers(ctx, log)
 
 	defer log.Info("job exiting")
 
 	periodicDone := make(chan struct{})
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
-	go j.snapper.Run(ctx, periodicDone)
+	periodicCtx, endTask := trace.WithTask(ctx, "snapshotting")
+	defer endTask()
+	go j.snapper.Run(periodicCtx, periodicDone)
 
 	invocationCount := 0
 outer:
@@ -108,8 +116,10 @@ outer:
 		case <-periodicDone:
 		}
 		invocationCount++
-		invLog := log.WithField("invocation", invocationCount)
+
-		j.doPrune(WithLogger(ctx, invLog))
+		invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", invocationCount))
+		j.doPrune(invocationCtx)
+		endSpan()
 	}
 }
 
@@ -133,9 +143,6 @@ type alwaysUpToDateReplicationCursorHistory struct {
 var _ pruner.History = (*alwaysUpToDateReplicationCursorHistory)(nil)
 
 func (h alwaysUpToDateReplicationCursorHistory) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
-	if req.GetGet() == nil {
-		return nil, fmt.Errorf("unsupported ReplicationCursor request: SnapJob only supports GETting a (faked) cursor")
-	}
 	fsvReq := &pdu.ListFilesystemVersionsReq{
 		Filesystem: req.GetFilesystem(),
 	}
@@ -160,9 +167,15 @@ func (h alwaysUpToDateReplicationCursorHistory) ListFilesystems(ctx context.Cont
 }
 
 func (j *SnapJob) doPrune(ctx context.Context) {
+	ctx, endSpan := trace.WithSpan(ctx, "snap-job-do-prune")
+	defer endSpan()
 	log := GetLogger(ctx)
-	ctx = logging.WithSubsystemLoggers(ctx, log)
+	sender := endpoint.NewSender(endpoint.SenderConfig{
-	sender := endpoint.NewSender(j.fsfilter)
+		JobID: j.name,
+		FSF:   j.fsfilter,
+		// FIXME encryption setting is irrelevant for SnapJob because the endpoint is only used as pruner.Target
+		Encrypt: &zfs.NilBool{B: true},
+	})
 	j.pruner = j.prunerFactory.BuildLocalPruner(ctx, sender, alwaysUpToDateReplicationCursorHistory{sender})
 	log.Info("start pruning")
 	j.pruner.Prune()

daemon/logging/build_logging.go

@@ -9,19 +9,10 @@ import (
 
 	"github.com/mattn/go-isatty"
 	"github.com/pkg/errors"
-
 	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/hooks"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
-	"github.com/zrepl/zrepl/daemon/pruner"
-	"github.com/zrepl/zrepl/daemon/snapper"
-	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/replication/driver"
-	"github.com/zrepl/zrepl/replication/logic"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/rpc/transportmux"
 	"github.com/zrepl/zrepl/tlsconf"
-	"github.com/zrepl/zrepl/transport"
 )
 
 func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error) {
@@ -69,8 +60,10 @@ func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error)
 type Subsystem string
 
 const (
+	SubsysMeta         Subsystem = "meta"
+	SubsysJob          Subsystem = "job"
 	SubsysReplication  Subsystem = "repl"
-	SubsyEndpoint      Subsystem = "endpoint"
+	SubsysEndpoint     Subsystem = "endpoint"
 	SubsysPruning      Subsystem = "pruning"
 	SubsysSnapshot     Subsystem = "snapshot"
 	SubsysHooks        Subsystem = "hook"
@@ -79,29 +72,100 @@ const (
 	SubsysRPC          Subsystem = "rpc"
 	SubsysRPCControl   Subsystem = "rpc.ctrl"
 	SubsysRPCData      Subsystem = "rpc.data"
+	SubsysZFSCmd       Subsystem = "zfs.cmd"
 )
 
-func WithSubsystemLoggers(ctx context.Context, log logger.Logger) context.Context {
+var AllSubsystems = []Subsystem{
-	ctx = logic.WithLogger(ctx, log.WithField(SubsysField, SubsysReplication))
+	SubsysMeta,
-	ctx = driver.WithLogger(ctx, log.WithField(SubsysField, SubsysReplication))
+	SubsysJob,
-	ctx = endpoint.WithLogger(ctx, log.WithField(SubsysField, SubsyEndpoint))
+	SubsysReplication,
-	ctx = pruner.WithLogger(ctx, log.WithField(SubsysField, SubsysPruning))
+	SubsysEndpoint,
-	ctx = snapper.WithLogger(ctx, log.WithField(SubsysField, SubsysSnapshot))
+	SubsysPruning,
-	ctx = hooks.WithLogger(ctx, log.WithField(SubsysField, SubsysHooks))
+	SubsysSnapshot,
-	ctx = transport.WithLogger(ctx, log.WithField(SubsysField, SubsysTransport))
+	SubsysHooks,
-	ctx = transportmux.WithLogger(ctx, log.WithField(SubsysField, SubsysTransportMux))
+	SubsysTransport,
-	ctx = rpc.WithLoggers(ctx,
+	SubsysTransportMux,
-		rpc.Loggers{
+	SubsysRPC,
-			General: log.WithField(SubsysField, SubsysRPC),
+	SubsysRPCControl,
-			Control: log.WithField(SubsysField, SubsysRPCControl),
+	SubsysRPCData,
-			Data:    log.WithField(SubsysField, SubsysRPCData),
+	SubsysZFSCmd,
-		},
-	)
-	return ctx
 }
 
-func LogSubsystem(log logger.Logger, subsys Subsystem) logger.Logger {
+type injectedField struct {
-	return log.ReplaceField(SubsysField, subsys)
+	field  string
+	value  interface{}
+	parent *injectedField
+}
+
+func WithInjectedField(ctx context.Context, field string, value interface{}) context.Context {
+	var parent *injectedField
+	parentI := ctx.Value(contextKeyInjectedField)
+	if parentI != nil {
+		parent = parentI.(*injectedField)
+	}
+	// TODO sanity-check `field` now
+	this := &injectedField{field, value, parent}
+	return context.WithValue(ctx, contextKeyInjectedField, this)
+}
+
+func iterInjectedFields(ctx context.Context, cb func(field string, value interface{})) {
+	injI := ctx.Value(contextKeyInjectedField)
+	if injI == nil {
+		return
+	}
+	inj := injI.(*injectedField)
+	for ; inj != nil; inj = inj.parent {
+		cb(inj.field, inj.value)
+	}
+}
+
+type SubsystemLoggers map[Subsystem]logger.Logger
+
+func SubsystemLoggersWithUniversalLogger(l logger.Logger) SubsystemLoggers {
+	loggers := make(SubsystemLoggers)
+	for _, s := range AllSubsystems {
+		loggers[s] = l
+	}
+	return loggers
+}
+
+func WithLoggers(ctx context.Context, loggers SubsystemLoggers) context.Context {
+	return context.WithValue(ctx, contextKeyLoggers, loggers)
+}
+
+func GetLoggers(ctx context.Context) SubsystemLoggers {
+	loggers, ok := ctx.Value(contextKeyLoggers).(SubsystemLoggers)
+	if !ok {
+		return nil
+	}
+	return loggers
+}
+
+func GetLogger(ctx context.Context, subsys Subsystem) logger.Logger {
+	return getLoggerImpl(ctx, subsys, true)
+}
+
+func getLoggerImpl(ctx context.Context, subsys Subsystem, panicIfEnded bool) logger.Logger {
+	loggers, ok := ctx.Value(contextKeyLoggers).(SubsystemLoggers)
+	if !ok || loggers == nil {
+		return logger.NewNullLogger()
+	}
+	l, ok := loggers[subsys]
+	if !ok {
+		return logger.NewNullLogger()
+	}
+
+	l = l.WithField(SubsysField, subsys)
+
+	l = l.WithField(SpanField, trace.GetSpanStackOrDefault(ctx, "NOSPAN"))
+
+	fields := make(logger.Fields)
+	iterInjectedFields(ctx, func(field string, value interface{}) {
+		fields[field] = value
+	})
+	l = l.WithFields(fields)
+
+	return l
 }
 
 func parseLogFormat(i interface{}) (f EntryFormatter, err error) {

daemon/logging/logging_context.go

@@ -0,0 +1,24 @@
+package logging
+
+import "context"
+
+type contextKey int
+
+const (
+	contextKeyLoggers contextKey = 1 + iota
+	contextKeyInjectedField
+)
+
+var contextKeys = []contextKey{
+	contextKeyLoggers,
+	contextKeyInjectedField,
+}
+
+func WithInherit(ctx, inheritFrom context.Context) context.Context {
+	for _, k := range contextKeys {
+		if v := inheritFrom.Value(k); v != nil {
+			ctx = context.WithValue(ctx, k, v) // no shadow
+		}
+	}
+	return ctx
+}

daemon/logging/logging_formatters.go

@@ -22,6 +22,7 @@ const (
 const (
 	JobField    string = "job"
 	SubsysField string = "subsystem"
+	SpanField   string = "span"
 )
 
 type MetadataFlags int64
@@ -85,7 +86,7 @@ func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
 		fmt.Fprintf(&line, "[%s]", col.Sprint(e.Level.Short()))
 	}
 
-	prefixFields := []string{JobField, SubsysField}
+	prefixFields := []string{JobField, SubsysField, SpanField}
 	prefixed := make(map[string]bool, len(prefixFields)+2)
 	for _, field := range prefixFields {
 		val, ok := e.Fields[field]
@@ -174,8 +175,8 @@ func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
 	}
 
 	// at least try and put job and task in front
-	prefixed := make(map[string]bool, 2)
+	prefixed := make(map[string]bool, 3)
-	prefix := []string{JobField, SubsysField}
+	prefix := []string{JobField, SubsysField, SpanField}
 	for _, pf := range prefix {
 		v, ok := e.Fields[pf]
 		if !ok {
@@ -211,7 +212,7 @@ func logfmtTryEncodeKeyval(enc *logfmt.Encoder, field, value interface{}) error
 	case logfmt.ErrUnsupportedValueType:
 		err := enc.EncodeKeyval(field, fmt.Sprintf("<%T>", value))
 		if err != nil {
-			return errors.Wrap(err, "cannot encode unsuuported value type Go type")
+			return errors.Wrap(err, "cannot encode unsupported value type Go type")
 		}
 		return nil
 	}

daemon/logging/logging_outlets.go

@@ -63,7 +63,7 @@ func NewTCPOutlet(formatter EntryFormatter, network, address string, tlsConfig *
 		return
 	}
 
-	entryChan := make(chan *bytes.Buffer, 1) // allow one message in flight while previos is in io.Copy()
+	entryChan := make(chan *bytes.Buffer, 1) // allow one message in flight while previous is in io.Copy()
 
 	o := &TCPOutlet{
 		formatter: formatter,

daemon/logging/trace/trace.go

@@ -0,0 +1,355 @@
+// package trace provides activity tracing via ctx through Tasks and Spans
+//
+// Basic Concepts
+//
+// Tracing can be used to identify where a piece of code spends its time.
+//
+// The Go standard library provides package runtime/trace which is useful to identify CPU bottlenecks or
+// to understand what happens inside the Go runtime.
+// However, it is not ideal for application level tracing, in particular if those traces should be understandable
+// to tech-savvy users (albeit not developers).
+//
+// This package provides the concept of Tasks and Spans to express what activity is happening within an application:
+//
+//  - Neither task nor span is really tangible but instead contained within the context.Context tree
+//  - Tasks represent concurrent activity (i.e. goroutines).
+//  - Spans represent a semantic stack trace within a task.
+//
+// As a consequence, whenever a context is propagated across goroutine boundary, you need to create a child task:
+//
+//   go func(ctx context.Context) {
+//     ctx, endTask = WithTask(ctx, "what-happens-inside-the-child-task")
+//     defer endTask()
+//     // ...
+//   }(ctx)
+//
+// Within the task, you can open up a hierarchy of spans.
+// In contrast to tasks, which have can multiple concurrently running child tasks,
+// spans must nest and not cross the goroutine boundary.
+//
+//  ctx, endSpan = WithSpan(ctx, "copy-dir")
+//  defer endSpan()
+//  for _, f := range dir.Files() {
+//    func() {
+//      ctx, endSpan := WithSpan(ctx, fmt.Sprintf("copy-file %q", f))
+//      defer endspan()
+//      b, _ := ioutil.ReadFile(f)
+//      _ = ioutil.WriteFile(f + ".copy", b, 0600)
+//    }()
+//  }
+//
+// In combination:
+//  ctx, endTask = WithTask(ctx, "copy-dirs")
+//  defer endTask()
+//  for i := range dirs {
+//    go func(dir string) {
+//      ctx, endTask := WithTask(ctx, "copy-dir")
+//      defer endTask()
+//      for _, f := range filesIn(dir) {
+//        func() {
+//          ctx, endSpan := WithSpan(ctx, fmt.Sprintf("copy-file %q", f))
+//          defer endspan()
+//          b, _ := ioutil.ReadFile(f)
+//          _ = ioutil.WriteFile(f + ".copy", b, 0600)
+//        }()
+//      }
+//    }()
+//  }
+//
+// Note that a span ends at the time you call endSpan - not before and not after that.
+// If you violate the stack-like nesting of spans by forgetting an endSpan() invocation,
+// the out-of-order endSpan() will panic.
+//
+// A similar rule applies to the endTask closure returned by WithTask:
+// If a task has live child tasks at the time you call endTask(), the call will panic.
+//
+// Recovering from endSpan() or endTask() panics will corrupt the trace stack and lead to corrupt tracefile output.
+//
+//
+// Best Practices For Naming Tasks And Spans
+//
+// Tasks should always have string constants as names, and must not contain the `#` character. WHy?
+// First, the visualization by chrome://tracing draws a horizontal bar for each task in the trace.
+// Also, the package appends `#NUM` for each concurrently running instance of a task name.
+// Note that the `#NUM` suffix will be reused if a task has ended, in order to avoid an
+// infinite number of horizontal bars in the visualization.
+//
+//
+// Chrome-compatible Tracefile Support
+//
+// The activity trace generated by usage of WithTask and WithSpan can be rendered to a JSON output file
+// that can be loaded into chrome://tracing .
+// Apart from function GetSpanStackOrDefault, this is the main benefit of this package.
+//
+// First, there is a convenience environment variable 'ZREPL_ACTIVITY_TRACE' that can be set to an output path.
+// From process start onward, a trace is written to that path.
+//
+// More consumers can attach to the activity trace through the ChrometraceClientWebsocketHandler websocket handler.
+//
+// If a write error is encountered with any consumer (including the env-var based one), the consumer is closed and
+// will not receive further trace output.
+package trace
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/zrepl/zrepl/util/chainlock"
+)
+
+var metrics struct {
+	activeTasks                          prometheus.Gauge
+	uniqueConcurrentTaskNameBitvecLength *prometheus.GaugeVec
+}
+
+func init() {
+	metrics.activeTasks = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "zrepl",
+		Subsystem: "trace",
+		Name:      "active_tasks",
+		Help:      "number of active (tracing-level) tasks in the daemon",
+	})
+	metrics.uniqueConcurrentTaskNameBitvecLength = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "zrepl",
+		Subsystem: "trace",
+		Name:      "unique_concurrent_task_name_bitvec_length",
+		Help:      "length of the bitvec used to find unique names for concurrent tasks",
+	}, []string{"task_name"})
+}
+
+func RegisterMetrics(r prometheus.Registerer) {
+	r.MustRegister(metrics.activeTasks)
+	r.MustRegister(metrics.uniqueConcurrentTaskNameBitvecLength)
+}
+
+var taskNamer = newUniqueTaskNamer(metrics.uniqueConcurrentTaskNameBitvecLength)
+
+type traceNode struct {
+	id         string
+	annotation string
+	parentTask *traceNode
+
+	mtx chainlock.L
+
+	activeChildTasks int32 // only for task nodes, insignificant for span nodes
+	parentSpan       *traceNode
+	activeChildSpan  *traceNode // nil if task or span doesn't have an active child span
+
+	startedAt time.Time
+	endedAt   time.Time
+}
+
+// Returned from WithTask or WithSpan.
+// Must be called once the task or span ends.
+// See package-level docs for nesting rules.
+// Wrong call order / forgetting to call it will result in panics.
+type DoneFunc func()
+
+var ErrTaskStillHasActiveChildTasks = fmt.Errorf("end task: task still has active child tasks")
+var ErrParentTaskAlreadyEnded = fmt.Errorf("create task: parent task already ended")
+
+// Start a new root task or create a child task of an existing task.
+//
+// This is required when starting a new goroutine and
+// passing an existing task context to it.
+//
+// taskName should be a constantand must not contain '#'
+//
+// The implementation ensures that,
+// if multiple tasks with the same name exist simultaneously,
+// a unique suffix is appended to uniquely identify the task opened with this function.
+func WithTask(ctx context.Context, taskName string) (context.Context, DoneFunc) {
+
+	var parentTask *traceNode
+	nodeI := ctx.Value(contextKeyTraceNode)
+	if nodeI != nil {
+		node := nodeI.(*traceNode)
+		if node.parentSpan != nil {
+			parentTask = node.parentTask
+		} else {
+			parentTask = node
+		}
+	}
+
+	taskName, taskNameDone := taskNamer.UniqueConcurrentTaskName(taskName)
+
+	this := &traceNode{
+		id:               genID(),
+		annotation:       taskName,
+		parentTask:       parentTask,
+		activeChildTasks: 0,
+		parentSpan:       nil,
+		activeChildSpan:  nil,
+
+		startedAt: time.Now(),
+		endedAt:   time.Time{},
+	}
+
+	if this.parentTask != nil {
+		this.parentTask.mtx.HoldWhile(func() {
+			if !this.parentTask.endedAt.IsZero() {
+				panic(ErrParentTaskAlreadyEnded)
+			}
+			this.parentTask.activeChildTasks++
+		})
+	}
+
+	ctx = context.WithValue(ctx, contextKeyTraceNode, this)
+
+	chrometraceBeginTask(this)
+
+	metrics.activeTasks.Inc()
+
+	endTaskFunc := func() {
+
+		// only hold locks while manipulating the tree
+		// (trace writer might block too long and unlike spans, tasks are updated concurrently)
+		alreadyEnded := func() (alreadyEnded bool) {
+			if this.parentTask != nil {
+				defer this.parentTask.mtx.Lock().Unlock()
+			}
+			defer this.mtx.Lock().Unlock()
+
+			if this.activeChildTasks != 0 {
+				panic(errors.Wrapf(ErrTaskStillHasActiveChildTasks, "end task: %v active child tasks", this.activeChildSpan))
+			}
+
+			// support idempotent task ends
+			if !this.endedAt.IsZero() {
+				return true
+			}
+			this.endedAt = time.Now()
+
+			if this.parentTask != nil {
+				this.parentTask.activeChildTasks--
+				if this.parentTask.activeChildTasks < 0 {
+					panic("impl error: parent task with negative activeChildTasks count")
+				}
+			}
+			return false
+		}()
+		if alreadyEnded {
+			return
+		}
+
+		chrometraceEndTask(this)
+
+		metrics.activeTasks.Dec()
+
+		taskNameDone()
+	}
+
+	return ctx, endTaskFunc
+}
+
+var ErrAlreadyActiveChildSpan = fmt.Errorf("create child span: span already has an active child span")
+var ErrSpanStillHasActiveChildSpan = fmt.Errorf("end span: span still has active child spans")
+
+// Start a new span.
+// Important: ctx must have an active task (see WithTask)
+func WithSpan(ctx context.Context, annotation string) (context.Context, DoneFunc) {
+	var parentSpan, parentTask *traceNode
+	nodeI := ctx.Value(contextKeyTraceNode)
+	if nodeI != nil {
+		parentSpan = nodeI.(*traceNode)
+		if parentSpan.parentSpan == nil {
+			parentTask = parentSpan
+		} else {
+			parentTask = parentSpan.parentTask
+		}
+	} else {
+		panic("must be called from within a task")
+	}
+
+	this := &traceNode{
+		id:              genID(),
+		annotation:      annotation,
+		parentTask:      parentTask,
+		parentSpan:      parentSpan,
+		activeChildSpan: nil,
+
+		startedAt: time.Now(),
+		endedAt:   time.Time{},
+	}
+
+	parentSpan.mtx.HoldWhile(func() {
+		if parentSpan.activeChildSpan != nil {
+			panic(ErrAlreadyActiveChildSpan)
+		}
+		parentSpan.activeChildSpan = this
+	})
+
+	ctx = context.WithValue(ctx, contextKeyTraceNode, this)
+	chrometraceBeginSpan(this)
+
+	endTaskFunc := func() {
+
+		defer parentSpan.mtx.Lock().Unlock()
+		if parentSpan.activeChildSpan != this && this.endedAt.IsZero() {
+			panic("impl error: activeChildSpan should not change while != nil because there can only be one")
+		}
+
+		defer this.mtx.Lock().Unlock()
+		if this.activeChildSpan != nil {
+			panic(ErrSpanStillHasActiveChildSpan)
+		}
+
+		if !this.endedAt.IsZero() {
+			return // support idempotent span ends
+		}
+
+		parentSpan.activeChildSpan = nil
+		this.endedAt = time.Now()
+
+		chrometraceEndSpan(this)
+	}
+
+	return ctx, endTaskFunc
+}
+
+func currentTaskNameAndSpanStack(this *traceNode) (taskName string, spanIdStack string) {
+
+	task := this.parentTask
+	if this.parentSpan == nil {
+		task = this
+	}
+
+	var spansInTask []*traceNode
+	for s := this; s != nil; s = s.parentSpan {
+		spansInTask = append(spansInTask, s)
+	}
+
+	var tasks []*traceNode
+	for t := task; t != nil; t = t.parentTask {
+		tasks = append(tasks, t)
+	}
+
+	var taskIdsRev []string
+	for i := len(tasks) - 1; i >= 0; i-- {
+		taskIdsRev = append(taskIdsRev, tasks[i].id)
+	}
+
+	var spanIdsRev []string
+	for i := len(spansInTask) - 1; i >= 0; i-- {
+		spanIdsRev = append(spanIdsRev, spansInTask[i].id)
+	}
+
+	taskStack := strings.Join(taskIdsRev, "$")
+	spanIdStack = fmt.Sprintf("%s$%s", taskStack, strings.Join(spanIdsRev, "."))
+
+	return task.annotation, spanIdStack
+}
+
+func GetSpanStackOrDefault(ctx context.Context, def string) string {
+	if nI := ctx.Value(contextKeyTraceNode); nI != nil {
+		n := nI.(*traceNode)
+		_, spanStack := currentTaskNameAndSpanStack(n)
+		return spanStack
+	} else {
+		return def
+	}
+}

daemon/logging/trace/trace_chrometrace.go

@@ -0,0 +1,230 @@
+package trace
+
+// The functions in this file are concerned with the generation
+// of trace files based on the information from WithTask and WithSpan.
+//
+// The emitted trace files are open-ended array of JSON objects
+// that follow the Chrome trace file format:
+//   https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview
+//
+// The emitted JSON can be loaded into Chrome's chrome://tracing view.
+//
+// The trace file can be written to a file whose path is specified in an env file,
+// and be written to web sockets established on ChrometraceHttpHandler
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"sync"
+	"sync/atomic"
+
+	"golang.org/x/net/websocket"
+
+	"github.com/zrepl/zrepl/util/envconst"
+)
+
+var chrometracePID string
+
+func init() {
+	var err error
+	chrometracePID, err = os.Hostname()
+	if err != nil {
+		panic(err)
+	}
+	chrometracePID = fmt.Sprintf("%q", chrometracePID)
+}
+
+type chrometraceEvent struct {
+	Cat                       string   `json:"cat,omitempty"`
+	Name                      string   `json:"name"`
+	Stack                     []string `json:"stack,omitempty"`
+	Phase                     string   `json:"ph"`
+	TimestampUnixMicroseconds int64    `json:"ts"`
+	DurationMicroseconds      int64    `json:"dur,omitempty"`
+	Pid                       string   `json:"pid"`
+	Tid                       string   `json:"tid"`
+	Id                        string   `json:"id,omitempty"`
+}
+
+func chrometraceBeginSpan(s *traceNode) {
+	taskName, _ := currentTaskNameAndSpanStack(s)
+	chrometraceWrite(chrometraceEvent{
+		Name:                      s.annotation,
+		Phase:                     "B",
+		TimestampUnixMicroseconds: s.startedAt.UnixNano() / 1000,
+		Pid:                       chrometracePID,
+		Tid:                       taskName,
+	})
+}
+
+func chrometraceEndSpan(s *traceNode) {
+	taskName, _ := currentTaskNameAndSpanStack(s)
+	chrometraceWrite(chrometraceEvent{
+		Name:                      s.annotation,
+		Phase:                     "E",
+		TimestampUnixMicroseconds: s.endedAt.UnixNano() / 1000,
+		Pid:                       chrometracePID,
+		Tid:                       taskName,
+	})
+}
+
+var chrometraceFlowId uint64
+
+func chrometraceBeginTask(s *traceNode) {
+	chrometraceBeginSpan(s)
+
+	if s.parentTask == nil {
+		return
+	}
+	// beginning of a task that has a parent
+	// => use flow events to link parent and child
+
+	flowId := atomic.AddUint64(&chrometraceFlowId, 1)
+	flowIdStr := fmt.Sprintf("%x", flowId)
+
+	parentTask, _ := currentTaskNameAndSpanStack(s.parentTask)
+	chrometraceWrite(chrometraceEvent{
+		Cat:                       "task", // seems to be necessary, otherwise the GUI shows some `indexOf` JS error
+		Name:                      "child-task",
+		Phase:                     "s",
+		TimestampUnixMicroseconds: s.startedAt.UnixNano() / 1000, // yes, the child's timestamp (=> from-point of the flow line is at right x-position of parent's bar)
+		Pid:                       chrometracePID,
+		Tid:                       parentTask,
+		Id:                        flowIdStr,
+	})
+
+	childTask, _ := currentTaskNameAndSpanStack(s)
+	if parentTask == childTask {
+		panic(parentTask)
+	}
+	chrometraceWrite(chrometraceEvent{
+		Cat:                       "task", // seems to be necessary, otherwise the GUI shows some `indexOf` JS error
+		Name:                      "child-task",
+		Phase:                     "f",
+		TimestampUnixMicroseconds: s.startedAt.UnixNano() / 1000,
+		Pid:                       chrometracePID,
+		Tid:                       childTask,
+		Id:                        flowIdStr,
+	})
+}
+
+func chrometraceEndTask(s *traceNode) {
+	chrometraceEndSpan(s)
+}
+
+type chrometraceConsumerRegistration struct {
+	w io.Writer
+	// errored must have capacity 1, the writer thread will send to it non-blocking, then close it
+	errored chan error
+}
+
+var chrometraceConsumers struct {
+	register  chan chrometraceConsumerRegistration
+	consumers map[chrometraceConsumerRegistration]bool
+	write     chan []byte
+}
+
+func init() {
+	chrometraceConsumers.register = make(chan chrometraceConsumerRegistration)
+	chrometraceConsumers.consumers = make(map[chrometraceConsumerRegistration]bool)
+	chrometraceConsumers.write = make(chan []byte)
+	go func() {
+		kickConsumer := func(c chrometraceConsumerRegistration, err error) {
+			debug("chrometrace kicking consumer %#v after error %v", c, err)
+			select {
+			case c.errored <- err:
+			default:
+			}
+			close(c.errored)
+			delete(chrometraceConsumers.consumers, c)
+		}
+		for {
+			select {
+			case reg := <-chrometraceConsumers.register:
+				debug("registered chrometrace consumer %#v", reg)
+				chrometraceConsumers.consumers[reg] = true
+				n, err := reg.w.Write([]byte("[\n"))
+				if err != nil {
+					kickConsumer(reg, err)
+				} else if n != 2 {
+					kickConsumer(reg, fmt.Errorf("short write: %v", n))
+				}
+				// successfully registered
+
+			case buf := <-chrometraceConsumers.write:
+				debug("chrometrace write request: %s", string(buf))
+				var r bytes.Reader
+				for c := range chrometraceConsumers.consumers {
+					r.Reset(buf)
+					n, err := io.Copy(c.w, &r)
+					debug("chrometrace wrote n=%v bytes to consumer %#v", n, c)
+					if err != nil {
+						kickConsumer(c, err)
+					}
+				}
+			}
+		}
+	}()
+}
+
+func chrometraceWrite(i interface{}) {
+	var buf bytes.Buffer
+	err := json.NewEncoder(&buf).Encode(i)
+	if err != nil {
+		panic(err)
+	}
+	buf.WriteString(",")
+	chrometraceConsumers.write <- buf.Bytes()
+}
+
+func ChrometraceClientWebsocketHandler(conn *websocket.Conn) {
+	defer conn.Close()
+
+	var wg sync.WaitGroup
+	defer wg.Wait()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		r := bufio.NewReader(conn)
+		_, _, _ = r.ReadLine() // ignore errors
+		conn.Close()
+	}()
+
+	errored := make(chan error, 1)
+	chrometraceConsumers.register <- chrometraceConsumerRegistration{
+		w:       conn,
+		errored: errored,
+	}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		<-errored
+		conn.Close()
+	}()
+}
+
+var chrometraceFileConsumerPath = envconst.String("ZREPL_ACTIVITY_TRACE", "")
+
+func init() {
+	if chrometraceFileConsumerPath != "" {
+		var err error
+		f, err := os.Create(chrometraceFileConsumerPath)
+		if err != nil {
+			panic(err)
+		}
+		errored := make(chan error, 1)
+		chrometraceConsumers.register <- chrometraceConsumerRegistration{
+			w:       f,
+			errored: errored,
+		}
+		go func() {
+			<-errored
+			f.Close()
+		}()
+	}
+}

daemon/logging/trace/trace_context.go

@@ -0,0 +1,27 @@
+package trace
+
+import "context"
+
+type contextKey int
+
+const (
+	contextKeyTraceNode contextKey = 1 + iota
+)
+
+var contextKeys = []contextKey{
+	contextKeyTraceNode,
+}
+
+// WithInherit inherits the task hierarchy from inheritFrom into ctx.
+// The returned context is a child of ctx, but its task and span are those of inheritFrom.
+//
+// Note that in most use cases, callers most likely want to call WithTask since it will most likely
+// be in some sort of connection handler context.
+func WithInherit(ctx, inheritFrom context.Context) context.Context {
+	for _, k := range contextKeys {
+		if v := inheritFrom.Value(k); v != nil {
+			ctx = context.WithValue(ctx, k, v) // no shadow
+		}
+	}
+	return ctx
+}

daemon/logging/trace/trace_convenience.go

@@ -0,0 +1,74 @@
+package trace
+
+import (
+	"context"
+	"fmt"
+	"runtime"
+	"strings"
+	"sync"
+)
+
+// use like this:
+//
+//	defer WithSpanFromStackUpdateCtx(&existingCtx)()
+//
+//
+func WithSpanFromStackUpdateCtx(ctx *context.Context) DoneFunc {
+	childSpanCtx, end := WithSpan(*ctx, getMyCallerOrPanic())
+	*ctx = childSpanCtx
+	return end
+}
+
+// derive task name from call stack (caller's name)
+func WithTaskFromStack(ctx context.Context) (context.Context, DoneFunc) {
+	return WithTask(ctx, getMyCallerOrPanic())
+}
+
+// derive task name from call stack (caller's name) and update *ctx
+// to point to be the child task ctx
+func WithTaskFromStackUpdateCtx(ctx *context.Context) DoneFunc {
+	child, end := WithTask(*ctx, getMyCallerOrPanic())
+	*ctx = child
+	return end
+}
+
+// create a task and a span within it in one call
+func WithTaskAndSpan(ctx context.Context, task string, span string) (context.Context, DoneFunc) {
+	ctx, endTask := WithTask(ctx, task)
+	ctx, endSpan := WithSpan(ctx, fmt.Sprintf("%s %s", task, span))
+	return ctx, func() {
+		endSpan()
+		endTask()
+	}
+}
+
+// create a span during which several child tasks are spawned using the `add` function
+func WithTaskGroup(ctx context.Context, taskGroup string) (_ context.Context, add func(f func(context.Context)), waitEnd DoneFunc) {
+	var wg sync.WaitGroup
+	ctx, endSpan := WithSpan(ctx, taskGroup)
+	add = func(f func(context.Context)) {
+		wg.Add(1)
+		defer wg.Done()
+		ctx, endTask := WithTask(ctx, taskGroup)
+		defer endTask()
+		f(ctx)
+	}
+	waitEnd = func() {
+		wg.Wait()
+		endSpan()
+	}
+	return ctx, add, waitEnd
+}
+
+func getMyCallerOrPanic() string {
+	pc, _, _, ok := runtime.Caller(2)
+	if !ok {
+		panic("cannot get caller")
+	}
+	details := runtime.FuncForPC(pc)
+	if ok && details != nil {
+		const prefix = "github.com/zrepl/zrepl"
+		return strings.TrimPrefix(strings.TrimPrefix(details.Name(), prefix), "/")
+	}
+	return ""
+}

daemon/logging/trace/trace_convenience_test.go

@@ -0,0 +1,16 @@
+package trace
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetCallerOrPanic(t *testing.T) {
+	withStackFromCtxMock := func() string {
+		return getMyCallerOrPanic()
+	}
+	ret := withStackFromCtxMock()
+	// zrepl prefix is stripped
+	assert.Equal(t, "daemon/logging/trace.TestGetCallerOrPanic", ret)
+}

daemon/logging/trace/trace_debug.go

@@ -0,0 +1,15 @@
+package trace
+
+import (
+	"fmt"
+	"os"
+)
+
+const debugEnabled = false
+
+func debug(format string, args ...interface{}) {
+	if !debugEnabled {
+		return
+	}
+	fmt.Fprintf(os.Stderr, format+"\n", args...)
+}

daemon/logging/trace/trace_genID.go

@@ -0,0 +1,47 @@
+package trace
+
+import (
+	"encoding/base64"
+	"math/rand"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/zrepl/zrepl/util/envconst"
+)
+
+var genIdPRNG = rand.New(rand.NewSource(1))
+
+func init() {
+	genIdPRNG.Seed(time.Now().UnixNano())
+	genIdPRNG.Seed(int64(os.Getpid()))
+}
+
+var genIdNumBytes = envconst.Int("ZREPL_TRACE_ID_NUM_BYTES", 3)
+
+func init() {
+	if genIdNumBytes < 1 {
+		panic("trace node id byte length must be at least 1")
+	}
+}
+
+func genID() string {
+	var out strings.Builder
+	enc := base64.NewEncoder(base64.RawStdEncoding, &out)
+	buf := make([]byte, genIdNumBytes)
+	for i := 0; i < len(buf); {
+		n, err := genIdPRNG.Read(buf[i:])
+		if err != nil {
+			panic(err)
+		}
+		i += n
+	}
+	n, err := enc.Write(buf[:])
+	if err != nil || n != len(buf) {
+		panic(err)
+	}
+	if err := enc.Close(); err != nil {
+		panic(err)
+	}
+	return out.String()
+}

daemon/logging/trace/trace_test.go

@@ -0,0 +1,172 @@
+package trace
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/gitchander/permutation"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRegularSpanUsage(t *testing.T) {
+	root, endRoot := WithTask(context.Background(), "root")
+	defer endRoot()
+
+	s1, endS1 := WithSpan(root, "parent")
+	s2, endS2 := WithSpan(s1, "child")
+	_, endS3 := WithSpan(s2, "grand-child")
+	require.NotPanics(t, func() { endS3() })
+	require.NotPanics(t, func() { endS2() })
+
+	// reuse
+	_, endS4 := WithSpan(s1, "child-2")
+	require.NotPanics(t, func() { endS4() })
+
+	// close parent
+	require.NotPanics(t, func() { endS1() })
+}
+
+func TestMultipleActiveChildSpansNotAllowed(t *testing.T) {
+	root, endRoot := WithTask(context.Background(), "root")
+	defer endRoot()
+
+	s1, _ := WithSpan(root, "s1")
+	_, endS2 := WithSpan(s1, "s1-child1")
+
+	require.PanicsWithValue(t, ErrAlreadyActiveChildSpan, func() {
+		_, _ = WithSpan(s1, "s1-child2")
+	})
+
+	endS2()
+
+	require.NotPanics(t, func() {
+		_, _ = WithSpan(s1, "s1-child2")
+	})
+}
+
+func TestForkingChildSpansNotAllowed(t *testing.T) {
+	root, endRoot := WithTask(context.Background(), "root")
+	defer endRoot()
+
+	s1, _ := WithSpan(root, "s1")
+	sc, endSC := WithSpan(s1, "s1-child")
+	_, _ = WithSpan(sc, "s1-child-child")
+
+	require.PanicsWithValue(t, ErrSpanStillHasActiveChildSpan, func() {
+		endSC()
+	})
+}
+
+func TestRegularTaskUsage(t *testing.T) {
+	// assert concurrent activities on different tasks can end in any order
+	closeOrder := []int{0, 1, 2}
+	closeOrders := permutation.New(permutation.IntSlice(closeOrder))
+	for closeOrders.Next() {
+		t.Run(fmt.Sprintf("%v", closeOrder), func(t *testing.T) {
+			root, endRoot := WithTask(context.Background(), "root")
+			defer endRoot()
+
+			c1, endC1 := WithTask(root, "c1")
+			defer endC1()
+			c2, endC2 := WithTask(root, "c2")
+			defer endC2()
+
+			// begin 3 concurrent activities
+			_, endAR := WithSpan(root, "aR")
+			_, endAC1 := WithSpan(c1, "aC1")
+			_, endAC2 := WithSpan(c2, "aC2")
+
+			endFuncs := []DoneFunc{endAR, endAC1, endAC2}
+			for _, i := range closeOrder {
+				require.NotPanics(t, func() {
+					endFuncs[i]()
+				}, "%v", i)
+			}
+		})
+	}
+}
+
+func TestTaskEndWithActiveChildTaskNotAllowed(t *testing.T) {
+	root, _ := WithTask(context.Background(), "root")
+	c, endC := WithTask(root, "child")
+	_, _ = WithTask(c, "grand-child")
+	func() {
+		defer func() {
+			r := recover()
+			require.NotNil(t, r)
+			err, ok := r.(error)
+			require.True(t, ok)
+			require.Equal(t, ErrTaskStillHasActiveChildTasks, errors.Cause(err))
+		}()
+		endC()
+	}()
+
+}
+
+func TestIdempotentEndTask(t *testing.T) {
+	_, end := WithTask(context.Background(), "root")
+	end()
+	require.NotPanics(t, func() { end() })
+}
+
+func TestCannotReuseEndedTask(t *testing.T) {
+	root, end := WithTask(context.Background(), "root")
+	end()
+	require.PanicsWithValue(t, ErrParentTaskAlreadyEnded, func() { WithTask(root, "child-after-parent-ended") })
+}
+
+func TestSpansPanicIfNoParentTask(t *testing.T) {
+	require.Panics(t, func() { WithSpan(context.Background(), "taskless-span") })
+}
+
+func TestIdempotentEndSpan(t *testing.T) {
+	root, _ := WithTask(context.Background(), "root")
+	_, end := WithSpan(root, "span")
+	end()
+	require.NotPanics(t, func() { end() })
+}
+
+func logAndGetTraceNode(t *testing.T, descr string, ctx context.Context) *traceNode {
+	n, ok := ctx.Value(contextKeyTraceNode).(*traceNode)
+	require.True(t, ok)
+	t.Logf("% 20s %p %#v", descr, n, n)
+	return n
+}
+
+func TestWhiteboxHierachy(t *testing.T) {
+	root, e1 := WithTask(context.Background(), "root")
+	rootN := logAndGetTraceNode(t, "root", root)
+	assert.Nil(t, rootN.parentTask)
+	assert.Nil(t, rootN.parentSpan)
+
+	child, e2 := WithSpan(root, "child")
+	childN := logAndGetTraceNode(t, "child", child)
+	assert.Equal(t, rootN, childN.parentTask)
+	assert.Equal(t, rootN, childN.parentSpan)
+
+	grandchild, e3 := WithSpan(child, "grandchild")
+	grandchildN := logAndGetTraceNode(t, "grandchild", grandchild)
+	assert.Equal(t, rootN, grandchildN.parentTask)
+	assert.Equal(t, childN, grandchildN.parentSpan)
+
+	gcTask, e4 := WithTask(grandchild, "grandchild-task")
+	gcTaskN := logAndGetTraceNode(t, "grandchild-task", gcTask)
+	assert.Equal(t, rootN, gcTaskN.parentTask)
+	assert.Nil(t, gcTaskN.parentSpan)
+
+	// it is allowed that a child task outlives the _span_ in which it was created
+	// (albeit not its parent task)
+	e3()
+	e2()
+	gcTaskSpan, e5 := WithSpan(gcTask, "granschild-task-span")
+	gcTaskSpanN := logAndGetTraceNode(t, "granschild-task-span", gcTaskSpan)
+	assert.Equal(t, gcTaskN, gcTaskSpanN.parentTask)
+	assert.Equal(t, gcTaskN, gcTaskSpanN.parentSpan)
+	e5()
+
+	e4()
+	e1()
+}

daemon/logging/trace/trace_unique_concurrent_task_namer.go

@@ -0,0 +1,61 @@
+package trace
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/willf/bitset"
+)
+
+type uniqueConcurrentTaskNamer struct {
+	mtx    sync.Mutex
+	active map[string]*bitset.BitSet
+
+	bitvecLengthGauge *prometheus.GaugeVec
+}
+
+// bitvecLengthGauge may be nil
+func newUniqueTaskNamer(bitvecLengthGauge *prometheus.GaugeVec) *uniqueConcurrentTaskNamer {
+	return &uniqueConcurrentTaskNamer{
+		active:            make(map[string]*bitset.BitSet),
+		bitvecLengthGauge: bitvecLengthGauge,
+	}
+}
+
+// appends `#%d` to `name` such that until `done` is called,
+// it is guaranteed that `#%d` is not returned a second time for the same `name`
+func (namer *uniqueConcurrentTaskNamer) UniqueConcurrentTaskName(name string) (uniqueName string, done func()) {
+	if strings.Contains(name, "#") {
+		panic(name)
+	}
+	namer.mtx.Lock()
+	act, ok := namer.active[name]
+	if !ok {
+		act = bitset.New(64) // FIXME magic const
+		namer.active[name] = act
+	}
+	id, ok := act.NextClear(0)
+	if !ok {
+		// if !ok, all bits are 1 and act.Len() returns the next bit
+		id = act.Len()
+		// FIXME unbounded growth without reclamation
+	}
+	act.Set(id)
+	namer.mtx.Unlock()
+
+	if namer.bitvecLengthGauge != nil {
+		namer.bitvecLengthGauge.WithLabelValues(name).Set(float64(act.Len()))
+	}
+
+	return fmt.Sprintf("%s#%d", name, id), func() {
+		namer.mtx.Lock()
+		defer namer.mtx.Unlock()
+		act, ok := namer.active[name]
+		if !ok {
+			panic("must be initialized upon entry")
+		}
+		act.Clear(id)
+	}
+}

daemon/logging/trace/trace_unique_concurrent_task_namer_test.go

@@ -0,0 +1,58 @@
+package trace
+
+import (
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/willf/bitset"
+)
+
+func TestBitsetFeaturesForUniqueConcurrentTaskNamer(t *testing.T) {
+	var b bitset.BitSet
+	require.Equal(t, uint(0), b.Len())
+	require.Equal(t, uint(0), b.Count())
+
+	b.Set(0)
+	require.Equal(t, uint(1), b.Len())
+	require.Equal(t, uint(1), b.Count())
+
+	b.Set(8)
+	require.Equal(t, uint(9), b.Len())
+	require.Equal(t, uint(2), b.Count())
+
+	b.Set(1)
+	require.Equal(t, uint(9), b.Len())
+	require.Equal(t, uint(3), b.Count())
+}
+
+func TestUniqueConcurrentTaskNamer(t *testing.T) {
+	namer := newUniqueTaskNamer(nil)
+
+	var wg sync.WaitGroup
+
+	const N = 8128
+	const Q = 23
+
+	var fails uint32
+	var m sync.Map
+	wg.Add(N)
+	for i := 0; i < N; i++ {
+		go func(i int) {
+			defer wg.Done()
+			name := fmt.Sprintf("%d", i/Q)
+			uniqueName, done := namer.UniqueConcurrentTaskName(name)
+			act, _ := m.LoadOrStore(uniqueName, i)
+			if act.(int) != i {
+				atomic.AddUint32(&fails, 1)
+			}
+			m.Delete(uniqueName)
+			done()
+		}(i)
+	}
+	wg.Wait()
+
+	require.Equal(t, uint32(0), fails)
+}

daemon/main.go

@@ -1,6 +1,8 @@
 package daemon
 
 import (
+	"context"
+
 	"github.com/zrepl/zrepl/cli"
 	"github.com/zrepl/zrepl/logger"
 )
@@ -10,7 +12,7 @@ type Logger = logger.Logger
 var DaemonCmd = &cli.Subcommand{
 	Use:   "daemon",
 	Short: "run the zrepl daemon",
-	Run: func(subcommand *cli.Subcommand, args []string) error {
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
-		return Run(subcommand.Config())
+		return Run(ctx, subcommand.Config())
 	},
 }

daemon/pprof.go

@@ -8,6 +8,9 @@ import (
 	"net"
 	"net/http/pprof"
 
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+	"golang.org/x/net/websocket"
+
 	"github.com/zrepl/zrepl/daemon/job"
 )
 
@@ -64,6 +67,7 @@ outer:
 			mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
 			mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
 			mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
+			mux.Handle("/debug/zrepl/activity-trace", websocket.Handler(trace.ChrometraceClientWebsocketHandler))
 			go func() {
 				err := http.Serve(s.listener, mux)
 				if ctx.Err() != nil {

daemon/prometheus.go

@@ -10,20 +10,24 @@ import (
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/rpc/dataconn/frameconn"
+	"github.com/zrepl/zrepl/util/tcpsock"
 	"github.com/zrepl/zrepl/zfs"
 )
 
 type prometheusJob struct {
-	listen string
+	listen   string
+	freeBind bool
 }
 
 func newPrometheusJobFromConfig(in *config.PrometheusMonitoring) (*prometheusJob, error) {
 	if _, _, err := net.SplitHostPort(in.Listen); err != nil {
 		return nil, err
 	}
-	return &prometheusJob{in.Listen}, nil
+	return &prometheusJob{in.Listen, in.ListenFreeBind}, nil
 }
 
 var prom struct {
@@ -46,6 +50,8 @@ func (j *prometheusJob) Status() *job.Status { return &job.Status{Type: job.Type
 
 func (j *prometheusJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
 
+func (j *prometheusJob) SenderConfig() *endpoint.SenderConfig { return nil }
+
 func (j *prometheusJob) RegisterMetrics(registerer prometheus.Registerer) {}
 
 func (j *prometheusJob) Run(ctx context.Context) {
@@ -60,7 +66,7 @@ func (j *prometheusJob) Run(ctx context.Context) {
 
 	log := job.GetLogger(ctx)
 
-	l, err := net.Listen("tcp", j.listen)
+	l, err := tcpsock.Listen(j.listen, j.freeBind)
 	if err != nil {
 		log.WithError(err).Error("cannot listen")
 		return
@@ -81,16 +87,19 @@ func (j *prometheusJob) Run(ctx context.Context) {
 }
 
 type prometheusJobOutlet struct {
-	jobName string
 }
 
 var _ logger.Outlet = prometheusJobOutlet{}
 
-func newPrometheusLogOutlet(jobName string) prometheusJobOutlet {
+func newPrometheusLogOutlet() prometheusJobOutlet {
-	return prometheusJobOutlet{jobName}
+	return prometheusJobOutlet{}
 }
 
 func (o prometheusJobOutlet) WriteEntry(entry logger.Entry) error {
-	prom.taskLogEntries.WithLabelValues(o.jobName, entry.Level.String()).Inc()
+	jobFieldVal, ok := entry.Fields[logging.JobField].(string)
+	if !ok {
+		jobFieldVal = "_nojobid"
+	}
+	prom.taskLogEntries.WithLabelValues(jobFieldVal, entry.Level.String()).Inc()
 	return nil
 }

daemon/pruner/pruner.go

@@ -12,19 +12,20 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/pruning"
 	"github.com/zrepl/zrepl/replication/logic/pdu"
 	"github.com/zrepl/zrepl/util/envconst"
 )
 
-// Try to keep it compatible with gitub.com/zrepl/zrepl/endpoint.Endpoint
+// Try to keep it compatible with github.com/zrepl/zrepl/endpoint.Endpoint
 type History interface {
 	ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error)
 	ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
 }
 
-// Try to keep it compatible with gitub.com/zrepl/zrepl/endpoint.Endpoint
+// Try to keep it compatible with github.com/zrepl/zrepl/endpoint.Endpoint
 type Target interface {
 	ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
 	ListFilesystemVersions(ctx context.Context, req *pdu.ListFilesystemVersionsReq) (*pdu.ListFilesystemVersionsRes, error)
@@ -35,17 +36,13 @@ type Logger = logger.Logger
 
 type contextKey int
 
-const contextKeyLogger contextKey = 0
+const (
-
+	contextKeyPruneSide contextKey = 1 + iota
-func WithLogger(ctx context.Context, log Logger) context.Context {
+)
-	return context.WithValue(ctx, contextKeyLogger, log)
-}
 
 func GetLogger(ctx context.Context) Logger {
-	if l, ok := ctx.Value(contextKeyLogger).(Logger); ok {
+	pruneSide := ctx.Value(contextKeyPruneSide).(string)
-		return l
+	return logging.GetLogger(ctx, logging.SubsysPruning).WithField("prune_side", pruneSide)
-	}
-	return logger.NewNullLogger()
 }
 
 type args struct {
@@ -138,7 +135,7 @@ func NewPrunerFactory(in config.PruningSenderReceiver, promPruneSecs *prometheus
 func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, receiver History) *Pruner {
 	p := &Pruner{
 		args: args{
-			WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "sender")),
+			context.WithValue(ctx, contextKeyPruneSide, "sender"),
 			target,
 			receiver,
 			f.senderRules,
@@ -154,7 +151,7 @@ func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, re
 func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target, receiver History) *Pruner {
 	p := &Pruner{
 		args: args{
-			WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "receiver")),
+			context.WithValue(ctx, contextKeyPruneSide, "receiver"),
 			target,
 			receiver,
 			f.receiverRules,
@@ -170,7 +167,7 @@ func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target,
 func (f *LocalPrunerFactory) BuildLocalPruner(ctx context.Context, target Target, receiver History) *Pruner {
 	p := &Pruner{
 		args: args{
-			ctx,
+			context.WithValue(ctx, contextKeyPruneSide, "local"),
 			target,
 			receiver,
 			f.keepRules,
@@ -412,9 +409,6 @@ tfss_loop:
 
 		rcReq := &pdu.ReplicationCursorReq{
 			Filesystem: tfs.Path,
-			Op: &pdu.ReplicationCursorReq_Get{
-				Get: &pdu.ReplicationCursorReq_GetOp{},
-			},
 		}
 		rc, err := receiver.ReplicationCursor(ctx, rcReq)
 		if err != nil {

daemon/snapper/snapper.go

@@ -8,11 +8,14 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/filters"
 	"github.com/zrepl/zrepl/daemon/hooks"
+	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/util/envconst"
 	"github.com/zrepl/zrepl/zfs"
 )
 
@@ -44,7 +47,6 @@ type snapProgress struct {
 
 type args struct {
 	ctx            context.Context
-	log            Logger
 	prefix         string
 	interval       time.Duration
 	fsf            *filters.DatasetMapFilter
@@ -101,23 +103,10 @@ func (s State) sf() state {
 type updater func(u func(*Snapper)) State
 type state func(a args, u updater) state
 
-type contextKey int
-
-const (
-	contextKeyLog contextKey = 0
-)
-
 type Logger = logger.Logger
 
-func WithLogger(ctx context.Context, log Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLog, log)
-}
-
 func getLogger(ctx context.Context) Logger {
-	if log, ok := ctx.Value(contextKeyLog).(Logger); ok {
+	return logging.GetLogger(ctx, logging.SubsysSnapshot)
-		return log
-	}
-	return logger.NewNullLogger()
 }
 
 func PeriodicFromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in *config.SnapshottingPeriodic) (*Snapper, error) {
@@ -145,13 +134,12 @@ func PeriodicFromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in *con
 }
 
 func (s *Snapper) Run(ctx context.Context, snapshotsTaken chan<- struct{}) {
-
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
 	getLogger(ctx).Debug("start")
 	defer getLogger(ctx).Debug("stop")
 
 	s.args.snapshotsTaken = snapshotsTaken
 	s.args.ctx = ctx
-	s.args.log = getLogger(ctx)
 	s.args.dryRun = false // for future expansion
 
 	u := func(u func(*Snapper)) State {
@@ -189,7 +177,7 @@ func onErr(err error, u updater) state {
 		case Snapshotting:
 			s.state = ErrorWait
 		}
-		s.args.log.WithError(err).WithField("pre_state", preState).WithField("post_state", s.state).Error("snapshotting error")
+		getLogger(s.args.ctx).WithError(err).WithField("pre_state", preState).WithField("post_state", s.state).Error("snapshotting error")
 	}).sf()
 }
 
@@ -208,7 +196,7 @@ func syncUp(a args, u updater) state {
 	if err != nil {
 		return onErr(err, u)
 	}
-	syncPoint, err := findSyncPoint(a.log, fss, a.prefix, a.interval)
+	syncPoint, err := findSyncPoint(a.ctx, fss, a.prefix, a.interval)
 	if err != nil {
 		return onErr(err, u)
 	}
@@ -265,18 +253,18 @@ func snapshot(a args, u updater) state {
 		suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
 		snapname := fmt.Sprintf("%s%s", a.prefix, suffix)
 
-		l := a.log.
+		ctx := logging.WithInjectedField(a.ctx, "fs", fs.ToString())
-			WithField("fs", fs.ToString()).
+		ctx = logging.WithInjectedField(ctx, "snap", snapname)
-			WithField("snap", snapname)
 
 		hookEnvExtra := hooks.Env{
 			hooks.EnvFS:       fs.ToString(),
 			hooks.EnvSnapshot: snapname,
 		}
 
-		jobCallback := hooks.NewCallbackHookForFilesystem("snapshot", fs, func(_ context.Context) (err error) {
+		jobCallback := hooks.NewCallbackHookForFilesystem("snapshot", fs, func(ctx context.Context) (err error) {
+			l := getLogger(ctx)
 			l.Debug("create snapshot")
-			err = zfs.ZFSSnapshot(fs, snapname, false) // TODO propagagte context to ZFSSnapshot
+			err = zfs.ZFSSnapshot(ctx, fs, snapname, false) // TODO propagate context to ZFSSnapshot
 			if err != nil {
 				l.WithError(err).Error("cannot create snapshot")
 			}
@@ -289,7 +277,7 @@ func snapshot(a args, u updater) state {
 		{
 			filteredHooks, err := a.hooks.CopyFilteredForFilesystem(fs)
 			if err != nil {
-				l.WithError(err).Error("unexpected filter error")
+				getLogger(ctx).WithError(err).Error("unexpected filter error")
 				fsHadErr = true
 				goto updateFSState
 			}
@@ -302,7 +290,7 @@ func snapshot(a args, u updater) state {
 			plan, planErr = hooks.NewPlan(&filteredHooks, hooks.PhaseSnapshot, jobCallback, hookEnvExtra)
 			if planErr != nil {
 				fsHadErr = true
-				l.WithError(planErr).Error("cannot create job hook plan")
+				getLogger(ctx).WithError(planErr).Error("cannot create job hook plan")
 				goto updateFSState
 			}
 		}
@@ -313,15 +301,14 @@ func snapshot(a args, u updater) state {
 			progress.state = SnapStarted
 		})
 		{
-			l := hooks.GetLogger(a.ctx).WithField("fs", fs.ToString()).WithField("snap", snapname)
+			getLogger(ctx).WithField("report", plan.Report().String()).Debug("begin run job plan")
-			l.WithField("report", plan.Report().String()).Debug("begin run job plan")
+			plan.Run(ctx, a.dryRun)
-			plan.Run(hooks.WithLogger(a.ctx, l), a.dryRun)
 			planReport = plan.Report()
 			fsHadErr = planReport.HadError() // not just fatal errors
 			if fsHadErr {
-				l.WithField("report", planReport.String()).Error("end run job plan with error")
+				getLogger(ctx).WithField("report", planReport.String()).Error("end run job plan with error")
 			} else {
-				l.WithField("report", planReport.String()).Info("end run job plan successful")
+				getLogger(ctx).WithField("report", planReport.String()).Info("end run job plan successful")
 			}
 		}
 
@@ -341,7 +328,7 @@ func snapshot(a args, u updater) state {
 	case a.snapshotsTaken <- struct{}{}:
 	default:
 		if a.snapshotsTaken != nil {
-			a.log.Warn("callback channel is full, discarding snapshot update event")
+			getLogger(a.ctx).Warn("callback channel is full, discarding snapshot update event")
 		}
 	}
 
@@ -354,7 +341,7 @@ func snapshot(a args, u updater) state {
 					break
 				}
 			}
-			a.log.WithField("hook", h.String()).WithField("hook_number", hookIdx+1).Warn("hook did not match any snapshotted filesystems")
+			getLogger(a.ctx).WithField("hook", h.String()).WithField("hook_number", hookIdx+1).Warn("hook did not match any snapshotted filesystems")
 		}
 	}
 
@@ -375,7 +362,7 @@ func wait(a args, u updater) state {
 		lastTick := snapper.lastInvocation
 		snapper.sleepUntil = lastTick.Add(a.interval)
 		sleepUntil = snapper.sleepUntil
-		log := a.log.WithField("sleep_until", sleepUntil).WithField("duration", a.interval)
+		log := getLogger(a.ctx).WithField("sleep_until", sleepUntil).WithField("duration", a.interval)
 		logFunc := log.Debug
 		if snapper.state == ErrorWait || snapper.state == SyncUpErrWait {
 			logFunc = log.Error
@@ -400,9 +387,19 @@ func listFSes(ctx context.Context, mf *filters.DatasetMapFilter) (fss []*zfs.Dat
 	return zfs.ZFSListMapping(ctx, mf)
 }
 
-func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval time.Duration) (syncPoint time.Time, err error) {
+var syncUpWarnNoSnapshotUntilSyncupMinDuration = envconst.Duration("ZREPL_SNAPPER_SYNCUP_WARN_MIN_DURATION", 1*time.Second)
+
+// see docs/snapshotting.rst
+func findSyncPoint(ctx context.Context, fss []*zfs.DatasetPath, prefix string, interval time.Duration) (syncPoint time.Time, err error) {
+
+	const (
+		prioHasVersions int = iota
+		prioNoVersions
+	)
+
 	type snapTime struct {
 		ds   *zfs.DatasetPath
+		prio int // lower is higher
 		time time.Time
 	}
 
@@ -411,55 +408,92 @@ func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval t
 	}
 
 	snaptimes := make([]snapTime, 0, len(fss))
+	hardErrs := 0
 
 	now := time.Now()
 
-	log.Debug("examine filesystem state")
+	getLogger(ctx).Debug("examine filesystem state to find sync point")
 	for _, d := range fss {
-
+		ctx := logging.WithInjectedField(ctx, "fs", d.ToString())
-		l := log.WithField("fs", d.ToString())
+		syncPoint, err := findSyncPointFSNextOptimalSnapshotTime(ctx, now, interval, prefix, d)
-
+		if err == findSyncPointFSNoFilesystemVersionsErr {
-		fsvs, err := zfs.ZFSListFilesystemVersions(d, filters.NewTypedPrefixFilter(prefix, zfs.Snapshot))
+			snaptimes = append(snaptimes, snapTime{
-		if err != nil {
+				ds:   d,
-			l.WithError(err).Error("cannot list filesystem versions")
+				prio: prioNoVersions,
-			continue
+				time: now,
-		}
+			})
-		if len(fsvs) <= 0 {
+		} else if err != nil {
-			l.WithField("prefix", prefix).Debug("no filesystem versions with prefix")
+			hardErrs++
-			continue
+			getLogger(ctx).WithError(err).Error("cannot determine optimal sync point for this filesystem")
+		} else {
+			getLogger(ctx).WithField("syncPoint", syncPoint).Debug("found optimal sync point for this filesystem")
+			snaptimes = append(snaptimes, snapTime{
+				ds:   d,
+				prio: prioHasVersions,
+				time: syncPoint,
+			})
 		}
+	}
 
-		// Sort versions by creation
+	if hardErrs == len(fss) {
-		sort.SliceStable(fsvs, func(i, j int) bool {
+		return time.Time{}, fmt.Errorf("hard errors in determining sync point for every matching filesystem")
-			return fsvs[i].CreateTXG < fsvs[j].CreateTXG
-		})
-
-		latest := fsvs[len(fsvs)-1]
-		l.WithField("creation", latest.Creation).
-			Debug("found latest snapshot")
-
-		since := now.Sub(latest.Creation)
-		if since < 0 {
-			l.WithField("snapshot", latest.Name).
-				WithField("creation", latest.Creation).
-				Error("snapshot is from the future")
-			continue
-		}
-		next := now
-		if since < interval {
-			next = latest.Creation.Add(interval)
-		}
-		snaptimes = append(snaptimes, snapTime{d, next})
 	}
 
 	if len(snaptimes) == 0 {
-		snaptimes = append(snaptimes, snapTime{nil, now})
+		panic("implementation error: loop must either inc hardErrs or add result to snaptimes")
 	}
 
+	// sort ascending by (prio,time)
+	// => those filesystems with versions win over those without any
 	sort.Slice(snaptimes, func(i, j int) bool {
-		return snaptimes[i].time.Before(snaptimes[j].time)
+		if snaptimes[i].prio == snaptimes[j].prio {
+			return snaptimes[i].time.Before(snaptimes[j].time)
+		}
+		return snaptimes[i].prio < snaptimes[j].prio
 	})
 
+	winnerSyncPoint := snaptimes[0].time
+	l := getLogger(ctx).WithField("syncPoint", winnerSyncPoint.String())
+	l.Info("determined sync point")
+	if winnerSyncPoint.Sub(now) > syncUpWarnNoSnapshotUntilSyncupMinDuration {
+		for _, st := range snaptimes {
+			if st.prio == prioNoVersions {
+				l.WithField("fs", st.ds.ToString()).Warn("filesystem will not be snapshotted until sync point")
+			}
+		}
+	}
+
 	return snaptimes[0].time, nil
 
 }
+
+var findSyncPointFSNoFilesystemVersionsErr = fmt.Errorf("no filesystem versions")
+
+func findSyncPointFSNextOptimalSnapshotTime(ctx context.Context, now time.Time, interval time.Duration, prefix string, d *zfs.DatasetPath) (time.Time, error) {
+
+	fsvs, err := zfs.ZFSListFilesystemVersions(ctx, d, zfs.ListFilesystemVersionsOptions{
+		Types:           zfs.Snapshots,
+		ShortnamePrefix: prefix,
+	})
+	if err != nil {
+		return time.Time{}, errors.Wrap(err, "list filesystem versions")
+	}
+	if len(fsvs) <= 0 {
+		return time.Time{}, findSyncPointFSNoFilesystemVersionsErr
+	}
+
+	// Sort versions by creation
+	sort.SliceStable(fsvs, func(i, j int) bool {
+		return fsvs[i].CreateTXG < fsvs[j].CreateTXG
+	})
+
+	latest := fsvs[len(fsvs)-1]
+	getLogger(ctx).WithField("creation", latest.Creation).Debug("found latest snapshot")
+
+	since := now.Sub(latest.Creation)
+	if since < 0 {
+		return time.Time{}, fmt.Errorf("snapshot %q is from the future: creation=%q now=%q", latest.ToAbsPath(d), latest.Creation, now)
+	}
+
+	return latest.Creation.Add(interval), nil
+}

docs/changelog.rst

@@ -27,6 +27,45 @@ We use the following annotations for classifying changes:
 * |bugfix| Change that fixes a bug, no regressions or incompatibilities expected.
 * |docs| Change to the documentation.
 
+0.3
+---
+
+This is a big one! Headlining features:
+
+* **Resumable Send & Recv Support**
+  No knobs required, automatically used where supported.
+* **Hold-Protected Send & Recv**
+  Automatic ZFS holds to ensure that we can always resume a replication step.
+* **Encrypted Send & Recv Support** for OpenZFS native encryption.
+  :ref:`Configurable <job-send-options>` at the job level, i.e., for all filesystems a job is responsible for.
+* **Receive-side hold on last received dataset**
+  The counterpart to the replication cursor bookmark on the send-side.
+  Ensures that incremental replication will always be possible between a sender and receiver.
+
+Actual changelog:
+
+* |break_config| **more restrictive job names than in prior zrepl versions**
+  Starting with this version, job names are going to be embedded into ZFS holds and bookmark names.
+
+* |break| |mig| replication cursor representation changed
+
+  * zrepl now manages the :ref:`replication cursor bookmark <replication-cursor-and-last-received-hold>` per job-filesystem tuple instead of a single replication cursor per filesystem.
+    In the future, this will permit multiple sending jobs to send from the same filesystems.
+  * ZFS does not allow bookmark renaming, thus we cannot migrate the old replication cursors.
+  * zrepl 0.3 will automatically create cursors in the new format for new replications, and warn if it still finds ones in the old format.
+  * Run ``zrepl migrate replication-cursor:v1-v2`` to safely destroy old-format cursors.
+    The migration will ensure that only those old-format cursors are destroyed that have been superseeded by new-format cursors.
+
+* |bugfix| missing logger context vars in control connection handlers
+* |bugfix| improved error messages on ``zfs send`` errors
+* |feature| New option ``listen_freebind`` (tcp, tls, prometheus listener)
+* |bugfix| |docs| snapshotting: clarify sync-up behavior and warn about filesystems
+  that will not be snapshotted until the sync-up phase is over
+* |docs| Document new replication features in the :ref:`config overview <overview-how-replication-works>` and :repomasterlink:`replication/design.md`.
+* |feature| documented subcommand to generate ``bash`` and ``zsh`` completions
+* **[MAINTAINER NOTICE]** New platform tests in this version, please make sure you run them for your distro!
+* **[MAINTAINER NOTICE]** Please add the shell completions to the zrepl packages.
+
 0.2.1
 -----
 
@@ -68,7 +107,7 @@ We use the following annotations for classifying changes:
    |  You can support maintenance and feature development through one of the following services:
    |  |Donate via Patreon| |Donate via Liberapay| |Donate via PayPal|
    |  Note that PayPal processing fees are relatively high for small donations.
-   |  For SEPA wire transfer and **commerical support**, please `contact Christian directly <https://cschwarz.com>`_.
+   |  For SEPA wire transfer and **commercial support**, please `contact Christian directly <https://cschwarz.com>`_.
 
 
 0.1.1

docs/configuration.rst

@@ -11,6 +11,7 @@ Configuration
     configuration/jobs
     configuration/transports
     configuration/filter_syntax
+    configuration/sendrecvoptions
     configuration/snapshotting
     configuration/prune
     configuration/logging

docs/configuration/filter_syntax.rst

@@ -5,7 +5,7 @@
 Filter Syntax
 =============
 
-For :ref:`source jobs <job-source>` and :ref:`push jobs <job-push>`, a filesystem filter must be defined (field ``filesystems``).
+For :ref:`source<job-source>`, :ref:`push<job-push>` and :ref:`snap<job-snap>` jobs, a filesystem filter must be defined (field ``filesystems``).
 A filter takes a filesystem path (in the ZFS filesystem hierarchy) as parameter and returns ``true`` (pass) or ``false`` (block).
 
 A filter is specified as a **YAML dictionary** with patterns as keys and booleans as values.

docs/configuration/jobs.rst

@@ -24,6 +24,8 @@ Job Type ``push``
       - |connect-transport|
     * - ``filesystems``
       - |filter-spec| for filesystems to be snapshotted and pushed to the sink
+    * - ``send``
+      - |send-options| 
     * - ``snapshotting``
       - |snapshotting-spec|
     * - ``pruning``
@@ -101,6 +103,8 @@ Job Type ``source``
       - |serve-transport|
     * - ``filesystems``
       - |filter-spec| for filesystems to be snapshotted and exposed to connecting clients
+    * - ``send``
+      - |send-options| 
     * - ``snapshotting``
       - |snapshotting-spec|
 

docs/configuration/logging.rst

@@ -202,7 +202,7 @@ The latter is particularly useful in combination with log aggregation services.
 .. WARNING::
 
     zrepl drops log messages to the TCP outlet if the underlying connection is not fast enough.
-    Note that TCP buffering in the kernel must first run full becfore messages are dropped.
+    Note that TCP buffering in the kernel must first run full before messages are dropped.
 
     Make sure to always configure a ``stdout`` outlet as the special error outlet to be informed about problems
     with the TCP outlet (see :ref:`above <logging-error-outlet>` ).

docs/configuration/monitoring.rst

@@ -14,7 +14,8 @@ Prometheus & Grafana
 
 zrepl can expose `Prometheus metrics <https://prometheus.io/docs/instrumenting/exposition_formats/>`_ via HTTP.
 The ``listen`` attribute is a `net.Listen <https://golang.org/pkg/net/#Listen>`_  string for tcp, e.g. ``:9091`` or ``127.0.0.1:9091``.
-The Prometheues monitoring job appears in the ``zrepl control`` job list and may be specified **at most once**.
+The ``listen_freebind`` attribute is :ref:`explained here <listen-freebind-explanation>`.
+The Prometheus monitoring job appears in the ``zrepl control`` job list and may be specified **at most once**.
 
 zrepl also ships with an importable `Grafana <https://grafana.com>`_ dashboard that consumes the Prometheus metrics:
 see :repomasterlink:`dist/grafana`.
@@ -30,6 +31,7 @@ The dashboard also contains some advice on which metrics are important to monito
       monitoring:
         - type: prometheus
           listen: ':9091'
+          listen_freebind: true # optional, default false
 
 
 

docs/configuration/overview.rst

@@ -26,9 +26,9 @@ Config File Structure
      type: push
    - ...
 
-zrepl is confgured using a single YAML configuration file with two main sections: ``global`` and ``jobs``.
+zrepl is configured using a single YAML configuration file with two main sections: ``global`` and ``jobs``.
 The ``global`` section is filled with sensible defaults and is covered later in this chapter.
-The ``jobs`` section is a list of jobs which we are goind to explain now.
+The ``jobs`` section is a list of jobs which we are going to explain now.
 
 .. _job-overview:
 
@@ -41,7 +41,7 @@ Jobs are identified by their ``name``, both in log files and the ``zrepl status`
 
 Replication always happens between a pair of jobs: one is the **active side**, and one the **passive side**.
 The active side connects to the passive side using a :ref:`transport <transport>` and starts executing the replication logic.
-The passive side responds to requests from the active side after checking its persmissions.
+The passive side responds to requests from the active side after checking its permissions.
 
 The following table shows how different job types can be combined to achieve **both push and pull mode setups**.
 Note that snapshot-creation denoted by "(snap)" is orthogonal to whether a job is active or passive.
@@ -90,6 +90,8 @@ It uses the client identity for access control:
 .. TIP::
    The implementation of the ``sink`` job requires that the connecting client identities be a valid ZFS filesystem name components.
 
+.. _overview-how-replication-works:
+
 How Replication Works
 ---------------------
 
@@ -107,9 +109,8 @@ The following steps take place during replication and can be monitored using the
     * Per filesystem, compute a diff between sender and receiver snapshots
     * Build a list of replication steps
 
-      * If possible, use incremental sends (``zfs send -i``)
+      * If possible, use incremental and resumable sends (``zfs send -i``)
       * Otherwise, use full send of most recent snapshot on sender
-      * Give up on filesystems that cannot be replicated without data loss
 
   * Retry on errors that are likely temporary (i.e. network failures).
   * Give up on filesystems where a permanent error was received over RPC.
@@ -119,22 +120,28 @@ The following steps take place during replication and can be monitored using the
   * Perform replication steps in the following order:
     Among all filesystems with pending replication steps, pick the filesystem whose next replication step's snapshot is the oldest.
   * Create placeholder filesystems on the receiving side to mirror the dataset paths on the sender to ``root_fs/${client_identity}``.
-  * After a successful replication step, update the replication cursor bookmark (see below).
+  * Acquire send-side *step-holds* on the step's `from` and `to` snapshots.
+  * Perform the replication step.
+  * Move the **replication cursor** bookmark on the sending side (see below).
+  * Move the **last-received-hold** on the receiving side (see below).
+  * Release the send-side step-holds.
    
 The idea behind the execution order of replication steps is that if the sender snapshots all filesystems simultaneously at fixed intervals, the receiver will have all filesystems snapshotted at time ``T1`` before the first snapshot at ``T2 = T1 + $interval`` is replicated.
 
-.. _replication-cursor-bookmark:
+.. _replication-cursor-and-last-received-hold:
 
-The **replication cursor bookmark** ``#zrepl_replication_cursor`` is kept per filesystem on the sending side of a replication setup:
+**Replication cursor** bookmark and **last-received-hold** are managed by zrepl to ensure that future replications can always be done incrementally:
-It is a bookmark of the most recent successfully replicated snapshot to the receiving side.
+the replication cursor is a send-side bookmark of the most recent successfully replicated snapshot,
-It is is used by the :ref:`not_replicated <prune-keep-not-replicated>` keep rule to identify all snapshots that have not yet been replicated to the receiving side.
+and the last-received-hold is a hold of that snapshot on the receiving side.
-Regardless of whether that keep rule is used, the bookmark ensures that replication can always continue incrementally.
+The replication cursor has the format ``#zrepl_CUSOR_G_<GUID>_J_<JOBNAME>``.
-Note that there is only one cursor bookmark per filesystem, which prohibits multiple jobs to replicate the same filesystem (:ref:`see below<jobs-multiple-jobs>`).
+The last-received-hold tag has the format ``#zrepl_last_received_J_<JOBNAME>``.
+Encoding the job name in the names ensures that multiple sending jobs can replicate the same filesystem to different receivers without interference.
+The ``zrepl holds list`` provides a listing of all bookmarks and holds managed by zrepl.
 
 .. _replication-placeholder-property:
 
 **Placeholder filesystems** on the receiving side are regular ZFS filesystems with the placeholder property ``zrepl:placeholder=on``.
-Placeholders allow the receiving side to mirror the sender's ZFS dataset hierachy without replicating every filesystem at every intermediary dataset path component.
+Placeholders allow the receiving side to mirror the sender's ZFS dataset hierarchy without replicating every filesystem at every intermediary dataset path component.
 Consider the following example: ``S/H/J`` shall be replicated to ``R/sink/job/S/H/J``, but neither ``S/H`` nor ``S`` shall be replicated.
 ZFS requires the existence of ``R/sink/job/S`` and ``R/sink/job/S/H`` in order to receive into ``R/sink/job/S/H/J``.
 Thus, zrepl creates the parent filesystems as placeholders on the receiving side.
@@ -144,9 +151,13 @@ The ``zrepl test placeholder`` command can be used to check whether a filesystem
 .. ATTENTION::
 
     Currently, zrepl does not replicate filesystem properties.
-    Whe receiving a filesystem, it is never mounted (`-u` flag)  and `mountpoint=none` is set.
+    When receiving a filesystem, it is never mounted (`-u` flag)  and `mountpoint=none` is set.
     This is temporary and being worked on :issue:`24`.
 
+.. NOTE::
+
+    More details can be found in the design document :repomasterlink:`replication/design.md`.
+
 
 .. _jobs-multiple-jobs:
 
@@ -170,8 +181,8 @@ No Overlapping
 
 Jobs run independently of each other.
 If two jobs match the same filesystem with their ``filesystems`` filter, they will operate on that filesystem independently and potentially in parallel.
-For example, if job A prunes snapshots that job B is planning to replicate, the replication will fail because B asssumed the snapshot to still be present.
+For example, if job A prunes snapshots that job B is planning to replicate, the replication will fail because B assumed the snapshot to still be present.
-More subtle race conditions can occur with the :ref:`replication cursor bookmark <replication-cursor-bookmark>`, which currently only exists once per filesystem.
+However, the next replication attempt will re-examine the situation from scratch and should work.
 
 N push jobs to 1 sink
 ~~~~~~~~~~~~~~~~~~~~~
@@ -187,5 +198,5 @@ Multiple pull jobs pulling from the same source have potential for race conditio
 each pull job prunes the source side independently, causing replication-prune and prune-prune races.
 
 There is currently no way for a pull job to filter which snapshots it should attempt to replicate.
-Thus, it is not possibe to just manually assert that the prune rules of all pull jobs are disjoint to avoid replication-prune and prune-prune races.
+Thus, it is not possible to just manually assert that the prune rules of all pull jobs are disjoint to avoid replication-prune and prune-prune races.
 

docs/configuration/prune.rst

@@ -4,7 +4,7 @@ Pruning Policies
 ================
 
 In zrepl, *pruning* means *destroying snapshots*.
-Pruning must happen on both sides of a replication or the systems would inevitable run out of disk space at some point.
+Pruning must happen on both sides of a replication or the systems would inevitably run out of disk space at some point.
 
 Typically, the requirements to temporal resolution and maximum retention time differ per side.
 For example, when using zrepl to back up a busy database server, you will want high temporal resolution (snapshots every 10 min) for the last 24h in case of administrative disasters, but cannot afford to store them for much longer because you might have high turnover volume in the database.
@@ -66,7 +66,7 @@ Policy ``not_replicated``
 
 ``not_replicated`` keeps all snapshots that have not been replicated to the receiving side.
 It only makes sense to specify this rule on a sender (source or push job).
-The state required to evaluate this rule is stored in the :ref:`replication cursor bookmark <replication-cursor-bookmark>` on the sending side.
+The state required to evaluate this rule is stored in the :ref:`replication cursor bookmark <replication-cursor-and-last-received-hold>` on the sending side.
 
 .. _prune-keep-retention-grid:
 
@@ -154,7 +154,7 @@ Policy ``regex``
            negate: true
            regex: "^zrepl_.*"
 
-``regex`` keeps all snapshots whose names are matched by the regular expressionin ``regex``.
+``regex`` keeps all snapshots whose names are matched by the regular expression in ``regex``.
 Like all other regular expression fields in prune policies, zrepl uses Go's `regexp.Regexp <https://golang.org/pkg/regexp/#Compile>`_ Perl-compatible regular expressions (`Syntax <https://golang.org/pkg/regexp/syntax>`_).
 The optional `negate` boolean field inverts the semantics: Use it if you want to keep all snapshots that *do not* match the given regex.
 

docs/configuration/sendrecvoptions.rst

@@ -0,0 +1,45 @@
+.. include:: ../global.rst.inc
+
+
+Send & Recv Options
+===================
+
+.. _job-send-options:
+
+Send Options
+~~~~~~~~~~~~
+
+::
+   
+   jobs:
+   - type: push
+     filesystems: ...
+     send:
+       encrypted: true
+     ...
+
+:ref:`Source<job-source>` and :ref:`push<job-push>` jobs have an optional ``send`` configuration section.
+
+``encryption`` option
+---------------------
+
+The ``encryption`` variable controls whether the matched filesystems are sent as `OpenZFS native encryption <http://open-zfs.org/wiki/ZFS-Native_Encryption>`_ raw sends.
+More specifically, if ``encryption=true``, zrepl
+
+* checks for any of the filesystems matched by ``filesystems`` whether the ZFS ``encryption`` property indicates that the filesystem is actually encrypted with ZFS native encryption and
+* invokes the ``zfs send`` subcommand with the ``-w`` option (raw sends) and
+* expects the receiving side to support OpenZFS native encryption (recv will fail otherwise)
+
+Filesystems matched by ``filesystems`` that are not encrypted are not sent and will cause error log messages.
+
+If ``encryption=false``, zrepl expects that filesystems matching ``filesystems`` are not encrypted or have loaded encryption keys.
+
+.. _job-recv-options:
+
+Recv Options
+~~~~~~~~~~~~
+
+:ref:`Sink<job-sink>` and :ref:`pull<job-pull>` jobs have an optional ``recv`` configuration section.
+However, there are currently no variables to configure there.
+
+

docs/configuration/snapshotting.rst

@@ -9,8 +9,15 @@ The ``push``, ``source`` and ``snap`` jobs can automatically take periodic snaps
 The snapshot names are composed of a user-defined prefix followed by a UTC date formatted like ``20060102_150405_000``.
 We use UTC because it will avoid name conflicts when switching time zones or between summer and winter time.
 
+When a job is started, the snapshotter attempts to get the snapshotting rhythms of the matched ``filesystems`` in sync because snapshotting all filesystems at the same time results in a more consistent backup.
+To find that sync point, the most recent snapshot, made by the snapshotter, in any of the matched ``filesystems`` is used.
+A filesystem that does not have snapshots by the snapshotter has lower priority than filesystem that do, and thus might not be snapshotted (and replicated) until it is snapshotted at the next sync point.
+
 For ``push`` jobs, replication is automatically triggered after all filesystems have been snapshotted.
 
+Note that the ``zrepl signal wakeup JOB`` subcommand does not trigger snapshotting.
+
+
 ::
 
     jobs:

docs/configuration/transports.rst

@@ -50,12 +50,18 @@ Serve
       serve:
         type: tcp
         listen: ":8888"
+        listen_freebind: true # optional, default false
         clients: {
           "192.168.122.123" : "mysql01"
           "192.168.122.123" : "mx01"
         }
       ...
 
+.. _listen-freebind-explanation:
+
+``listen_freebind`` controls whether the socket is allowed to bind to non-local or unconfigured IP addresses (Linux ``IP_FREEBIND`` , FreeBSD ``IP_BINDANY``).
+Enable this option if you want to ``listen`` on a specific IP address that might not yet be configured when the zrepl daemon starts.
+
 Connect
 ~~~~~~~
 
@@ -101,6 +107,7 @@ Serve
         serve:
           type: tls
           listen: ":8888"
+          listen_freebind: true # optional, default false
           ca:   /etc/zrepl/ca.crt
           cert: /etc/zrepl/prod.fullchain
           key:  /etc/zrepl/prod.key
@@ -110,6 +117,7 @@ Serve
 
 The ``ca`` field specified the certificate authority used to validate client certificates.
 The ``client_cns`` list specifies a list of accepted client common names (which are also the client identities for this transport).
+The ``listen_freebind`` field is :ref:`explained here <listen-freebind-explanation>`.
 
 Connect
 ~~~~~~~

docs/global.rst.inc

@@ -21,6 +21,8 @@
 
 .. |serve-transport| replace:: :ref:`serve specification<transport>`
 .. |connect-transport| replace:: :ref:`connect specification<transport>`
+.. |send-options| replace:: :ref:`send options<job-send-options>`, e.g. for encrypted sends
+.. |recv-options| replace:: :ref:`recv options<job-recv-options>`
 .. |snapshotting-spec| replace:: :ref:`snapshotting specification <job-snapshotting-spec>`
 .. |pruning-spec| replace:: :ref:`pruning specification <prune>`
 .. |filter-spec| replace:: :ref:`filter specification<pattern-filter>`

docs/index.rst

@@ -57,9 +57,11 @@ Main Features
   * Advanced replication features
 
     * [x] Automatic retries for temporary network errors
-    * [ ] Resumable send & receive
+    * [x] Automatic resumable send & receive
+    * [x] Automatic ZFS holds during send & receive
+    * [x] Automatic bookmark \& hold management for guaranteed incremental send & recv 
+    * [x] Encrypted raw send & receive to untrusted receivers (OpenZFS native encryption)
     * [ ] Compressed send & receive
-    * [ ] Raw encrypted send & receive
 
 * **Automatic snapshot management**
 
@@ -83,6 +85,7 @@ Main Features
 * **Maintainable implementation in Go**
 
   * [x] Cross platform
+  * [x] Dynamic feature checking
   * [x] Type safe & testable code
 
 

docs/installation.rst

@@ -52,6 +52,9 @@ The following list may be incomplete, feel free to submit a PR with an update:
     * - OmniOS
       - ``pkg install zrepl``
       - Available since `r151030 <https://pkg.omniosce.org/r151030/extra/en/search.shtml?token=zrepl&action=Search>`_
+    * - Void Linux
+      - ``xbps-install zrepl``
+      - Available since `a88a2a4 <https://github.com/void-linux/void-packages/commit/a88a2a4d7bf56072dadf61ab56b8424e39155890>`_
     * - Others
       -
       - Use `binary releases`_ or build from source.
@@ -98,9 +101,9 @@ and serves as a reference for build dependencies and procedure:
 
 ::
 
-    git clone https://github.com/zrepl/zrepl.git
+    git clone https://github.com/zrepl/zrepl.git && \
-    cd zrepl
+    cd zrepl && \
-    sudo docker build -t zrepl_build -f build.Dockerfile .
+    sudo docker build -t zrepl_build -f build.Dockerfile . && \
     sudo docker run -it --rm \
         -v "${PWD}:/src" \
         --user "$(id -u):$(id -g)" \
@@ -124,7 +127,7 @@ Either way, all build results are located in the ``artifacts/`` directory.
 
 .. NOTE::
 
-    It is your job to install the apropriate binary in the zrepl users's ``$PATH``, e.g. ``/usr/local/bin/zrepl``.
+    It is your job to install the appropriate binary in the zrepl users's ``$PATH``, e.g. ``/usr/local/bin/zrepl``.
     Otherwise, the examples in the :ref:`tutorial` may need to be adjusted.
 
 What next?

docs/supporters.rst

@@ -6,7 +6,7 @@
 
 zrepl is a spare-time project primarily developed by `Christian Schwarz <https://cschwarz.com>`_.
 You can support maintenance and feature development through one of the services listed above.
-For SEPA wire transfer and **commerical support**, please `contact Christian directly <https://cschwarz.com>`_.
+For SEPA wire transfer and **commercial support**, please `contact Christian directly <https://cschwarz.com>`_.
 
 **Thanks for your support!**
 
@@ -32,7 +32,11 @@ We would like to thank the following people and organizations for supporting zre
 
    <div class="fa fa-code" style="width: 1em;"></div>
 
+* |supporter-gold| `Mateusz Kwiatkowski (runhyve.app) <https://runhyve.app>`_
+* |supporter-std| `Gaelan D'costa <https://www.robot-disco.net>`_
 * |supporter-std| `Tenzin Lhakhang <https://confluence.tenzin.io>`_
+* |supporter-std| `Lapo Luchini <https://github.com/lapo-luchini>`_
+* |supporter-std| `F. Schmid <https://github.com/tdschmidl>`_
 * |supporter-gold| `MNX.io <https://mnx.io>`_
 * |supporter-std| `Marshall Clyburn <https://github.com/mdclyburn>`_
 * |supporter-code| `Ross Williams <https://github.com/overhacked>`_
@@ -40,4 +44,5 @@ We would like to thank the following people and organizations for supporting zre
 * |supporter-code| `Justin Scholz <https://github.com/JMoVS>`_
 * |supporter-code| `InsanePrawn <https://github.com/InsanePrawn>`_
 * |supporter-code| `Ben Woods <https://www.freshports.org/sysutils/zrepl/>`_
+* |supporter-code| `Janis Streib <https://github.com/janisstreib>`_
 * |supporter-code| `Anton Schirg <https://github.com/antonxy>`_

docs/tutorial.rst

@@ -52,9 +52,9 @@ Follow the :ref:`OS-specific installation instructions <installation>` and come
 Generate TLS Certificates
 -------------------------
 
-We use the `TLS client authentication transport <transport-tcp+tlsclientauth>` to protect our data on the wire.
+We use the :ref:`TLS client authentication transport <transport-tcp+tlsclientauth>` to protect our data on the wire.
 To get things going quickly, we skip setting up a CA and generate two self-signed certificates as described :ref:`here <transport-tcp+tlsclientauth-2machineopenssl>`.
-Again, for convenience, We generate the key pairs on our local machine and distribute them using ssh:
+For convenience, we generate the key pairs on our local machine and distribute them using ssh:
 
 .. code-block:: bash
    :emphasize-lines: 6,13
@@ -138,7 +138,7 @@ We define a corresponding **sink job** named ``sink`` in ``/etc/zrepl/zrepl.yml`
 Apply Configuration Changes
 ---------------------------
 
-We use ``zrepl configcheck`` before to catch any configuration errors: no output indicates that everything is fine.
+We use ``zrepl configcheck`` to catch any configuration errors: no output indicates that everything is fine.
 If that is the case, restart the zrepl daemon on **both** ``prod`` and ``backups`` using ``service zrepl restart`` or ``systemctl restart zrepl``.
 
 
@@ -175,4 +175,3 @@ Congratulations, you have a working push backup. Where to go next?
 
 * Read more about :ref:`configuration format, options & job types <configuration_toc>`
 * Configure :ref:`logging <logging>` \& :ref:`monitoring <monitoring>`.
-

docs/usage.rst

@@ -38,6 +38,8 @@ CLI Overview
     * - ``zrepl migrate``
       - | perform on-disk state / ZFS property migrations
         | (see :ref:`changelog <changelog>` for details)
+    * - ``zrepl zfs-abstractions``
+      - list and remove zrepl's abstractions on top of ZFS, e.g. holds and step bookmarks (see :ref:`overview <replication-cursor-and-last-received-hold>` )
 
 .. _usage-zrepl-daemon:
 
@@ -48,7 +50,7 @@ zrepl daemon
 All actual work zrepl does is performed by a daemon process.
 The daemon supports structured :ref:`logging <logging>` and provides :ref:`monitoring endpoints <monitoring>`.
 
-When installating from a package, the package maintainer should have provided an init script / systemd.service file.
+When installing from a package, the package maintainer should have provided an init script / systemd.service file.
 You should thus be able to start zrepl daemon using your init system.
 
 Alternatively, or for running zrepl in the foreground, simply execute ``zrepl daemon``.
@@ -73,6 +75,6 @@ The daemon exits as soon as all jobs have reported shut down.
 Systemd Unit File
 ~~~~~~~~~~~~~~~~~
 
-A systemd service defintion template is available in :repomasterlink:`dist/systemd`.
+A systemd service definition template is available in :repomasterlink:`dist/systemd`.
 Note that some of the options only work on recent versions of systemd.
 Any help & improvements are very welcome, see :issue:`145`.

endpoint/context.go

@@ -3,25 +3,18 @@ package endpoint
 import (
 	"context"
 
+	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
 )
 
 type contextKey int
 
 const (
-	contextKeyLogger contextKey = iota
+	ClientIdentityKey contextKey = iota
-	ClientIdentityKey
 )
 
 type Logger = logger.Logger
 
-func WithLogger(ctx context.Context, log Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLogger, log)
-}
-
 func getLogger(ctx context.Context) Logger {
-	if l, ok := ctx.Value(contextKeyLogger).(Logger); ok {
+	return logging.GetLogger(ctx, logging.SubsysEndpoint)
-		return l
-	}
-	return logger.NewNullLogger()
 }

endpoint/endpoint.go

@@ -2,26 +2,57 @@
 package endpoint
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"path"
+	"sync"
 
 	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/replication/logic/pdu"
+	"github.com/zrepl/zrepl/util/chainedio"
 	"github.com/zrepl/zrepl/util/chainlock"
 	"github.com/zrepl/zrepl/util/envconst"
 	"github.com/zrepl/zrepl/util/semaphore"
 	"github.com/zrepl/zrepl/zfs"
 )
 
+type SenderConfig struct {
+	FSF     zfs.DatasetFilter
+	Encrypt *zfs.NilBool
+	JobID   JobID
+}
+
+func (c *SenderConfig) Validate() error {
+	c.JobID.MustValidate()
+	if err := c.Encrypt.Validate(); err != nil {
+		return errors.Wrap(err, "`Encrypt` field invalid")
+	}
+	if _, err := StepHoldTag(c.JobID); err != nil {
+		return fmt.Errorf("JobID cannot be used for hold tag: %s", err)
+	}
+	return nil
+}
+
 // Sender implements replication.ReplicationEndpoint for a sending side
 type Sender struct {
 	FSFilter zfs.DatasetFilter
+	encrypt  *zfs.NilBool
+	jobId    JobID
 }
 
-func NewSender(fsf zfs.DatasetFilter) *Sender {
+func NewSender(conf SenderConfig) *Sender {
-	return &Sender{FSFilter: fsf}
+	if err := conf.Validate(); err != nil {
+		panic("invalid config" + err.Error())
+	}
+	return &Sender{
+		FSFilter: conf.FSF,
+		encrypt:  conf.Encrypt,
+		jobId:    conf.JobID,
+	}
 }
 
 func (s *Sender) filterCheckFS(fs string) (*zfs.DatasetPath, error) {
@@ -43,16 +74,23 @@ func (s *Sender) filterCheckFS(fs string) (*zfs.DatasetPath, error) {
 }
 
 func (s *Sender) ListFilesystems(ctx context.Context, r *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	fss, err := zfs.ZFSListMapping(ctx, s.FSFilter)
 	if err != nil {
 		return nil, err
 	}
 	rfss := make([]*pdu.Filesystem, len(fss))
 	for i := range fss {
+		encEnabled, err := zfs.ZFSGetEncryptionEnabled(ctx, fss[i].ToString())
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot get filesystem encryption status")
+		}
 		rfss[i] = &pdu.Filesystem{
 			Path: fss[i].ToString(),
-			// FIXME: not supporting ResumeToken yet
+			// ResumeToken does not make sense from Sender
 			IsPlaceholder: false, // sender FSs are never placeholders
+			IsEncrypted:   encEnabled,
 		}
 	}
 	res := &pdu.ListFilesystemRes{Filesystems: rfss}
@@ -60,11 +98,13 @@ func (s *Sender) ListFilesystems(ctx context.Context, r *pdu.ListFilesystemReq)
 }
 
 func (s *Sender) ListFilesystemVersions(ctx context.Context, r *pdu.ListFilesystemVersionsReq) (*pdu.ListFilesystemVersionsRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	lp, err := s.filterCheckFS(r.GetFilesystem())
 	if err != nil {
 		return nil, err
 	}
-	fsvs, err := zfs.ZFSListFilesystemVersions(lp, nil)
+	fsvs, err := zfs.ZFSListFilesystemVersions(ctx, lp, zfs.ListFilesystemVersionsOptions{})
 	if err != nil {
 		return nil, err
 	}
@@ -77,13 +117,176 @@ func (s *Sender) ListFilesystemVersions(ctx context.Context, r *pdu.ListFilesyst
 
 }
 
+func (p *Sender) HintMostRecentCommonAncestor(ctx context.Context, r *pdu.HintMostRecentCommonAncestorReq) (*pdu.HintMostRecentCommonAncestorRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
+	fsp, err := p.filterCheckFS(r.GetFilesystem())
+	if err != nil {
+		return nil, err
+	}
+	fs := fsp.ToString()
+
+	log := getLogger(ctx).WithField("fs", fs).WithField("hinted_most_recent", fmt.Sprintf("%#v", r.GetSenderVersion()))
+
+	log.WithField("full_hint", r).Debug("full hint")
+
+	if r.GetSenderVersion() == nil {
+		// no common ancestor found, likely due to failed prior replication attempt
+		// => release stale step holds to prevent them from accumulating
+		//    (they can accumulate on initial replication because each inital replication step might hold a different `to`)
+		// => replication cursors cannot accumulate because we always _move_ the replication cursor
+		log.Debug("releasing all step holds on the filesystem")
+		TryReleaseStepStaleFS(ctx, fs, p.jobId)
+		return &pdu.HintMostRecentCommonAncestorRes{}, nil
+	}
+
+	// we were hinted a specific common ancestor
+
+	mostRecentVersion, err := sendArgsFromPDUAndValidateExistsAndGetVersion(ctx, fs, r.GetSenderVersion())
+	if err != nil {
+		msg := "HintMostRecentCommonAncestor rpc with nonexistent most recent version"
+		log.Warn(msg)
+		return nil, errors.Wrap(err, msg)
+	}
+
+	// move replication cursor to this position
+	destroyedCursors, err := MoveReplicationCursor(ctx, fs, mostRecentVersion, p.jobId)
+	if err == zfs.ErrBookmarkCloningNotSupported {
+		log.Debug("not creating replication cursor from bookmark because ZFS does not support it")
+		// fallthrough
+	} else if err != nil {
+		return nil, errors.Wrap(err, "cannot set replication cursor to hinted version")
+	}
+
+	// take care of stale step holds
+	log.WithField("step-holds-cleanup-mode", senderHintMostRecentCommonAncestorStepCleanupMode).
+		Debug("taking care of possibly stale step holds")
+	doStepCleanup := false
+	var stepCleanupSince *CreateTXGRangeBound
+	switch senderHintMostRecentCommonAncestorStepCleanupMode {
+	case StepCleanupNoCleanup:
+		doStepCleanup = false
+	case StepCleanupRangeSinceUnbounded:
+		doStepCleanup = true
+		stepCleanupSince = nil
+	case StepCleanupRangeSinceReplicationCursor:
+		doStepCleanup = true
+		// Use the destroyed replication cursors as indicator how far the previous replication got.
+		// To be precise: We limit the amount of visisted snapshots to exactly those snapshots
+		// created since the last successful replication cursor movement (i.e. last successful replication step)
+		//
+		// If we crash now, we'll leak the step we are about to release, but the performance gain
+		// of limiting the amount of snapshots we visit makes up for that.
+		// Users have the `zrepl holds release-stale` command to cleanup leaked step holds.
+		for _, destroyed := range destroyedCursors {
+			if stepCleanupSince == nil {
+				stepCleanupSince = &CreateTXGRangeBound{
+					CreateTXG: destroyed.GetCreateTXG(),
+					Inclusive: &zfs.NilBool{B: true},
+				}
+			} else if destroyed.GetCreateTXG() < stepCleanupSince.CreateTXG {
+				stepCleanupSince.CreateTXG = destroyed.GetCreateTXG()
+			}
+		}
+	default:
+		panic(senderHintMostRecentCommonAncestorStepCleanupMode)
+	}
+	if !doStepCleanup {
+		log.Info("skipping cleanup of prior invocations' step holds due to environment variable setting")
+	} else {
+		if err := ReleaseStepCummulativeInclusive(ctx, fs, stepCleanupSince, mostRecentVersion, p.jobId); err != nil {
+			return nil, errors.Wrap(err, "cannot cleanup prior invocation's step holds and bookmarks")
+		} else {
+			log.Info("step hold cleanup done")
+		}
+	}
+
+	return &pdu.HintMostRecentCommonAncestorRes{}, nil
+}
+
+type HintMostRecentCommonAncestorStepCleanupMode struct{ string }
+
+var (
+	StepCleanupRangeSinceReplicationCursor = HintMostRecentCommonAncestorStepCleanupMode{"range-since-replication-cursor"}
+	StepCleanupRangeSinceUnbounded         = HintMostRecentCommonAncestorStepCleanupMode{"range-since-unbounded"}
+	StepCleanupNoCleanup                   = HintMostRecentCommonAncestorStepCleanupMode{"no-cleanup"}
+)
+
+func (m HintMostRecentCommonAncestorStepCleanupMode) String() string { return string(m.string) }
+func (m *HintMostRecentCommonAncestorStepCleanupMode) Set(s string) error {
+	switch s {
+	case StepCleanupRangeSinceReplicationCursor.String():
+		*m = StepCleanupRangeSinceReplicationCursor
+	case StepCleanupRangeSinceUnbounded.String():
+		*m = StepCleanupRangeSinceUnbounded
+	case StepCleanupNoCleanup.String():
+		*m = StepCleanupNoCleanup
+	default:
+		return fmt.Errorf("unknown step cleanup mode %q", s)
+	}
+	return nil
+}
+
+var senderHintMostRecentCommonAncestorStepCleanupMode = *envconst.Var("ZREPL_ENDPOINT_SENDER_HINT_MOST_RECENT_STEP_HOLD_CLEANUP_MODE", &StepCleanupRangeSinceReplicationCursor).(*HintMostRecentCommonAncestorStepCleanupMode)
+
 var maxConcurrentZFSSendSemaphore = semaphore.New(envconst.Int64("ZREPL_ENDPOINT_MAX_CONCURRENT_SEND", 10))
 
-func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, zfs.StreamCopier, error) {
+func uncheckedSendArgsFromPDU(fsv *pdu.FilesystemVersion) *zfs.ZFSSendArgVersion {
+	if fsv == nil {
+		return nil
+	}
+	return &zfs.ZFSSendArgVersion{RelName: fsv.GetRelName(), GUID: fsv.Guid}
+}
+
+func sendArgsFromPDUAndValidateExistsAndGetVersion(ctx context.Context, fs string, fsv *pdu.FilesystemVersion) (v zfs.FilesystemVersion, err error) {
+	sendArgs := uncheckedSendArgsFromPDU(fsv)
+	if sendArgs == nil {
+		return v, errors.New("must not be nil")
+	}
+	version, err := sendArgs.ValidateExistsAndGetVersion(ctx, fs)
+	if err != nil {
+		return v, err
+	}
+	return version, nil
+}
+
+func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	_, err := s.filterCheckFS(r.Filesystem)
 	if err != nil {
 		return nil, nil, err
 	}
+	switch r.Encrypted {
+	case pdu.Tri_DontCare:
+		// use s.encrypt setting
+		// ok, fallthrough outer
+	case pdu.Tri_False:
+		if s.encrypt.B {
+			return nil, nil, errors.New("only encrypted sends allowed (send -w + encryption!= off), but unencrypted send requested")
+		}
+		// fallthrough outer
+	case pdu.Tri_True:
+		if !s.encrypt.B {
+			return nil, nil, errors.New("only unencrypted sends allowed, but encrypted send requested")
+		}
+		// fallthrough outer
+	default:
+		return nil, nil, fmt.Errorf("unknown pdu.Tri variant %q", r.Encrypted)
+	}
+
+	sendArgsUnvalidated := zfs.ZFSSendArgsUnvalidated{
+		FS:          r.Filesystem,
+		From:        uncheckedSendArgsFromPDU(r.GetFrom()), // validated by zfs.ZFSSendDry / zfs.ZFSSend
+		To:          uncheckedSendArgsFromPDU(r.GetTo()),   // validated by zfs.ZFSSendDry / zfs.ZFSSend
+		Encrypted:   s.encrypt,
+		ResumeToken: r.ResumeToken, // nil or not nil, depending on decoding success
+	}
+
+	sendArgs, err := sendArgsUnvalidated.Validate(ctx)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "validate send arguments")
+	}
 
 	getLogger(ctx).Debug("acquire concurrent send semaphore")
 	// TODO use try-acquire and fail with resource-exhaustion rpc status
@@ -95,29 +298,168 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, zfs.St
 	}
 	defer guard.Release()
 
-	si, err := zfs.ZFSSendDry(r.Filesystem, r.From, r.To, "")
+	si, err := zfs.ZFSSendDry(ctx, sendArgs)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errors.Wrap(err, "zfs send dry failed")
 	}
 
+	// From now on, assume that sendArgs has been validated by ZFSSendDry
+	// (because validation involves shelling out, it's actually a little expensive)
+
 	var expSize int64 = 0      // protocol says 0 means no estimate
 	if si.SizeEstimate != -1 { // but si returns -1 for no size estimate
 		expSize = si.SizeEstimate
 	}
-	res := &pdu.SendRes{ExpectedSize: expSize}
+	res := &pdu.SendRes{
+		ExpectedSize:    expSize,
+		UsedResumeToken: r.ResumeToken != "",
+	}
 
 	if r.DryRun {
 		return res, nil, nil
 	}
 
-	streamCopier, err := zfs.ZFSSend(ctx, r.Filesystem, r.From, r.To, "")
+	// update replication cursor
-	if err != nil {
+	if sendArgs.From != nil {
-		return nil, nil, err
+		// For all but the first replication, this should always be a no-op because SendCompleted already moved the cursor
+		_, err = MoveReplicationCursor(ctx, sendArgs.FS, sendArgs.FromVersion, s.jobId)
+		if err == zfs.ErrBookmarkCloningNotSupported {
+			getLogger(ctx).Debug("not creating replication cursor from bookmark because ZFS does not support it")
+			// fallthrough
+		} else if err != nil {
+			return nil, nil, errors.Wrap(err, "cannot set replication cursor to `from` version before starting send")
+		}
 	}
-	return res, streamCopier, nil
+
+	// make sure `From` doesn't go away in order to make this step resumable
+	if sendArgs.From != nil {
+		_, err := HoldStep(ctx, sendArgs.FS, *sendArgs.FromVersion, s.jobId)
+		if err == zfs.ErrBookmarkCloningNotSupported {
+			getLogger(ctx).Debug("not creating step bookmark because ZFS does not support it")
+			// fallthrough
+		} else if err != nil {
+			return nil, nil, errors.Wrapf(err, "cannot hold `from` version %q before starting send", *sendArgs.FromVersion)
+		}
+	}
+	// make sure `To` doesn't go away in order to make this step resumable
+	_, err = HoldStep(ctx, sendArgs.FS, sendArgs.ToVersion, s.jobId)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "cannot hold `to` version %q before starting send", sendArgs.ToVersion)
+	}
+
+	// step holds & replication cursor released / moved forward in s.SendCompleted => s.moveCursorAndReleaseSendHolds
+
+	sendStream, err := zfs.ZFSSend(ctx, sendArgs)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "zfs send failed")
+	}
+	return res, sendStream, nil
+}
+
+func (p *Sender) SendCompleted(ctx context.Context, r *pdu.SendCompletedReq) (*pdu.SendCompletedRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
+	orig := r.GetOriginalReq() // may be nil, always use proto getters
+	fsp, err := p.filterCheckFS(orig.GetFilesystem())
+	if err != nil {
+		return nil, err
+	}
+	fs := fsp.ToString()
+
+	var from *zfs.FilesystemVersion
+	if orig.GetFrom() != nil {
+		f, err := sendArgsFromPDUAndValidateExistsAndGetVersion(ctx, fs, orig.GetFrom()) // no shadow
+		if err != nil {
+			return nil, errors.Wrap(err, "validate `from` exists")
+		}
+		from = &f
+	}
+	to, err := sendArgsFromPDUAndValidateExistsAndGetVersion(ctx, fs, orig.GetTo())
+	if err != nil {
+		return nil, errors.Wrap(err, "validate `to` exists")
+	}
+
+	log := func(ctx context.Context) Logger {
+		log := getLogger(ctx).WithField("to_guid", to.Guid).
+			WithField("fs", fs).
+			WithField("to", to.RelName)
+		if from != nil {
+			log = log.WithField("from", from.RelName).WithField("from_guid", from.Guid)
+		}
+		return log
+	}
+
+	log(ctx).Debug("move replication cursor to most recent common version")
+	destroyedCursors, err := MoveReplicationCursor(ctx, fs, to, p.jobId)
+	if err != nil {
+		if err == zfs.ErrBookmarkCloningNotSupported {
+			log(ctx).Debug("not setting replication cursor, bookmark cloning not supported")
+		} else {
+			msg := "cannot move replication cursor, keeping hold on `to` until successful"
+			log(ctx).WithError(err).Error(msg)
+			err = errors.Wrap(err, msg)
+			// it is correct to not release the hold if we can't move the cursor!
+			return &pdu.SendCompletedRes{}, err
+		}
+	} else {
+		log(ctx).Info("successfully moved replication cursor")
+	}
+
+	// kick off releasing of step holds / bookmarks
+	// if we fail to release them, don't bother the caller:
+	// they are merely an implementation detail on the sender for better resumability
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		ctx, endTask := trace.WithTask(ctx, "release-step-hold-to")
+		defer endTask()
+
+		log(ctx).Debug("release step-hold of or step-bookmark on `to`")
+		err = ReleaseStep(ctx, fs, to, p.jobId)
+		if err != nil {
+			log(ctx).WithError(err).Error("cannot release step-holds on or destroy step-bookmark of `to`")
+		} else {
+			log(ctx).Info("successfully released step-holds on or destroyed step-bookmark of `to`")
+		}
+
+	}()
+	go func() {
+		defer wg.Done()
+		ctx, endTask := trace.WithTask(ctx, "release-step-hold-from")
+		defer endTask()
+
+		if from == nil {
+			return
+		}
+		log(ctx).Debug("release step-hold of or step-bookmark on `from`")
+		err := ReleaseStep(ctx, fs, *from, p.jobId)
+		if err != nil {
+			if dne, ok := err.(*zfs.DatasetDoesNotExist); ok {
+				// If bookmark cloning is not supported, `from` might be the old replication cursor
+				// and thus have already been destroyed by MoveReplicationCursor above
+				// In that case, nonexistence of `from` is not an error, otherwise it is.
+				for _, c := range destroyedCursors {
+					if c.GetFullPath() == dne.Path {
+						log(ctx).Info("`from` was a replication cursor and has already been destroyed")
+						return
+					}
+				}
+				// fallthrough
+			}
+			log(ctx).WithError(err).Error("cannot release step-holds on or destroy step-bookmark of `from`")
+		} else {
+			log(ctx).Info("successfully released step-holds on or destroyed step-bookmark of `from`")
+		}
+	}()
+	wg.Wait()
+
+	return &pdu.SendCompletedRes{}, nil
 }
 
 func (p *Sender) DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	dp, err := p.filterCheckFS(req.Filesystem)
 	if err != nil {
 		return nil, err
@@ -126,6 +468,8 @@ func (p *Sender) DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshots
 }
 
 func (p *Sender) Ping(ctx context.Context, req *pdu.PingReq) (*pdu.PingRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	res := pdu.PingRes{
 		Echo: req.GetMessage(),
 	}
@@ -133,41 +477,36 @@ func (p *Sender) Ping(ctx context.Context, req *pdu.PingReq) (*pdu.PingRes, erro
 }
 
 func (p *Sender) PingDataconn(ctx context.Context, req *pdu.PingReq) (*pdu.PingRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	return p.Ping(ctx, req)
 }
 
 func (p *Sender) WaitForConnectivity(ctx context.Context) error {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	return nil
 }
 
 func (p *Sender) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	dp, err := p.filterCheckFS(req.Filesystem)
 	if err != nil {
 		return nil, err
 	}
 
-	switch op := req.Op.(type) {
+	cursor, err := GetMostRecentReplicationCursorOfJob(ctx, dp.ToString(), p.jobId)
-	case *pdu.ReplicationCursorReq_Get:
+	if err != nil {
-		cursor, err := zfs.ZFSGetReplicationCursor(dp)
+		return nil, err
-		if err != nil {
-			return nil, err
-		}
-		if cursor == nil {
-			return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Notexist{Notexist: true}}, nil
-		}
-		return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Guid{Guid: cursor.Guid}}, nil
-	case *pdu.ReplicationCursorReq_Set:
-		guid, err := zfs.ZFSSetReplicationCursor(dp, op.Set.Snapshot)
-		if err != nil {
-			return nil, err
-		}
-		return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Guid{Guid: guid}}, nil
-	default:
-		return nil, errors.Errorf("unknown op %T", op)
 	}
+	if cursor == nil {
+		return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Notexist{Notexist: true}}, nil
+	}
+	return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Guid{Guid: cursor.Guid}}, nil
 }
 
-func (p *Sender) Receive(ctx context.Context, r *pdu.ReceiveReq, receive zfs.StreamCopier) (*pdu.ReceiveRes, error) {
+func (p *Sender) Receive(ctx context.Context, r *pdu.ReceiveReq, _ io.ReadCloser) (*pdu.ReceiveRes, error) {
 	return nil, fmt.Errorf("sender does not implement Receive()")
 }
 
@@ -183,22 +522,42 @@ type FSMap interface { // FIXME unused
 	AsFilter() FSFilter
 }
 
+type ReceiverConfig struct {
+	JobID JobID
+
+	RootWithoutClientComponent *zfs.DatasetPath // TODO use
+	AppendClientIdentity       bool
+
+	UpdateLastReceivedHold bool
+}
+
+func (c *ReceiverConfig) copyIn() {
+	c.RootWithoutClientComponent = c.RootWithoutClientComponent.Copy()
+}
+
+func (c *ReceiverConfig) Validate() error {
+	c.JobID.MustValidate()
+	if c.RootWithoutClientComponent.Length() <= 0 {
+		return errors.New("RootWithoutClientComponent must not be an empty dataset path")
+	}
+	return nil
+}
+
 // Receiver implements replication.ReplicationEndpoint for a receiving side
 type Receiver struct {
-	rootWithoutClientComponent *zfs.DatasetPath
+	conf ReceiverConfig // validated
-	appendClientIdentity       bool
 
 	recvParentCreationMtx *chainlock.L
 }
 
-func NewReceiver(rootDataset *zfs.DatasetPath, appendClientIdentity bool) *Receiver {
+func NewReceiver(config ReceiverConfig) *Receiver {
-	if rootDataset.Length() <= 0 {
+	config.copyIn()
-		panic(fmt.Sprintf("root dataset must not be an empty path: %v", rootDataset))
+	if err := config.Validate(); err != nil {
+		panic(err)
 	}
 	return &Receiver{
-		rootWithoutClientComponent: rootDataset.Copy(),
+		conf:                  config,
-		appendClientIdentity:       appendClientIdentity,
+		recvParentCreationMtx: chainlock.New(),
-		recvParentCreationMtx:      chainlock.New(),
 	}
 }
 
@@ -221,8 +580,8 @@ func clientRoot(rootFS *zfs.DatasetPath, clientIdentity string) (*zfs.DatasetPat
 }
 
 func (s *Receiver) clientRootFromCtx(ctx context.Context) *zfs.DatasetPath {
-	if !s.appendClientIdentity {
+	if !s.conf.AppendClientIdentity {
-		return s.rootWithoutClientComponent.Copy()
+		return s.conf.RootWithoutClientComponent.Copy()
 	}
 
 	clientIdentity, ok := ctx.Value(ClientIdentityKey).(string)
@@ -230,7 +589,7 @@ func (s *Receiver) clientRootFromCtx(ctx context.Context) *zfs.DatasetPath {
 		panic(fmt.Sprintf("ClientIdentityKey context value must be set"))
 	}
 
-	clientRoot, err := clientRoot(s.rootWithoutClientComponent, clientIdentity)
+	clientRoot, err := clientRoot(s.conf.RootWithoutClientComponent, clientIdentity)
 	if err != nil {
 		panic(fmt.Sprintf("ClientIdentityContextKey must have been validated before invoking Receiver: %s", err))
 	}
@@ -262,6 +621,15 @@ func (f subroot) MapToLocal(fs string) (*zfs.DatasetPath, error) {
 }
 
 func (s *Receiver) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
+	// first make sure that root_fs is imported
+	if rphs, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, s.conf.RootWithoutClientComponent); err != nil {
+		return nil, errors.Wrap(err, "cannot determine whether root_fs exists")
+	} else if !rphs.FSExists {
+		return nil, errors.New("root_fs does not exist")
+	}
+
 	root := s.clientRootFromCtx(ctx)
 	filtered, err := zfs.ZFSListMapping(ctx, subroot{root})
 	if err != nil {
@@ -271,7 +639,7 @@ func (s *Receiver) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemR
 	fss := make([]*pdu.Filesystem, 0, len(filtered))
 	for _, a := range filtered {
 		l := getLogger(ctx).WithField("fs", a)
-		ph, err := zfs.ZFSGetFilesystemPlaceholderState(a)
+		ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, a)
 		if err != nil {
 			l.WithError(err).Error("error getting placeholder state")
 			return nil, errors.Wrapf(err, "cannot get placeholder state for fs %q", a)
@@ -282,8 +650,27 @@ func (s *Receiver) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemR
 			err := errors.Errorf("inconsistent placeholder state: filesystem %q must exist in this context", a.ToString())
 			return nil, err
 		}
+		token, err := zfs.ZFSGetReceiveResumeTokenOrEmptyStringIfNotSupported(ctx, a)
+		if err != nil {
+			l.WithError(err).Error("cannot get receive resume token")
+			return nil, err
+		}
+		encEnabled, err := zfs.ZFSGetEncryptionEnabled(ctx, a.ToString())
+		if err != nil {
+			l.WithError(err).Error("cannot get encryption enabled status")
+			return nil, err
+		}
+		l.WithField("receive_resume_token", token).Debug("receive resume token")
+
 		a.TrimPrefix(root)
-		fss = append(fss, &pdu.Filesystem{Path: a.ToString(), IsPlaceholder: ph.IsPlaceholder})
+
+		fs := &pdu.Filesystem{
+			Path:          a.ToString(),
+			IsPlaceholder: ph.IsPlaceholder,
+			ResumeToken:   token,
+			IsEncrypted:   encEnabled,
+		}
+		fss = append(fss, fs)
 	}
 	if len(fss) == 0 {
 		getLogger(ctx).Debug("no filesystems found")
@@ -293,13 +680,16 @@ func (s *Receiver) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemR
 }
 
 func (s *Receiver) ListFilesystemVersions(ctx context.Context, req *pdu.ListFilesystemVersionsReq) (*pdu.ListFilesystemVersionsRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	root := s.clientRootFromCtx(ctx)
 	lp, err := subroot{root}.MapToLocal(req.GetFilesystem())
 	if err != nil {
 		return nil, err
 	}
+	// TODO share following code with sender
 
-	fsvs, err := zfs.ZFSListFilesystemVersions(lp, nil)
+	fsvs, err := zfs.ZFSListFilesystemVersions(ctx, lp, zfs.ListFilesystemVersionsOptions{})
 	if err != nil {
 		return nil, err
 	}
@@ -313,6 +703,8 @@ func (s *Receiver) ListFilesystemVersions(ctx context.Context, req *pdu.ListFile
 }
 
 func (s *Receiver) Ping(ctx context.Context, req *pdu.PingReq) (*pdu.PingRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	res := pdu.PingRes{
 		Echo: req.GetMessage(),
 	}
@@ -320,31 +712,45 @@ func (s *Receiver) Ping(ctx context.Context, req *pdu.PingReq) (*pdu.PingRes, er
 }
 
 func (s *Receiver) PingDataconn(ctx context.Context, req *pdu.PingReq) (*pdu.PingRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
 	return s.Ping(ctx, req)
 }
 
 func (s *Receiver) WaitForConnectivity(ctx context.Context) error {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
 	return nil
 }
 
-func (s *Receiver) ReplicationCursor(context.Context, *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
+func (s *Receiver) ReplicationCursor(ctx context.Context, _ *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
 	return nil, fmt.Errorf("ReplicationCursor not implemented for Receiver")
 }
 
-func (s *Receiver) Send(ctx context.Context, req *pdu.SendReq) (*pdu.SendRes, zfs.StreamCopier, error) {
+func (s *Receiver) Send(ctx context.Context, req *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
 	return nil, nil, fmt.Errorf("receiver does not implement Send()")
 }
 
 var maxConcurrentZFSRecvSemaphore = semaphore.New(envconst.Int64("ZREPL_ENDPOINT_MAX_CONCURRENT_RECV", 10))
 
-func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs.StreamCopier) (*pdu.ReceiveRes, error) {
+func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive io.ReadCloser) (*pdu.ReceiveRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	getLogger(ctx).Debug("incoming Receive")
 	defer receive.Close()
 
 	root := s.clientRootFromCtx(ctx)
 	lp, err := subroot{root}.MapToLocal(req.Filesystem)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "`Filesystem` invalid")
+	}
+
+	to := uncheckedSendArgsFromPDU(req.GetTo())
+	if to == nil {
+		return nil, errors.New("`To` must not be nil")
+	}
+	if !to.IsSnapshot() {
+		return nil, errors.New("`To` must be a snapshot")
 	}
 
 	// create placeholder parent filesystems as appropriate
@@ -355,9 +761,9 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 	//       ZFS dataset hierarchy subtrees.
 	var visitErr error
 	func() {
-		getLogger(ctx).Debug("begin aquire recvParentCreationMtx")
+		getLogger(ctx).Debug("begin acquire recvParentCreationMtx")
 		defer s.recvParentCreationMtx.Lock().Unlock()
-		getLogger(ctx).Debug("end aquire recvParentCreationMtx")
+		getLogger(ctx).Debug("end acquire recvParentCreationMtx")
 		defer getLogger(ctx).Debug("release recvParentCreationMtx")
 
 		f := zfs.NewDatasetPathForest()
@@ -367,7 +773,7 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 			if v.Path.Equal(lp) {
 				return false
 			}
-			ph, err := zfs.ZFSGetFilesystemPlaceholderState(v.Path)
+			ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, v.Path)
 			getLogger(ctx).
 				WithField("fs", v.Path.ToString()).
 				WithField("placeholder_state", fmt.Sprintf("%#v", ph)).
@@ -380,18 +786,18 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 			}
 
 			if !ph.FSExists {
-				if s.rootWithoutClientComponent.HasPrefix(v.Path) {
+				if s.conf.RootWithoutClientComponent.HasPrefix(v.Path) {
 					if v.Path.Length() == 1 {
 						visitErr = fmt.Errorf("pool %q not imported", v.Path.ToString())
 					} else {
-						visitErr = fmt.Errorf("root_fs %q does not exist", s.rootWithoutClientComponent.ToString())
+						visitErr = fmt.Errorf("root_fs %q does not exist", s.conf.RootWithoutClientComponent.ToString())
 					}
 					getLogger(ctx).WithError(visitErr).Error("placeholders are only created automatically below root_fs")
 					return false
 				}
 				l := getLogger(ctx).WithField("placeholder_fs", v.Path)
 				l.Debug("create placeholder filesystem")
-				err := zfs.ZFSCreatePlaceholderFilesystem(v.Path)
+				err := zfs.ZFSCreatePlaceholderFilesystem(ctx, v.Path)
 				if err != nil {
 					l.WithError(err).Error("cannot create placeholder filesystem")
 					visitErr = err
@@ -408,21 +814,41 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 		return nil, visitErr
 	}
 
+	log := getLogger(ctx).WithField("proto_fs", req.GetFilesystem()).WithField("local_fs", lp.ToString())
+
 	// determine whether we need to rollback the filesystem / change its placeholder state
 	var clearPlaceholderProperty bool
 	var recvOpts zfs.RecvOptions
-	ph, err := zfs.ZFSGetFilesystemPlaceholderState(lp)
+	ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, lp)
-	if err == nil && ph.FSExists && ph.IsPlaceholder {
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot get placeholder state")
+	}
+	log.WithField("placeholder_state", fmt.Sprintf("%#v", ph)).Debug("placeholder state")
+	if ph.FSExists && ph.IsPlaceholder {
 		recvOpts.RollbackAndForceRecv = true
 		clearPlaceholderProperty = true
 	}
+
 	if clearPlaceholderProperty {
-		if err := zfs.ZFSSetPlaceholder(lp, false); err != nil {
+		log.Info("clearing placeholder property")
+		if err := zfs.ZFSSetPlaceholder(ctx, lp, false); err != nil {
 			return nil, fmt.Errorf("cannot clear placeholder property for forced receive: %s", err)
 		}
 	}
 
-	getLogger(ctx).Debug("acquire concurrent recv semaphore")
+	if req.ClearResumeToken && ph.FSExists {
+		log.Info("clearing resume token")
+		if err := zfs.ZFSRecvClearResumeToken(ctx, lp.ToString()); err != nil {
+			return nil, errors.Wrap(err, "cannot clear resume token")
+		}
+	}
+
+	recvOpts.SavePartialRecvState, err = zfs.ResumeRecvSupported(ctx, lp)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot determine whether we can use resumable send & recv")
+	}
+
+	log.Debug("acquire concurrent recv semaphore")
 	// TODO use try-acquire and fail with resource-exhaustion rpc status
 	// => would require handling on the client-side
 	// => this is a dataconn endpoint, doesn't have the status code semantics of gRPC
@@ -432,19 +858,114 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 	}
 	defer guard.Release()
 
-	getLogger(ctx).WithField("opts", fmt.Sprintf("%#v", recvOpts)).Debug("start receive command")
+	log.Info("peeking 1M ahead")
+	var peek bytes.Buffer
+	var MaxPeek = envconst.Int64("ZREPL_ENDPOINT_RECV_PEEK_SIZE", 1<<20)
+	if _, err := io.Copy(&peek, io.LimitReader(receive, MaxPeek)); err != nil {
+		log.WithError(err).Error("cannot read peek-buffer from send stream")
+	}
+	var peekCopy bytes.Buffer
+	if n, err := peekCopy.Write(peek.Bytes()); err != nil || n != peek.Len() {
+		panic(peek.Len())
+	}
 
-	if err := zfs.ZFSRecv(ctx, lp.ToString(), receive, recvOpts); err != nil {
+	log.WithField("opts", fmt.Sprintf("%#v", recvOpts)).Debug("start receive command")
-		getLogger(ctx).
+
+	snapFullPath := to.FullPath(lp.ToString())
+	if err := zfs.ZFSRecv(ctx, lp.ToString(), to, chainedio.NewChainedReader(&peek, receive), recvOpts); err != nil {
+
+		// best-effort rollback of placeholder state if the recv didn't start
+		_, resumableStatePresent := err.(*zfs.RecvFailedWithResumeTokenErr)
+		disablePlaceholderRestoration := envconst.Bool("ZREPL_ENDPOINT_DISABLE_PLACEHOLDER_RESTORATION", false)
+		placeholderRestored := !ph.IsPlaceholder
+		if !disablePlaceholderRestoration && !resumableStatePresent && recvOpts.RollbackAndForceRecv && ph.FSExists && ph.IsPlaceholder && clearPlaceholderProperty {
+			log.Info("restoring placeholder property")
+			if phErr := zfs.ZFSSetPlaceholder(ctx, lp, true); phErr != nil {
+				log.WithError(phErr).Error("cannot restore placeholder property after failed receive, subsequent replications will likely fail with a different error")
+				// fallthrough
+			} else {
+				placeholderRestored = true
+			}
+			// fallthrough
+		}
+
+		// deal with failing initial encrypted send & recv
+		if _, ok := err.(*zfs.RecvDestroyOrOverwriteEncryptedErr); ok && ph.IsPlaceholder && placeholderRestored {
+			msg := `cannot automatically replace placeholder filesystem with incoming send stream - please see receive-side log for details`
+			err := errors.New(msg)
+			log.Error(msg)
+
+			log.Error(`zrepl creates placeholder filesystems on the receiving side of a replication to match the sending side's dataset hierarchy`)
+			log.Error(`zrepl uses zfs receive -F to replace those placeholders with incoming full sends`)
+			log.Error(`OpenZFS native encryption prohibits zfs receive -F for encrypted filesystems`)
+			log.Error(`the current zrepl placeholder filesystem concept is thus incompatible with OpenZFS native encryption`)
+
+			tempStartFullRecvFS := lp.Copy().ToString() + ".zrepl.initial-recv"
+			tempStartFullRecvFSDP, dpErr := zfs.NewDatasetPath(tempStartFullRecvFS)
+			if dpErr != nil {
+				log.WithError(dpErr).Error("cannot determine temporary filesystem name for initial encrypted recv workaround")
+				return nil, err // yes, err, not dpErr
+			}
+
+			log := log.WithField("temp_recv_fs", tempStartFullRecvFS)
+			log.Error(`as a workaround, zrepl will now attempt to re-receive the beginning of the stream into a temporary filesystem temp_recv_fs`)
+			log.Error(`if that step succeeds: shut down zrepl and use 'zfs rename' to swap temp_recv_fs with local_fs, then restart zrepl`)
+			log.Error(`replication will then resume using resumable send+recv`)
+
+			tempPH, phErr := zfs.ZFSGetFilesystemPlaceholderState(ctx, tempStartFullRecvFSDP)
+			if phErr != nil {
+				log.WithError(phErr).Error("cannot determine placeholder state of temp_recv_fs")
+				return nil, err // yes, err, not dpErr
+			}
+			if tempPH.FSExists {
+				log.Error("temp_recv_fs already exists, assuming a (partial) initial recv to that filesystem has already been done")
+				return nil, err
+			}
+
+			recvOpts.RollbackAndForceRecv = false
+			recvOpts.SavePartialRecvState = true
+			rerecvErr := zfs.ZFSRecv(ctx, tempStartFullRecvFS, to, chainedio.NewChainedReader(&peekCopy), recvOpts)
+			if _, isResumable := rerecvErr.(*zfs.RecvFailedWithResumeTokenErr); rerecvErr == nil || isResumable {
+				log.Error("completed re-receive into temporary filesystem temp_recv_fs, now shut down zrepl and use zfs rename to swap temp_recv_fs with local_fs")
+			} else {
+				log.WithError(rerecvErr).Error("failed to receive the beginning of the stream into temporary filesystem temp_recv_fs")
+				log.Error("we advise you to collect the error log and current configuration, open an issue on GitHub, and revert to your previous configuration in the meantime")
+			}
+
+			log.Error(`if you would like to see improvements to this situation, please open an issue on GitHub`)
+			return nil, err
+		}
+
+		log.
 			WithError(err).
-			WithField("opts", recvOpts).
+			WithField("opts", fmt.Sprintf("%#v", recvOpts)).
 			Error("zfs receive failed")
+
 		return nil, err
 	}
+
+	// validate that we actually received what the sender claimed
+	toRecvd, err := to.ValidateExistsAndGetVersion(ctx, lp.ToString())
+	if err != nil {
+		msg := "receive request's `To` version does not match what we received in the stream"
+		log.WithError(err).WithField("snap", snapFullPath).Error(msg)
+		log.Error("aborting recv request, but keeping received snapshot for inspection")
+		return nil, errors.Wrap(err, msg)
+	}
+
+	if s.conf.UpdateLastReceivedHold {
+		log.Debug("move last-received-hold")
+		if err := MoveLastReceivedHold(ctx, lp.ToString(), toRecvd, s.conf.JobID); err != nil {
+			return nil, errors.Wrap(err, "cannot move last-received-hold")
+		}
+	}
+
 	return &pdu.ReceiveRes{}, nil
 }
 
 func (s *Receiver) DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	root := s.clientRootFromCtx(ctx)
 	lp, err := subroot{root}.MapToLocal(req.Filesystem)
 	if err != nil {
@@ -453,6 +974,22 @@ func (s *Receiver) DestroySnapshots(ctx context.Context, req *pdu.DestroySnapsho
 	return doDestroySnapshots(ctx, lp, req.Snapshots)
 }
 
+func (p *Receiver) HintMostRecentCommonAncestor(ctx context.Context, r *pdu.HintMostRecentCommonAncestorReq) (*pdu.HintMostRecentCommonAncestorRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+	// we don't move last-received-hold as part of this hint
+	// because that wouldn't give us any benefit wrt resumability.
+	//
+	// Other reason: the replication logic that issues this RPC would require refactoring
+	// to include the receiver's FilesystemVersion in the request)
+	return &pdu.HintMostRecentCommonAncestorRes{}, nil
+}
+
+func (p *Receiver) SendCompleted(ctx context.Context, _ *pdu.SendCompletedReq) (*pdu.SendCompletedRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
+	return &pdu.SendCompletedRes{}, nil
+}
+
 func doDestroySnapshots(ctx context.Context, lp *zfs.DatasetPath, snaps []*pdu.FilesystemVersion) (*pdu.DestroySnapshotsRes, error) {
 	reqs := make([]*zfs.DestroySnapOp, len(snaps))
 	ress := make([]*pdu.DestroySnapshotRes, len(snaps))
@@ -471,7 +1008,7 @@ func doDestroySnapshots(ctx context.Context, lp *zfs.DatasetPath, snaps []*pdu.F
 			ErrOut:     &errs[i],
 		}
 	}
-	zfs.ZFSDestroyFilesystemVersions(reqs)
+	zfs.ZFSDestroyFilesystemVersions(ctx, reqs)
 	for i := range reqs {
 		if errs[i] != nil {
 			if de, ok := errs[i].(*zfs.DestroySnapshotsError); ok && len(de.Reason) == 1 {

endpoint/endpoint_zfs_abstraction.go

@@ -0,0 +1,840 @@
+package endpoint
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+	"github.com/zrepl/zrepl/util/envconst"
+	"github.com/zrepl/zrepl/util/semaphore"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type AbstractionType string
+
+// Implementation note:
+// There are a lot of exhaustive switches on AbstractionType in the code base.
+// When adding a new abstraction type, make sure to search and update them!
+const (
+	AbstractionStepBookmark                AbstractionType = "step-bookmark"
+	AbstractionStepHold                    AbstractionType = "step-hold"
+	AbstractionLastReceivedHold            AbstractionType = "last-received-hold"
+	AbstractionReplicationCursorBookmarkV1 AbstractionType = "replication-cursor-bookmark-v1"
+	AbstractionReplicationCursorBookmarkV2 AbstractionType = "replication-cursor-bookmark-v2"
+)
+
+var AbstractionTypesAll = map[AbstractionType]bool{
+	AbstractionStepBookmark:                true,
+	AbstractionStepHold:                    true,
+	AbstractionLastReceivedHold:            true,
+	AbstractionReplicationCursorBookmarkV1: true,
+	AbstractionReplicationCursorBookmarkV2: true,
+}
+
+// Implementation Note:
+// Whenever you add a new accessor, adjust AbstractionJSON.MarshalJSON accordingly
+type Abstraction interface {
+	GetType() AbstractionType
+	GetFS() string
+	GetName() string
+	GetFullPath() string
+	GetJobID() *JobID // may return nil if the abstraction does not have a JobID
+	GetCreateTXG() uint64
+	GetFilesystemVersion() zfs.FilesystemVersion
+	String() string
+	// destroy the abstraction: either releases the hold or destroys the bookmark
+	Destroy(context.Context) error
+	json.Marshaler
+}
+
+func (t AbstractionType) Validate() error {
+	switch t {
+	case AbstractionStepBookmark:
+		return nil
+	case AbstractionStepHold:
+		return nil
+	case AbstractionLastReceivedHold:
+		return nil
+	case AbstractionReplicationCursorBookmarkV1:
+		return nil
+	case AbstractionReplicationCursorBookmarkV2:
+		return nil
+	default:
+		return errors.Errorf("unknown abstraction type %q", t)
+	}
+}
+
+func (t AbstractionType) MustValidate() error {
+	if err := t.Validate(); err != nil {
+		panic(err)
+	}
+	return nil
+}
+
+type AbstractionJSON struct{ Abstraction }
+
+var _ json.Marshaler = (*AbstractionJSON)(nil)
+
+func (a AbstractionJSON) MarshalJSON() ([]byte, error) {
+	type S struct {
+		Type              AbstractionType
+		FS                string
+		Name              string
+		FullPath          string
+		JobID             *JobID // may return nil if the abstraction does not have a JobID
+		CreateTXG         uint64
+		FilesystemVersion zfs.FilesystemVersion
+		String            string
+	}
+	v := S{
+		Type:              a.Abstraction.GetType(),
+		FS:                a.Abstraction.GetFS(),
+		Name:              a.Abstraction.GetName(),
+		FullPath:          a.Abstraction.GetFullPath(),
+		JobID:             a.Abstraction.GetJobID(),
+		CreateTXG:         a.Abstraction.GetCreateTXG(),
+		FilesystemVersion: a.Abstraction.GetFilesystemVersion(),
+		String:            a.Abstraction.String(),
+	}
+	return json.Marshal(v)
+}
+
+type AbstractionTypeSet map[AbstractionType]bool
+
+func AbstractionTypeSetFromStrings(sts []string) (AbstractionTypeSet, error) {
+	ats := make(map[AbstractionType]bool, len(sts))
+	for i, t := range sts {
+		at := AbstractionType(t)
+		if err := at.Validate(); err != nil {
+			return nil, errors.Wrapf(err, "invalid abstraction type #%d %q", i+1, t)
+		}
+		ats[at] = true
+	}
+	return ats, nil
+}
+
+func (s AbstractionTypeSet) ContainsAll(q AbstractionTypeSet) bool {
+	for k := range q {
+		if _, ok := s[k]; !ok {
+			return false
+		}
+	}
+	return true
+}
+
+func (s AbstractionTypeSet) ContainsAnyOf(q AbstractionTypeSet) bool {
+	for k := range q {
+		if _, ok := s[k]; ok {
+			return true
+		}
+	}
+	return false
+}
+
+func (s AbstractionTypeSet) String() string {
+	sts := make([]string, 0, len(s))
+	for i := range s {
+		sts = append(sts, string(i))
+	}
+	sts = sort.StringSlice(sts)
+	return strings.Join(sts, ",")
+}
+
+func (s AbstractionTypeSet) Validate() error {
+	for k := range s {
+		if err := k.Validate(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type BookmarkExtractor func(fs *zfs.DatasetPath, v zfs.FilesystemVersion) Abstraction
+
+// returns nil if the abstraction type is not bookmark-based
+func (t AbstractionType) BookmarkExtractor() BookmarkExtractor {
+	switch t {
+	case AbstractionStepBookmark:
+		return StepBookmarkExtractor
+	case AbstractionReplicationCursorBookmarkV1:
+		return ReplicationCursorV1Extractor
+	case AbstractionReplicationCursorBookmarkV2:
+		return ReplicationCursorV2Extractor
+	case AbstractionStepHold:
+		return nil
+	case AbstractionLastReceivedHold:
+		return nil
+	default:
+		panic(fmt.Sprintf("unimpl: %q", t))
+	}
+}
+
+type HoldExtractor = func(fs *zfs.DatasetPath, v zfs.FilesystemVersion, tag string) Abstraction
+
+// returns nil if the abstraction type is not hold-based
+func (t AbstractionType) HoldExtractor() HoldExtractor {
+	switch t {
+	case AbstractionStepBookmark:
+		return nil
+	case AbstractionReplicationCursorBookmarkV1:
+		return nil
+	case AbstractionReplicationCursorBookmarkV2:
+		return nil
+	case AbstractionStepHold:
+		return StepHoldExtractor
+	case AbstractionLastReceivedHold:
+		return LastReceivedHoldExtractor
+	default:
+		panic(fmt.Sprintf("unimpl: %q", t))
+	}
+}
+
+type ListZFSHoldsAndBookmarksQuery struct {
+	FS ListZFSHoldsAndBookmarksQueryFilesystemFilter
+	// What abstraction types should match (any contained in the set)
+	What AbstractionTypeSet
+
+	// The output for the query must satisfy _all_ (AND) requirements of all fields in this query struct.
+
+	// if not nil: JobID of the hold or bookmark in question must be equal
+	// else: JobID of the hold or bookmark can be any value
+	JobID *JobID
+
+	// zero-value means any CreateTXG is acceptable
+	CreateTXG CreateTXGRange
+
+	// Number of concurrently queried filesystems. Must be >= 1
+	Concurrency int64
+}
+
+type CreateTXGRangeBound struct {
+	CreateTXG uint64
+	Inclusive *zfs.NilBool // must not be nil
+}
+
+// A non-empty range of CreateTXGs
+//
+// If both Since and Until are nil, any CreateTXG is acceptable
+type CreateTXGRange struct {
+	// if not nil: The hold's snapshot or the bookmark's createtxg must be greater than (or equal) Since
+	// else: CreateTXG of the hold or bookmark can be any value accepted by Until
+	Since *CreateTXGRangeBound
+	// if not nil: The hold's snapshot or the bookmark's createtxg must be less than (or equal) Until
+	// else: CreateTXG of the hold or bookmark can be any value accepted by Since
+	Until *CreateTXGRangeBound
+}
+
+// FS == nil XOR Filter == nil
+type ListZFSHoldsAndBookmarksQueryFilesystemFilter struct {
+	FS     *string
+	Filter zfs.DatasetFilter
+}
+
+func (q *ListZFSHoldsAndBookmarksQuery) Validate() error {
+	if err := q.FS.Validate(); err != nil {
+		return errors.Wrap(err, "FS")
+	}
+	if q.JobID != nil {
+		q.JobID.MustValidate() // FIXME
+	}
+	if err := q.CreateTXG.Validate(); err != nil {
+		return errors.Wrap(err, "CreateTXGRange")
+	}
+	if err := q.What.Validate(); err != nil {
+		return err
+	}
+	if q.Concurrency < 1 {
+		return errors.New("Concurrency must be >= 1")
+	}
+	return nil
+}
+
+var createTXGRangeBoundAllowCreateTXG0 = envconst.Bool("ZREPL_ENDPOINT_LIST_ABSTRACTIONS_QUERY_CREATETXG_RANGE_BOUND_ALLOW_0", false)
+
+func (i *CreateTXGRangeBound) Validate() error {
+	if err := i.Inclusive.Validate(); err != nil {
+		return errors.Wrap(err, "Inclusive")
+	}
+	if i.CreateTXG == 0 && !createTXGRangeBoundAllowCreateTXG0 {
+		return errors.New("CreateTXG must be non-zero")
+	}
+	return nil
+
+}
+
+func (f *ListZFSHoldsAndBookmarksQueryFilesystemFilter) Validate() error {
+	if f == nil {
+		return nil
+	}
+	fsSet := f.FS != nil
+	filterSet := f.Filter != nil
+	if fsSet && filterSet || !fsSet && !filterSet {
+		return fmt.Errorf("must set FS or Filter field, but fsIsSet=%v and filterIsSet=%v", fsSet, filterSet)
+	}
+	if fsSet {
+		if err := zfs.EntityNamecheck(*f.FS, zfs.EntityTypeFilesystem); err != nil {
+			return errors.Wrap(err, "FS invalid")
+		}
+	}
+	return nil
+}
+
+func (f *ListZFSHoldsAndBookmarksQueryFilesystemFilter) Filesystems(ctx context.Context) ([]string, error) {
+	if err := f.Validate(); err != nil {
+		panic(err)
+	}
+	if f.FS != nil {
+		return []string{*f.FS}, nil
+	}
+	if f.Filter != nil {
+		dps, err := zfs.ZFSListMapping(ctx, f.Filter)
+		if err != nil {
+			return nil, err
+		}
+		fss := make([]string, len(dps))
+		for i, dp := range dps {
+			fss[i] = dp.ToString()
+		}
+		return fss, nil
+	}
+	panic("unreachable")
+}
+
+func (r *CreateTXGRange) Validate() error {
+	if r.Since != nil {
+		if err := r.Since.Validate(); err != nil {
+			return errors.Wrap(err, "Since")
+		}
+	}
+	if r.Until != nil {
+		if err := r.Until.Validate(); err != nil {
+			return errors.Wrap(err, "Until")
+		}
+	}
+	if _, err := r.effectiveBounds(); err != nil {
+		return errors.Wrapf(err, "specified range %s is semantically invalid", r)
+	}
+	return nil
+}
+
+// inclusive-inclusive bounds
+type effectiveBounds struct {
+	sinceInclusive uint64
+	sinceUnbounded bool
+	untilInclusive uint64
+	untilUnbounded bool
+}
+
+// callers must have validated r.Since and r.Until before calling this method
+func (r *CreateTXGRange) effectiveBounds() (bounds effectiveBounds, err error) {
+
+	bounds.sinceUnbounded = r.Since == nil
+	bounds.untilUnbounded = r.Until == nil
+
+	if r.Since == nil && r.Until == nil {
+		return bounds, nil
+	}
+
+	if r.Since != nil {
+		bounds.sinceInclusive = r.Since.CreateTXG
+		if !r.Since.Inclusive.B {
+			if r.Since.CreateTXG == math.MaxUint64 {
+				return bounds, errors.Errorf("Since-exclusive (%v) must be less than math.MaxUint64 (%v)",
+					r.Since.CreateTXG, uint64(math.MaxUint64))
+			}
+			bounds.sinceInclusive++
+		}
+	}
+
+	if r.Until != nil {
+		bounds.untilInclusive = r.Until.CreateTXG
+		if !r.Until.Inclusive.B {
+			if r.Until.CreateTXG == 0 {
+				return bounds, errors.Errorf("Until-exclusive (%v) must be greater than 0", r.Until.CreateTXG)
+			}
+			bounds.untilInclusive--
+		}
+	}
+
+	if !bounds.sinceUnbounded && !bounds.untilUnbounded {
+		if bounds.sinceInclusive >= bounds.untilInclusive {
+			return bounds, errors.Errorf("effective range bounds are [%v,%v] which is empty or invalid", bounds.sinceInclusive, bounds.untilInclusive)
+		}
+		// fallthrough
+	}
+
+	return bounds, nil
+}
+
+func (r *CreateTXGRange) String() string {
+	var buf strings.Builder
+	if r.Since == nil {
+		fmt.Fprintf(&buf, "~")
+	} else {
+		if err := r.Since.Inclusive.Validate(); err != nil {
+			fmt.Fprintf(&buf, "?")
+		} else if r.Since.Inclusive.B {
+			fmt.Fprintf(&buf, "[")
+		} else {
+			fmt.Fprintf(&buf, "(")
+		}
+		fmt.Fprintf(&buf, "%d", r.Since.CreateTXG)
+	}
+
+	fmt.Fprintf(&buf, ",")
+
+	if r.Until == nil {
+		fmt.Fprintf(&buf, "~")
+	} else {
+		fmt.Fprintf(&buf, "%d", r.Until.CreateTXG)
+		if err := r.Until.Inclusive.Validate(); err != nil {
+			fmt.Fprintf(&buf, "?")
+		} else if r.Until.Inclusive.B {
+			fmt.Fprintf(&buf, "]")
+		} else {
+			fmt.Fprintf(&buf, ")")
+		}
+	}
+
+	return buf.String()
+}
+
+// panics if not .Validate()
+func (r *CreateTXGRange) IsUnbounded() bool {
+	if err := r.Validate(); err != nil {
+		panic(err)
+	}
+	bounds, err := r.effectiveBounds()
+	if err != nil {
+		panic(err)
+	}
+	return bounds.sinceUnbounded && bounds.untilUnbounded
+}
+
+// panics if not .Validate()
+func (r *CreateTXGRange) Contains(qCreateTxg uint64) bool {
+	if err := r.Validate(); err != nil {
+		panic(err)
+	}
+
+	bounds, err := r.effectiveBounds()
+	if err != nil {
+		panic(err)
+	}
+
+	sinceMatches := bounds.sinceUnbounded || bounds.sinceInclusive <= qCreateTxg
+	untilMatches := bounds.untilUnbounded || qCreateTxg <= bounds.untilInclusive
+
+	return sinceMatches && untilMatches
+}
+
+type ListAbstractionsError struct {
+	FS   string
+	Snap string
+	What string
+	Err  error
+}
+
+func (e ListAbstractionsError) Error() string {
+	if e.FS == "" {
+		return fmt.Sprintf("list endpoint abstractions: %s: %s", e.What, e.Err)
+	} else {
+		v := e.FS
+		if e.Snap != "" {
+			v = fmt.Sprintf("%s@%s", e.FS, e.Snap)
+		}
+		return fmt.Sprintf("list endpoint abstractions on %q: %s: %s", v, e.What, e.Err)
+	}
+}
+
+type putListAbstractionErr func(err error, fs string, what string)
+type putListAbstraction func(a Abstraction)
+
+type ListAbstractionsErrors []ListAbstractionsError
+
+func (e ListAbstractionsErrors) Error() string {
+	if len(e) == 0 {
+		panic(e)
+	}
+	if len(e) == 1 {
+		return fmt.Sprintf("list endpoint abstractions: %s", e[0])
+	}
+	msgs := make([]string, len(e))
+	for i := range e {
+		msgs[i] = e[i].Error()
+	}
+	return fmt.Sprintf("list endpoint abstractions: multiple errors:\n%s", strings.Join(msgs, "\n"))
+}
+
+func ListAbstractions(ctx context.Context, query ListZFSHoldsAndBookmarksQuery) (out []Abstraction, outErrs []ListAbstractionsError, err error) {
+	outChan, outErrsChan, err := ListAbstractionsStreamed(ctx, query)
+	if err != nil {
+		return nil, nil, err
+	}
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for a := range outChan {
+			out = append(out, a)
+		}
+	}()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for err := range outErrsChan {
+			outErrs = append(outErrs, err)
+		}
+	}()
+	wg.Wait()
+	return out, outErrs, nil
+}
+
+// if err != nil, the returned channels are both nil
+// if err == nil, both channels must be fully drained by the caller to avoid leaking goroutines
+func ListAbstractionsStreamed(ctx context.Context, query ListZFSHoldsAndBookmarksQuery) (<-chan Abstraction, <-chan ListAbstractionsError, error) {
+
+	// impl note: structure the query processing in such a way that
+	// a minimum amount of zfs shell-outs needs to be done
+
+	if err := query.Validate(); err != nil {
+		return nil, nil, errors.Wrap(err, "validate query")
+	}
+
+	fss, err := query.FS.Filesystems(ctx)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "list filesystems")
+	}
+
+	outErrs := make(chan ListAbstractionsError)
+	out := make(chan Abstraction)
+
+	errCb := func(err error, fs string, what string) {
+		outErrs <- ListAbstractionsError{Err: err, FS: fs, What: what}
+	}
+	emitAbstraction := func(a Abstraction) {
+		jobIdMatches := query.JobID == nil || a.GetJobID() == nil || *a.GetJobID() == *query.JobID
+
+		createTXGMatches := query.CreateTXG.Contains(a.GetCreateTXG())
+
+		if jobIdMatches && createTXGMatches {
+			out <- a
+		}
+	}
+
+	sem := semaphore.New(int64(query.Concurrency))
+	ctx, endTask := trace.WithTask(ctx, "list-abstractions-streamed-producer")
+	go func() {
+		defer endTask()
+		defer close(out)
+		defer close(outErrs)
+
+		_, add, wait := trace.WithTaskGroup(ctx, "list-abstractions-impl-fs")
+		defer wait()
+		for i := range fss {
+			add(func(ctx context.Context) {
+				g, err := sem.Acquire(ctx)
+				if err != nil {
+					errCb(err, fss[i], err.Error())
+					return
+				}
+				func() {
+					defer g.Release()
+					listAbstractionsImplFS(ctx, fss[i], &query, emitAbstraction, errCb)
+				}()
+			})
+		}
+	}()
+
+	return out, outErrs, nil
+}
+
+func listAbstractionsImplFS(ctx context.Context, fs string, query *ListZFSHoldsAndBookmarksQuery, emitCandidate putListAbstraction, errCb putListAbstractionErr) {
+	fsp, err := zfs.NewDatasetPath(fs)
+	if err != nil {
+		panic(err)
+	}
+
+	if len(query.What) == 0 {
+		return
+	}
+
+	whatTypes := zfs.VersionTypeSet{}
+	for what := range query.What {
+		if e := what.BookmarkExtractor(); e != nil {
+			whatTypes[zfs.Bookmark] = true
+		}
+		if e := what.HoldExtractor(); e != nil {
+			whatTypes[zfs.Snapshot] = true
+		}
+	}
+	fsvs, err := zfs.ZFSListFilesystemVersions(ctx, fsp, zfs.ListFilesystemVersionsOptions{
+		Types: whatTypes,
+	})
+	if err != nil {
+		errCb(err, fs, "list filesystem versions")
+		return
+	}
+
+	for at := range query.What {
+		bmE := at.BookmarkExtractor()
+		holdE := at.HoldExtractor()
+		if bmE == nil && holdE == nil || bmE != nil && holdE != nil {
+			panic("implementation error: extractors misconfigured for " + at)
+		}
+		for _, v := range fsvs {
+			var a Abstraction
+			if v.Type == zfs.Bookmark && bmE != nil {
+				a = bmE(fsp, v)
+			}
+			if v.Type == zfs.Snapshot && holdE != nil && query.CreateTXG.Contains(v.GetCreateTXG()) && (!v.UserRefs.Valid || v.UserRefs.Value > 0) {
+				holds, err := zfs.ZFSHolds(ctx, fsp.ToString(), v.Name)
+				if err != nil {
+					errCb(err, v.ToAbsPath(fsp), "get hold on snap")
+					continue
+				}
+				for _, tag := range holds {
+					a = holdE(fsp, v, tag)
+				}
+			}
+			if a != nil {
+				emitCandidate(a)
+			}
+		}
+	}
+}
+
+type BatchDestroyResult struct {
+	Abstraction
+	DestroyErr error
+}
+
+var _ json.Marshaler = (*BatchDestroyResult)(nil)
+
+func (r BatchDestroyResult) MarshalJSON() ([]byte, error) {
+	err := ""
+	if r.DestroyErr != nil {
+		err = r.DestroyErr.Error()
+	}
+	s := struct {
+		Abstraction AbstractionJSON
+		DestroyErr  string
+	}{
+		AbstractionJSON{r.Abstraction},
+		err,
+	}
+	return json.Marshal(s)
+}
+
+func BatchDestroy(ctx context.Context, abs []Abstraction) <-chan BatchDestroyResult {
+	// hold-based batching: per snapshot
+	// bookmark-based batching: none possible via CLI
+	// => not worth the trouble for now, will be worth it once we start using channel programs
+	// => TODO: actual batching using channel programs
+	res := make(chan BatchDestroyResult, len(abs))
+	go func() {
+		for _, a := range abs {
+			res <- BatchDestroyResult{
+				a,
+				a.Destroy(ctx),
+			}
+		}
+		close(res)
+	}()
+	return res
+}
+
+type StalenessInfo struct {
+	ConstructedWithQuery ListZFSHoldsAndBookmarksQuery
+	Live                 []Abstraction
+	Stale                []Abstraction
+}
+
+type fsAndJobId struct {
+	fs    string
+	jobId JobID
+}
+
+type ListStaleQueryError struct {
+	error
+}
+
+// returns *ListStaleQueryError if the given query cannot be used for determining staleness info
+func ListStale(ctx context.Context, q ListZFSHoldsAndBookmarksQuery) (*StalenessInfo, error) {
+	if !q.CreateTXG.IsUnbounded() {
+		// we must determine the most recent step per FS, can't allow that
+		return nil, &ListStaleQueryError{errors.New("ListStale cannot have Until != nil set on query")}
+	}
+
+	// if asking for step holds, must also as for step bookmarks (same kind of abstraction)
+	// as well as replication cursor bookmarks (for firstNotStale)
+	ifAnyThenAll := AbstractionTypeSet{
+		AbstractionStepHold:                    true,
+		AbstractionStepBookmark:                true,
+		AbstractionReplicationCursorBookmarkV2: true,
+	}
+	if q.What.ContainsAnyOf(ifAnyThenAll) && !q.What.ContainsAll(ifAnyThenAll) {
+		return nil, &ListStaleQueryError{errors.Errorf("ListStale requires query to ask for all of %s", ifAnyThenAll.String())}
+	}
+
+	// ----------------- done validating query for listStaleFiltering -----------------------
+
+	qAbs, absErr, err := ListAbstractions(ctx, q)
+	if err != nil {
+		return nil, err
+	}
+	if len(absErr) > 0 {
+		// can't go on here because we can't determine the most recent step
+		return nil, ListAbstractionsErrors(absErr)
+	}
+
+	si := listStaleFiltering(qAbs, q.CreateTXG.Since)
+	si.ConstructedWithQuery = q
+	return si, nil
+}
+
+type fsAjobAtype struct {
+	fsAndJobId
+	Type AbstractionType
+}
+
+// For step holds and bookmarks, only those older than the most recent replication cursor
+// of their (filesystem,job) is considered because younger ones cannot be stale by definition
+// (if we destroy them, we might actually lose the hold on the `To` for an ongoing incremental replication)
+//
+// For replication cursors and last-received-holds, only the most recent one is kept.
+//
+// the returned StalenessInfo.ConstructedWithQuery is not set
+func listStaleFiltering(abs []Abstraction, sinceBound *CreateTXGRangeBound) *StalenessInfo {
+
+	var noJobId []Abstraction
+	by := make(map[fsAjobAtype][]Abstraction)
+	for _, a := range abs {
+		if a.GetJobID() == nil {
+			noJobId = append(noJobId, a)
+			continue
+		}
+		faj := fsAjobAtype{fsAndJobId{a.GetFS(), *a.GetJobID()}, a.GetType()}
+		l := by[faj]
+		l = append(l, a)
+		by[faj] = l
+	}
+
+	type stepFirstNotStaleCandidate struct {
+		cursor *Abstraction
+		step   *Abstraction
+	}
+	stepFirstNotStaleCandidates := make(map[fsAndJobId]stepFirstNotStaleCandidate) // empty map => will always return nil
+	for _, a := range abs {
+		if a.GetJobID() == nil {
+			continue // already put those into always-live list noJobId in above loop
+		}
+		key := fsAndJobId{a.GetFS(), *a.GetJobID()}
+		c := stepFirstNotStaleCandidates[key]
+
+		switch a.GetType() {
+		// stepFirstNotStaleCandidate.cursor
+		case AbstractionReplicationCursorBookmarkV2:
+			if c.cursor == nil || (*c.cursor).GetCreateTXG() < a.GetCreateTXG() {
+				a := a
+				c.cursor = &a
+			}
+
+		// stepFirstNotStaleCandidate.step
+		case AbstractionStepBookmark:
+			fallthrough
+		case AbstractionStepHold:
+			if c.step == nil || (*c.step).GetCreateTXG() < a.GetCreateTXG() {
+				a := a
+				c.step = &a
+			}
+
+		// not interested in the others
+		default:
+			continue // not relevant
+
+		}
+
+		stepFirstNotStaleCandidates[key] = c
+	}
+
+	ret := &StalenessInfo{
+		Live:  noJobId,
+		Stale: []Abstraction{},
+	}
+
+	for k := range by {
+		l := by[k]
+
+		if k.Type == AbstractionStepHold || k.Type == AbstractionStepBookmark {
+			// all older than the most recent cursor are stale, others are always live
+
+			// if we don't have a replication cursor yet, use untilBound = nil
+			// to consider all steps stale (...at first)
+			var untilBound *CreateTXGRangeBound
+			{
+				sfnsc := stepFirstNotStaleCandidates[k.fsAndJobId]
+
+				// if there's a replication cursor, use it as a cutoff between live and stale
+				// if there's none, we are in initial replication and only need to keep
+				// the most recent step hold live, since that's what our initial replication strategy
+				// uses (both initially and on resume)
+				// (FIXME hardcoded replication strategy)
+				if sfnsc.cursor != nil {
+					untilBound = &CreateTXGRangeBound{
+						CreateTXG: (*sfnsc.cursor).GetCreateTXG(),
+						// if we have a cursor, can throw away step hold on both From and To
+						Inclusive: &zfs.NilBool{B: true},
+					}
+				} else if sfnsc.step != nil {
+					untilBound = &CreateTXGRangeBound{
+						CreateTXG: (*sfnsc.step).GetCreateTXG(),
+						// if we don't have a cursor, the step most recent step hold is our
+						// initial replication cursor and it's possibly still live (interrupted initial replication)
+						Inclusive: &zfs.NilBool{B: false},
+					}
+				} else {
+					untilBound = nil // consider everything stale
+				}
+			}
+			staleRange := CreateTXGRange{
+				Since: sinceBound,
+				Until: untilBound,
+			}
+
+			// partition by staleRange
+			for _, a := range l {
+				if staleRange.Contains(a.GetCreateTXG()) {
+					ret.Stale = append(ret.Stale, a)
+				} else {
+					ret.Live = append(ret.Live, a)
+				}
+			}
+
+		} else if k.Type == AbstractionReplicationCursorBookmarkV2 || k.Type == AbstractionLastReceivedHold {
+			// all but the most recent are stale by definition (we always _move_ them)
+			// NOTE: must not use firstNotStale in this branch, not computed for these types
+
+			// sort descending (highest createtxg first), then cut off
+			sort.Slice(l, func(i, j int) bool {
+				return l[i].GetCreateTXG() > l[j].GetCreateTXG()
+			})
+			if len(l) > 0 {
+				ret.Live = append(ret.Live, l[0])
+				ret.Stale = append(ret.Stale, l[1:]...)
+			}
+		} else {
+			ret.Live = append(ret.Live, l...)
+		}
+	}
+
+	return ret
+
+}

endpoint/endpoint_zfs_abstraction_cursor_and_last_received_hold.go

@@ -0,0 +1,386 @@
+package endpoint
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"sort"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/util/errorarray"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+const replicationCursorBookmarkNamePrefix = "zrepl_CURSOR"
+
+func ReplicationCursorBookmarkName(fs string, guid uint64, id JobID) (string, error) {
+	return replicationCursorBookmarkNameImpl(fs, guid, id.String())
+}
+
+func replicationCursorBookmarkNameImpl(fs string, guid uint64, jobid string) (string, error) {
+	return makeJobAndGuidBookmarkName(replicationCursorBookmarkNamePrefix, fs, guid, jobid)
+}
+
+var ErrV1ReplicationCursor = fmt.Errorf("bookmark name is a v1-replication cursor")
+
+//err != nil always means that the bookmark is not a valid replication bookmark
+//
+// Returns ErrV1ReplicationCursor as error if the bookmark is a v1 replication cursor
+func ParseReplicationCursorBookmarkName(fullname string) (uint64, JobID, error) {
+
+	// check for legacy cursors
+	{
+		if err := zfs.EntityNamecheck(fullname, zfs.EntityTypeBookmark); err != nil {
+			return 0, JobID{}, errors.Wrap(err, "parse replication cursor bookmark name")
+		}
+		_, _, name, err := zfs.DecomposeVersionString(fullname)
+		if err != nil {
+			return 0, JobID{}, errors.Wrap(err, "parse replication cursor bookmark name: decompose version string")
+		}
+		const V1ReplicationCursorBookmarkName = "zrepl_replication_cursor"
+		if name == V1ReplicationCursorBookmarkName {
+			return 0, JobID{}, ErrV1ReplicationCursor
+		}
+		// fallthrough to main parser
+	}
+
+	guid, jobID, err := parseJobAndGuidBookmarkName(fullname, replicationCursorBookmarkNamePrefix)
+	if err != nil {
+		err = errors.Wrap(err, "parse replication cursor bookmark name") // no shadow
+	}
+	return guid, jobID, err
+}
+
+// may return nil for both values, indicating there is no cursor
+func GetMostRecentReplicationCursorOfJob(ctx context.Context, fs string, jobID JobID) (*zfs.FilesystemVersion, error) {
+	fsp, err := zfs.NewDatasetPath(fs)
+	if err != nil {
+		return nil, err
+	}
+	candidates, err := GetReplicationCursors(ctx, fsp, jobID)
+	if err != nil || len(candidates) == 0 {
+		return nil, err
+	}
+
+	sort.Slice(candidates, func(i, j int) bool {
+		return candidates[i].CreateTXG < candidates[j].CreateTXG
+	})
+
+	mostRecent := candidates[len(candidates)-1]
+	return &mostRecent, nil
+}
+
+func GetReplicationCursors(ctx context.Context, dp *zfs.DatasetPath, jobID JobID) ([]zfs.FilesystemVersion, error) {
+
+	fs := dp.ToString()
+	q := ListZFSHoldsAndBookmarksQuery{
+		FS: ListZFSHoldsAndBookmarksQueryFilesystemFilter{FS: &fs},
+		What: map[AbstractionType]bool{
+			AbstractionReplicationCursorBookmarkV1: true,
+			AbstractionReplicationCursorBookmarkV2: true,
+		},
+		JobID:       &jobID,
+		CreateTXG:   CreateTXGRange{},
+		Concurrency: 1,
+	}
+	abs, absErr, err := ListAbstractions(ctx, q)
+	if err != nil {
+		return nil, errors.Wrap(err, "get replication cursor: list bookmarks and holds")
+	}
+	if len(absErr) > 0 {
+		return nil, ListAbstractionsErrors(absErr)
+	}
+
+	var v1, v2 []Abstraction
+	for _, a := range abs {
+		switch a.GetType() {
+		case AbstractionReplicationCursorBookmarkV1:
+			v1 = append(v1, a)
+		case AbstractionReplicationCursorBookmarkV2:
+			v2 = append(v2, a)
+		default:
+			panic("unexpected abstraction: " + a.GetType())
+		}
+	}
+
+	if len(v1) > 0 {
+		getLogger(ctx).WithField("bookmark", v1).
+			Warn("found v1-replication cursor bookmarks, consider running migration 'replication-cursor:v1-v2' after successful replication with this zrepl version")
+	}
+
+	candidates := make([]zfs.FilesystemVersion, 0)
+	for _, v := range v2 {
+		candidates = append(candidates, v.GetFilesystemVersion())
+	}
+
+	return candidates, nil
+}
+
+type ReplicationCursorTarget interface {
+	IsSnapshot() bool
+	GetGuid() uint64
+	GetCreateTXG() uint64
+	ToSendArgVersion() zfs.ZFSSendArgVersion
+}
+
+// `target` is validated before replication cursor is set. if validation fails, the cursor is not moved.
+//
+// returns ErrBookmarkCloningNotSupported if version is a bookmark and bookmarking bookmarks is not supported by ZFS
+func MoveReplicationCursor(ctx context.Context, fs string, target ReplicationCursorTarget, jobID JobID) (destroyedCursors []Abstraction, err error) {
+
+	if !target.IsSnapshot() {
+		return nil, zfs.ErrBookmarkCloningNotSupported
+	}
+
+	bookmarkname, err := ReplicationCursorBookmarkName(fs, target.GetGuid(), jobID)
+	if err != nil {
+		return nil, errors.Wrap(err, "determine replication cursor name")
+	}
+
+	// idempotently create bookmark (guid is encoded in it, hence we'll most likely add a new one
+	// cleanup the old one afterwards
+
+	err = zfs.ZFSBookmark(ctx, fs, target.ToSendArgVersion(), bookmarkname)
+	if err != nil {
+		if err == zfs.ErrBookmarkCloningNotSupported {
+			return nil, err // TODO go1.13 use wrapping
+		}
+		return nil, errors.Wrapf(err, "cannot create bookmark")
+	}
+
+	destroyedCursors, err = DestroyObsoleteReplicationCursors(ctx, fs, target, jobID)
+	if err != nil {
+		return nil, errors.Wrap(err, "destroy obsolete replication cursors")
+	}
+
+	return destroyedCursors, nil
+}
+
+type ReplicationCursor interface {
+	GetCreateTXG() uint64
+}
+
+func DestroyObsoleteReplicationCursors(ctx context.Context, fs string, current ReplicationCursor, jobID JobID) (_ []Abstraction, err error) {
+
+	q := ListZFSHoldsAndBookmarksQuery{
+		FS: ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &fs,
+		},
+		What: AbstractionTypeSet{
+			AbstractionReplicationCursorBookmarkV2: true,
+		},
+		JobID: &jobID,
+		CreateTXG: CreateTXGRange{
+			Since: nil,
+			Until: &CreateTXGRangeBound{
+				CreateTXG: current.GetCreateTXG(),
+				Inclusive: &zfs.NilBool{B: false},
+			},
+		},
+		Concurrency: 1,
+	}
+	abs, absErr, err := ListAbstractions(ctx, q)
+	if err != nil {
+		return nil, errors.Wrap(err, "list abstractions")
+	}
+	if len(absErr) > 0 {
+		return nil, errors.Wrap(ListAbstractionsErrors(absErr), "list abstractions")
+	}
+
+	var destroyed []Abstraction
+	var errs []error
+	for res := range BatchDestroy(ctx, abs) {
+		log := getLogger(ctx).
+			WithField("replication_cursor_bookmark", res.Abstraction)
+		if res.DestroyErr != nil {
+			errs = append(errs, res.DestroyErr)
+			log.WithError(err).
+				Error("cannot destroy obsolete replication cursor bookmark")
+		} else {
+			destroyed = append(destroyed, res.Abstraction)
+			log.Info("destroyed obsolete replication cursor bookmark")
+		}
+	}
+	if len(errs) == 0 {
+		return destroyed, nil
+	} else {
+		return destroyed, errorarray.Wrap(errs, "destroy obsolete replication cursor")
+	}
+}
+
+var lastReceivedHoldTagRE = regexp.MustCompile("^zrepl_last_received_J_(.+)$")
+
+// err != nil always means that the bookmark is not a step bookmark
+func ParseLastReceivedHoldTag(tag string) (JobID, error) {
+	match := lastReceivedHoldTagRE.FindStringSubmatch(tag)
+	if match == nil {
+		return JobID{}, errors.Errorf("parse last-received-hold tag: does not match regex %s", lastReceivedHoldTagRE.String())
+	}
+	jobId, err := MakeJobID(match[1])
+	if err != nil {
+		return JobID{}, errors.Wrap(err, "parse last-received-hold tag: invalid job id field")
+	}
+	return jobId, nil
+}
+
+func LastReceivedHoldTag(jobID JobID) (string, error) {
+	return lastReceivedHoldImpl(jobID.String())
+}
+
+func lastReceivedHoldImpl(jobid string) (string, error) {
+	tag := fmt.Sprintf("zrepl_last_received_J_%s", jobid)
+	if err := zfs.ValidHoldTag(tag); err != nil {
+		return "", err
+	}
+	return tag, nil
+}
+
+func MoveLastReceivedHold(ctx context.Context, fs string, to zfs.FilesystemVersion, jobID JobID) error {
+
+	if !to.IsSnapshot() {
+		return errors.Errorf("last-received-hold: target must be a snapshot: %s", to.FullPath(fs))
+	}
+
+	tag, err := LastReceivedHoldTag(jobID)
+	if err != nil {
+		return errors.Wrap(err, "last-received-hold: hold tag")
+	}
+
+	// we never want to be without a hold
+	// => hold new one before releasing old hold
+
+	err = zfs.ZFSHold(ctx, fs, to, tag)
+	if err != nil {
+		return errors.Wrap(err, "last-received-hold: hold newly received")
+	}
+
+	q := ListZFSHoldsAndBookmarksQuery{
+		What: AbstractionTypeSet{
+			AbstractionLastReceivedHold: true,
+		},
+		FS: ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &fs,
+		},
+		JobID: &jobID,
+		CreateTXG: CreateTXGRange{
+			Since: nil,
+			Until: &CreateTXGRangeBound{
+				CreateTXG: to.GetCreateTXG(),
+				Inclusive: &zfs.NilBool{B: false},
+			},
+		},
+		Concurrency: 1,
+	}
+	abs, absErrs, err := ListAbstractions(ctx, q)
+	if err != nil {
+		return errors.Wrap(err, "last-received-hold: list")
+	}
+	if len(absErrs) > 0 {
+		return errors.Wrap(ListAbstractionsErrors(absErrs), "last-received-hold: list")
+	}
+
+	getLogger(ctx).WithField("last-received-holds", fmt.Sprintf("%s", abs)).Debug("releasing last-received-holds")
+
+	var errs []error
+	for res := range BatchDestroy(ctx, abs) {
+		log := getLogger(ctx).
+			WithField("last-received-hold", res.Abstraction)
+		if res.DestroyErr != nil {
+			errs = append(errs, res.DestroyErr)
+			log.WithError(err).
+				Error("cannot release last-received-hold")
+		} else {
+			log.Info("released last-received-hold")
+		}
+	}
+	if len(errs) == 0 {
+		return nil
+	} else {
+		return errorarray.Wrap(errs, "last-received-hold: release")
+	}
+}
+
+func ReplicationCursorV2Extractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion) (_ Abstraction) {
+	if v.Type != zfs.Bookmark {
+		panic("impl error")
+	}
+	fullname := v.ToAbsPath(fs)
+	guid, jobid, err := ParseReplicationCursorBookmarkName(fullname)
+	if err == nil {
+		if guid != v.Guid {
+			// TODO log this possibly tinkered-with bookmark
+			return nil
+		}
+		return &bookmarkBasedAbstraction{
+			Type:              AbstractionReplicationCursorBookmarkV2,
+			FS:                fs.ToString(),
+			FilesystemVersion: v,
+			JobID:             jobid,
+		}
+	}
+	return nil
+}
+
+func ReplicationCursorV1Extractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion) (_ Abstraction) {
+	if v.Type != zfs.Bookmark {
+		panic("impl error")
+	}
+	fullname := v.ToAbsPath(fs)
+	_, _, err := ParseReplicationCursorBookmarkName(fullname)
+	if err == ErrV1ReplicationCursor {
+		return &ReplicationCursorV1{
+			Type:              AbstractionReplicationCursorBookmarkV1,
+			FS:                fs.ToString(),
+			FilesystemVersion: v,
+		}
+	}
+	return nil
+}
+
+var _ HoldExtractor = LastReceivedHoldExtractor
+
+func LastReceivedHoldExtractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion, holdTag string) Abstraction {
+	var err error
+
+	if v.Type != zfs.Snapshot {
+		panic("impl error")
+	}
+
+	jobID, err := ParseLastReceivedHoldTag(holdTag)
+	if err == nil {
+		return &holdBasedAbstraction{
+			Type:              AbstractionLastReceivedHold,
+			FS:                fs.ToString(),
+			FilesystemVersion: v,
+			Tag:               holdTag,
+			JobID:             jobID,
+		}
+	}
+	return nil
+}
+
+type ReplicationCursorV1 struct {
+	Type AbstractionType
+	FS   string
+	zfs.FilesystemVersion
+}
+
+func (c ReplicationCursorV1) GetType() AbstractionType                    { return c.Type }
+func (c ReplicationCursorV1) GetFS() string                               { return c.FS }
+func (c ReplicationCursorV1) GetFullPath() string                         { return fmt.Sprintf("%s#%s", c.FS, c.GetName()) }
+func (c ReplicationCursorV1) GetJobID() *JobID                            { return nil }
+func (c ReplicationCursorV1) GetFilesystemVersion() zfs.FilesystemVersion { return c.FilesystemVersion }
+func (c ReplicationCursorV1) MarshalJSON() ([]byte, error) {
+	return json.Marshal(AbstractionJSON{c})
+}
+func (c ReplicationCursorV1) String() string {
+	return fmt.Sprintf("%s %s", c.Type, c.GetFullPath())
+}
+func (c ReplicationCursorV1) Destroy(ctx context.Context) error {
+	if err := zfs.ZFSDestroyIdempotent(ctx, c.GetFullPath()); err != nil {
+		return errors.Wrapf(err, "destroy %s %s: zfs", c.Type, c.GetFullPath())
+	}
+	return nil
+}

endpoint/endpoint_zfs_abstraction_step.go

@@ -0,0 +1,286 @@
+package endpoint
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/util/errorarray"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var stepHoldTagRE = regexp.MustCompile("^zrepl_STEP_J_(.+)")
+
+func StepHoldTag(jobid JobID) (string, error) {
+	return stepHoldTagImpl(jobid.String())
+}
+
+func stepHoldTagImpl(jobid string) (string, error) {
+	t := fmt.Sprintf("zrepl_STEP_J_%s", jobid)
+	if err := zfs.ValidHoldTag(t); err != nil {
+		return "", err
+	}
+	return t, nil
+}
+
+// err != nil always means that the bookmark is not a step bookmark
+func ParseStepHoldTag(tag string) (JobID, error) {
+	match := stepHoldTagRE.FindStringSubmatch(tag)
+	if match == nil {
+		return JobID{}, fmt.Errorf("parse hold tag: match regex %q", stepHoldTagRE)
+	}
+	jobID, err := MakeJobID(match[1])
+	if err != nil {
+		return JobID{}, errors.Wrap(err, "parse hold tag: invalid job id field")
+	}
+	return jobID, nil
+}
+
+const stepBookmarkNamePrefix = "zrepl_STEP"
+
+// v must be validated by caller
+func StepBookmarkName(fs string, guid uint64, id JobID) (string, error) {
+	return stepBookmarkNameImpl(fs, guid, id.String())
+}
+
+func stepBookmarkNameImpl(fs string, guid uint64, jobid string) (string, error) {
+	return makeJobAndGuidBookmarkName(stepBookmarkNamePrefix, fs, guid, jobid)
+}
+
+// name is the full bookmark name, including dataset path
+//
+// err != nil always means that the bookmark is not a step bookmark
+func ParseStepBookmarkName(fullname string) (guid uint64, jobID JobID, err error) {
+	guid, jobID, err = parseJobAndGuidBookmarkName(fullname, stepBookmarkNamePrefix)
+	if err != nil {
+		err = errors.Wrap(err, "parse step bookmark name") // no shadow!
+	}
+	return guid, jobID, err
+}
+
+// idempotently hold / step-bookmark `version`
+//
+// returns ErrBookmarkCloningNotSupported if version is a bookmark and bookmarking bookmarks is not supported by ZFS
+func HoldStep(ctx context.Context, fs string, v zfs.FilesystemVersion, jobID JobID) (Abstraction, error) {
+	if v.IsSnapshot() {
+
+		tag, err := StepHoldTag(jobID)
+		if err != nil {
+			return nil, errors.Wrap(err, "step hold tag")
+		}
+
+		if err := zfs.ZFSHold(ctx, fs, v, tag); err != nil {
+			return nil, errors.Wrap(err, "step hold: zfs")
+		}
+
+		return &holdBasedAbstraction{
+			Type:              AbstractionStepHold,
+			FS:                fs,
+			Tag:               tag,
+			JobID:             jobID,
+			FilesystemVersion: v,
+		}, nil
+	}
+
+	if !v.IsBookmark() {
+		panic(fmt.Sprintf("version must bei either snapshot or bookmark, got %#v", v))
+	}
+
+	bmname, err := StepBookmarkName(fs, v.Guid, jobID)
+	if err != nil {
+		return nil, errors.Wrap(err, "create step bookmark: determine bookmark name")
+	}
+	// idempotently create bookmark
+	err = zfs.ZFSBookmark(ctx, fs, v.ToSendArgVersion(), bmname)
+	if err != nil {
+		if err == zfs.ErrBookmarkCloningNotSupported {
+			// TODO we could actually try to find a local snapshot that has the requested GUID
+			// 		however, the replication algorithm prefers snapshots anyways, so this quest
+			// 		is most likely not going to be successful. Also, there's the possibility that
+			//      the caller might want to filter what snapshots are eligibile, and this would
+			//      complicate things even further.
+			return nil, err // TODO go1.13 use wrapping
+		}
+		return nil, errors.Wrap(err, "create step bookmark: zfs")
+	}
+	return &bookmarkBasedAbstraction{
+		Type:              AbstractionStepBookmark,
+		FS:                fs,
+		FilesystemVersion: v,
+		JobID:             jobID,
+	}, nil
+}
+
+// idempotently release the step-hold on v if v is a snapshot
+// or idempotently destroy the step-bookmark  of v if v is a bookmark
+//
+// note that this operation leaves v itself untouched, unless v is the step-bookmark itself, in which case v is destroyed
+//
+// returns an instance of *zfs.DatasetDoesNotExist if `v` does not exist
+func ReleaseStep(ctx context.Context, fs string, v zfs.FilesystemVersion, jobID JobID) error {
+
+	if v.IsSnapshot() {
+		tag, err := StepHoldTag(jobID)
+		if err != nil {
+			return errors.Wrap(err, "step release tag")
+		}
+
+		if err := zfs.ZFSRelease(ctx, tag, v.FullPath(fs)); err != nil {
+			return errors.Wrap(err, "step release: zfs")
+		}
+
+		return nil
+	}
+	if !v.IsBookmark() {
+		panic(fmt.Sprintf("impl error: expecting version to be a bookmark, got %#v", v))
+	}
+
+	bmname, err := StepBookmarkName(fs, v.Guid, jobID)
+	if err != nil {
+		return errors.Wrap(err, "step release: determine bookmark name")
+	}
+	// idempotently destroy bookmark
+
+	if err := zfs.ZFSDestroyIdempotent(ctx, bmname); err != nil {
+		return errors.Wrap(err, "step release: bookmark destroy: zfs")
+	}
+
+	return nil
+}
+
+// release {step holds, step bookmarks} earlier and including `mostRecent`
+func ReleaseStepCummulativeInclusive(ctx context.Context, fs string, since *CreateTXGRangeBound, mostRecent zfs.FilesystemVersion, jobID JobID) error {
+	q := ListZFSHoldsAndBookmarksQuery{
+		What: AbstractionTypeSet{
+			AbstractionStepHold:     true,
+			AbstractionStepBookmark: true,
+		},
+		FS: ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &fs,
+		},
+		JobID: &jobID,
+		CreateTXG: CreateTXGRange{
+			Since: since,
+			Until: &CreateTXGRangeBound{
+				CreateTXG: mostRecent.CreateTXG,
+				Inclusive: &zfs.NilBool{B: true},
+			},
+		},
+		Concurrency: 1,
+	}
+	abs, absErrs, err := ListAbstractions(ctx, q)
+	if err != nil {
+		return errors.Wrap(err, "step release cummulative: list")
+	}
+	if len(absErrs) > 0 {
+		return errors.Wrap(ListAbstractionsErrors(absErrs), "step release cummulative: list")
+	}
+
+	getLogger(ctx).WithField("step_holds_and_bookmarks", fmt.Sprintf("%s", abs)).Debug("releasing step holds and bookmarks")
+
+	var errs []error
+	for res := range BatchDestroy(ctx, abs) {
+		log := getLogger(ctx).
+			WithField("step_hold_or_bookmark", res.Abstraction)
+		if res.DestroyErr != nil {
+			errs = append(errs, res.DestroyErr)
+			log.WithError(err).
+				Error("cannot release step hold or bookmark")
+		} else {
+			log.Info("released step hold or bookmark")
+		}
+	}
+	if len(errs) == 0 {
+		return nil
+	} else {
+		return errorarray.Wrap(errs, "step release cummulative: release")
+	}
+}
+
+func TryReleaseStepStaleFS(ctx context.Context, fs string, jobID JobID) {
+
+	q := ListZFSHoldsAndBookmarksQuery{
+		FS: ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &fs,
+		},
+		JobID: &jobID,
+		What: AbstractionTypeSet{
+			AbstractionStepHold:                    true,
+			AbstractionStepBookmark:                true,
+			AbstractionReplicationCursorBookmarkV2: true,
+		},
+		Concurrency: 1,
+	}
+	staleness, err := ListStale(ctx, q)
+	if _, ok := err.(*ListStaleQueryError); ok {
+		panic(err)
+	} else if err != nil {
+		getLogger(ctx).WithError(err).Error("cannot list stale step holds and bookmarks")
+		return
+	}
+	for _, s := range staleness.Stale {
+		getLogger(ctx).WithField("stale_step_hold_or_bookmark", s).Info("batch-destroying stale step hold or bookmark")
+	}
+	for res := range BatchDestroy(ctx, staleness.Stale) {
+		if res.DestroyErr != nil {
+			getLogger(ctx).
+				WithField("stale_step_hold_or_bookmark", res.Abstraction).
+				WithError(res.DestroyErr).
+				Error("cannot destroy stale step-hold or bookmark")
+		} else {
+			getLogger(ctx).
+				WithField("stale_step_hold_or_bookmark", res.Abstraction).
+				WithError(res.DestroyErr).
+				Info("destroyed stale step-hold or bookmark")
+		}
+	}
+
+}
+
+var _ BookmarkExtractor = StepBookmarkExtractor
+
+func StepBookmarkExtractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion) (_ Abstraction) {
+	if v.Type != zfs.Bookmark {
+		panic("impl error")
+	}
+
+	fullname := v.ToAbsPath(fs)
+
+	guid, jobid, err := ParseStepBookmarkName(fullname)
+	if guid != v.Guid {
+		// TODO log this possibly tinkered-with bookmark
+		return nil
+	}
+	if err == nil {
+		bm := &bookmarkBasedAbstraction{
+			Type:              AbstractionStepBookmark,
+			FS:                fs.ToString(),
+			FilesystemVersion: v,
+			JobID:             jobid,
+		}
+		return bm
+	}
+	return nil
+}
+
+var _ HoldExtractor = StepHoldExtractor
+
+func StepHoldExtractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion, holdTag string) Abstraction {
+	if v.Type != zfs.Snapshot {
+		panic("impl error")
+	}
+
+	jobID, err := ParseStepHoldTag(holdTag)
+	if err == nil {
+		return &holdBasedAbstraction{
+			Type:              AbstractionStepHold,
+			FS:                fs.ToString(),
+			Tag:               holdTag,
+			FilesystemVersion: v,
+			JobID:             jobID,
+		}
+	}
+	return nil
+}

endpoint/endpoint_zfs_abstraction_test.go

@@ -0,0 +1,234 @@
+package endpoint
+
+import (
+	"fmt"
+	"math"
+	"runtime/debug"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func TestCreateTXGRange(t *testing.T) {
+
+	type testCaseExpectation struct {
+		input  uint64
+		expect bool
+	}
+	type testCase struct {
+		name                     string
+		config                   *CreateTXGRange
+		configAllowZeroCreateTXG bool
+		expectInvalid            bool
+		expectString             string
+		expect                   []testCaseExpectation
+	}
+
+	tcs := []testCase{
+		{
+			name:          "unbounded",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Since: nil,
+				Until: nil,
+			},
+			expectString: "~,~",
+			expect: []testCaseExpectation{
+				{0, true},
+				{math.MaxUint64, true},
+				{1, true},
+				{math.MaxUint64 - 1, true},
+			},
+		},
+		{
+			name:          "wrong order obvious",
+			expectInvalid: true,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{23, &zfs.NilBool{B: true}},
+				Until: &CreateTXGRangeBound{20, &zfs.NilBool{B: true}},
+			},
+			expectString: "[23,20]",
+		},
+		{
+			name:          "wrong order edge-case could also be empty",
+			expectInvalid: true,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{23, &zfs.NilBool{B: false}},
+				Until: &CreateTXGRangeBound{22, &zfs.NilBool{B: true}},
+			},
+			expectString: "(23,22]",
+		},
+		{
+			name:          "empty",
+			expectInvalid: true,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
+				Until: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
+			},
+			expectString: "(2,2)",
+		},
+		{
+			name:          "inclusive-since-exclusive-until",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: true}},
+				Until: &CreateTXGRangeBound{5, &zfs.NilBool{B: false}},
+			},
+			expectString: "[2,5)",
+			expect: []testCaseExpectation{
+				{0, false},
+				{1, false},
+				{2, true},
+				{3, true},
+				{4, true},
+				{5, false},
+				{6, false},
+			},
+		},
+		{
+			name:          "exclusive-since-inclusive-until",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
+				Until: &CreateTXGRangeBound{5, &zfs.NilBool{B: true}},
+			},
+			expectString: "(2,5]",
+			expect: []testCaseExpectation{
+				{0, false},
+				{1, false},
+				{2, false},
+				{3, true},
+				{4, true},
+				{5, true},
+				{6, false},
+			},
+		},
+		{
+			name:          "zero-createtxg-not-allowed-because-likely-programmer-error",
+			expectInvalid: true,
+			config: &CreateTXGRange{
+				Since: nil,
+				Until: &CreateTXGRangeBound{0, &zfs.NilBool{B: true}},
+			},
+			expectString: "~,0]",
+		},
+		{
+			name:          "half-open-no-until",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
+				Until: nil,
+			},
+			expectString: "(2,~",
+			expect: []testCaseExpectation{
+				{0, false},
+				{1, false},
+				{2, false},
+				{3, true},
+				{4, true},
+				{5, true},
+				{6, true},
+			},
+		},
+		{
+			name:          "half-open-no-since",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Since: nil,
+				Until: &CreateTXGRangeBound{4, &zfs.NilBool{B: true}},
+			},
+			expectString: "~,4]",
+			expect: []testCaseExpectation{
+				{0, true},
+				{1, true},
+				{2, true},
+				{3, true},
+				{4, true},
+				{5, false},
+			},
+		},
+		{
+			name:          "edgeSince",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{math.MaxUint64, &zfs.NilBool{B: true}},
+				Until: nil,
+			},
+			expectString: "[18446744073709551615,~",
+			expect: []testCaseExpectation{
+				{math.MaxUint64, true},
+				{math.MaxUint64 - 1, false},
+				{0, false},
+				{1, false},
+			},
+		},
+		{
+			name:          "edgeSinceNegative",
+			expectInvalid: true,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{math.MaxUint64, &zfs.NilBool{B: false}},
+				Until: nil,
+			},
+			expectString: "(18446744073709551615,~",
+		},
+		{
+			name:          "edgeUntil",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Until: &CreateTXGRangeBound{0, &zfs.NilBool{B: true}},
+			},
+			configAllowZeroCreateTXG: true,
+			expectString:             "~,0]",
+			expect: []testCaseExpectation{
+				{0, true},
+				{math.MaxUint64, false},
+				{1, false},
+			},
+		},
+		{
+			name:                     "edgeUntilNegative",
+			expectInvalid:            true,
+			configAllowZeroCreateTXG: true,
+			config: &CreateTXGRange{
+				Until: &CreateTXGRangeBound{0, &zfs.NilBool{B: false}},
+			},
+			expectString: "~,0)",
+		},
+	}
+
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			require.True(t, tc.expectInvalid != (len(tc.expect) > 0), "invalid test config: must either expect invalid or have expectations: %s", tc.name)
+			require.NotEmpty(t, tc.expectString)
+			assert.Equal(t, tc.expectString, tc.config.String())
+
+			save := createTXGRangeBoundAllowCreateTXG0
+			createTXGRangeBoundAllowCreateTXG0 = tc.configAllowZeroCreateTXG
+			defer func() {
+				createTXGRangeBoundAllowCreateTXG0 = save
+			}()
+
+			if tc.expectInvalid {
+				t.Run(tc.name, func(t *testing.T) {
+					assert.Error(t, tc.config.Validate())
+				})
+			} else {
+				for i, e := range tc.expect {
+					t.Run(fmt.Sprint(i), func(t *testing.T) {
+						defer func() {
+							v := recover()
+							if v != nil {
+								t.Fatalf("should not panic: %T %v\n%s", v, v, debug.Stack())
+							}
+						}()
+						assert.Equal(t, e.expect, tc.config.Contains(e.input))
+					})
+				}
+			}
+		})
+	}
+
+}

endpoint/endpoint_zfs_helpers_repr.go

@@ -0,0 +1,58 @@
+package endpoint
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+// returns the short name (no fs# prefix)
+func makeJobAndGuidBookmarkName(prefix string, fs string, guid uint64, jobid string) (string, error) {
+	bmname := fmt.Sprintf(prefix+"_G_%016x_J_%s", guid, jobid)
+	if err := zfs.EntityNamecheck(fmt.Sprintf("%s#%s", fs, bmname), zfs.EntityTypeBookmark); err != nil {
+		return "", err
+	}
+	return bmname, nil
+}
+
+var jobAndGuidBookmarkRE = regexp.MustCompile(`(.+)_G_([0-9a-f]{16})_J_(.+)$`)
+
+func parseJobAndGuidBookmarkName(fullname string, prefix string) (guid uint64, jobID JobID, _ error) {
+
+	if len(prefix) == 0 {
+		panic("prefix must not be empty")
+	}
+
+	if err := zfs.EntityNamecheck(fullname, zfs.EntityTypeBookmark); err != nil {
+		return 0, JobID{}, err
+	}
+
+	_, _, name, err := zfs.DecomposeVersionString(fullname)
+	if err != nil {
+		return 0, JobID{}, errors.Wrap(err, "decompose bookmark name")
+	}
+
+	match := jobAndGuidBookmarkRE.FindStringSubmatch(name)
+	if match == nil {
+		return 0, JobID{}, errors.Errorf("bookmark name does not match regex %q", jobAndGuidBookmarkRE.String())
+	}
+	if match[1] != prefix {
+		return 0, JobID{}, errors.Errorf("prefix component does not match: expected %q, got %q", prefix, match[1])
+	}
+
+	guid, err = strconv.ParseUint(match[2], 16, 64)
+	if err != nil {
+		return 0, JobID{}, errors.Wrapf(err, "parse guid component: %q", match[2])
+	}
+
+	jobID, err = MakeJobID(match[3])
+	if err != nil {
+		return 0, JobID{}, errors.Wrapf(err, "parse jobid component: %q", match[3])
+	}
+
+	return guid, jobID, nil
+}

endpoint/endpoint_zfs_helpers_repr_test.go

@@ -0,0 +1,51 @@
+package endpoint
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseJobAndGuidBookmarkName(t *testing.T) {
+
+	type Case struct {
+		input     string
+		expectErr bool
+
+		guid  uint64
+		jobid string
+	}
+
+	cases := []Case{
+		{
+			`p1/sync#zrepl_CURSOR_G_932f3a7089080ce2_J_push with legitimate name`,
+			false, 0x932f3a7089080ce2, "push with legitimate name",
+		},
+		{
+			input:     `p1/sync#zrepl_CURSOR_G_932f3a7089_J_push with legitimate name`,
+			expectErr: true,
+		},
+		{
+			input:     `p1/sync#zrepl_CURSOR_G_932f3a7089080ce2_J_push with il\tlegitimate name`,
+			expectErr: true,
+		},
+		{
+			input:     `p1/sync#otherprefix_G_932f3a7089080ce2_J_push with legitimate name`,
+			expectErr: true,
+		},
+	}
+
+	for i := range cases {
+		t.Run(cases[i].input, func(t *testing.T) {
+			guid, jobid, err := parseJobAndGuidBookmarkName(cases[i].input, replicationCursorBookmarkNamePrefix)
+			if cases[i].expectErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, cases[i].guid, guid)
+				assert.Equal(t, MustMakeJobID(cases[i].jobid), jobid)
+			}
+		})
+	}
+
+}

endpoint/endpoint_zfs_helpers_types.go

@@ -0,0 +1,74 @@
+package endpoint
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type bookmarkBasedAbstraction struct {
+	Type AbstractionType
+	FS   string
+	zfs.FilesystemVersion
+	JobID JobID
+}
+
+func (b bookmarkBasedAbstraction) GetType() AbstractionType { return b.Type }
+func (b bookmarkBasedAbstraction) GetFS() string            { return b.FS }
+func (b bookmarkBasedAbstraction) GetJobID() *JobID         { return &b.JobID }
+func (b bookmarkBasedAbstraction) GetFullPath() string {
+	return fmt.Sprintf("%s#%s", b.FS, b.Name) // TODO use zfs.FilesystemVersion.ToAbsPath
+}
+func (b bookmarkBasedAbstraction) MarshalJSON() ([]byte, error) {
+	return json.Marshal(AbstractionJSON{b})
+}
+func (b bookmarkBasedAbstraction) String() string {
+	return fmt.Sprintf("%s %s", b.Type, b.GetFullPath())
+}
+
+func (b bookmarkBasedAbstraction) GetFilesystemVersion() zfs.FilesystemVersion {
+	return b.FilesystemVersion
+}
+
+func (b bookmarkBasedAbstraction) Destroy(ctx context.Context) error {
+	if err := zfs.ZFSDestroyIdempotent(ctx, b.GetFullPath()); err != nil {
+		return errors.Wrapf(err, "destroy %s: zfs", b)
+	}
+	return nil
+}
+
+type holdBasedAbstraction struct {
+	Type AbstractionType
+	FS   string
+	zfs.FilesystemVersion
+	Tag   string
+	JobID JobID
+}
+
+func (h holdBasedAbstraction) GetType() AbstractionType { return h.Type }
+func (h holdBasedAbstraction) GetFS() string            { return h.FS }
+func (h holdBasedAbstraction) GetJobID() *JobID         { return &h.JobID }
+func (h holdBasedAbstraction) GetFullPath() string {
+	return fmt.Sprintf("%s@%s", h.FS, h.GetName()) // TODO use zfs.FilesystemVersion.ToAbsPath
+}
+func (h holdBasedAbstraction) MarshalJSON() ([]byte, error) {
+	return json.Marshal(AbstractionJSON{h})
+}
+func (h holdBasedAbstraction) String() string {
+	return fmt.Sprintf("%s %q on %s", h.Type, h.Tag, h.GetFullPath())
+}
+
+func (h holdBasedAbstraction) GetFilesystemVersion() zfs.FilesystemVersion {
+	return h.FilesystemVersion
+}
+
+func (h holdBasedAbstraction) Destroy(ctx context.Context) error {
+	if err := zfs.ZFSRelease(ctx, h.Tag, h.GetFullPath()); err != nil {
+		return errors.Wrapf(err, "release %s: zfs", h)
+	}
+	return nil
+}

endpoint/jobid.go

@@ -0,0 +1,78 @@
+package endpoint
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+// JobID instances returned by MakeJobID() guarantee their JobID.String()
+// can be used in ZFS dataset names and hold tags.
+type JobID struct {
+	jid string
+}
+
+func MakeJobID(s string) (JobID, error) {
+	if len(s) == 0 {
+		return JobID{}, fmt.Errorf("must not be empty string")
+	}
+
+	if err := zfs.ComponentNamecheck(s); err != nil {
+		return JobID{}, errors.Wrap(err, "must be usable as a dataset path component")
+	}
+
+	if _, err := stepBookmarkNameImpl("pool/ds", 0xface601d, s); err != nil {
+		// note that this might still fail due to total maximum name length, but we can't enforce that
+		return JobID{}, errors.Wrap(err, "must be usable for a step bookmark")
+	}
+
+	if _, err := stepHoldTagImpl(s); err != nil {
+		return JobID{}, errors.Wrap(err, "must be usable for a step hold tag")
+	}
+
+	if _, err := lastReceivedHoldImpl(s); err != nil {
+		return JobID{}, errors.Wrap(err, "must be usable as a last-received-hold tag")
+	}
+
+	// FIXME replication cursor bookmark name
+
+	_, err := zfs.NewDatasetPath(s)
+	if err != nil {
+		return JobID{}, fmt.Errorf("must be usable in a ZFS dataset path: %s", err)
+	}
+
+	return JobID{s}, nil
+}
+
+func MustMakeJobID(s string) JobID {
+	jid, err := MakeJobID(s)
+	if err != nil {
+		panic(err)
+	}
+	return jid
+}
+
+func (j JobID) expectInitialized() {
+	if j.jid == "" {
+		panic("use of uninitialized JobID")
+	}
+}
+
+func (j JobID) String() string {
+	j.expectInitialized()
+	return j.jid
+}
+
+var _ json.Marshaler = JobID{}
+var _ json.Unmarshaler = (*JobID)(nil)
+
+func (j JobID) MarshalJSON() ([]byte, error) { return json.Marshal(j.jid) }
+
+func (j *JobID) UnmarshalJSON(b []byte) error {
+	return json.Unmarshal(b, &j.jid)
+}
+
+func (j JobID) MustValidate() { j.expectInitialized() }

go.mod

@@ -5,6 +5,7 @@ go 1.12
 require (
 	github.com/fatih/color v1.7.0
 	github.com/gdamore/tcell v1.2.0
+	github.com/gitchander/permutation v0.0.0-20181107151852-9e56b92e9909
 	github.com/go-logfmt/logfmt v0.4.0
 	github.com/go-sql-driver/mysql v1.4.1-0.20190907122137-b2c03bcae3d4
 	github.com/golang/protobuf v1.3.2
@@ -21,32 +22,20 @@ require (
 	github.com/onsi/gomega v1.7.0 // indirect
 	github.com/pkg/errors v0.8.1
 	github.com/pkg/profile v1.2.1
-	github.com/problame/go-netssh v0.0.0-20191026123024-f34099f4f6b1
+	github.com/problame/go-netssh v0.0.0-20191209123953-18d8aa6923c7
 	github.com/prometheus/client_golang v1.2.1
+	github.com/prometheus/common v0.7.0
 	github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 // go1.12 thinks it needs this
 	github.com/spf13/cobra v0.0.2
-	github.com/spf13/pflag v1.0.3
+	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.4.0
+	github.com/willf/bitset v1.1.10
 	github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d
 	github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // go1.12 thinks it needs this
-	github.com/zrepl/yaml-config v0.0.0-20190928121844-af7ca3f8448f
+	github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd
 	golang.org/x/net v0.0.0-20190613194153-d28f0bde5980
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	golang.org/x/sys v0.0.0-20191010194322-b09406accb47
+	golang.org/x/sys v0.0.0-20191026070338-33540a1f6037
+	gonum.org/v1/gonum v0.7.0 // indirect
 	google.golang.org/grpc v1.17.0
 )
-
-// invalid dates in transitive dependencies (first validated in Go 1.13, didn't fail in earlier Go versions)
-replace (
-	github.com/go-critic/go-critic v0.0.0-20181204210945-1df300866540 => github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540
-	github.com/go-critic/go-critic v0.0.0-20181204210945-ee9bf5809ead => github.com/go-critic/go-critic v0.3.5-0.20190210220443-ee9bf5809ead
-	github.com/golangci/errcheck v0.0.0-20181003203344-ef45e06d44b6 => github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6
-	github.com/golangci/go-tools v0.0.0-20180109140146-35a9f45a5db0 => github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0
-	github.com/golangci/go-tools v0.0.0-20180109140146-af6baa5dc196 => github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196
-	github.com/golangci/gofmt v0.0.0-20181105071733-0b8337e80d98 => github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98
-	github.com/golangci/gosec v0.0.0-20180901114220-66fb7fc33547 => github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547
-	github.com/golangci/ineffassign v0.0.0-20180808204949-42439a7714cc => github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc
-	github.com/golangci/lint-1 v0.0.0-20180610141402-ee948d087217 => github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217
-	golang.org/x/tools v0.0.0-20190125232054-379209517ffe => golang.org/x/tools v0.0.0-20190205201329-379209517ffe
-	mvdan.cc/unparam v0.0.0-20190124213536-fbb59629db34 => mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34
-)

go.sum

@@ -5,9 +5,12 @@ github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q
 github.com/OpenPeeDeeP/depguard v0.0.0-20180806142446-a69c782687b2/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
 github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alvaroloes/enumer v1.1.1/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
@@ -28,6 +31,7 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ftrvxmtrx/fd v0.0.0-20150925145434-c6d800382fff h1:zk1wwii7uXmI0znwU+lqg+wFL9G5+vm5I+9rv2let60=
@@ -36,6 +40,8 @@ github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdk
 github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
 github.com/gdamore/tcell v1.2.0 h1:ikixzsxc8K8o3V2/CEmyoEW8mJZaNYQQ3NP3VIQdUe4=
 github.com/gdamore/tcell v1.2.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM=
+github.com/gitchander/permutation v0.0.0-20181107151852-9e56b92e9909 h1:9NC8seTx6/zRmMTAdsHj/uOMi0EGHGQtjyLafBjk77Q=
+github.com/gitchander/permutation v0.0.0-20181107151852-9e56b92e9909/go.mod h1:lP+DW8LR6Rw3ru9Vo2/y/3iiLaLWmofYql/va+7zJOk=
 github.com/go-critic/go-critic v0.3.4/go.mod h1:AHR42Lk/E/aOznsrYdMYeIQS5RH10HZHSqP+rD6AJrc=
 github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -70,6 +76,7 @@ github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
@@ -122,6 +129,7 @@ github.com/jinzhu/copier v0.0.0-20170922082739-db4671f3a9b8/go.mod h1:yL958EeXv8
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -202,6 +210,8 @@ github.com/problame/go-netssh v0.0.0-20190110232351-09d6bc45d284 h1:GX8YFBfSL6Gl
 github.com/problame/go-netssh v0.0.0-20190110232351-09d6bc45d284/go.mod h1:XMR9fjP5/wMyFK0Ifk9ZHDFem4Rukm+ChvoYr2nxrgw=
 github.com/problame/go-netssh v0.0.0-20191026123024-f34099f4f6b1 h1:HH8yzlaZq/A8xdJaSj/eu32yFFySWAJMTmOQPAGwfYg=
 github.com/problame/go-netssh v0.0.0-20191026123024-f34099f4f6b1/go.mod h1:JRs3nyBjvOv/9YHC+Vlw9z1Jfzl5s5t0BNnTzmclj1c=
+github.com/problame/go-netssh v0.0.0-20191209123953-18d8aa6923c7 h1:Oik8tLrLhoMq4fduDRuNNOAQ+q0M0dXkjAYLUf4+mAc=
+github.com/problame/go-netssh v0.0.0-20191209123953-18d8aa6923c7/go.mod h1:Ba6RaFpK1+IHWs1wLZ6sCBuXkt4iAVLeOG5GinXHusM=
 github.com/prometheus/client_golang v0.0.0-20180410130117-e11c6ff8170b/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
@@ -236,6 +246,7 @@ github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOms
 github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
 github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
@@ -251,6 +262,8 @@ github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -268,6 +281,8 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
 github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
 github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc=
+github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d h1:yJIizrfO599ot2kQ6Af1enICnwBD3XoxgX3MrMwot2M=
 github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
@@ -277,12 +292,19 @@ github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcm
 github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
 github.com/zrepl/yaml-config v0.0.0-20190928121844-af7ca3f8448f h1:3MuiGfgMHCSwKUcsuI7ODbi50j+evTB7SsoOBMNC5Fk=
 github.com/zrepl/yaml-config v0.0.0-20190928121844-af7ca3f8448f/go.mod h1:JmNwisZzOvW4GfpfLvhZ+gtyKLsIiA+WC+wNKJGJaFg=
+github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd h1:SSo67WLS+99QESvbW8Meibz7zCrxshP71U9dH5KOCXM=
+github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd/go.mod h1:JmNwisZzOvW4GfpfLvhZ+gtyKLsIiA+WC+wNKJGJaFg=
 github.com/zrepl/zrepl v0.2.0/go.mod h1:M3Zv2IGSO8iYpUjsZD6ayZ2LHy7zyMfzet9XatKOrZ8=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2 h1:y102fOLFqhV41b+4GPiJoa0k/x+pJcEi2/HB1Y5T6fU=
+golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -315,24 +337,34 @@ golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756 h1:9nuHUbU8dRnRRfj9KjWUVrJeo
 golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
 golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181205014116-22934f0fdb62/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
+gonum.org/v1/gonum v0.7.0 h1:Hdks0L0hgznZLG9nzXb8vZ0rRvqNvAcgAp84y7Mwkgw=
+gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM=
+gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
+gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
 google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898 h1:yvw+zsSmSM02Z5H3ZdEV7B7Ql7eFrjQTnmByJvK+3J8=
 google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
@@ -340,7 +372,9 @@ google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9M
 google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -359,5 +393,6 @@ mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIa
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
 mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
 mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe/go.mod h1:BnhuWBAqxH3+J5bDybdxgw5ZfS+DsVd4iylsKQePN8o=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
 sourcegraph.com/sqs/pbtypes v1.0.0/go.mod h1:3AciMUv4qUuRHRHhOG4TZOB+72GdPVz5k+c648qsFS4=

main.go

@@ -17,6 +17,7 @@ func init() {
 	cli.AddSubcommand(client.PprofCmd)
 	cli.AddSubcommand(client.TestCmd)
 	cli.AddSubcommand(client.MigrateCmd)
+	cli.AddSubcommand(client.ZFSAbstractionsCmd)
 }
 
 func main() {

platformtest/harness/harness.go

@@ -4,11 +4,15 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"os"
 	"path/filepath"
+	"regexp"
 	"time"
 
 	"github.com/fatih/color"
 	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
@@ -16,19 +20,38 @@ import (
 	"github.com/zrepl/zrepl/platformtest/tests"
 )
 
-func main() {
+var bold = color.New(color.Bold)
+var boldRed = color.New(color.Bold, color.FgHiRed)
+var boldGreen = color.New(color.Bold, color.FgHiGreen)
 
-	var args struct {
+var args struct {
-		createArgs            platformtest.ZpoolCreateArgs
+	createArgs            platformtest.ZpoolCreateArgs
-		stopAndKeepPoolOnFail bool
+	stopAndKeepPoolOnFail bool
+
+	run   string
+	runRE *regexp.Regexp
+}
+
+func main() {
+	if err := doMain(); err != nil {
+		os.Exit(1)
 	}
+}
+
+var exitWithErr = fmt.Errorf("exit with error")
+
+func doMain() error {
+
 	flag.StringVar(&args.createArgs.PoolName, "poolname", "", "")
 	flag.StringVar(&args.createArgs.ImagePath, "imagepath", "", "")
-	flag.Int64Var(&args.createArgs.ImageSize, "imagesize", 100*(1<<20), "")
+	flag.Int64Var(&args.createArgs.ImageSize, "imagesize", 200*(1<<20), "")
-	flag.StringVar(&args.createArgs.Mountpoint, "mountpoint", "none", "")
+	flag.StringVar(&args.createArgs.Mountpoint, "mountpoint", "", "")
 	flag.BoolVar(&args.stopAndKeepPoolOnFail, "failure.stop-and-keep-pool", false, "if a test case fails, stop test execution and keep pool as it was when the test failed")
+	flag.StringVar(&args.run, "run", "", "")
 	flag.Parse()
 
+	args.runRE = regexp.MustCompile(args.run)
+
 	outlets := logger.NewOutlets()
 	outlet, level, err := logging.ParseOutlet(config.LoggingOutletEnum{Ret: &config.StdoutLoggingOutlet{
 		LoggingOutletCommon: config.LoggingOutletCommon{
@@ -46,51 +69,137 @@ func main() {
 		logger.Error(err.Error())
 		panic(err)
 	}
-	ctx := platformtest.WithLogger(context.Background(), logger)
+	ctx := context.Background()
+	defer trace.WithTaskFromStackUpdateCtx(&ctx)()
+	ctx = platformtest.WithLogger(ctx, logger)
 	ex := platformtest.NewEx(logger)
 
-	bold := color.New(color.Bold)
+	type invocation struct {
-	boldRed := color.New(color.Bold, color.FgHiRed)
+		runFunc tests.Case
-	boldGreen := color.New(color.Bold, color.FgHiGreen)
+		result  *testCaseResult
+	}
+
+	invocations := make([]*invocation, 0, len(tests.Cases))
 	for _, c := range tests.Cases {
-		// ATTENTION future parallelism must pass c by value into closure!
+		if args.runRE.MatchString(c.String()) {
-		err := func() (err error) {
+			invocations = append(invocations, &invocation{runFunc: c})
-			var oErr = &err
-			bold.Printf("BEGIN TEST CASE %s\n", c)
-			pool, err := platformtest.CreateOrReplaceZpool(ctx, ex, args.createArgs)
-			if err != nil {
-				return errors.Wrap(err, "create test pool")
-			}
-			defer func() {
-				if err := recover(); err != nil {
-					*oErr = errors.Errorf("panic while running test: %v", err) // shadow
-					if args.stopAndKeepPoolOnFail {
-						return
-					}
-					// fallthrough
-				}
-				if err := pool.Destroy(ctx, ex); err != nil {
-					fmt.Printf("error destroying test pool: %s", err)
-				}
-			}()
-			ctx := &platformtest.Context{
-				Context:     ctx,
-				RootDataset: filepath.Join(pool.Name(), "rootds"),
-			}
-			c(ctx)
-			return nil
-		}()
-		if err != nil {
-			boldRed.Printf("TEST CASE FAILED WITH ERROR:\n")
-			fmt.Printf("%+v\n", err) // print with stack trace
-			if args.stopAndKeepPoolOnFail {
-				bold.Printf("STOPPING TEST RUN AT FAILING TEST PER USER REQUEST\n")
-				return
-			}
-		} else {
-			boldGreen.Printf("DONE  TEST CASE %s\n", c)
 		}
+	}
+
+	for _, inv := range invocations {
+
+		bold.Printf("BEGIN TEST CASE %s\n", inv.runFunc.String())
+
+		pool, err := platformtest.CreateOrReplaceZpool(ctx, ex, args.createArgs)
+		if err != nil {
+			panic(errors.Wrap(err, "create test pool"))
+		}
+
+		ctx := &platformtest.Context{
+			Context:     ctx,
+			RootDataset: filepath.Join(pool.Name(), "rootds"),
+		}
+
+		res := runTestCase(ctx, ex, inv.runFunc)
+		inv.result = res
+		if res.failed {
+			fmt.Printf("%+v\n", res.failedStack) // print with stack trace
+		}
+
+		if res.failed && args.stopAndKeepPoolOnFail {
+			boldRed.Printf("STOPPING TEST RUN AT FAILING TEST PER USER REQUEST\n")
+			return exitWithErr
+		}
+
+		if err := pool.Destroy(ctx, ex); err != nil {
+			panic(fmt.Sprintf("error destroying test pool: %s", err))
+		}
+
+		if res.failed {
+			boldRed.Printf("TEST FAILED\n")
+		} else if res.skipped {
+			bold.Printf("TEST SKIPPED\n")
+		} else if res.succeeded {
+			boldGreen.Printf("TEST PASSED\n")
+		} else {
+			panic("unreachable")
+		}
+
 		fmt.Println()
 	}
 
+	var summary struct {
+		succ, fail, skip []*invocation
+	}
+	for _, inv := range invocations {
+		var bucket *[]*invocation
+		if inv.result.failed {
+			bucket = &summary.fail
+		} else if inv.result.skipped {
+			bucket = &summary.skip
+		} else if inv.result.succeeded {
+			bucket = &summary.succ
+		} else {
+			panic("unreachable")
+		}
+		*bucket = append(*bucket, inv)
+	}
+	printBucket := func(bucketName string, c *color.Color, bucket []*invocation) {
+		c.Printf("%s:", bucketName)
+		if len(bucket) == 0 {
+			fmt.Printf(" []\n")
+			return
+		}
+		fmt.Printf("\n")
+		for _, inv := range bucket {
+			fmt.Printf("  %s\n", inv.runFunc.String())
+		}
+	}
+	printBucket("PASSING TESTS", boldGreen, summary.succ)
+	printBucket("SKIPPED TESTS", bold, summary.skip)
+	printBucket("FAILED TESTS", boldRed, summary.fail)
+
+	if len(summary.fail) > 0 {
+		return errors.New("at least one test failed")
+	}
+	return nil
+}
+
+type testCaseResult struct {
+	// oneof
+	failed, skipped, succeeded bool
+
+	failedStack error // has stack inside, valid if failed=true
+}
+
+func runTestCase(ctx *platformtest.Context, ex platformtest.Execer, c tests.Case) *testCaseResult {
+
+	// run case
+	var panicked = false
+	var panicValue interface{} = nil
+	var panicStack error
+	func() {
+		defer func() {
+			if item := recover(); item != nil {
+				panicValue = item
+				panicked = true
+				panicStack = errors.Errorf("panic while running test: %v", panicValue)
+			}
+		}()
+		c(ctx)
+	}()
+
+	if panicked {
+		switch panicValue {
+		case platformtest.SkipNowSentinel:
+			return &testCaseResult{skipped: true}
+		case platformtest.FailNowSentinel:
+			return &testCaseResult{failed: true, failedStack: panicStack}
+		default:
+			return &testCaseResult{failed: true, failedStack: panicStack}
+		}
+	} else {
+		return &testCaseResult{succeeded: true}
+	}
+
 }

platformtest/platformtest.go

@@ -2,6 +2,7 @@ package platformtest
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -12,13 +13,26 @@ type Context struct {
 	RootDataset string
 }
 
+var FailNowSentinel = fmt.Errorf("platformtest: FailNow called on context")
+
+var SkipNowSentinel = fmt.Errorf("platformtest: SkipNow called on context")
+
 var _ assert.TestingT = (*Context)(nil)
 var _ require.TestingT = (*Context)(nil)
 
+func (c *Context) Logf(format string, args ...interface{}) {
+	msg := fmt.Sprintf(format, args...)
+	GetLog(c).Info(msg)
+}
+
 func (c *Context) Errorf(format string, args ...interface{}) {
 	GetLog(c).Printf(format, args...)
 }
 
 func (c *Context) FailNow() {
-	panic(nil)
+	panic(FailNowSentinel)
+}
+
+func (c *Context) SkipNow() {
+	panic(SkipNowSentinel)
 }

platformtest/platformtest_ops.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"strings"
@@ -23,6 +24,7 @@ type Stmt interface {
 type Op string
 
 const (
+	Comment         Op = "#"
 	AssertExists    Op = "!E"
 	AssertNotExists Op = "!N"
 	Add             Op = "+"
@@ -46,8 +48,9 @@ func (o *DestroyRootOp) Run(ctx context.Context, e Execer) error {
 }
 
 type FSOp struct {
-	Op   Op
+	Op        Op
-	Path string
+	Path      string
+	Encrypted bool // only for Op=Add
 }
 
 func (o *FSOp) Run(ctx context.Context, e Execer) error {
@@ -57,7 +60,22 @@ func (o *FSOp) Run(ctx context.Context, e Execer) error {
 	case AssertNotExists:
 		return e.RunExpectFailureNoOutput(ctx, "zfs", "get", "-H", "name", o.Path)
 	case Add:
-		return e.RunExpectSuccessNoOutput(ctx, "zfs", "create", o.Path)
+		opts := []string{"create"}
+		if o.Encrypted {
+			const passphraseFilePath = "/tmp/zreplplatformtest.encryption.passphrase"
+			const passphrase = "foobar2342"
+			err := ioutil.WriteFile(passphraseFilePath, []byte(passphrase), 0600)
+			if err != nil {
+				panic(err)
+			}
+			opts = append(opts,
+				"-o", "encryption=on",
+				"-o", "keylocation=file:///"+passphraseFilePath,
+				"-o", "keyformat=passphrase",
+			)
+		}
+		opts = append(opts, o.Path)
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", opts...)
 	case Del:
 		return e.RunExpectSuccessNoOutput(ctx, "zfs", "destroy", o.Path)
 	default:
@@ -85,6 +103,26 @@ func (o *SnapOp) Run(ctx context.Context, e Execer) error {
 	}
 }
 
+type BookmarkOp struct {
+	Op       Op
+	Existing string
+	Bookmark string
+}
+
+func (o *BookmarkOp) Run(ctx context.Context, e Execer) error {
+	switch o.Op {
+	case Add:
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", "bookmark", o.Existing, o.Bookmark)
+	case Del:
+		if o.Existing != "" {
+			panic("existing must be empty for destroy, got " + o.Existing)
+		}
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", "destroy", o.Bookmark)
+	default:
+		panic(o.Op)
+	}
+}
+
 type RunOp struct {
 	RootDS string
 	Script string
@@ -163,8 +201,8 @@ func splitQuotedWords(data []byte, atEOF bool) (advance int, token []byte, err e
 				// unescaped quote, end of this string
 				// remove backslash-escapes
 				withBackslash := data[begin+1 : end]
-				withoutBaskslash := bytes.Replace(withBackslash, []byte("\\\""), []byte("\""), -1)
+				withoutBackslash := bytes.Replace(withBackslash, []byte("\\\""), []byte("\""), -1)
-				return end + 1, withoutBaskslash, nil
+				return end + 1, withoutBackslash, nil
 			} else {
 				// continue to next quote
 				end += 1
@@ -238,18 +276,46 @@ nextLine:
 			op = AssertExists
 		case string(AssertNotExists):
 			op = AssertNotExists
+		case string(Comment):
+			op = Comment
+			continue
 		default:
 			return nil, &LineError{scan.Text(), fmt.Sprintf("invalid op %q", comps.Text())}
 		}
 
-		// FS / SNAP
+		// FS / SNAP / BOOKMARK
 		if err := expectMoreTokens(); err != nil {
 			return nil, err
 		}
 		if strings.ContainsAny(comps.Text(), "@") {
 			stmts = append(stmts, &SnapOp{Op: op, Path: fmt.Sprintf("%s/%s", rootds, comps.Text())})
+		} else if strings.ContainsAny(comps.Text(), "#") {
+			bookmark := fmt.Sprintf("%s/%s", rootds, comps.Text())
+			if err := expectMoreTokens(); err != nil {
+				return nil, err
+			}
+			existing := fmt.Sprintf("%s/%s", rootds, comps.Text())
+			stmts = append(stmts, &BookmarkOp{Op: op, Existing: existing, Bookmark: bookmark})
 		} else {
-			stmts = append(stmts, &FSOp{Op: op, Path: fmt.Sprintf("%s/%s", rootds, comps.Text())})
+			// FS
+			fs := comps.Text()
+			var encrypted bool = false
+			if op == Add {
+				if comps.Scan() {
+					t := comps.Text()
+					switch t {
+					case "encrypted":
+						encrypted = true
+					default:
+						panic(fmt.Sprintf("unexpected token %q", t))
+					}
+				}
+			}
+			stmts = append(stmts, &FSOp{
+				Op:        op,
+				Path:      fmt.Sprintf("%s/%s", rootds, fs),
+				Encrypted: encrypted,
+			})
 		}
 
 		if comps.Scan() {

platformtest/platformtest_zpool.go

@@ -2,10 +2,12 @@ package platformtest
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/zfs"
 )
 
@@ -28,11 +30,11 @@ func (a ZpoolCreateArgs) Validate() error {
 	if a.ImageSize < minImageSize {
 		return errors.Errorf("ImageSize must be > %v, got %v", minImageSize, a.ImageSize)
 	}
-	if a.Mountpoint != "none" {
+	if a.Mountpoint == "" || a.Mountpoint[0] != '/' {
-		return errors.Errorf("Mountpoint must be \"none\"")
+		return errors.Errorf("Mountpoint must be an absolute path to a directory")
 	}
 	if a.PoolName == "" {
-		return errors.Errorf("PoolName must not be emtpy")
+		return errors.Errorf("PoolName must not be empty")
 	}
 	return nil
 }
@@ -43,7 +45,7 @@ func CreateOrReplaceZpool(ctx context.Context, e Execer, args ZpoolCreateArgs) (
 	}
 
 	// export pool if it already exists (idempotence)
-	if _, err := zfs.ZFSGetRawAnySource(args.PoolName, []string{"name"}); err != nil {
+	if _, err := zfs.ZFSGetRawAnySource(ctx, args.PoolName, []string{"name"}); err != nil {
 		if _, ok := err.(*zfs.DatasetDoesNotExist); ok {
 			// we'll create it shortly
 		} else {
@@ -68,7 +70,10 @@ func CreateOrReplaceZpool(ctx context.Context, e Execer, args ZpoolCreateArgs) (
 	image.Close()
 
 	// create the pool
-	err = e.RunExpectSuccessNoOutput(ctx, "zpool", "create", "-f", "-O", "mountpoint=none", args.PoolName, args.ImagePath)
+	err = e.RunExpectSuccessNoOutput(ctx, "zpool", "create", "-f",
+		"-O", fmt.Sprintf("mountpoint=%s", args.Mountpoint),
+		args.PoolName, args.ImagePath,
+	)
 	if err != nil {
 		return nil, errors.Wrap(err, "zpool create")
 	}

platformtest/tests/batchDestroy.go

@@ -32,7 +32,7 @@ func BatchDestroy(ctx *platformtest.Context) {
 			Name:       "2",
 		},
 	}
-	zfs.ZFSDestroyFilesystemVersions(reqs)
+	zfs.ZFSDestroyFilesystemVersions(ctx, reqs)
 	if *reqs[0].ErrOut != nil {
 		panic("expecting no error")
 	}

platformtest/tests/getNonexistent.go

@@ -17,14 +17,14 @@ func GetNonexistent(ctx *platformtest.Context) {
 	`)
 
 	// test raw
-	_, err := zfs.ZFSGetRawAnySource(fmt.Sprintf("%s/foo bar", ctx.RootDataset), []string{"name"})
+	_, err := zfs.ZFSGetRawAnySource(ctx, fmt.Sprintf("%s/foo bar", ctx.RootDataset), []string{"name"})
 	if err != nil {
 		panic(err)
 	}
 
 	// test nonexistent filesystem
 	nonexistent := fmt.Sprintf("%s/nonexistent filesystem", ctx.RootDataset)
-	props, err := zfs.ZFSGetRawAnySource(nonexistent, []string{"name"})
+	props, err := zfs.ZFSGetRawAnySource(ctx, nonexistent, []string{"name"})
 	if err == nil {
 		panic(props)
 	}
@@ -37,7 +37,7 @@ func GetNonexistent(ctx *platformtest.Context) {
 
 	// test nonexistent snapshot
 	nonexistent = fmt.Sprintf("%s/foo bar@non existent", ctx.RootDataset)
-	props, err = zfs.ZFSGetRawAnySource(nonexistent, []string{"name"})
+	props, err = zfs.ZFSGetRawAnySource(ctx, nonexistent, []string{"name"})
 	if err == nil {
 		panic(props)
 	}
@@ -50,7 +50,7 @@ func GetNonexistent(ctx *platformtest.Context) {
 
 	// test nonexistent bookmark
 	nonexistent = fmt.Sprintf("%s/foo bar#non existent", ctx.RootDataset)
-	props, err = zfs.ZFSGetRawAnySource(nonexistent, []string{"name"})
+	props, err = zfs.ZFSGetRawAnySource(ctx, nonexistent, []string{"name"})
 	if err == nil {
 		panic(props)
 	}

platformtest/tests/helpers.go

@@ -0,0 +1,176 @@
+package tests
+
+import (
+	"io"
+	"math/rand"
+	"os"
+	"path"
+	"sort"
+	"strings"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/util/limitio"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func sendArgVersion(ctx *platformtest.Context, fs, relName string) zfs.ZFSSendArgVersion {
+	guid, err := zfs.ZFSGetGUID(ctx, fs, relName)
+	if err != nil {
+		panic(err)
+	}
+	return zfs.ZFSSendArgVersion{
+		RelName: relName,
+		GUID:    guid,
+	}
+}
+
+func fsversion(ctx *platformtest.Context, fs, relname string) zfs.FilesystemVersion {
+	v, err := zfs.ZFSGetFilesystemVersion(ctx, fs+relname)
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
+func mustDatasetPath(fs string) *zfs.DatasetPath {
+	p, err := zfs.NewDatasetPath(fs)
+	if err != nil {
+		panic(err)
+	}
+	return p
+}
+
+func mustSnapshot(ctx *platformtest.Context, snap string) {
+	if err := zfs.EntityNamecheck(snap, zfs.EntityTypeSnapshot); err != nil {
+		panic(err)
+	}
+	comps := strings.Split(snap, "@")
+	if len(comps) != 2 {
+		panic(comps)
+	}
+	err := zfs.ZFSSnapshot(ctx, mustDatasetPath(comps[0]), comps[1], false)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func mustGetFilesystemVersion(ctx *platformtest.Context, snapOrBookmark string) zfs.FilesystemVersion {
+	v, err := zfs.ZFSGetFilesystemVersion(ctx, snapOrBookmark)
+	check(err)
+	return v
+}
+
+func check(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+var dummyDataRand = rand.New(rand.NewSource(99))
+
+func writeDummyData(path string, numBytes int64) {
+	r := io.LimitReader(dummyDataRand, numBytes)
+	d, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
+	check(err)
+	defer d.Close()
+	_, err = io.Copy(d, r)
+	check(err)
+}
+
+type dummySnapshotSituation struct {
+	sendFS       string
+	dummyDataLen int64
+	snapA        *zfs.ZFSSendArgVersion
+	snapB        *zfs.ZFSSendArgVersion
+}
+
+type resumeSituation struct {
+	sendArgs         zfs.ZFSSendArgsUnvalidated
+	recvOpts         zfs.RecvOptions
+	sendErr, recvErr error
+	recvErrDecoded   *zfs.RecvFailedWithResumeTokenErr
+}
+
+func makeDummyDataSnapshots(ctx *platformtest.Context, sendFS string) (situation dummySnapshotSituation) {
+
+	situation.sendFS = sendFS
+	sendFSMount, err := zfs.ZFSGetMountpoint(ctx, sendFS)
+	require.NoError(ctx, err)
+	require.True(ctx, sendFSMount.Mounted)
+
+	const dummyLen = int64(10 * (1 << 20))
+	situation.dummyDataLen = dummyLen
+
+	writeDummyData(path.Join(sendFSMount.Mountpoint, "dummy_data"), dummyLen)
+	mustSnapshot(ctx, sendFS+"@a snapshot")
+	snapA := sendArgVersion(ctx, sendFS, "@a snapshot")
+	situation.snapA = &snapA
+
+	writeDummyData(path.Join(sendFSMount.Mountpoint, "dummy_data"), dummyLen)
+	mustSnapshot(ctx, sendFS+"@b snapshot")
+	snapB := sendArgVersion(ctx, sendFS, "@b snapshot")
+	situation.snapB = &snapB
+
+	return situation
+}
+
+func makeResumeSituation(ctx *platformtest.Context, src dummySnapshotSituation, recvFS string, sendArgs zfs.ZFSSendArgsUnvalidated, recvOptions zfs.RecvOptions) *resumeSituation {
+
+	situation := &resumeSituation{}
+
+	situation.sendArgs = sendArgs
+	situation.recvOpts = recvOptions
+	require.True(ctx, recvOptions.SavePartialRecvState, "this method would be pointless otherwise")
+	require.Equal(ctx, sendArgs.FS, src.sendFS)
+	sendArgsValidated, err := sendArgs.Validate(ctx)
+	situation.sendErr = err
+	if err != nil {
+		return situation
+	}
+
+	copier, err := zfs.ZFSSend(ctx, sendArgsValidated)
+	situation.sendErr = err
+	if err != nil {
+		return situation
+	}
+
+	limitedCopier := limitio.ReadCloser(copier, src.dummyDataLen/2)
+	defer limitedCopier.Close()
+
+	require.NotNil(ctx, sendArgs.To)
+	err = zfs.ZFSRecv(ctx, recvFS, sendArgs.To, limitedCopier, recvOptions)
+	situation.recvErr = err
+	ctx.Logf("zfs recv exit with %T %s", err, err)
+
+	require.NotNil(ctx, err)
+	resumeErr, ok := err.(*zfs.RecvFailedWithResumeTokenErr)
+	require.True(ctx, ok)
+	situation.recvErrDecoded = resumeErr
+
+	return situation
+}
+
+func versionRelnamesSorted(versions []zfs.FilesystemVersion) []string {
+	var vstrs []string
+	for _, v := range versions {
+		vstrs = append(vstrs, v.RelName())
+	}
+	sort.Strings(vstrs)
+	return vstrs
+}
+
+func datasetToStringSortedTrimPrefix(prefix *zfs.DatasetPath, paths []*zfs.DatasetPath) []string {
+	var pstrs []string
+	for _, p := range paths {
+		trimmed := p.Copy()
+		trimmed.TrimPrefix(prefix)
+		if trimmed.Length() == 0 {
+			continue
+		}
+		pstrs = append(pstrs, trimmed.ToString())
+	}
+	sort.Strings(pstrs)
+	return pstrs
+}

platformtest/tests/idempotentBookmark.go

@@ -0,0 +1,59 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func IdempotentBookmark(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@a snap"
+		+  "foo bar@another snap"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", ctx.RootDataset)
+
+	asnap := sendArgVersion(ctx, fs, "@a snap")
+	anotherSnap := sendArgVersion(ctx, fs, "@another snap")
+
+	err := zfs.ZFSBookmark(ctx, fs, asnap, "a bookmark")
+	if err != nil {
+		panic(err)
+	}
+
+	// do it again, should be idempotent
+	err = zfs.ZFSBookmark(ctx, fs, asnap, "a bookmark")
+	if err != nil {
+		panic(err)
+	}
+
+	// should fail for another snapshot
+	err = zfs.ZFSBookmark(ctx, fs, anotherSnap, "a bookmark")
+	if err == nil {
+		panic(err)
+	}
+	if _, ok := err.(*zfs.BookmarkExists); !ok {
+		panic(fmt.Sprintf("has type %T", err))
+	}
+
+	// destroy the snapshot
+	if err := zfs.ZFSDestroy(ctx, fmt.Sprintf("%s@a snap", fs)); err != nil {
+		panic(err)
+	}
+
+	// do it again, should fail with special error type
+	err = zfs.ZFSBookmark(ctx, fs, asnap, "a bookmark")
+	if err == nil {
+		panic(err)
+	}
+	if _, ok := err.(*zfs.DatasetDoesNotExist); !ok {
+		panic(fmt.Sprintf("has type %T", err))
+	}
+
+}

platformtest/tests/idempotentDestroy.go

@@ -0,0 +1,75 @@
+package tests
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func IdempotentDestroy(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@a snap"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", ctx.RootDataset)
+	asnap := sendArgVersion(ctx, fs, "@a snap")
+	err := zfs.ZFSBookmark(ctx, fs, asnap, "a bookmark")
+	if err != nil {
+		panic(err)
+	}
+
+	type testCase struct {
+		description, path string
+	}
+
+	cases := []testCase{
+		{"snapshot", fmt.Sprintf("%s@a snap", fs)},
+		{"bookmark", fmt.Sprintf("%s#a bookmark", fs)},
+		{"filesystem", fs},
+	}
+
+	for i := range cases {
+		func() {
+			c := cases[i]
+
+			log.Printf("SUBBEGIN testing idempotent destroy %q for path %q", c.description, c.path)
+
+			log.Println("destroy existing")
+			err = zfs.ZFSDestroy(ctx, c.path)
+			if err != nil {
+				panic(err)
+			}
+			log.Println("destroy again, non-idempotently, must error")
+			err = zfs.ZFSDestroy(ctx, c.path)
+			if _, ok := err.(*zfs.DatasetDoesNotExist); !ok {
+				panic(fmt.Sprintf("%T: %s", err, err))
+			}
+			log.Println("destroy again, idempotently, must not error")
+			err = zfs.ZFSDestroyIdempotent(ctx, c.path)
+			if err != nil {
+				panic(err)
+			}
+
+			log.Println("SUBEND")
+
+		}()
+	}
+
+	// also test idempotent destroy for cases where the parent dataset does not exist
+	err = zfs.ZFSDestroyIdempotent(ctx, fmt.Sprintf("%s/not foo bar@nonexistent snapshot", ctx.RootDataset))
+	if err != nil {
+		panic(err)
+	}
+
+	err = zfs.ZFSDestroyIdempotent(ctx, fmt.Sprintf("%s/not foo bar#nonexistent bookmark", ctx.RootDataset))
+	if err != nil {
+		panic(err)
+	}
+
+}

platformtest/tests/idempotentHold.go

@@ -0,0 +1,37 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func IdempotentHold(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@1"
+	`)
+	defer platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		R zfs release zrepl_platformtest "${ROOTDS}/foo bar@1"
+		-  "foo bar@1"
+		-  "foo bar"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", ctx.RootDataset)
+	v1 := fsversion(ctx, fs, "@1")
+
+	tag := "zrepl_platformtest"
+	err := zfs.ZFSHold(ctx, fs, v1, tag)
+	if err != nil {
+		panic(err)
+	}
+
+	err = zfs.ZFSHold(ctx, fs, v1, tag)
+	if err != nil {
+		panic(err)
+	}
+}