trace: test for main API, fix bugs discovered by them

fixes #320
refs #318

Makefile

@@ -23,7 +23,8 @@ GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
 GO_ENV_VARS := GO111MODULE=on
 GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
 GO_MOD_READONLY := -mod=readonly
-GO_BUILDFLAGS := $(GO_MOD_READONLY)
+GO_EXTRA_BUILDFLAGS :=
+GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
 GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
 GOLANGCI_LINT := golangci-lint
 ifneq ($(GOARM),)
@@ -65,6 +66,7 @@ wrapup-and-checksum:
 		-acf $(NOARCH_TARBALL) \
 		$(ARTIFACTDIR)/docs/html \
 		$(ARTIFACTDIR)/bash_completion \
+		$(ARTIFACTDIR)/_zrepl.zsh_completion \
 		$(ARTIFACTDIR)/go_env.txt \
 		dist \
 		config/samples
@@ -159,7 +161,7 @@ platformtest: # do not track dependency on platformtest-bin to allow build of pl
 		$(ZREPL_PLATFORMTEST_ARGS)
 
 ##################### NOARCH #####################
-.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
+.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
 
 
 $(ARTIFACTDIR):
@@ -167,12 +169,16 @@ $(ARTIFACTDIR):
 $(ARTIFACTDIR)/docs: $(ARTIFACTDIR)
 	mkdir -p "$@"
 
-noarch: $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_env.txt docs
+noarch: $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs
 	# pass
 
 $(ARTIFACTDIR)/bash_completion:
 	$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
-	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) bashcomp "$@"
+	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion bash "$@"
+
+$(ARTIFACTDIR)/_zrepl.zsh_completion:
+	$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
+	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion zsh "$@"
 
 $(ARTIFACTDIR)/go_env.txt:
 	$(GO_ENV_VARS) $(GO) env > $@

cli/cli.go

@@ -1,11 +1,13 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 	"os"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"
 )
@@ -19,29 +21,55 @@ var rootCmd = &cobra.Command{
 	Short: "One-stop ZFS replication solution",
 }
 
-var bashcompCmd = &cobra.Command{
+func init() {
-	Use:   "bashcomp path/to/out/file",
+	rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
-	Short: "generate bash completions",
+}
-	Run: func(cmd *cobra.Command, args []string) {
+
-		if len(args) != 1 {
+var genCompletionCmd = &cobra.Command{
-			fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
+	Use:   "gencompletion",
-			err := cmd.Usage()
+	Short: "generate shell auto-completions",
-			if err != nil {
+}
-				panic(err)
+
-			}
+type completionCmdInfo struct {
-			os.Exit(1)
+	genFunc func(outpath string) error
-		}
+	help    string
-		if err := rootCmd.GenBashCompletionFile(args[0]); err != nil {
+}
-			fmt.Fprintf(os.Stderr, "error generating bash completion: %s", err)
+
-			os.Exit(1)
+var completionCmdMap = map[string]completionCmdInfo{
-		}
+	"zsh": {
+		rootCmd.GenZshCompletionFile,
+		"  save to file `_zrepl` in your zsh's $fpath",
+	},
+	"bash": {
+		rootCmd.GenBashCompletionFile,
+		"  save to a path and source that path in your .bashrc",
 	},
-	Hidden: true,
 }
 
 func init() {
-	rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
+	for sh, info := range completionCmdMap {
-	rootCmd.AddCommand(bashcompCmd)
+		sh, info := sh, info
+		genCompletionCmd.AddCommand(&cobra.Command{
+			Use:     fmt.Sprintf("%s path/to/out/file", sh),
+			Short:   fmt.Sprintf("generate %s completions", sh),
+			Example: info.help,
+			Run: func(cmd *cobra.Command, args []string) {
+				if len(args) != 1 {
+					fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
+					err := cmd.Usage()
+					if err != nil {
+						panic(err)
+					}
+					os.Exit(1)
+				}
+				if err := info.genFunc(args[0]); err != nil {
+					fmt.Fprintf(os.Stderr, "error generating %s completion: %s", sh, err)
+					os.Exit(1)
+				}
+			},
+		})
+	}
+	rootCmd.AddCommand(genCompletionCmd)
 }
 
 type Subcommand struct {
@@ -49,7 +77,7 @@ type Subcommand struct {
 	Short            string
 	Example          string
 	NoRequireConfig  bool
-	Run              func(subcommand *Subcommand, args []string) error
+	Run              func(ctx context.Context, subcommand *Subcommand, args []string) error
 	SetupFlags       func(f *pflag.FlagSet)
 	SetupSubcommands func() []*Subcommand
 
@@ -70,7 +98,11 @@ func (s *Subcommand) Config() *config.Config {
 
 func (s *Subcommand) run(cmd *cobra.Command, args []string) {
 	s.tryParseConfig()
-	err := s.Run(s, args)
+	ctx := context.Background()
+	endTask := trace.WithTaskFromStackUpdateCtx(&ctx)
+	defer endTask()
+	err := s.Run(ctx, s, args)
+	endTask()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%s\n", err)
 		os.Exit(1)

client/configcheck.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -29,7 +30,7 @@ var ConfigcheckCmd = &cli.Subcommand{
 		f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]")
 		f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]")
 	},
-	Run: func(subcommand *cli.Subcommand, args []string) error {
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
 		formatMap := map[string]func(interface{}){
 			"": func(i interface{}) {},
 			"pretty": func(i interface{}) {

client/holds.go

@@ -1,87 +0,0 @@
-package client
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"os"
-	"strings"
-
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-var (
-	HoldsCmd = &cli.Subcommand{
-		Use:   "holds",
-		Short: "manage holds & step bookmarks",
-		SetupSubcommands: func() []*cli.Subcommand {
-			return holdsList
-		},
-	}
-)
-
-var holdsList = []*cli.Subcommand{
-	&cli.Subcommand{
-		Use:             "list [FSFILTER]",
-		Run:             doHoldsList,
-		NoRequireConfig: true,
-		Short: `
-FSFILTER SYNTAX:
-representation of a 'filesystems' filter statement on the command line
-		`,
-	},
-}
-
-func fsfilterFromCliArg(arg string) (zfs.DatasetFilter, error) {
-	mappings := strings.Split(arg, ",")
-	f := filters.NewDatasetMapFilter(len(mappings), true)
-	for _, m := range mappings {
-		thisMappingErr := fmt.Errorf("expecting comma-separated list of <dataset-pattern>:<ok|!> pairs, got %q", m)
-		lhsrhs := strings.SplitN(m, ":", 2)
-		if len(lhsrhs) != 2 {
-			return nil, thisMappingErr
-		}
-		err := f.Add(lhsrhs[0], lhsrhs[1])
-		if err != nil {
-			return nil, fmt.Errorf("%s: %s", thisMappingErr, err)
-		}
-	}
-	return f.AsFilter(), nil
-}
-
-func doHoldsList(sc *cli.Subcommand, args []string) error {
-	var err error
-	ctx := context.Background()
-
-	if len(args) > 1 {
-		return errors.New("this subcommand takes at most one argument")
-	}
-
-	var filter zfs.DatasetFilter
-	if len(args) == 0 {
-		filter = zfs.NoFilter()
-	} else {
-		filter, err = fsfilterFromCliArg(args[0])
-		if err != nil {
-			return errors.Wrap(err, "cannot parse filesystem filter args")
-		}
-	}
-
-	listing, err := endpoint.ListZFSHoldsAndBookmarks(ctx, filter)
-	if err != nil {
-		return err // context clear by invocation of command
-	}
-
-	enc := json.NewEncoder(os.Stdout)
-	enc.SetIndent("  ", "  ")
-	if err := enc.Encode(listing); err != nil {
-		panic(err)
-	}
-
-	return nil
-}

client/migrate.go

@@ -48,14 +48,13 @@ var migratePlaceholder0_1Args struct {
 	dryRun bool
 }
 
-func doMigratePlaceholder0_1(sc *cli.Subcommand, args []string) error {
+func doMigratePlaceholder0_1(ctx context.Context, sc *cli.Subcommand, args []string) error {
 	if len(args) != 0 {
 		return fmt.Errorf("migration does not take arguments, got %v", args)
 	}
 
 	cfg := sc.Config()
 
-	ctx := context.Background()
 	allFSS, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
 	if err != nil {
 		return errors.Wrap(err, "cannot list filesystems")
@@ -124,7 +123,7 @@ var fail = color.New(color.FgRed)
 
 var migrateReplicationCursorSkipSentinel = fmt.Errorf("skipping this filesystem")
 
-func doMigrateReplicationCursor(sc *cli.Subcommand, args []string) error {
+func doMigrateReplicationCursor(ctx context.Context, sc *cli.Subcommand, args []string) error {
 	if len(args) != 0 {
 		return fmt.Errorf("migration does not take arguments, got %v", args)
 	}
@@ -137,8 +136,6 @@ func doMigrateReplicationCursor(sc *cli.Subcommand, args []string) error {
 		return fmt.Errorf("exiting migration after error")
 	}
 
-	ctx := context.Background()
-
 	v1cursorJobs := make([]job.Job, 0, len(cfg.Jobs))
 	for i, j := range cfg.Jobs {
 		if jobs[i].Name() != j.Name() {
@@ -211,17 +208,15 @@ func doMigrateReplicationCursorFS(ctx context.Context, v1CursorJobs []job.Job, f
 	}
 	fmt.Printf("identified owning job %q\n", owningJob.Name())
 
-	versions, err := zfs.ZFSListFilesystemVersions(fs, nil)
+	bookmarks, err := zfs.ZFSListFilesystemVersions(ctx, fs, zfs.ListFilesystemVersionsOptions{
+		Types: zfs.Bookmarks,
+	})
 	if err != nil {
 		return errors.Wrapf(err, "list filesystem versions of %q", fs.ToString())
 	}
 
 	var oldCursor *zfs.FilesystemVersion
-	for i, fsv := range versions {
+	for i, fsv := range bookmarks {
-		if fsv.Type != zfs.Bookmark {
-			continue
-		}
-
 		_, _, err := endpoint.ParseReplicationCursorBookmarkName(fsv.ToAbsPath(fs))
 		if err != endpoint.ErrV1ReplicationCursor {
 			continue
@@ -232,7 +227,7 @@ func doMigrateReplicationCursorFS(ctx context.Context, v1CursorJobs []job.Job, f
 			return errors.Wrap(err, "multiple filesystem versions identified as v1 replication cursors")
 		}
 
-		oldCursor = &versions[i]
+		oldCursor = &bookmarks[i]
 
 	}
 

client/pprof.go

@@ -1,67 +1,10 @@
 package client
 
-import (
+import "github.com/zrepl/zrepl/cli"
-	"errors"
-	"log"
-	"os"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon"
-)
-
-var pprofArgs struct {
-	daemon.PprofServerControlMsg
-}
 
 var PprofCmd = &cli.Subcommand{
-	Use:   "pprof off | [on TCP_LISTEN_ADDRESS]",
+	Use: "pprof",
-	Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
+	SetupSubcommands: func() []*cli.Subcommand {
-	Run: func(subcommand *cli.Subcommand, args []string) error {
+		return []*cli.Subcommand{PprofListenCmd, pprofActivityTraceCmd}
-		if len(args) < 1 {
-			goto enargs
-		}
-		switch args[0] {
-		case "on":
-			pprofArgs.Run = true
-			if len(args) != 2 {
-				return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
-			}
-			pprofArgs.HttpListenAddress = args[1]
-		case "off":
-			if len(args) != 1 {
-				goto enargs
-			}
-			pprofArgs.Run = false
-		}
-
-		RunPProf(subcommand.Config())
-		return nil
-	enargs:
-		return errors.New("invalid number of positional arguments")
-
 	},
 }
-
-func RunPProf(conf *config.Config) {
-	log := log.New(os.Stderr, "", 0)
-
-	die := func() {
-		log.Printf("exiting after error")
-		os.Exit(1)
-	}
-
-	log.Printf("connecting to zrepl daemon")
-
-	httpc, err := controlHttpClient(conf.Global.Control.SockPath)
-	if err != nil {
-		log.Printf("error creating http client: %s", err)
-		die()
-	}
-	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofArgs.PprofServerControlMsg, struct{}{})
-	if err != nil {
-		log.Printf("error sending control message: %s", err)
-		die()
-	}
-	log.Printf("finished")
-}

client/pprof_activity_trace.go

@@ -0,0 +1,44 @@
+package client
+
+import (
+	"context"
+	"io"
+	"log"
+	"os"
+
+	"golang.org/x/net/websocket"
+
+	"github.com/zrepl/zrepl/cli"
+)
+
+var pprofActivityTraceCmd = &cli.Subcommand{
+	Use:   "activity-trace ZREPL_PPROF_HOST:ZREPL_PPROF_PORT",
+	Short: "attach to zrepl daemon with activated pprof listener and dump an activity-trace to stdout",
+	Run:   runPProfActivityTrace,
+}
+
+func runPProfActivityTrace(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+	log := log.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Printf("exiting after error")
+		os.Exit(1)
+	}
+
+	if len(args) != 1 {
+		log.Printf("exactly one positional argument is required")
+		die()
+	}
+
+	url := "ws://" + args[0] + "/debug/zrepl/activity-trace" // FIXME dont' repeat that
+
+	log.Printf("attaching to activity trace stream %s", url)
+	ws, err := websocket.Dial(url, "", url)
+	if err != nil {
+		log.Printf("error: %s", err)
+		die()
+	}
+
+	_, err = io.Copy(os.Stdout, ws)
+	return err
+}

client/pprof_listen.go

@@ -0,0 +1,68 @@
+package client
+
+import (
+	"context"
+	"errors"
+	"log"
+	"os"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+)
+
+var pprofListenCmd struct {
+	daemon.PprofServerControlMsg
+}
+
+var PprofListenCmd = &cli.Subcommand{
+	Use:   "listen off | [on TCP_LISTEN_ADDRESS]",
+	Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		if len(args) < 1 {
+			goto enargs
+		}
+		switch args[0] {
+		case "on":
+			pprofListenCmd.Run = true
+			if len(args) != 2 {
+				return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
+			}
+			pprofListenCmd.HttpListenAddress = args[1]
+		case "off":
+			if len(args) != 1 {
+				goto enargs
+			}
+			pprofListenCmd.Run = false
+		}
+
+		RunPProf(subcommand.Config())
+		return nil
+	enargs:
+		return errors.New("invalid number of positional arguments")
+
+	},
+}
+
+func RunPProf(conf *config.Config) {
+	log := log.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Printf("exiting after error")
+		os.Exit(1)
+	}
+
+	log.Printf("connecting to zrepl daemon")
+
+	httpc, err := controlHttpClient(conf.Global.Control.SockPath)
+	if err != nil {
+		log.Printf("error creating http client: %s", err)
+		die()
+	}
+	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofListenCmd.PprofServerControlMsg, struct{}{})
+	if err != nil {
+		log.Printf("error sending control message: %s", err)
+		die()
+	}
+	log.Printf("finished")
+}

client/signal.go

@@ -1,6 +1,8 @@
 package client
 
 import (
+	"context"
+
 	"github.com/pkg/errors"
 
 	"github.com/zrepl/zrepl/cli"
@@ -11,7 +13,7 @@ import (
 var SignalCmd = &cli.Subcommand{
 	Use:   "signal [wakeup|reset] JOB",
 	Short: "wake up a job from wait state or abort its current invocation",
-	Run: func(subcommand *cli.Subcommand, args []string) error {
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
 		return runSignalCmd(subcommand.Config(), args)
 	},
 }

client/status.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"math"
@@ -180,7 +181,7 @@ var StatusCmd = &cli.Subcommand{
 	Run: runStatus,
 }
 
-func runStatus(s *cli.Subcommand, args []string) error {
+func runStatus(ctx context.Context, s *cli.Subcommand, args []string) error {
 	httpc, err := controlHttpClient(s.Config().Global.Control.SockPath)
 	if err != nil {
 		return err

client/stdinserver.go

@@ -17,7 +17,7 @@ import (
 var StdinserverCmd = &cli.Subcommand{
 	Use:   "stdinserver CLIENT_IDENTITY",
 	Short: "stdinserver transport mode (started from authorized_keys file as forced command)",
-	Run: func(subcommand *cli.Subcommand, args []string) error {
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
 		return runStdinserver(subcommand.Config(), args)
 	},
 }

client/testcmd.go

@@ -39,7 +39,7 @@ var testFilter = &cli.Subcommand{
 	Run: runTestFilterCmd,
 }
 
-func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
+func runTestFilterCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
 
 	if testFilterArgs.job == "" {
 		return fmt.Errorf("must specify --job flag")
@@ -49,7 +49,6 @@ func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
 	}
 
 	conf := subcommand.Config()
-	ctx := context.Background()
 
 	var confFilter config.FilesystemsFilter
 	job, err := conf.Job(testFilterArgs.job)
@@ -137,10 +136,9 @@ var testPlaceholder = &cli.Subcommand{
 	Run: runTestPlaceholder,
 }
 
-func runTestPlaceholder(subcommand *cli.Subcommand, args []string) error {
+func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
 
 	var checkDPs []*zfs.DatasetPath
-	ctx := context.Background()
 
 	// all actions first
 	if testPlaceholderArgs.all {
@@ -197,11 +195,11 @@ var testDecodeResumeToken = &cli.Subcommand{
 	Run: runTestDecodeResumeTokenCmd,
 }
 
-func runTestDecodeResumeTokenCmd(subcommand *cli.Subcommand, args []string) error {
+func runTestDecodeResumeTokenCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
 	if testDecodeResumeTokenArgs.token == "" {
 		return fmt.Errorf("token argument must be specified")
 	}
-	token, err := zfs.ParseResumeToken(context.Background(), testDecodeResumeTokenArgs.token)
+	token, err := zfs.ParseResumeToken(ctx, testDecodeResumeTokenArgs.token)
 	if err != nil {
 		return err
 	}

client/version.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"context"
 	"fmt"
 	"os"
 
@@ -25,7 +26,7 @@ var VersionCmd = &cli.Subcommand{
 	SetupFlags: func(f *pflag.FlagSet) {
 		f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)")
 	},
-	Run: func(subcommand *cli.Subcommand, args []string) error {
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
 		versionArgs.Config = subcommand.Config()
 		versionArgs.ConfigErr = subcommand.ConfigParsingError()
 		return runVersionCmd()

client/zfsabstractions.go

@@ -0,0 +1,147 @@
+package client
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var (
+	ZFSAbstractionsCmd = &cli.Subcommand{
+		Use:   "zfs-abstraction",
+		Short: "manage abstractions that zrepl builds on top of ZFS",
+		SetupSubcommands: func() []*cli.Subcommand {
+			return []*cli.Subcommand{
+				zabsCmdList,
+				zabsCmdReleaseAll,
+				zabsCmdReleaseStale,
+				zabsCmdCreate,
+			}
+		},
+	}
+)
+
+// a common set of CLI flags that map to the fields of an
+// endpoint.ListZFSHoldsAndBookmarksQuery
+type zabsFilterFlags struct {
+	Filesystems FilesystemsFilterFlag
+	Job         JobIDFlag
+	Types       AbstractionTypesFlag
+	Concurrency int64
+}
+
+// produce a query from the CLI flags
+func (f zabsFilterFlags) Query() (endpoint.ListZFSHoldsAndBookmarksQuery, error) {
+	q := endpoint.ListZFSHoldsAndBookmarksQuery{
+		FS:          f.Filesystems.FlagValue(),
+		What:        f.Types.FlagValue(),
+		JobID:       f.Job.FlagValue(),
+		Concurrency: f.Concurrency,
+	}
+	return q, q.Validate()
+}
+
+func (f *zabsFilterFlags) registerZabsFilterFlags(s *pflag.FlagSet, verb string) {
+	// Note: the default value is defined in the .FlagValue methods
+	s.Var(&f.Filesystems, "fs", fmt.Sprintf("only %s holds on the specified filesystem [default: all filesystems] [comma-separated list of <dataset-pattern>:<ok|!> pairs]", verb))
+	s.Var(&f.Job, "job", fmt.Sprintf("only %s holds created by the specified job [default: any job]", verb))
+
+	variants := make([]string, 0, len(endpoint.AbstractionTypesAll))
+	for v := range endpoint.AbstractionTypesAll {
+		variants = append(variants, string(v))
+	}
+	variants = sort.StringSlice(variants)
+	variantsJoined := strings.Join(variants, "|")
+	s.Var(&f.Types, "type", fmt.Sprintf("only %s holds of the specified type [default: all] [comma-separated list of %s]", verb, variantsJoined))
+
+	s.Int64VarP(&f.Concurrency, "concurrency", "p", 1, "number of concurrently queried filesystems")
+}
+
+type JobIDFlag struct{ J *endpoint.JobID }
+
+func (f *JobIDFlag) Set(s string) error {
+	if len(s) == 0 {
+		*f = JobIDFlag{J: nil}
+		return nil
+	}
+
+	jobID, err := endpoint.MakeJobID(s)
+	if err != nil {
+		return err
+	}
+	*f = JobIDFlag{J: &jobID}
+	return nil
+}
+func (f JobIDFlag) Type() string               { return "job-ID" }
+func (f JobIDFlag) String() string             { return fmt.Sprint(f.J) }
+func (f JobIDFlag) FlagValue() *endpoint.JobID { return f.J }
+
+type AbstractionTypesFlag map[endpoint.AbstractionType]bool
+
+func (f *AbstractionTypesFlag) Set(s string) error {
+	ats, err := endpoint.AbstractionTypeSetFromStrings(strings.Split(s, ","))
+	if err != nil {
+		return err
+	}
+	*f = AbstractionTypesFlag(ats)
+	return nil
+}
+func (f AbstractionTypesFlag) Type() string { return "abstraction-type" }
+func (f AbstractionTypesFlag) String() string {
+	return endpoint.AbstractionTypeSet(f).String()
+}
+func (f AbstractionTypesFlag) FlagValue() map[endpoint.AbstractionType]bool {
+	if len(f) > 0 {
+		return f
+	}
+	return endpoint.AbstractionTypesAll
+}
+
+type FilesystemsFilterFlag struct {
+	F endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter
+}
+
+func (flag *FilesystemsFilterFlag) Set(s string) error {
+	mappings := strings.Split(s, ",")
+	if len(mappings) == 1 && !strings.Contains(mappings[0], ":") {
+		flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &mappings[0],
+		}
+		return nil
+	}
+
+	f := filters.NewDatasetMapFilter(len(mappings), true)
+	for _, m := range mappings {
+		thisMappingErr := fmt.Errorf("expecting comma-separated list of <dataset-pattern>:<ok|!> pairs, got %q", m)
+		lhsrhs := strings.SplitN(m, ":", 2)
+		if len(lhsrhs) != 2 {
+			return thisMappingErr
+		}
+		err := f.Add(lhsrhs[0], lhsrhs[1])
+		if err != nil {
+			return fmt.Errorf("%s: %s", thisMappingErr, err)
+		}
+	}
+	flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+		Filter: f,
+	}
+	return nil
+}
+func (flag FilesystemsFilterFlag) Type() string { return "filesystem filter spec" }
+func (flag FilesystemsFilterFlag) String() string {
+	return fmt.Sprintf("%v", flag.F)
+}
+func (flag FilesystemsFilterFlag) FlagValue() endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter {
+	var z FilesystemsFilterFlag
+	if flag == z {
+		return endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{Filter: zfs.NoFilter()}
+	}
+	return flag.F
+}

client/zfsabstractions_create.go

@@ -0,0 +1,14 @@
+package client
+
+import "github.com/zrepl/zrepl/cli"
+
+var zabsCmdCreate = &cli.Subcommand{
+	Use:             "create",
+	NoRequireConfig: true,
+	Short:           `create zrepl ZFS abstractions (mostly useful for debugging & development, users should not need to use this command)`,
+	SetupSubcommands: func() []*cli.Subcommand {
+		return []*cli.Subcommand{
+			zabsCmdCreateStepHold,
+		}
+	},
+}

client/zfsabstractions_create_step_hold.go

@@ -0,0 +1,58 @@
+package client
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var zabsCreateStepHoldFlags struct {
+	target string
+	jobid  JobIDFlag
+}
+
+var zabsCmdCreateStepHold = &cli.Subcommand{
+	Use:             "step",
+	Run:             doZabsCreateStep,
+	NoRequireConfig: true,
+	Short:           `create a step hold or bookmark`,
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVarP(&zabsCreateStepHoldFlags.target, "target", "t", "", "snapshot to be held / bookmark to be held")
+		f.VarP(&zabsCreateStepHoldFlags.jobid, "jobid", "j", "jobid for which the hold is installed")
+	},
+}
+
+func doZabsCreateStep(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	if len(args) > 0 {
+		return errors.New("subcommand takes no arguments")
+	}
+
+	f := &zabsCreateStepHoldFlags
+
+	fs, _, _, err := zfs.DecomposeVersionString(f.target)
+	if err != nil {
+		return errors.Wrapf(err, "%q invalid target", f.target)
+	}
+
+	if f.jobid.FlagValue() == nil {
+		return errors.Errorf("jobid must be set")
+	}
+
+	v, err := zfs.ZFSGetFilesystemVersion(ctx, f.target)
+	if err != nil {
+		return errors.Wrapf(err, "get info about target %q", f.target)
+	}
+
+	step, err := endpoint.HoldStep(ctx, fs, v, *f.jobid.FlagValue())
+	if err != nil {
+		return errors.Wrap(err, "create step hold")
+	}
+	fmt.Println(step.String())
+	return nil
+}

client/zfsabstractions_list.go

@@ -0,0 +1,99 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/util/chainlock"
+)
+
+var zabsListFlags struct {
+	Filter zabsFilterFlags
+	Json   bool
+}
+
+var zabsCmdList = &cli.Subcommand{
+	Use:             "list",
+	Short:           `list zrepl ZFS abstractions`,
+	Run:             doZabsList,
+	NoRequireConfig: true,
+	SetupFlags: func(f *pflag.FlagSet) {
+		zabsListFlags.Filter.registerZabsFilterFlags(f, "list")
+		f.BoolVar(&zabsListFlags.Json, "json", false, "emit JSON")
+	},
+}
+
+func doZabsList(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsListFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	abstractions, errors, err := endpoint.ListAbstractionsStreamed(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+
+	var line chainlock.L
+	var wg sync.WaitGroup
+	defer wg.Wait()
+	wg.Add(1)
+
+	// print results
+	go func() {
+		defer wg.Done()
+		enc := json.NewEncoder(os.Stdout)
+		for a := range abstractions {
+			func() {
+				defer line.Lock().Unlock()
+				if zabsListFlags.Json {
+					enc.SetIndent("", "  ")
+					if err := enc.Encode(abstractions); err != nil {
+						panic(err)
+					}
+					fmt.Println()
+				} else {
+					fmt.Println(a)
+				}
+			}()
+		}
+	}()
+
+	// print errors to stderr
+	errorColor := color.New(color.FgRed)
+	var errorsSlice []endpoint.ListAbstractionsError
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for err := range errors {
+			func() {
+				defer line.Lock().Unlock()
+				errorsSlice = append(errorsSlice, err)
+				errorColor.Fprintf(os.Stderr, "%s\n", err)
+			}()
+		}
+	}()
+	wg.Wait()
+	if len(errorsSlice) > 0 {
+		errorColor.Add(color.Bold).Fprintf(os.Stderr, "there were errors in listing the abstractions")
+		return fmt.Errorf("")
+	} else {
+		return nil
+	}
+
+}

client/zfsabstractions_release.go

@@ -0,0 +1,143 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+)
+
+// shared between release-all and release-step
+var zabsReleaseFlags struct {
+	Filter zabsFilterFlags
+	Json   bool
+	DryRun bool
+}
+
+func registerZabsReleaseFlags(s *pflag.FlagSet) {
+	zabsReleaseFlags.Filter.registerZabsFilterFlags(s, "release")
+	s.BoolVar(&zabsReleaseFlags.Json, "json", false, "emit json instead of pretty-printed")
+	s.BoolVar(&zabsReleaseFlags.DryRun, "dry-run", false, "do a dry-run")
+}
+
+var zabsCmdReleaseAll = &cli.Subcommand{
+	Use:             "release-all",
+	Run:             doZabsReleaseAll,
+	NoRequireConfig: true,
+	Short:           `(DANGEROUS) release ALL zrepl ZFS abstractions (mostly useful for uninstalling zrepl completely or for "de-zrepl-ing" a filesystem)`,
+	SetupFlags:      registerZabsReleaseFlags,
+}
+
+var zabsCmdReleaseStale = &cli.Subcommand{
+	Use:             "release-stale",
+	Run:             doZabsReleaseStale,
+	NoRequireConfig: true,
+	Short:           `release stale zrepl ZFS abstractions (useful if zrepl has a bug and does not do it by itself)`,
+	SetupFlags:      registerZabsReleaseFlags,
+}
+
+func doZabsReleaseAll(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsReleaseFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	abstractions, listErrors, err := endpoint.ListAbstractions(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+	if len(listErrors) > 0 {
+		color.New(color.FgRed).Fprintf(os.Stderr, "there were errors in listing the abstractions:\n%s\n", listErrors)
+		// proceed anyways with rest of abstractions
+	}
+
+	return doZabsRelease_Common(ctx, abstractions)
+}
+
+func doZabsReleaseStale(ctx context.Context, sc *cli.Subcommand, args []string) error {
+
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsReleaseFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	stalenessInfo, err := endpoint.ListStale(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+
+	return doZabsRelease_Common(ctx, stalenessInfo.Stale)
+}
+
+func doZabsRelease_Common(ctx context.Context, destroy []endpoint.Abstraction) error {
+
+	if zabsReleaseFlags.DryRun {
+		if zabsReleaseFlags.Json {
+			m, err := json.MarshalIndent(destroy, "", "  ")
+			if err != nil {
+				panic(err)
+			}
+			if _, err := os.Stdout.Write(m); err != nil {
+				panic(err)
+			}
+			fmt.Println()
+		} else {
+			for _, a := range destroy {
+				fmt.Printf("would destroy %s\n", a)
+			}
+		}
+		return nil
+	}
+
+	outcome := endpoint.BatchDestroy(ctx, destroy)
+	hadErr := false
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	colorErr := color.New(color.FgRed)
+	printfSuccess := color.New(color.FgGreen).FprintfFunc()
+	printfSection := color.New(color.Bold).FprintfFunc()
+
+	for res := range outcome {
+		hadErr = hadErr || res.DestroyErr != nil
+		if zabsReleaseFlags.Json {
+			err := enc.Encode(res)
+			if err != nil {
+				colorErr.Fprintf(os.Stderr, "cannot marshal there were errors in destroying the abstractions")
+			}
+		} else {
+			printfSection(os.Stdout, "destroy %s ...", res.Abstraction)
+			if res.DestroyErr != nil {
+				colorErr.Fprintf(os.Stdout, " failed:\n%s\n", res.DestroyErr)
+			} else {
+				printfSuccess(os.Stdout, " OK\n")
+			}
+		}
+	}
+
+	if hadErr {
+		colorErr.Add(color.Bold).Fprintf(os.Stderr, "there were errors in destroying the abstractions")
+		return fmt.Errorf("")
+	} else {
+		return nil
+	}
+}

daemon/daemon.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/job"
@@ -23,9 +24,8 @@ import (
 	"github.com/zrepl/zrepl/zfs/zfscmd"
 )
 
-func Run(conf *config.Config) error {
+func Run(ctx context.Context, conf *config.Config) error {
-
+	ctx, cancel := context.WithCancel(ctx)
-	ctx, cancel := context.WithCancel(context.Background())
 
 	defer cancel()
 	sigChan := make(chan os.Signal, 1)
@@ -39,6 +39,7 @@ func Run(conf *config.Config) error {
 	if err != nil {
 		return errors.Wrap(err, "cannot build logging from config")
 	}
+	outlets.Add(newPrometheusLogOutlet(), logger.Debug)
 
 	confJobs, err := job.JobsFromConfig(conf)
 	if err != nil {
@@ -48,14 +49,14 @@ func Run(conf *config.Config) error {
 	log := logger.NewLogger(outlets, 1*time.Second)
 	log.Info(version.NewZreplVersionInformation().String())
 
+	ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
+
 	for _, job := range confJobs {
 		if IsInternalJobName(job.Name()) {
 			panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME
 		}
 	}
 
-	ctx = job.WithLogger(ctx, log)
-
 	jobs := newJobs()
 
 	// start control socket
@@ -84,6 +85,7 @@ func Run(conf *config.Config) error {
 
 	// register global (=non job-local) metrics
 	zfscmd.RegisterMetrics(prometheus.DefaultRegisterer)
+	trace.RegisterMetrics(prometheus.DefaultRegisterer)
 
 	log.Info("starting daemon")
 
@@ -98,6 +100,8 @@ func Run(conf *config.Config) error {
 	case <-ctx.Done():
 		log.WithError(ctx.Err()).Info("context finished")
 	}
+	log.Info("waiting for jobs to finish")
+	<-jobs.wait()
 	log.Info("daemon exiting")
 	return nil
 }
@@ -120,14 +124,11 @@ func newJobs() *jobs {
 	}
 }
 
-const (
-	logJobField string = "job"
-)
-
 func (s *jobs) wait() <-chan struct{} {
 	ch := make(chan struct{})
 	go func() {
 		s.wg.Wait()
+		close(ch)
 	}()
 	return ch
 }
@@ -202,9 +203,8 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
 	s.m.Lock()
 	defer s.m.Unlock()
 
-	jobLog := job.GetLogger(ctx).
+	ctx = logging.WithInjectedField(ctx, logging.JobField, j.Name())
-		WithField(logJobField, j.Name()).
+
-		WithOutlet(newPrometheusLogOutlet(j.Name()), logger.Debug)
 	jobName := j.Name()
 	if !internal && IsInternalJobName(jobName) {
 		panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName))
@@ -219,7 +219,6 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
 	j.RegisterMetrics(prometheus.DefaultRegisterer)
 
 	s.jobs[jobName] = j
-	ctx = job.WithLogger(ctx, jobLog)
 	ctx = zfscmd.WithJobID(ctx, j.Name())
 	ctx, wakeup := wakeup.Context(ctx)
 	ctx, resetFunc := reset.Context(ctx)
@@ -229,8 +228,8 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
 	s.wg.Add(1)
 	go func() {
 		defer s.wg.Done()
-		jobLog.Info("starting job")
+		job.GetLogger(ctx).Info("starting job")
-		defer jobLog.Info("job exited")
+		defer job.GetLogger(ctx).Info("job exited")
 		j.Run(ctx)
 	}()
 }

daemon/filters/fsvfilter.go

@@ -1,41 +0,0 @@
-package filters
-
-import (
-	"strings"
-
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type AnyFSVFilter struct{}
-
-func NewAnyFSVFilter() AnyFSVFilter {
-	return AnyFSVFilter{}
-}
-
-var _ zfs.FilesystemVersionFilter = AnyFSVFilter{}
-
-func (AnyFSVFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) {
-	return true, nil
-}
-
-type PrefixFilter struct {
-	prefix    string
-	fstype    zfs.VersionType
-	fstypeSet bool // optionals anyone?
-}
-
-var _ zfs.FilesystemVersionFilter = &PrefixFilter{}
-
-func NewPrefixFilter(prefix string) *PrefixFilter {
-	return &PrefixFilter{prefix: prefix}
-}
-
-func NewTypedPrefixFilter(prefix string, versionType zfs.VersionType) *PrefixFilter {
-	return &PrefixFilter{prefix, versionType, true}
-}
-
-func (f *PrefixFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) {
-	fstypeMatches := (!f.fstypeSet || t == f.fstype)
-	prefixMatches := strings.HasPrefix(name, f.prefix)
-	return fstypeMatches && prefixMatches, nil
-}

daemon/hooks/hook_logging.go

@@ -6,29 +6,17 @@ import (
 	"context"
 	"sync"
 
+	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/util/envconst"
 )
 
-type contextKey int
-
-const (
-	contextKeyLog contextKey = 0
-)
-
 type Logger = logger.Logger
 
-func WithLogger(ctx context.Context, log Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLog, log)
-}
-
 func GetLogger(ctx context.Context) Logger { return getLogger(ctx) }
 
 func getLogger(ctx context.Context) Logger {
-	if log, ok := ctx.Value(contextKeyLog).(Logger); ok {
+	return logging.GetLogger(ctx, logging.SubsysHooks)
-		return log
-	}
-	return logger.NewNullLogger()
 }
 
 const MAX_HOOK_LOG_SIZE_DEFAULT int = 1 << 20

daemon/hooks/hooks_test.go

@@ -10,9 +10,11 @@ import (
 	"text/template"
 
 	"github.com/stretchr/testify/require"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/hooks"
+	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/zfs"
 )
@@ -69,6 +71,9 @@ func curry(f comparisonAssertionFunc, expected interface{}, right bool) (ret val
 }
 
 func TestHooks(t *testing.T) {
+	ctx, end := trace.WithTaskFromStack(context.Background())
+	defer end()
+
 	testFSName := "testpool/testdataset"
 	testSnapshotName := "testsnap"
 
@@ -418,9 +423,8 @@ jobs:
 
 			cbReached = false
 
-			ctx := context.Background()
 			if testing.Verbose() && !tt.SuppressOutput {
-				ctx = hooks.WithLogger(ctx, log)
+				ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
 			}
 			plan.Run(ctx, false)
 			report := plan.Report()

daemon/job/active.go

@@ -8,12 +8,13 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/log"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/filters"
 	"github.com/zrepl/zrepl/daemon/job/reset"
 	"github.com/zrepl/zrepl/daemon/job/wakeup"
-	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/daemon/pruner"
 	"github.com/zrepl/zrepl/daemon/snapper"
 	"github.com/zrepl/zrepl/endpoint"
@@ -79,7 +80,7 @@ func (a *ActiveSide) updateTasks(u func(*activeSideTasks)) activeSideTasks {
 }
 
 type activeMode interface {
-	ConnectEndpoints(rpcLoggers rpc.Loggers, connecter transport.Connecter)
+	ConnectEndpoints(ctx context.Context, connecter transport.Connecter)
 	DisconnectEndpoints()
 	SenderReceiver() (logic.Sender, logic.Receiver)
 	Type() Type
@@ -98,14 +99,14 @@ type modePush struct {
 	snapper       *snapper.PeriodicOrManual
 }
 
-func (m *modePush) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Connecter) {
+func (m *modePush) ConnectEndpoints(ctx context.Context, connecter transport.Connecter) {
 	m.setupMtx.Lock()
 	defer m.setupMtx.Unlock()
 	if m.receiver != nil || m.sender != nil {
 		panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
 	}
 	m.sender = endpoint.NewSender(*m.senderConfig)
-	m.receiver = rpc.NewClient(connecter, loggers)
+	m.receiver = rpc.NewClient(connecter, rpc.GetLoggersOrPanic(ctx))
 }
 
 func (m *modePush) DisconnectEndpoints() {
@@ -176,14 +177,14 @@ type modePull struct {
 	interval       config.PositiveDurationOrManual
 }
 
-func (m *modePull) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Connecter) {
+func (m *modePull) ConnectEndpoints(ctx context.Context, connecter transport.Connecter) {
 	m.setupMtx.Lock()
 	defer m.setupMtx.Unlock()
 	if m.receiver != nil || m.sender != nil {
 		panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
 	}
 	m.receiver = endpoint.NewReceiver(m.receiverConfig)
-	m.sender = rpc.NewClient(connecter, loggers)
+	m.sender = rpc.NewClient(connecter, rpc.GetLoggersOrPanic(ctx))
 }
 
 func (m *modePull) DisconnectEndpoints() {
@@ -376,15 +377,18 @@ func (j *ActiveSide) SenderConfig() *endpoint.SenderConfig {
 }
 
 func (j *ActiveSide) Run(ctx context.Context) {
+	ctx, endTask := trace.WithTaskAndSpan(ctx, "active-side-job", j.Name())
+	defer endTask()
 	log := GetLogger(ctx)
-	ctx = logging.WithSubsystemLoggers(ctx, log)
 
 	defer log.Info("job exiting")
 
 	periodicDone := make(chan struct{})
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
-	go j.mode.RunPeriodic(ctx, periodicDone)
+	periodicCtx, endTask := trace.WithTask(ctx, "periodic")
+	defer endTask()
+	go j.mode.RunPeriodic(periodicCtx, periodicDone)
 
 	invocationCount := 0
 outer:
@@ -400,17 +404,15 @@ outer:
 		case <-periodicDone:
 		}
 		invocationCount++
-		invLog := log.WithField("invocation", invocationCount)
+		invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", invocationCount))
-		j.do(WithLogger(ctx, invLog))
+		j.do(invocationCtx)
+		endSpan()
 	}
 }
 
 func (j *ActiveSide) do(ctx context.Context) {
 
-	log := GetLogger(ctx)
+	j.mode.ConnectEndpoints(ctx, j.connecter)
-	ctx = logging.WithSubsystemLoggers(ctx, log)
-	loggers := rpc.GetLoggersOrPanic(ctx) // filled by WithSubsystemLoggers
-	j.mode.ConnectEndpoints(loggers, j.connecter)
 	defer j.mode.DisconnectEndpoints()
 
 	// allow cancellation of an invocation (this function)
@@ -433,20 +435,22 @@ func (j *ActiveSide) do(ctx context.Context) {
 			return
 		default:
 		}
+		ctx, endSpan := trace.WithSpan(ctx, "replication")
 		ctx, repCancel := context.WithCancel(ctx)
 		var repWait driver.WaitFunc
 		j.updateTasks(func(tasks *activeSideTasks) {
 			// reset it
 			*tasks = activeSideTasks{}
-			tasks.replicationCancel = repCancel
+			tasks.replicationCancel = func() { repCancel(); endSpan() }
 			tasks.replicationReport, repWait = replication.Do(
 				ctx, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver, j.mode.PlannerPolicy()),
 			)
 			tasks.state = ActiveSideReplicating
 		})
-		log.Info("start replication")
+		GetLogger(ctx).Info("start replication")
 		repWait(true) // wait blocking
 		repCancel()   // always cancel to free up context resources
+		endSpan()
 	}
 
 	{
@@ -455,16 +459,18 @@ func (j *ActiveSide) do(ctx context.Context) {
 			return
 		default:
 		}
+		ctx, endSpan := trace.WithSpan(ctx, "prune_sender")
 		ctx, senderCancel := context.WithCancel(ctx)
 		tasks := j.updateTasks(func(tasks *activeSideTasks) {
 			tasks.prunerSender = j.prunerFactory.BuildSenderPruner(ctx, sender, sender)
-			tasks.prunerSenderCancel = senderCancel
+			tasks.prunerSenderCancel = func() { senderCancel(); endSpan() }
 			tasks.state = ActiveSidePruneSender
 		})
-		log.Info("start pruning sender")
+		GetLogger(ctx).Info("start pruning sender")
 		tasks.prunerSender.Prune()
-		log.Info("finished pruning sender")
+		GetLogger(ctx).Info("finished pruning sender")
 		senderCancel()
+		endSpan()
 	}
 	{
 		select {
@@ -472,16 +478,18 @@ func (j *ActiveSide) do(ctx context.Context) {
 			return
 		default:
 		}
+		ctx, endSpan := trace.WithSpan(ctx, "prune_recever")
 		ctx, receiverCancel := context.WithCancel(ctx)
 		tasks := j.updateTasks(func(tasks *activeSideTasks) {
 			tasks.prunerReceiver = j.prunerFactory.BuildReceiverPruner(ctx, receiver, sender)
-			tasks.prunerReceiverCancel = receiverCancel
+			tasks.prunerReceiverCancel = func() { receiverCancel(); endSpan() }
 			tasks.state = ActiveSidePruneReceiver
 		})
-		log.Info("start pruning receiver")
+		GetLogger(ctx).Info("start pruning receiver")
 		tasks.prunerReceiver.Prune()
-		log.Info("finished pruning receiver")
+		GetLogger(ctx).Info("finished pruning receiver")
 		receiverCancel()
+		endSpan()
 	}
 
 	j.updateTasks(func(tasks *activeSideTasks) {

daemon/job/job.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 
+	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/zfs"
@@ -14,21 +15,8 @@ import (
 
 type Logger = logger.Logger
 
-type contextKey int
-
-const (
-	contextKeyLog contextKey = iota
-)
-
 func GetLogger(ctx context.Context) Logger {
-	if l, ok := ctx.Value(contextKeyLog).(Logger); ok {
+	return logging.GetLogger(ctx, logging.SubsysJob)
-		return l
-	}
-	return logger.NewNullLogger()
-}
-
-func WithLogger(ctx context.Context, l Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLog, l)
 }
 
 type Job interface {

daemon/job/passive.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/filters"
@@ -164,12 +165,14 @@ func (j *PassiveSide) SenderConfig() *endpoint.SenderConfig {
 func (*PassiveSide) RegisterMetrics(registerer prometheus.Registerer) {}
 
 func (j *PassiveSide) Run(ctx context.Context) {
-
+	ctx, endTask := trace.WithTaskAndSpan(ctx, "passive-side-job", j.Name())
+	defer endTask()
 	log := GetLogger(ctx)
 	defer log.Info("job exiting")
-	ctx = logging.WithSubsystemLoggers(ctx, log)
 	{
-		ctx, cancel := context.WithCancel(ctx) // shadowing
+		ctx, endTask := trace.WithTask(ctx, "periodic") // shadowing
+		defer endTask()
+		ctx, cancel := context.WithCancel(ctx)
 		defer cancel()
 		go j.mode.RunPeriodic(ctx)
 	}
@@ -179,8 +182,14 @@ func (j *PassiveSide) Run(ctx context.Context) {
 		panic(fmt.Sprintf("implementation error: j.mode.Handler() returned nil: %#v", j))
 	}
 
-	ctxInterceptor := func(handlerCtx context.Context) context.Context {
+	ctxInterceptor := func(handlerCtx context.Context, info rpc.HandlerContextInterceptorData, handler func(ctx context.Context)) {
-		return logging.WithSubsystemLoggers(handlerCtx, log)
+		// the handlerCtx is clean => need to inherit logging and tracing config from job context
+		handlerCtx = logging.WithInherit(handlerCtx, ctx)
+		handlerCtx = trace.WithInherit(handlerCtx, ctx)
+
+		handlerCtx, endTask := trace.WithTaskAndSpan(handlerCtx, "handler", fmt.Sprintf("job=%q client=%q method=%q", j.Name(), info.ClientIdentity(), info.FullMethod()))
+		defer endTask()
+		handler(handlerCtx)
 	}
 
 	rpcLoggers := rpc.GetLoggersOrPanic(ctx) // WithSubsystemLoggers above

daemon/job/snapjob.go

@@ -2,15 +2,16 @@ package job
 
 import (
 	"context"
+	"fmt"
 	"sort"
 
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/filters"
 	"github.com/zrepl/zrepl/daemon/job/wakeup"
-	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/daemon/pruner"
 	"github.com/zrepl/zrepl/daemon/snapper"
 	"github.com/zrepl/zrepl/endpoint"
@@ -89,15 +90,18 @@ func (j *SnapJob) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
 func (j *SnapJob) SenderConfig() *endpoint.SenderConfig { return nil }
 
 func (j *SnapJob) Run(ctx context.Context) {
+	ctx, endTask := trace.WithTaskAndSpan(ctx, "snap-job", j.Name())
+	defer endTask()
 	log := GetLogger(ctx)
-	ctx = logging.WithSubsystemLoggers(ctx, log)
 
 	defer log.Info("job exiting")
 
 	periodicDone := make(chan struct{})
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
-	go j.snapper.Run(ctx, periodicDone)
+	periodicCtx, endTask := trace.WithTask(ctx, "snapshotting")
+	defer endTask()
+	go j.snapper.Run(periodicCtx, periodicDone)
 
 	invocationCount := 0
 outer:
@@ -112,8 +116,10 @@ outer:
 		case <-periodicDone:
 		}
 		invocationCount++
-		invLog := log.WithField("invocation", invocationCount)
+
-		j.doPrune(WithLogger(ctx, invLog))
+		invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", invocationCount))
+		j.doPrune(invocationCtx)
+		endSpan()
 	}
 }
 
@@ -161,8 +167,9 @@ func (h alwaysUpToDateReplicationCursorHistory) ListFilesystems(ctx context.Cont
 }
 
 func (j *SnapJob) doPrune(ctx context.Context) {
+	ctx, endSpan := trace.WithSpan(ctx, "snap-job-do-prune")
+	defer endSpan()
 	log := GetLogger(ctx)
-	ctx = logging.WithSubsystemLoggers(ctx, log)
 	sender := endpoint.NewSender(endpoint.SenderConfig{
 		JobID: j.name,
 		FSF:   j.fsfilter,

daemon/logging/build_logging.go

@@ -9,20 +9,10 @@ import (
 
 	"github.com/mattn/go-isatty"
 	"github.com/pkg/errors"
-
 	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/hooks"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
-	"github.com/zrepl/zrepl/daemon/pruner"
-	"github.com/zrepl/zrepl/daemon/snapper"
-	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/replication/driver"
-	"github.com/zrepl/zrepl/replication/logic"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/rpc/transportmux"
 	"github.com/zrepl/zrepl/tlsconf"
-	"github.com/zrepl/zrepl/transport"
-	"github.com/zrepl/zrepl/zfs/zfscmd"
 )
 
 func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error) {
@@ -70,6 +60,8 @@ func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error)
 type Subsystem string
 
 const (
+	SubsysMeta         Subsystem = "meta"
+	SubsysJob          Subsystem = "job"
 	SubsysReplication  Subsystem = "repl"
 	SubsysEndpoint     Subsystem = "endpoint"
 	SubsysPruning      Subsystem = "pruning"
@@ -83,28 +75,97 @@ const (
 	SubsysZFSCmd       Subsystem = "zfs.cmd"
 )
 
-func WithSubsystemLoggers(ctx context.Context, log logger.Logger) context.Context {
+var AllSubsystems = []Subsystem{
-	ctx = logic.WithLogger(ctx, log.WithField(SubsysField, SubsysReplication))
+	SubsysMeta,
-	ctx = driver.WithLogger(ctx, log.WithField(SubsysField, SubsysReplication))
+	SubsysJob,
-	ctx = endpoint.WithLogger(ctx, log.WithField(SubsysField, SubsysEndpoint))
+	SubsysReplication,
-	ctx = pruner.WithLogger(ctx, log.WithField(SubsysField, SubsysPruning))
+	SubsysEndpoint,
-	ctx = snapper.WithLogger(ctx, log.WithField(SubsysField, SubsysSnapshot))
+	SubsysPruning,
-	ctx = hooks.WithLogger(ctx, log.WithField(SubsysField, SubsysHooks))
+	SubsysSnapshot,
-	ctx = transport.WithLogger(ctx, log.WithField(SubsysField, SubsysTransport))
+	SubsysHooks,
-	ctx = transportmux.WithLogger(ctx, log.WithField(SubsysField, SubsysTransportMux))
+	SubsysTransport,
-	ctx = zfscmd.WithLogger(ctx, log.WithField(SubsysField, SubsysZFSCmd))
+	SubsysTransportMux,
-	ctx = rpc.WithLoggers(ctx,
+	SubsysRPC,
-		rpc.Loggers{
+	SubsysRPCControl,
-			General: log.WithField(SubsysField, SubsysRPC),
+	SubsysRPCData,
-			Control: log.WithField(SubsysField, SubsysRPCControl),
+	SubsysZFSCmd,
-			Data:    log.WithField(SubsysField, SubsysRPCData),
-		},
-	)
-	return ctx
 }
 
-func LogSubsystem(log logger.Logger, subsys Subsystem) logger.Logger {
+type injectedField struct {
-	return log.ReplaceField(SubsysField, subsys)
+	field  string
+	value  interface{}
+	parent *injectedField
+}
+
+func WithInjectedField(ctx context.Context, field string, value interface{}) context.Context {
+	var parent *injectedField
+	parentI := ctx.Value(contextKeyInjectedField)
+	if parentI != nil {
+		parent = parentI.(*injectedField)
+	}
+	// TODO sanity-check `field` now
+	this := &injectedField{field, value, parent}
+	return context.WithValue(ctx, contextKeyInjectedField, this)
+}
+
+func iterInjectedFields(ctx context.Context, cb func(field string, value interface{})) {
+	injI := ctx.Value(contextKeyInjectedField)
+	if injI == nil {
+		return
+	}
+	inj := injI.(*injectedField)
+	for ; inj != nil; inj = inj.parent {
+		cb(inj.field, inj.value)
+	}
+}
+
+type SubsystemLoggers map[Subsystem]logger.Logger
+
+func SubsystemLoggersWithUniversalLogger(l logger.Logger) SubsystemLoggers {
+	loggers := make(SubsystemLoggers)
+	for _, s := range AllSubsystems {
+		loggers[s] = l
+	}
+	return loggers
+}
+
+func WithLoggers(ctx context.Context, loggers SubsystemLoggers) context.Context {
+	return context.WithValue(ctx, contextKeyLoggers, loggers)
+}
+
+func GetLoggers(ctx context.Context) SubsystemLoggers {
+	loggers, ok := ctx.Value(contextKeyLoggers).(SubsystemLoggers)
+	if !ok {
+		return nil
+	}
+	return loggers
+}
+
+func GetLogger(ctx context.Context, subsys Subsystem) logger.Logger {
+	return getLoggerImpl(ctx, subsys, true)
+}
+
+func getLoggerImpl(ctx context.Context, subsys Subsystem, panicIfEnded bool) logger.Logger {
+	loggers, ok := ctx.Value(contextKeyLoggers).(SubsystemLoggers)
+	if !ok || loggers == nil {
+		return logger.NewNullLogger()
+	}
+	l, ok := loggers[subsys]
+	if !ok {
+		return logger.NewNullLogger()
+	}
+
+	l = l.WithField(SubsysField, subsys)
+
+	l = l.WithField(SpanField, trace.GetSpanStackOrDefault(ctx, "NOSPAN"))
+
+	fields := make(logger.Fields)
+	iterInjectedFields(ctx, func(field string, value interface{}) {
+		fields[field] = value
+	})
+	l = l.WithFields(fields)
+
+	return l
 }
 
 func parseLogFormat(i interface{}) (f EntryFormatter, err error) {

daemon/logging/logging_context.go

@@ -0,0 +1,24 @@
+package logging
+
+import "context"
+
+type contextKey int
+
+const (
+	contextKeyLoggers contextKey = 1 + iota
+	contextKeyInjectedField
+)
+
+var contextKeys = []contextKey{
+	contextKeyLoggers,
+	contextKeyInjectedField,
+}
+
+func WithInherit(ctx, inheritFrom context.Context) context.Context {
+	for _, k := range contextKeys {
+		if v := inheritFrom.Value(k); v != nil {
+			ctx = context.WithValue(ctx, k, v) // no shadow
+		}
+	}
+	return ctx
+}

daemon/logging/logging_formatters.go

@@ -22,6 +22,7 @@ const (
 const (
 	JobField    string = "job"
 	SubsysField string = "subsystem"
+	SpanField   string = "span"
 )
 
 type MetadataFlags int64
@@ -85,7 +86,7 @@ func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
 		fmt.Fprintf(&line, "[%s]", col.Sprint(e.Level.Short()))
 	}
 
-	prefixFields := []string{JobField, SubsysField}
+	prefixFields := []string{JobField, SubsysField, SpanField}
 	prefixed := make(map[string]bool, len(prefixFields)+2)
 	for _, field := range prefixFields {
 		val, ok := e.Fields[field]
@@ -174,8 +175,8 @@ func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
 	}
 
 	// at least try and put job and task in front
-	prefixed := make(map[string]bool, 2)
+	prefixed := make(map[string]bool, 3)
-	prefix := []string{JobField, SubsysField}
+	prefix := []string{JobField, SubsysField, SpanField}
 	for _, pf := range prefix {
 		v, ok := e.Fields[pf]
 		if !ok {

daemon/logging/trace/trace.go

@@ -0,0 +1,355 @@
+// package trace provides activity tracing via ctx through Tasks and Spans
+//
+// Basic Concepts
+//
+// Tracing can be used to identify where a piece of code spends its time.
+//
+// The Go standard library provides package runtime/trace which is useful to identify CPU bottlenecks or
+// to understand what happens inside the Go runtime.
+// However, it is not ideal for application level tracing, in particular if those traces should be understandable
+// to tech-savvy users (albeit not developers).
+//
+// This package provides the concept of Tasks and Spans to express what activity is happening within an application:
+//
+//  - Neither task nor span is really tangible but instead contained within the context.Context tree
+//  - Tasks represent concurrent activity (i.e. goroutines).
+//  - Spans represent a semantic stack trace within a task.
+//
+// As a consequence, whenever a context is propagated across goroutine boundary, you need to create a child task:
+//
+//   go func(ctx context.Context) {
+//     ctx, endTask = WithTask(ctx, "what-happens-inside-the-child-task")
+//     defer endTask()
+//     // ...
+//   }(ctx)
+//
+// Within the task, you can open up a hierarchy of spans.
+// In contrast to tasks, which have can multiple concurrently running child tasks,
+// spans must nest and not cross the goroutine boundary.
+//
+//  ctx, endSpan = WithSpan(ctx, "copy-dir")
+//  defer endSpan()
+//  for _, f := range dir.Files() {
+//    func() {
+//      ctx, endSpan := WithSpan(ctx, fmt.Sprintf("copy-file %q", f))
+//      defer endspan()
+//      b, _ := ioutil.ReadFile(f)
+//      _ = ioutil.WriteFile(f + ".copy", b, 0600)
+//    }()
+//  }
+//
+// In combination:
+//  ctx, endTask = WithTask(ctx, "copy-dirs")
+//  defer endTask()
+//  for i := range dirs {
+//    go func(dir string) {
+//      ctx, endTask := WithTask(ctx, "copy-dir")
+//      defer endTask()
+//      for _, f := range filesIn(dir) {
+//        func() {
+//          ctx, endSpan := WithSpan(ctx, fmt.Sprintf("copy-file %q", f))
+//          defer endspan()
+//          b, _ := ioutil.ReadFile(f)
+//          _ = ioutil.WriteFile(f + ".copy", b, 0600)
+//        }()
+//      }
+//    }()
+//  }
+//
+// Note that a span ends at the time you call endSpan - not before and not after that.
+// If you violate the stack-like nesting of spans by forgetting an endSpan() invocation,
+// the out-of-order endSpan() will panic.
+//
+// A similar rule applies to the endTask closure returned by WithTask:
+// If a task has live child tasks at the time you call endTask(), the call will panic.
+//
+// Recovering from endSpan() or endTask() panics will corrupt the trace stack and lead to corrupt tracefile output.
+//
+//
+// Best Practices For Naming Tasks And Spans
+//
+// Tasks should always have string constants as names, and must not contain the `#` character. WHy?
+// First, the visualization by chrome://tracing draws a horizontal bar for each task in the trace.
+// Also, the package appends `#NUM` for each concurrently running instance of a task name.
+// Note that the `#NUM` suffix will be reused if a task has ended, in order to avoid an
+// infinite number of horizontal bars in the visualization.
+//
+//
+// Chrome-compatible Tracefile Support
+//
+// The activity trace generated by usage of WithTask and WithSpan can be rendered to a JSON output file
+// that can be loaded into chrome://tracing .
+// Apart from function GetSpanStackOrDefault, this is the main benefit of this package.
+//
+// First, there is a convenience environment variable 'ZREPL_ACTIVITY_TRACE' that can be set to an output path.
+// From process start onward, a trace is written to that path.
+//
+// More consumers can attach to the activity trace through the ChrometraceClientWebsocketHandler websocket handler.
+//
+// If a write error is encountered with any consumer (including the env-var based one), the consumer is closed and
+// will not receive further trace output.
+package trace
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/zrepl/zrepl/util/chainlock"
+)
+
+var metrics struct {
+	activeTasks                          prometheus.Gauge
+	uniqueConcurrentTaskNameBitvecLength *prometheus.GaugeVec
+}
+
+func init() {
+	metrics.activeTasks = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "zrepl",
+		Subsystem: "trace",
+		Name:      "active_tasks",
+		Help:      "number of active (tracing-level) tasks in the daemon",
+	})
+	metrics.uniqueConcurrentTaskNameBitvecLength = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "zrepl",
+		Subsystem: "trace",
+		Name:      "unique_concurrent_task_name_bitvec_length",
+		Help:      "length of the bitvec used to find unique names for concurrent tasks",
+	}, []string{"task_name"})
+}
+
+func RegisterMetrics(r prometheus.Registerer) {
+	r.MustRegister(metrics.activeTasks)
+	r.MustRegister(metrics.uniqueConcurrentTaskNameBitvecLength)
+}
+
+var taskNamer = newUniqueTaskNamer(metrics.uniqueConcurrentTaskNameBitvecLength)
+
+type traceNode struct {
+	id         string
+	annotation string
+	parentTask *traceNode
+
+	mtx chainlock.L
+
+	activeChildTasks int32 // only for task nodes, insignificant for span nodes
+	parentSpan       *traceNode
+	activeChildSpan  *traceNode // nil if task or span doesn't have an active child span
+
+	startedAt time.Time
+	endedAt   time.Time
+}
+
+// Returned from WithTask or WithSpan.
+// Must be called once the task or span ends.
+// See package-level docs for nesting rules.
+// Wrong call order / forgetting to call it will result in panics.
+type DoneFunc func()
+
+var ErrTaskStillHasActiveChildTasks = fmt.Errorf("end task: task still has active child tasks")
+var ErrParentTaskAlreadyEnded = fmt.Errorf("create task: parent task already ended")
+
+// Start a new root task or create a child task of an existing task.
+//
+// This is required when starting a new goroutine and
+// passing an existing task context to it.
+//
+// taskName should be a constantand must not contain '#'
+//
+// The implementation ensures that,
+// if multiple tasks with the same name exist simultaneously,
+// a unique suffix is appended to uniquely identify the task opened with this function.
+func WithTask(ctx context.Context, taskName string) (context.Context, DoneFunc) {
+
+	var parentTask *traceNode
+	nodeI := ctx.Value(contextKeyTraceNode)
+	if nodeI != nil {
+		node := nodeI.(*traceNode)
+		if node.parentSpan != nil {
+			parentTask = node.parentTask
+		} else {
+			parentTask = node
+		}
+	}
+
+	taskName, taskNameDone := taskNamer.UniqueConcurrentTaskName(taskName)
+
+	this := &traceNode{
+		id:               genID(),
+		annotation:       taskName,
+		parentTask:       parentTask,
+		activeChildTasks: 0,
+		parentSpan:       nil,
+		activeChildSpan:  nil,
+
+		startedAt: time.Now(),
+		endedAt:   time.Time{},
+	}
+
+	if this.parentTask != nil {
+		this.parentTask.mtx.HoldWhile(func() {
+			if !this.parentTask.endedAt.IsZero() {
+				panic(ErrParentTaskAlreadyEnded)
+			}
+			this.parentTask.activeChildTasks++
+		})
+	}
+
+	ctx = context.WithValue(ctx, contextKeyTraceNode, this)
+
+	chrometraceBeginTask(this)
+
+	metrics.activeTasks.Inc()
+
+	endTaskFunc := func() {
+
+		// only hold locks while manipulating the tree
+		// (trace writer might block too long and unlike spans, tasks are updated concurrently)
+		alreadyEnded := func() (alreadyEnded bool) {
+			if this.parentTask != nil {
+				defer this.parentTask.mtx.Lock().Unlock()
+			}
+			defer this.mtx.Lock().Unlock()
+
+			if this.activeChildTasks != 0 {
+				panic(errors.Wrapf(ErrTaskStillHasActiveChildTasks, "end task: %v active child tasks", this.activeChildSpan))
+			}
+
+			// support idempotent task ends
+			if !this.endedAt.IsZero() {
+				return true
+			}
+			this.endedAt = time.Now()
+
+			if this.parentTask != nil {
+				this.parentTask.activeChildTasks--
+				if this.parentTask.activeChildTasks < 0 {
+					panic("impl error: parent task with negative activeChildTasks count")
+				}
+			}
+			return false
+		}()
+		if alreadyEnded {
+			return
+		}
+
+		chrometraceEndTask(this)
+
+		metrics.activeTasks.Dec()
+
+		taskNameDone()
+	}
+
+	return ctx, endTaskFunc
+}
+
+var ErrAlreadyActiveChildSpan = fmt.Errorf("create child span: span already has an active child span")
+var ErrSpanStillHasActiveChildSpan = fmt.Errorf("end span: span still has active child spans")
+
+// Start a new span.
+// Important: ctx must have an active task (see WithTask)
+func WithSpan(ctx context.Context, annotation string) (context.Context, DoneFunc) {
+	var parentSpan, parentTask *traceNode
+	nodeI := ctx.Value(contextKeyTraceNode)
+	if nodeI != nil {
+		parentSpan = nodeI.(*traceNode)
+		if parentSpan.parentSpan == nil {
+			parentTask = parentSpan
+		} else {
+			parentTask = parentSpan.parentTask
+		}
+	} else {
+		panic("must be called from within a task")
+	}
+
+	this := &traceNode{
+		id:              genID(),
+		annotation:      annotation,
+		parentTask:      parentTask,
+		parentSpan:      parentSpan,
+		activeChildSpan: nil,
+
+		startedAt: time.Now(),
+		endedAt:   time.Time{},
+	}
+
+	parentSpan.mtx.HoldWhile(func() {
+		if parentSpan.activeChildSpan != nil {
+			panic(ErrAlreadyActiveChildSpan)
+		}
+		parentSpan.activeChildSpan = this
+	})
+
+	ctx = context.WithValue(ctx, contextKeyTraceNode, this)
+	chrometraceBeginSpan(this)
+
+	endTaskFunc := func() {
+
+		defer parentSpan.mtx.Lock().Unlock()
+		if parentSpan.activeChildSpan != this && this.endedAt.IsZero() {
+			panic("impl error: activeChildSpan should not change while != nil because there can only be one")
+		}
+
+		defer this.mtx.Lock().Unlock()
+		if this.activeChildSpan != nil {
+			panic(ErrSpanStillHasActiveChildSpan)
+		}
+
+		if !this.endedAt.IsZero() {
+			return // support idempotent span ends
+		}
+
+		parentSpan.activeChildSpan = nil
+		this.endedAt = time.Now()
+
+		chrometraceEndSpan(this)
+	}
+
+	return ctx, endTaskFunc
+}
+
+func currentTaskNameAndSpanStack(this *traceNode) (taskName string, spanIdStack string) {
+
+	task := this.parentTask
+	if this.parentSpan == nil {
+		task = this
+	}
+
+	var spansInTask []*traceNode
+	for s := this; s != nil; s = s.parentSpan {
+		spansInTask = append(spansInTask, s)
+	}
+
+	var tasks []*traceNode
+	for t := task; t != nil; t = t.parentTask {
+		tasks = append(tasks, t)
+	}
+
+	var taskIdsRev []string
+	for i := len(tasks) - 1; i >= 0; i-- {
+		taskIdsRev = append(taskIdsRev, tasks[i].id)
+	}
+
+	var spanIdsRev []string
+	for i := len(spansInTask) - 1; i >= 0; i-- {
+		spanIdsRev = append(spanIdsRev, spansInTask[i].id)
+	}
+
+	taskStack := strings.Join(taskIdsRev, "$")
+	spanIdStack = fmt.Sprintf("%s$%s", taskStack, strings.Join(spanIdsRev, "."))
+
+	return task.annotation, spanIdStack
+}
+
+func GetSpanStackOrDefault(ctx context.Context, def string) string {
+	if nI := ctx.Value(contextKeyTraceNode); nI != nil {
+		n := nI.(*traceNode)
+		_, spanStack := currentTaskNameAndSpanStack(n)
+		return spanStack
+	} else {
+		return def
+	}
+}

daemon/logging/trace/trace_chrometrace.go

@@ -0,0 +1,230 @@
+package trace
+
+// The functions in this file are concerned with the generation
+// of trace files based on the information from WithTask and WithSpan.
+//
+// The emitted trace files are open-ended array of JSON objects
+// that follow the Chrome trace file format:
+//   https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview
+//
+// The emitted JSON can be loaded into Chrome's chrome://tracing view.
+//
+// The trace file can be written to a file whose path is specified in an env file,
+// and be written to web sockets established on ChrometraceHttpHandler
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"sync"
+	"sync/atomic"
+
+	"golang.org/x/net/websocket"
+
+	"github.com/zrepl/zrepl/util/envconst"
+)
+
+var chrometracePID string
+
+func init() {
+	var err error
+	chrometracePID, err = os.Hostname()
+	if err != nil {
+		panic(err)
+	}
+	chrometracePID = fmt.Sprintf("%q", chrometracePID)
+}
+
+type chrometraceEvent struct {
+	Cat                       string   `json:"cat,omitempty"`
+	Name                      string   `json:"name"`
+	Stack                     []string `json:"stack,omitempty"`
+	Phase                     string   `json:"ph"`
+	TimestampUnixMicroseconds int64    `json:"ts"`
+	DurationMicroseconds      int64    `json:"dur,omitempty"`
+	Pid                       string   `json:"pid"`
+	Tid                       string   `json:"tid"`
+	Id                        string   `json:"id,omitempty"`
+}
+
+func chrometraceBeginSpan(s *traceNode) {
+	taskName, _ := currentTaskNameAndSpanStack(s)
+	chrometraceWrite(chrometraceEvent{
+		Name:                      s.annotation,
+		Phase:                     "B",
+		TimestampUnixMicroseconds: s.startedAt.UnixNano() / 1000,
+		Pid:                       chrometracePID,
+		Tid:                       taskName,
+	})
+}
+
+func chrometraceEndSpan(s *traceNode) {
+	taskName, _ := currentTaskNameAndSpanStack(s)
+	chrometraceWrite(chrometraceEvent{
+		Name:                      s.annotation,
+		Phase:                     "E",
+		TimestampUnixMicroseconds: s.endedAt.UnixNano() / 1000,
+		Pid:                       chrometracePID,
+		Tid:                       taskName,
+	})
+}
+
+var chrometraceFlowId uint64
+
+func chrometraceBeginTask(s *traceNode) {
+	chrometraceBeginSpan(s)
+
+	if s.parentTask == nil {
+		return
+	}
+	// beginning of a task that has a parent
+	// => use flow events to link parent and child
+
+	flowId := atomic.AddUint64(&chrometraceFlowId, 1)
+	flowIdStr := fmt.Sprintf("%x", flowId)
+
+	parentTask, _ := currentTaskNameAndSpanStack(s.parentTask)
+	chrometraceWrite(chrometraceEvent{
+		Cat:                       "task", // seems to be necessary, otherwise the GUI shows some `indexOf` JS error
+		Name:                      "child-task",
+		Phase:                     "s",
+		TimestampUnixMicroseconds: s.startedAt.UnixNano() / 1000, // yes, the child's timestamp (=> from-point of the flow line is at right x-position of parent's bar)
+		Pid:                       chrometracePID,
+		Tid:                       parentTask,
+		Id:                        flowIdStr,
+	})
+
+	childTask, _ := currentTaskNameAndSpanStack(s)
+	if parentTask == childTask {
+		panic(parentTask)
+	}
+	chrometraceWrite(chrometraceEvent{
+		Cat:                       "task", // seems to be necessary, otherwise the GUI shows some `indexOf` JS error
+		Name:                      "child-task",
+		Phase:                     "f",
+		TimestampUnixMicroseconds: s.startedAt.UnixNano() / 1000,
+		Pid:                       chrometracePID,
+		Tid:                       childTask,
+		Id:                        flowIdStr,
+	})
+}
+
+func chrometraceEndTask(s *traceNode) {
+	chrometraceEndSpan(s)
+}
+
+type chrometraceConsumerRegistration struct {
+	w io.Writer
+	// errored must have capacity 1, the writer thread will send to it non-blocking, then close it
+	errored chan error
+}
+
+var chrometraceConsumers struct {
+	register  chan chrometraceConsumerRegistration
+	consumers map[chrometraceConsumerRegistration]bool
+	write     chan []byte
+}
+
+func init() {
+	chrometraceConsumers.register = make(chan chrometraceConsumerRegistration)
+	chrometraceConsumers.consumers = make(map[chrometraceConsumerRegistration]bool)
+	chrometraceConsumers.write = make(chan []byte)
+	go func() {
+		kickConsumer := func(c chrometraceConsumerRegistration, err error) {
+			debug("chrometrace kicking consumer %#v after error %v", c, err)
+			select {
+			case c.errored <- err:
+			default:
+			}
+			close(c.errored)
+			delete(chrometraceConsumers.consumers, c)
+		}
+		for {
+			select {
+			case reg := <-chrometraceConsumers.register:
+				debug("registered chrometrace consumer %#v", reg)
+				chrometraceConsumers.consumers[reg] = true
+				n, err := reg.w.Write([]byte("[\n"))
+				if err != nil {
+					kickConsumer(reg, err)
+				} else if n != 2 {
+					kickConsumer(reg, fmt.Errorf("short write: %v", n))
+				}
+				// successfully registered
+
+			case buf := <-chrometraceConsumers.write:
+				debug("chrometrace write request: %s", string(buf))
+				var r bytes.Reader
+				for c := range chrometraceConsumers.consumers {
+					r.Reset(buf)
+					n, err := io.Copy(c.w, &r)
+					debug("chrometrace wrote n=%v bytes to consumer %#v", n, c)
+					if err != nil {
+						kickConsumer(c, err)
+					}
+				}
+			}
+		}
+	}()
+}
+
+func chrometraceWrite(i interface{}) {
+	var buf bytes.Buffer
+	err := json.NewEncoder(&buf).Encode(i)
+	if err != nil {
+		panic(err)
+	}
+	buf.WriteString(",")
+	chrometraceConsumers.write <- buf.Bytes()
+}
+
+func ChrometraceClientWebsocketHandler(conn *websocket.Conn) {
+	defer conn.Close()
+
+	var wg sync.WaitGroup
+	defer wg.Wait()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		r := bufio.NewReader(conn)
+		_, _, _ = r.ReadLine() // ignore errors
+		conn.Close()
+	}()
+
+	errored := make(chan error, 1)
+	chrometraceConsumers.register <- chrometraceConsumerRegistration{
+		w:       conn,
+		errored: errored,
+	}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		<-errored
+		conn.Close()
+	}()
+}
+
+var chrometraceFileConsumerPath = envconst.String("ZREPL_ACTIVITY_TRACE", "")
+
+func init() {
+	if chrometraceFileConsumerPath != "" {
+		var err error
+		f, err := os.Create(chrometraceFileConsumerPath)
+		if err != nil {
+			panic(err)
+		}
+		errored := make(chan error, 1)
+		chrometraceConsumers.register <- chrometraceConsumerRegistration{
+			w:       f,
+			errored: errored,
+		}
+		go func() {
+			<-errored
+			f.Close()
+		}()
+	}
+}

daemon/logging/trace/trace_context.go

@@ -0,0 +1,27 @@
+package trace
+
+import "context"
+
+type contextKey int
+
+const (
+	contextKeyTraceNode contextKey = 1 + iota
+)
+
+var contextKeys = []contextKey{
+	contextKeyTraceNode,
+}
+
+// WithInherit inherits the task hierarchy from inheritFrom into ctx.
+// The returned context is a child of ctx, but its task and span are those of inheritFrom.
+//
+// Note that in most use cases, callers most likely want to call WithTask since it will most likely
+// be in some sort of connection handler context.
+func WithInherit(ctx, inheritFrom context.Context) context.Context {
+	for _, k := range contextKeys {
+		if v := inheritFrom.Value(k); v != nil {
+			ctx = context.WithValue(ctx, k, v) // no shadow
+		}
+	}
+	return ctx
+}

daemon/logging/trace/trace_convenience.go

@@ -0,0 +1,74 @@
+package trace
+
+import (
+	"context"
+	"fmt"
+	"runtime"
+	"strings"
+	"sync"
+)
+
+// use like this:
+//
+//	defer WithSpanFromStackUpdateCtx(&existingCtx)()
+//
+//
+func WithSpanFromStackUpdateCtx(ctx *context.Context) DoneFunc {
+	childSpanCtx, end := WithSpan(*ctx, getMyCallerOrPanic())
+	*ctx = childSpanCtx
+	return end
+}
+
+// derive task name from call stack (caller's name)
+func WithTaskFromStack(ctx context.Context) (context.Context, DoneFunc) {
+	return WithTask(ctx, getMyCallerOrPanic())
+}
+
+// derive task name from call stack (caller's name) and update *ctx
+// to point to be the child task ctx
+func WithTaskFromStackUpdateCtx(ctx *context.Context) DoneFunc {
+	child, end := WithTask(*ctx, getMyCallerOrPanic())
+	*ctx = child
+	return end
+}
+
+// create a task and a span within it in one call
+func WithTaskAndSpan(ctx context.Context, task string, span string) (context.Context, DoneFunc) {
+	ctx, endTask := WithTask(ctx, task)
+	ctx, endSpan := WithSpan(ctx, fmt.Sprintf("%s %s", task, span))
+	return ctx, func() {
+		endSpan()
+		endTask()
+	}
+}
+
+// create a span during which several child tasks are spawned using the `add` function
+func WithTaskGroup(ctx context.Context, taskGroup string) (_ context.Context, add func(f func(context.Context)), waitEnd DoneFunc) {
+	var wg sync.WaitGroup
+	ctx, endSpan := WithSpan(ctx, taskGroup)
+	add = func(f func(context.Context)) {
+		wg.Add(1)
+		defer wg.Done()
+		ctx, endTask := WithTask(ctx, taskGroup)
+		defer endTask()
+		f(ctx)
+	}
+	waitEnd = func() {
+		wg.Wait()
+		endSpan()
+	}
+	return ctx, add, waitEnd
+}
+
+func getMyCallerOrPanic() string {
+	pc, _, _, ok := runtime.Caller(2)
+	if !ok {
+		panic("cannot get caller")
+	}
+	details := runtime.FuncForPC(pc)
+	if ok && details != nil {
+		const prefix = "github.com/zrepl/zrepl"
+		return strings.TrimPrefix(strings.TrimPrefix(details.Name(), prefix), "/")
+	}
+	return ""
+}

daemon/logging/trace/trace_convenience_test.go

@@ -0,0 +1,16 @@
+package trace
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetCallerOrPanic(t *testing.T) {
+	withStackFromCtxMock := func() string {
+		return getMyCallerOrPanic()
+	}
+	ret := withStackFromCtxMock()
+	// zrepl prefix is stripped
+	assert.Equal(t, "daemon/logging/trace.TestGetCallerOrPanic", ret)
+}

daemon/logging/trace/trace_debug.go

@@ -0,0 +1,15 @@
+package trace
+
+import (
+	"fmt"
+	"os"
+)
+
+const debugEnabled = false
+
+func debug(format string, args ...interface{}) {
+	if !debugEnabled {
+		return
+	}
+	fmt.Fprintf(os.Stderr, format+"\n", args...)
+}

daemon/logging/trace/trace_genID.go

@@ -0,0 +1,47 @@
+package trace
+
+import (
+	"encoding/base64"
+	"math/rand"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/zrepl/zrepl/util/envconst"
+)
+
+var genIdPRNG = rand.New(rand.NewSource(1))
+
+func init() {
+	genIdPRNG.Seed(time.Now().UnixNano())
+	genIdPRNG.Seed(int64(os.Getpid()))
+}
+
+var genIdNumBytes = envconst.Int("ZREPL_TRACE_ID_NUM_BYTES", 3)
+
+func init() {
+	if genIdNumBytes < 1 {
+		panic("trace node id byte length must be at least 1")
+	}
+}
+
+func genID() string {
+	var out strings.Builder
+	enc := base64.NewEncoder(base64.RawStdEncoding, &out)
+	buf := make([]byte, genIdNumBytes)
+	for i := 0; i < len(buf); {
+		n, err := genIdPRNG.Read(buf[i:])
+		if err != nil {
+			panic(err)
+		}
+		i += n
+	}
+	n, err := enc.Write(buf[:])
+	if err != nil || n != len(buf) {
+		panic(err)
+	}
+	if err := enc.Close(); err != nil {
+		panic(err)
+	}
+	return out.String()
+}

daemon/logging/trace/trace_test.go

@@ -0,0 +1,172 @@
+package trace
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/gitchander/permutation"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRegularSpanUsage(t *testing.T) {
+	root, endRoot := WithTask(context.Background(), "root")
+	defer endRoot()
+
+	s1, endS1 := WithSpan(root, "parent")
+	s2, endS2 := WithSpan(s1, "child")
+	_, endS3 := WithSpan(s2, "grand-child")
+	require.NotPanics(t, func() { endS3() })
+	require.NotPanics(t, func() { endS2() })
+
+	// reuse
+	_, endS4 := WithSpan(s1, "child-2")
+	require.NotPanics(t, func() { endS4() })
+
+	// close parent
+	require.NotPanics(t, func() { endS1() })
+}
+
+func TestMultipleActiveChildSpansNotAllowed(t *testing.T) {
+	root, endRoot := WithTask(context.Background(), "root")
+	defer endRoot()
+
+	s1, _ := WithSpan(root, "s1")
+	_, endS2 := WithSpan(s1, "s1-child1")
+
+	require.PanicsWithValue(t, ErrAlreadyActiveChildSpan, func() {
+		_, _ = WithSpan(s1, "s1-child2")
+	})
+
+	endS2()
+
+	require.NotPanics(t, func() {
+		_, _ = WithSpan(s1, "s1-child2")
+	})
+}
+
+func TestForkingChildSpansNotAllowed(t *testing.T) {
+	root, endRoot := WithTask(context.Background(), "root")
+	defer endRoot()
+
+	s1, _ := WithSpan(root, "s1")
+	sc, endSC := WithSpan(s1, "s1-child")
+	_, _ = WithSpan(sc, "s1-child-child")
+
+	require.PanicsWithValue(t, ErrSpanStillHasActiveChildSpan, func() {
+		endSC()
+	})
+}
+
+func TestRegularTaskUsage(t *testing.T) {
+	// assert concurrent activities on different tasks can end in any order
+	closeOrder := []int{0, 1, 2}
+	closeOrders := permutation.New(permutation.IntSlice(closeOrder))
+	for closeOrders.Next() {
+		t.Run(fmt.Sprintf("%v", closeOrder), func(t *testing.T) {
+			root, endRoot := WithTask(context.Background(), "root")
+			defer endRoot()
+
+			c1, endC1 := WithTask(root, "c1")
+			defer endC1()
+			c2, endC2 := WithTask(root, "c2")
+			defer endC2()
+
+			// begin 3 concurrent activities
+			_, endAR := WithSpan(root, "aR")
+			_, endAC1 := WithSpan(c1, "aC1")
+			_, endAC2 := WithSpan(c2, "aC2")
+
+			endFuncs := []DoneFunc{endAR, endAC1, endAC2}
+			for _, i := range closeOrder {
+				require.NotPanics(t, func() {
+					endFuncs[i]()
+				}, "%v", i)
+			}
+		})
+	}
+}
+
+func TestTaskEndWithActiveChildTaskNotAllowed(t *testing.T) {
+	root, _ := WithTask(context.Background(), "root")
+	c, endC := WithTask(root, "child")
+	_, _ = WithTask(c, "grand-child")
+	func() {
+		defer func() {
+			r := recover()
+			require.NotNil(t, r)
+			err, ok := r.(error)
+			require.True(t, ok)
+			require.Equal(t, ErrTaskStillHasActiveChildTasks, errors.Cause(err))
+		}()
+		endC()
+	}()
+
+}
+
+func TestIdempotentEndTask(t *testing.T) {
+	_, end := WithTask(context.Background(), "root")
+	end()
+	require.NotPanics(t, func() { end() })
+}
+
+func TestCannotReuseEndedTask(t *testing.T) {
+	root, end := WithTask(context.Background(), "root")
+	end()
+	require.PanicsWithValue(t, ErrParentTaskAlreadyEnded, func() { WithTask(root, "child-after-parent-ended") })
+}
+
+func TestSpansPanicIfNoParentTask(t *testing.T) {
+	require.Panics(t, func() { WithSpan(context.Background(), "taskless-span") })
+}
+
+func TestIdempotentEndSpan(t *testing.T) {
+	root, _ := WithTask(context.Background(), "root")
+	_, end := WithSpan(root, "span")
+	end()
+	require.NotPanics(t, func() { end() })
+}
+
+func logAndGetTraceNode(t *testing.T, descr string, ctx context.Context) *traceNode {
+	n, ok := ctx.Value(contextKeyTraceNode).(*traceNode)
+	require.True(t, ok)
+	t.Logf("% 20s %p %#v", descr, n, n)
+	return n
+}
+
+func TestWhiteboxHierachy(t *testing.T) {
+	root, e1 := WithTask(context.Background(), "root")
+	rootN := logAndGetTraceNode(t, "root", root)
+	assert.Nil(t, rootN.parentTask)
+	assert.Nil(t, rootN.parentSpan)
+
+	child, e2 := WithSpan(root, "child")
+	childN := logAndGetTraceNode(t, "child", child)
+	assert.Equal(t, rootN, childN.parentTask)
+	assert.Equal(t, rootN, childN.parentSpan)
+
+	grandchild, e3 := WithSpan(child, "grandchild")
+	grandchildN := logAndGetTraceNode(t, "grandchild", grandchild)
+	assert.Equal(t, rootN, grandchildN.parentTask)
+	assert.Equal(t, childN, grandchildN.parentSpan)
+
+	gcTask, e4 := WithTask(grandchild, "grandchild-task")
+	gcTaskN := logAndGetTraceNode(t, "grandchild-task", gcTask)
+	assert.Equal(t, rootN, gcTaskN.parentTask)
+	assert.Nil(t, gcTaskN.parentSpan)
+
+	// it is allowed that a child task outlives the _span_ in which it was created
+	// (albeit not its parent task)
+	e3()
+	e2()
+	gcTaskSpan, e5 := WithSpan(gcTask, "granschild-task-span")
+	gcTaskSpanN := logAndGetTraceNode(t, "granschild-task-span", gcTaskSpan)
+	assert.Equal(t, gcTaskN, gcTaskSpanN.parentTask)
+	assert.Equal(t, gcTaskN, gcTaskSpanN.parentSpan)
+	e5()
+
+	e4()
+	e1()
+}

daemon/logging/trace/trace_unique_concurrent_task_namer.go

@@ -0,0 +1,61 @@
+package trace
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/willf/bitset"
+)
+
+type uniqueConcurrentTaskNamer struct {
+	mtx    sync.Mutex
+	active map[string]*bitset.BitSet
+
+	bitvecLengthGauge *prometheus.GaugeVec
+}
+
+// bitvecLengthGauge may be nil
+func newUniqueTaskNamer(bitvecLengthGauge *prometheus.GaugeVec) *uniqueConcurrentTaskNamer {
+	return &uniqueConcurrentTaskNamer{
+		active:            make(map[string]*bitset.BitSet),
+		bitvecLengthGauge: bitvecLengthGauge,
+	}
+}
+
+// appends `#%d` to `name` such that until `done` is called,
+// it is guaranteed that `#%d` is not returned a second time for the same `name`
+func (namer *uniqueConcurrentTaskNamer) UniqueConcurrentTaskName(name string) (uniqueName string, done func()) {
+	if strings.Contains(name, "#") {
+		panic(name)
+	}
+	namer.mtx.Lock()
+	act, ok := namer.active[name]
+	if !ok {
+		act = bitset.New(64) // FIXME magic const
+		namer.active[name] = act
+	}
+	id, ok := act.NextClear(0)
+	if !ok {
+		// if !ok, all bits are 1 and act.Len() returns the next bit
+		id = act.Len()
+		// FIXME unbounded growth without reclamation
+	}
+	act.Set(id)
+	namer.mtx.Unlock()
+
+	if namer.bitvecLengthGauge != nil {
+		namer.bitvecLengthGauge.WithLabelValues(name).Set(float64(act.Len()))
+	}
+
+	return fmt.Sprintf("%s#%d", name, id), func() {
+		namer.mtx.Lock()
+		defer namer.mtx.Unlock()
+		act, ok := namer.active[name]
+		if !ok {
+			panic("must be initialized upon entry")
+		}
+		act.Clear(id)
+	}
+}

daemon/logging/trace/trace_unique_concurrent_task_namer_test.go

@@ -0,0 +1,58 @@
+package trace
+
+import (
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/willf/bitset"
+)
+
+func TestBitsetFeaturesForUniqueConcurrentTaskNamer(t *testing.T) {
+	var b bitset.BitSet
+	require.Equal(t, uint(0), b.Len())
+	require.Equal(t, uint(0), b.Count())
+
+	b.Set(0)
+	require.Equal(t, uint(1), b.Len())
+	require.Equal(t, uint(1), b.Count())
+
+	b.Set(8)
+	require.Equal(t, uint(9), b.Len())
+	require.Equal(t, uint(2), b.Count())
+
+	b.Set(1)
+	require.Equal(t, uint(9), b.Len())
+	require.Equal(t, uint(3), b.Count())
+}
+
+func TestUniqueConcurrentTaskNamer(t *testing.T) {
+	namer := newUniqueTaskNamer(nil)
+
+	var wg sync.WaitGroup
+
+	const N = 8128
+	const Q = 23
+
+	var fails uint32
+	var m sync.Map
+	wg.Add(N)
+	for i := 0; i < N; i++ {
+		go func(i int) {
+			defer wg.Done()
+			name := fmt.Sprintf("%d", i/Q)
+			uniqueName, done := namer.UniqueConcurrentTaskName(name)
+			act, _ := m.LoadOrStore(uniqueName, i)
+			if act.(int) != i {
+				atomic.AddUint32(&fails, 1)
+			}
+			m.Delete(uniqueName)
+			done()
+		}(i)
+	}
+	wg.Wait()
+
+	require.Equal(t, uint32(0), fails)
+}

daemon/main.go

@@ -1,6 +1,8 @@
 package daemon
 
 import (
+	"context"
+
 	"github.com/zrepl/zrepl/cli"
 	"github.com/zrepl/zrepl/logger"
 )
@@ -10,7 +12,7 @@ type Logger = logger.Logger
 var DaemonCmd = &cli.Subcommand{
 	Use:   "daemon",
 	Short: "run the zrepl daemon",
-	Run: func(subcommand *cli.Subcommand, args []string) error {
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
-		return Run(subcommand.Config())
+		return Run(ctx, subcommand.Config())
 	},
 }

daemon/pprof.go

@@ -8,6 +8,9 @@ import (
 	"net"
 	"net/http/pprof"
 
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+	"golang.org/x/net/websocket"
+
 	"github.com/zrepl/zrepl/daemon/job"
 )
 
@@ -64,6 +67,7 @@ outer:
 			mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
 			mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
 			mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
+			mux.Handle("/debug/zrepl/activity-trace", websocket.Handler(trace.ChrometraceClientWebsocketHandler))
 			go func() {
 				err := http.Serve(s.listener, mux)
 				if ctx.Err() != nil {

daemon/prometheus.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/rpc/dataconn/frameconn"
@@ -86,16 +87,19 @@ func (j *prometheusJob) Run(ctx context.Context) {
 }
 
 type prometheusJobOutlet struct {
-	jobName string
 }
 
 var _ logger.Outlet = prometheusJobOutlet{}
 
-func newPrometheusLogOutlet(jobName string) prometheusJobOutlet {
+func newPrometheusLogOutlet() prometheusJobOutlet {
-	return prometheusJobOutlet{jobName}
+	return prometheusJobOutlet{}
 }
 
 func (o prometheusJobOutlet) WriteEntry(entry logger.Entry) error {
-	prom.taskLogEntries.WithLabelValues(o.jobName, entry.Level.String()).Inc()
+	jobFieldVal, ok := entry.Fields[logging.JobField].(string)
+	if !ok {
+		jobFieldVal = "_nojobid"
+	}
+	prom.taskLogEntries.WithLabelValues(jobFieldVal, entry.Level.String()).Inc()
 	return nil
 }

daemon/pruner/pruner.go

@@ -12,6 +12,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/pruning"
 	"github.com/zrepl/zrepl/replication/logic/pdu"
@@ -35,17 +36,13 @@ type Logger = logger.Logger
 
 type contextKey int
 
-const contextKeyLogger contextKey = 0
+const (
-
+	contextKeyPruneSide contextKey = 1 + iota
-func WithLogger(ctx context.Context, log Logger) context.Context {
+)
-	return context.WithValue(ctx, contextKeyLogger, log)
-}
 
 func GetLogger(ctx context.Context) Logger {
-	if l, ok := ctx.Value(contextKeyLogger).(Logger); ok {
+	pruneSide := ctx.Value(contextKeyPruneSide).(string)
-		return l
+	return logging.GetLogger(ctx, logging.SubsysPruning).WithField("prune_side", pruneSide)
-	}
-	return logger.NewNullLogger()
 }
 
 type args struct {
@@ -138,7 +135,7 @@ func NewPrunerFactory(in config.PruningSenderReceiver, promPruneSecs *prometheus
 func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, receiver History) *Pruner {
 	p := &Pruner{
 		args: args{
-			WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "sender")),
+			context.WithValue(ctx, contextKeyPruneSide, "sender"),
 			target,
 			receiver,
 			f.senderRules,
@@ -154,7 +151,7 @@ func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, re
 func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target, receiver History) *Pruner {
 	p := &Pruner{
 		args: args{
-			WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "receiver")),
+			context.WithValue(ctx, contextKeyPruneSide, "receiver"),
 			target,
 			receiver,
 			f.receiverRules,
@@ -170,7 +167,7 @@ func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target,
 func (f *LocalPrunerFactory) BuildLocalPruner(ctx context.Context, target Target, receiver History) *Pruner {
 	p := &Pruner{
 		args: args{
-			ctx,
+			context.WithValue(ctx, contextKeyPruneSide, "local"),
 			target,
 			receiver,
 			f.keepRules,

daemon/snapper/snapper.go

@@ -8,10 +8,12 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/filters"
 	"github.com/zrepl/zrepl/daemon/hooks"
+	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/util/envconst"
 	"github.com/zrepl/zrepl/zfs"
@@ -45,7 +47,6 @@ type snapProgress struct {
 
 type args struct {
 	ctx            context.Context
-	log            Logger
 	prefix         string
 	interval       time.Duration
 	fsf            *filters.DatasetMapFilter
@@ -102,23 +103,10 @@ func (s State) sf() state {
 type updater func(u func(*Snapper)) State
 type state func(a args, u updater) state
 
-type contextKey int
-
-const (
-	contextKeyLog contextKey = 0
-)
-
 type Logger = logger.Logger
 
-func WithLogger(ctx context.Context, log Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLog, log)
-}
-
 func getLogger(ctx context.Context) Logger {
-	if log, ok := ctx.Value(contextKeyLog).(Logger); ok {
+	return logging.GetLogger(ctx, logging.SubsysSnapshot)
-		return log
-	}
-	return logger.NewNullLogger()
 }
 
 func PeriodicFromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in *config.SnapshottingPeriodic) (*Snapper, error) {
@@ -146,13 +134,12 @@ func PeriodicFromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in *con
 }
 
 func (s *Snapper) Run(ctx context.Context, snapshotsTaken chan<- struct{}) {
-
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
 	getLogger(ctx).Debug("start")
 	defer getLogger(ctx).Debug("stop")
 
 	s.args.snapshotsTaken = snapshotsTaken
 	s.args.ctx = ctx
-	s.args.log = getLogger(ctx)
 	s.args.dryRun = false // for future expansion
 
 	u := func(u func(*Snapper)) State {
@@ -190,7 +177,7 @@ func onErr(err error, u updater) state {
 		case Snapshotting:
 			s.state = ErrorWait
 		}
-		s.args.log.WithError(err).WithField("pre_state", preState).WithField("post_state", s.state).Error("snapshotting error")
+		getLogger(s.args.ctx).WithError(err).WithField("pre_state", preState).WithField("post_state", s.state).Error("snapshotting error")
 	}).sf()
 }
 
@@ -209,7 +196,7 @@ func syncUp(a args, u updater) state {
 	if err != nil {
 		return onErr(err, u)
 	}
-	syncPoint, err := findSyncPoint(a.log, fss, a.prefix, a.interval)
+	syncPoint, err := findSyncPoint(a.ctx, fss, a.prefix, a.interval)
 	if err != nil {
 		return onErr(err, u)
 	}
@@ -266,18 +253,18 @@ func snapshot(a args, u updater) state {
 		suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
 		snapname := fmt.Sprintf("%s%s", a.prefix, suffix)
 
-		l := a.log.
+		ctx := logging.WithInjectedField(a.ctx, "fs", fs.ToString())
-			WithField("fs", fs.ToString()).
+		ctx = logging.WithInjectedField(ctx, "snap", snapname)
-			WithField("snap", snapname)
 
 		hookEnvExtra := hooks.Env{
 			hooks.EnvFS:       fs.ToString(),
 			hooks.EnvSnapshot: snapname,
 		}
 
-		jobCallback := hooks.NewCallbackHookForFilesystem("snapshot", fs, func(_ context.Context) (err error) {
+		jobCallback := hooks.NewCallbackHookForFilesystem("snapshot", fs, func(ctx context.Context) (err error) {
+			l := getLogger(ctx)
 			l.Debug("create snapshot")
-			err = zfs.ZFSSnapshot(a.ctx, fs, snapname, false) // TODO propagate context to ZFSSnapshot
+			err = zfs.ZFSSnapshot(ctx, fs, snapname, false) // TODO propagate context to ZFSSnapshot
 			if err != nil {
 				l.WithError(err).Error("cannot create snapshot")
 			}
@@ -290,7 +277,7 @@ func snapshot(a args, u updater) state {
 		{
 			filteredHooks, err := a.hooks.CopyFilteredForFilesystem(fs)
 			if err != nil {
-				l.WithError(err).Error("unexpected filter error")
+				getLogger(ctx).WithError(err).Error("unexpected filter error")
 				fsHadErr = true
 				goto updateFSState
 			}
@@ -303,7 +290,7 @@ func snapshot(a args, u updater) state {
 			plan, planErr = hooks.NewPlan(&filteredHooks, hooks.PhaseSnapshot, jobCallback, hookEnvExtra)
 			if planErr != nil {
 				fsHadErr = true
-				l.WithError(planErr).Error("cannot create job hook plan")
+				getLogger(ctx).WithError(planErr).Error("cannot create job hook plan")
 				goto updateFSState
 			}
 		}
@@ -314,15 +301,14 @@ func snapshot(a args, u updater) state {
 			progress.state = SnapStarted
 		})
 		{
-			l := hooks.GetLogger(a.ctx).WithField("fs", fs.ToString()).WithField("snap", snapname)
+			getLogger(ctx).WithField("report", plan.Report().String()).Debug("begin run job plan")
-			l.WithField("report", plan.Report().String()).Debug("begin run job plan")
+			plan.Run(ctx, a.dryRun)
-			plan.Run(hooks.WithLogger(a.ctx, l), a.dryRun)
 			planReport = plan.Report()
 			fsHadErr = planReport.HadError() // not just fatal errors
 			if fsHadErr {
-				l.WithField("report", planReport.String()).Error("end run job plan with error")
+				getLogger(ctx).WithField("report", planReport.String()).Error("end run job plan with error")
 			} else {
-				l.WithField("report", planReport.String()).Info("end run job plan successful")
+				getLogger(ctx).WithField("report", planReport.String()).Info("end run job plan successful")
 			}
 		}
 
@@ -342,7 +328,7 @@ func snapshot(a args, u updater) state {
 	case a.snapshotsTaken <- struct{}{}:
 	default:
 		if a.snapshotsTaken != nil {
-			a.log.Warn("callback channel is full, discarding snapshot update event")
+			getLogger(a.ctx).Warn("callback channel is full, discarding snapshot update event")
 		}
 	}
 
@@ -355,7 +341,7 @@ func snapshot(a args, u updater) state {
 					break
 				}
 			}
-			a.log.WithField("hook", h.String()).WithField("hook_number", hookIdx+1).Warn("hook did not match any snapshotted filesystems")
+			getLogger(a.ctx).WithField("hook", h.String()).WithField("hook_number", hookIdx+1).Warn("hook did not match any snapshotted filesystems")
 		}
 	}
 
@@ -376,7 +362,7 @@ func wait(a args, u updater) state {
 		lastTick := snapper.lastInvocation
 		snapper.sleepUntil = lastTick.Add(a.interval)
 		sleepUntil = snapper.sleepUntil
-		log := a.log.WithField("sleep_until", sleepUntil).WithField("duration", a.interval)
+		log := getLogger(a.ctx).WithField("sleep_until", sleepUntil).WithField("duration", a.interval)
 		logFunc := log.Debug
 		if snapper.state == ErrorWait || snapper.state == SyncUpErrWait {
 			logFunc = log.Error
@@ -404,7 +390,7 @@ func listFSes(ctx context.Context, mf *filters.DatasetMapFilter) (fss []*zfs.Dat
 var syncUpWarnNoSnapshotUntilSyncupMinDuration = envconst.Duration("ZREPL_SNAPPER_SYNCUP_WARN_MIN_DURATION", 1*time.Second)
 
 // see docs/snapshotting.rst
-func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval time.Duration) (syncPoint time.Time, err error) {
+func findSyncPoint(ctx context.Context, fss []*zfs.DatasetPath, prefix string, interval time.Duration) (syncPoint time.Time, err error) {
 
 	const (
 		prioHasVersions int = iota
@@ -426,10 +412,10 @@ func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval t
 
 	now := time.Now()
 
-	log.Debug("examine filesystem state to find sync point")
+	getLogger(ctx).Debug("examine filesystem state to find sync point")
 	for _, d := range fss {
-		l := log.WithField("fs", d.ToString())
+		ctx := logging.WithInjectedField(ctx, "fs", d.ToString())
-		syncPoint, err := findSyncPointFSNextOptimalSnapshotTime(l, now, interval, prefix, d)
+		syncPoint, err := findSyncPointFSNextOptimalSnapshotTime(ctx, now, interval, prefix, d)
 		if err == findSyncPointFSNoFilesystemVersionsErr {
 			snaptimes = append(snaptimes, snapTime{
 				ds:   d,
@@ -438,9 +424,9 @@ func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval t
 			})
 		} else if err != nil {
 			hardErrs++
-			l.WithError(err).Error("cannot determine optimal sync point for this filesystem")
+			getLogger(ctx).WithError(err).Error("cannot determine optimal sync point for this filesystem")
 		} else {
-			l.WithField("syncPoint", syncPoint).Debug("found optimal sync point for this filesystem")
+			getLogger(ctx).WithField("syncPoint", syncPoint).Debug("found optimal sync point for this filesystem")
 			snaptimes = append(snaptimes, snapTime{
 				ds:   d,
 				prio: prioHasVersions,
@@ -467,7 +453,7 @@ func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval t
 	})
 
 	winnerSyncPoint := snaptimes[0].time
-	l := log.WithField("syncPoint", winnerSyncPoint.String())
+	l := getLogger(ctx).WithField("syncPoint", winnerSyncPoint.String())
 	l.Info("determined sync point")
 	if winnerSyncPoint.Sub(now) > syncUpWarnNoSnapshotUntilSyncupMinDuration {
 		for _, st := range snaptimes {
@@ -483,9 +469,12 @@ func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval t
 
 var findSyncPointFSNoFilesystemVersionsErr = fmt.Errorf("no filesystem versions")
 
-func findSyncPointFSNextOptimalSnapshotTime(l Logger, now time.Time, interval time.Duration, prefix string, d *zfs.DatasetPath) (time.Time, error) {
+func findSyncPointFSNextOptimalSnapshotTime(ctx context.Context, now time.Time, interval time.Duration, prefix string, d *zfs.DatasetPath) (time.Time, error) {
 
-	fsvs, err := zfs.ZFSListFilesystemVersions(d, filters.NewTypedPrefixFilter(prefix, zfs.Snapshot))
+	fsvs, err := zfs.ZFSListFilesystemVersions(ctx, d, zfs.ListFilesystemVersionsOptions{
+		Types:           zfs.Snapshots,
+		ShortnamePrefix: prefix,
+	})
 	if err != nil {
 		return time.Time{}, errors.Wrap(err, "list filesystem versions")
 	}
@@ -499,7 +488,7 @@ func findSyncPointFSNextOptimalSnapshotTime(l Logger, now time.Time, interval ti
 	})
 
 	latest := fsvs[len(fsvs)-1]
-	l.WithField("creation", latest.Creation).Debug("found latest snapshot")
+	getLogger(ctx).WithField("creation", latest.Creation).Debug("found latest snapshot")
 
 	since := now.Sub(latest.Creation)
 	if since < 0 {

docs/changelog.rst

@@ -62,7 +62,9 @@ Actual changelog:
 * |bugfix| |docs| snapshotting: clarify sync-up behavior and warn about filesystems
   that will not be snapshotted until the sync-up phase is over
 * |docs| Document new replication features in the :ref:`config overview <overview-how-replication-works>` and :repomasterlink:`replication/design.md`.
+* |feature| documented subcommand to generate ``bash`` and ``zsh`` completions
 * **[MAINTAINER NOTICE]** New platform tests in this version, please make sure you run them for your distro!
+* **[MAINTAINER NOTICE]** Please add the shell completions to the zrepl packages.
 
 0.2.1
 -----

docs/configuration/overview.rst

@@ -120,7 +120,7 @@ The following steps take place during replication and can be monitored using the
   * Perform replication steps in the following order:
     Among all filesystems with pending replication steps, pick the filesystem whose next replication step's snapshot is the oldest.
   * Create placeholder filesystems on the receiving side to mirror the dataset paths on the sender to ``root_fs/${client_identity}``.
-  * Acquire send-side step-holds on the step's `from` and `to` snapshots.
+  * Acquire send-side *step-holds* on the step's `from` and `to` snapshots.
   * Perform the replication step.
   * Move the **replication cursor** bookmark on the sending side (see below).
   * Move the **last-received-hold** on the receiving side (see below).

docs/usage.rst

@@ -38,6 +38,8 @@ CLI Overview
     * - ``zrepl migrate``
       - | perform on-disk state / ZFS property migrations
         | (see :ref:`changelog <changelog>` for details)
+    * - ``zrepl zfs-abstractions``
+      - list and remove zrepl's abstractions on top of ZFS, e.g. holds and step bookmarks (see :ref:`overview <replication-cursor-and-last-received-hold>` )
 
 .. _usage-zrepl-daemon:
 

endpoint/context.go

@@ -3,25 +3,18 @@ package endpoint
 import (
 	"context"
 
+	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
 )
 
 type contextKey int
 
 const (
-	contextKeyLogger contextKey = iota
+	ClientIdentityKey contextKey = iota
-	ClientIdentityKey
 )
 
 type Logger = logger.Logger
 
-func WithLogger(ctx context.Context, log Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLogger, log)
-}
-
 func getLogger(ctx context.Context) Logger {
-	if l, ok := ctx.Value(contextKeyLogger).(Logger); ok {
+	return logging.GetLogger(ctx, logging.SubsysEndpoint)
-		return l
-	}
-	return logger.NewNullLogger()
 }

endpoint/endpoint.go

@@ -2,14 +2,18 @@
 package endpoint
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"path"
 	"sync"
 
 	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/replication/logic/pdu"
+	"github.com/zrepl/zrepl/util/chainedio"
 	"github.com/zrepl/zrepl/util/chainlock"
 	"github.com/zrepl/zrepl/util/envconst"
 	"github.com/zrepl/zrepl/util/semaphore"
@@ -70,6 +74,8 @@ func (s *Sender) filterCheckFS(fs string) (*zfs.DatasetPath, error) {
 }
 
 func (s *Sender) ListFilesystems(ctx context.Context, r *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	fss, err := zfs.ZFSListMapping(ctx, s.FSFilter)
 	if err != nil {
 		return nil, err
@@ -92,11 +98,13 @@ func (s *Sender) ListFilesystems(ctx context.Context, r *pdu.ListFilesystemReq)
 }
 
 func (s *Sender) ListFilesystemVersions(ctx context.Context, r *pdu.ListFilesystemVersionsReq) (*pdu.ListFilesystemVersionsRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	lp, err := s.filterCheckFS(r.GetFilesystem())
 	if err != nil {
 		return nil, err
 	}
-	fsvs, err := zfs.ZFSListFilesystemVersions(lp, nil)
+	fsvs, err := zfs.ZFSListFilesystemVersions(ctx, lp, zfs.ListFilesystemVersionsOptions{})
 	if err != nil {
 		return nil, err
 	}
@@ -110,34 +118,117 @@ func (s *Sender) ListFilesystemVersions(ctx context.Context, r *pdu.ListFilesyst
 }
 
 func (p *Sender) HintMostRecentCommonAncestor(ctx context.Context, r *pdu.HintMostRecentCommonAncestorReq) (*pdu.HintMostRecentCommonAncestorRes, error) {
-	var err error
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
 
-	fs := r.GetFilesystem()
+	fsp, err := p.filterCheckFS(r.GetFilesystem())
-	mostRecent, err := sendArgsFromPDUAndValidateExists(ctx, fs, r.GetSenderVersion())
+	if err != nil {
+		return nil, err
+	}
+	fs := fsp.ToString()
+
+	log := getLogger(ctx).WithField("fs", fs).WithField("hinted_most_recent", fmt.Sprintf("%#v", r.GetSenderVersion()))
+
+	log.WithField("full_hint", r).Debug("full hint")
+
+	if r.GetSenderVersion() == nil {
+		// no common ancestor found, likely due to failed prior replication attempt
+		// => release stale step holds to prevent them from accumulating
+		//    (they can accumulate on initial replication because each inital replication step might hold a different `to`)
+		// => replication cursors cannot accumulate because we always _move_ the replication cursor
+		log.Debug("releasing all step holds on the filesystem")
+		TryReleaseStepStaleFS(ctx, fs, p.jobId)
+		return &pdu.HintMostRecentCommonAncestorRes{}, nil
+	}
+
+	// we were hinted a specific common ancestor
+
+	mostRecentVersion, err := sendArgsFromPDUAndValidateExistsAndGetVersion(ctx, fs, r.GetSenderVersion())
 	if err != nil {
 		msg := "HintMostRecentCommonAncestor rpc with nonexistent most recent version"
-		getLogger(ctx).WithField("fs", fs).WithField("hinted_most_recent", fmt.Sprintf("%#v", mostRecent)).
+		log.Warn(msg)
-			Warn(msg)
 		return nil, errors.Wrap(err, msg)
 	}
 
 	// move replication cursor to this position
-	_, err = MoveReplicationCursor(ctx, fs, mostRecent, p.jobId)
+	destroyedCursors, err := MoveReplicationCursor(ctx, fs, mostRecentVersion, p.jobId)
 	if err == zfs.ErrBookmarkCloningNotSupported {
-		getLogger(ctx).Debug("not creating replication cursor from bookmark because ZFS does not support it")
+		log.Debug("not creating replication cursor from bookmark because ZFS does not support it")
 		// fallthrough
 	} else if err != nil {
 		return nil, errors.Wrap(err, "cannot set replication cursor to hinted version")
 	}
 
-	// cleanup previous steps
+	// take care of stale step holds
-	if err := ReleaseStepAll(ctx, fs, mostRecent, p.jobId); err != nil {
+	log.WithField("step-holds-cleanup-mode", senderHintMostRecentCommonAncestorStepCleanupMode).
-		return nil, errors.Wrap(err, "cannot cleanup prior invocation's step holds and bookmarks")
+		Debug("taking care of possibly stale step holds")
+	doStepCleanup := false
+	var stepCleanupSince *CreateTXGRangeBound
+	switch senderHintMostRecentCommonAncestorStepCleanupMode {
+	case StepCleanupNoCleanup:
+		doStepCleanup = false
+	case StepCleanupRangeSinceUnbounded:
+		doStepCleanup = true
+		stepCleanupSince = nil
+	case StepCleanupRangeSinceReplicationCursor:
+		doStepCleanup = true
+		// Use the destroyed replication cursors as indicator how far the previous replication got.
+		// To be precise: We limit the amount of visisted snapshots to exactly those snapshots
+		// created since the last successful replication cursor movement (i.e. last successful replication step)
+		//
+		// If we crash now, we'll leak the step we are about to release, but the performance gain
+		// of limiting the amount of snapshots we visit makes up for that.
+		// Users have the `zrepl holds release-stale` command to cleanup leaked step holds.
+		for _, destroyed := range destroyedCursors {
+			if stepCleanupSince == nil {
+				stepCleanupSince = &CreateTXGRangeBound{
+					CreateTXG: destroyed.GetCreateTXG(),
+					Inclusive: &zfs.NilBool{B: true},
+				}
+			} else if destroyed.GetCreateTXG() < stepCleanupSince.CreateTXG {
+				stepCleanupSince.CreateTXG = destroyed.GetCreateTXG()
+			}
+		}
+	default:
+		panic(senderHintMostRecentCommonAncestorStepCleanupMode)
+	}
+	if !doStepCleanup {
+		log.Info("skipping cleanup of prior invocations' step holds due to environment variable setting")
+	} else {
+		if err := ReleaseStepCummulativeInclusive(ctx, fs, stepCleanupSince, mostRecentVersion, p.jobId); err != nil {
+			return nil, errors.Wrap(err, "cannot cleanup prior invocation's step holds and bookmarks")
+		} else {
+			log.Info("step hold cleanup done")
+		}
 	}
 
 	return &pdu.HintMostRecentCommonAncestorRes{}, nil
 }
 
+type HintMostRecentCommonAncestorStepCleanupMode struct{ string }
+
+var (
+	StepCleanupRangeSinceReplicationCursor = HintMostRecentCommonAncestorStepCleanupMode{"range-since-replication-cursor"}
+	StepCleanupRangeSinceUnbounded         = HintMostRecentCommonAncestorStepCleanupMode{"range-since-unbounded"}
+	StepCleanupNoCleanup                   = HintMostRecentCommonAncestorStepCleanupMode{"no-cleanup"}
+)
+
+func (m HintMostRecentCommonAncestorStepCleanupMode) String() string { return string(m.string) }
+func (m *HintMostRecentCommonAncestorStepCleanupMode) Set(s string) error {
+	switch s {
+	case StepCleanupRangeSinceReplicationCursor.String():
+		*m = StepCleanupRangeSinceReplicationCursor
+	case StepCleanupRangeSinceUnbounded.String():
+		*m = StepCleanupRangeSinceUnbounded
+	case StepCleanupNoCleanup.String():
+		*m = StepCleanupNoCleanup
+	default:
+		return fmt.Errorf("unknown step cleanup mode %q", s)
+	}
+	return nil
+}
+
+var senderHintMostRecentCommonAncestorStepCleanupMode = *envconst.Var("ZREPL_ENDPOINT_SENDER_HINT_MOST_RECENT_STEP_HOLD_CLEANUP_MODE", &StepCleanupRangeSinceReplicationCursor).(*HintMostRecentCommonAncestorStepCleanupMode)
+
 var maxConcurrentZFSSendSemaphore = semaphore.New(envconst.Int64("ZREPL_ENDPOINT_MAX_CONCURRENT_SEND", 10))
 
 func uncheckedSendArgsFromPDU(fsv *pdu.FilesystemVersion) *zfs.ZFSSendArgVersion {
@@ -147,18 +238,20 @@ func uncheckedSendArgsFromPDU(fsv *pdu.FilesystemVersion) *zfs.ZFSSendArgVersion
 	return &zfs.ZFSSendArgVersion{RelName: fsv.GetRelName(), GUID: fsv.Guid}
 }
 
-func sendArgsFromPDUAndValidateExists(ctx context.Context, fs string, fsv *pdu.FilesystemVersion) (*zfs.ZFSSendArgVersion, error) {
+func sendArgsFromPDUAndValidateExistsAndGetVersion(ctx context.Context, fs string, fsv *pdu.FilesystemVersion) (v zfs.FilesystemVersion, err error) {
-	v := uncheckedSendArgsFromPDU(fsv)
+	sendArgs := uncheckedSendArgsFromPDU(fsv)
-	if v == nil {
+	if sendArgs == nil {
-		return nil, errors.New("must not be nil")
+		return v, errors.New("must not be nil")
 	}
-	if err := v.ValidateExists(ctx, fs); err != nil {
+	version, err := sendArgs.ValidateExistsAndGetVersion(ctx, fs)
-		return nil, err
+	if err != nil {
+		return v, err
 	}
-	return v, nil
+	return version, nil
 }
 
-func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, zfs.StreamCopier, error) {
+func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
 
 	_, err := s.filterCheckFS(r.Filesystem)
 	if err != nil {
@@ -182,7 +275,7 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, zfs.St
 		return nil, nil, fmt.Errorf("unknown pdu.Tri variant %q", r.Encrypted)
 	}
 
-	sendArgs := zfs.ZFSSendArgs{
+	sendArgsUnvalidated := zfs.ZFSSendArgsUnvalidated{
 		FS:          r.Filesystem,
 		From:        uncheckedSendArgsFromPDU(r.GetFrom()), // validated by zfs.ZFSSendDry / zfs.ZFSSend
 		To:          uncheckedSendArgsFromPDU(r.GetTo()),   // validated by zfs.ZFSSendDry / zfs.ZFSSend
@@ -190,6 +283,11 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, zfs.St
 		ResumeToken: r.ResumeToken, // nil or not nil, depending on decoding success
 	}
 
+	sendArgs, err := sendArgsUnvalidated.Validate(ctx)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "validate send arguments")
+	}
+
 	getLogger(ctx).Debug("acquire concurrent send semaphore")
 	// TODO use try-acquire and fail with resource-exhaustion rpc status
 	// => would require handling on the client-side
@@ -224,7 +322,7 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, zfs.St
 	// update replication cursor
 	if sendArgs.From != nil {
 		// For all but the first replication, this should always be a no-op because SendCompleted already moved the cursor
-		_, err = MoveReplicationCursor(ctx, sendArgs.FS, sendArgs.From, s.jobId)
+		_, err = MoveReplicationCursor(ctx, sendArgs.FS, sendArgs.FromVersion, s.jobId)
 		if err == zfs.ErrBookmarkCloningNotSupported {
 			getLogger(ctx).Debug("not creating replication cursor from bookmark because ZFS does not support it")
 			// fallthrough
@@ -235,67 +333,76 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, zfs.St
 
 	// make sure `From` doesn't go away in order to make this step resumable
 	if sendArgs.From != nil {
-		err := HoldStep(ctx, sendArgs.FS, sendArgs.From, s.jobId)
+		_, err := HoldStep(ctx, sendArgs.FS, *sendArgs.FromVersion, s.jobId)
 		if err == zfs.ErrBookmarkCloningNotSupported {
 			getLogger(ctx).Debug("not creating step bookmark because ZFS does not support it")
 			// fallthrough
 		} else if err != nil {
-			return nil, nil, errors.Wrap(err, "cannot create step bookmark")
+			return nil, nil, errors.Wrapf(err, "cannot hold `from` version %q before starting send", *sendArgs.FromVersion)
 		}
 	}
 	// make sure `To` doesn't go away in order to make this step resumable
-	err = HoldStep(ctx, sendArgs.FS, sendArgs.To, s.jobId)
+	_, err = HoldStep(ctx, sendArgs.FS, sendArgs.ToVersion, s.jobId)
 	if err != nil {
-		return nil, nil, errors.Wrapf(err, "cannot hold `to` version %q before starting send", sendArgs.To.RelName)
+		return nil, nil, errors.Wrapf(err, "cannot hold `to` version %q before starting send", sendArgs.ToVersion)
 	}
 
 	// step holds & replication cursor released / moved forward in s.SendCompleted => s.moveCursorAndReleaseSendHolds
 
-	streamCopier, err := zfs.ZFSSend(ctx, sendArgs)
+	sendStream, err := zfs.ZFSSend(ctx, sendArgs)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "zfs send failed")
 	}
-	return res, streamCopier, nil
+	return res, sendStream, nil
 }
 
 func (p *Sender) SendCompleted(ctx context.Context, r *pdu.SendCompletedReq) (*pdu.SendCompletedRes, error) {
-	orig := r.GetOriginalReq() // may be nil, always use proto getters
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
-	fs := orig.GetFilesystem()
 
-	var err error
+	orig := r.GetOriginalReq() // may be nil, always use proto getters
-	var from *zfs.ZFSSendArgVersion
+	fsp, err := p.filterCheckFS(orig.GetFilesystem())
+	if err != nil {
+		return nil, err
+	}
+	fs := fsp.ToString()
+
+	var from *zfs.FilesystemVersion
 	if orig.GetFrom() != nil {
-		from, err = sendArgsFromPDUAndValidateExists(ctx, fs, orig.GetFrom()) // no shadow
+		f, err := sendArgsFromPDUAndValidateExistsAndGetVersion(ctx, fs, orig.GetFrom()) // no shadow
 		if err != nil {
 			return nil, errors.Wrap(err, "validate `from` exists")
 		}
+		from = &f
 	}
-	to, err := sendArgsFromPDUAndValidateExists(ctx, fs, orig.GetTo())
+	to, err := sendArgsFromPDUAndValidateExistsAndGetVersion(ctx, fs, orig.GetTo())
 	if err != nil {
 		return nil, errors.Wrap(err, "validate `to` exists")
 	}
 
-	log := getLogger(ctx).WithField("to_guid", to.GUID).
+	log := func(ctx context.Context) Logger {
-		WithField("fs", fs).
+		log := getLogger(ctx).WithField("to_guid", to.Guid).
-		WithField("to", to.RelName)
+			WithField("fs", fs).
-	if from != nil {
+			WithField("to", to.RelName)
-		log = log.WithField("from", from.RelName).WithField("from_guid", from.GUID)
+		if from != nil {
+			log = log.WithField("from", from.RelName).WithField("from_guid", from.Guid)
+		}
+		return log
 	}
 
-	log.Debug("move replication cursor to most recent common version")
+	log(ctx).Debug("move replication cursor to most recent common version")
 	destroyedCursors, err := MoveReplicationCursor(ctx, fs, to, p.jobId)
 	if err != nil {
 		if err == zfs.ErrBookmarkCloningNotSupported {
-			log.Debug("not setting replication cursor, bookmark cloning not supported")
+			log(ctx).Debug("not setting replication cursor, bookmark cloning not supported")
 		} else {
 			msg := "cannot move replication cursor, keeping hold on `to` until successful"
-			log.WithError(err).Error(msg)
+			log(ctx).WithError(err).Error(msg)
 			err = errors.Wrap(err, msg)
 			// it is correct to not release the hold if we can't move the cursor!
 			return &pdu.SendCompletedRes{}, err
 		}
 	} else {
-		log.Info("successfully moved replication cursor")
+		log(ctx).Info("successfully moved replication cursor")
 	}
 
 	// kick off releasing of step holds / bookmarks
@@ -305,42 +412,44 @@ func (p *Sender) SendCompleted(ctx context.Context, r *pdu.SendCompletedReq) (*p
 	wg.Add(2)
 	go func() {
 		defer wg.Done()
-		log.Debug("release step-hold of or step-bookmark on `to`")
+		ctx, endTask := trace.WithTask(ctx, "release-step-hold-to")
+		defer endTask()
+
+		log(ctx).Debug("release step-hold of or step-bookmark on `to`")
 		err = ReleaseStep(ctx, fs, to, p.jobId)
 		if err != nil {
-			log.WithError(err).Error("cannot release step-holds on or destroy step-bookmark of `to`")
+			log(ctx).WithError(err).Error("cannot release step-holds on or destroy step-bookmark of `to`")
 		} else {
-			log.Info("successfully released step-holds on or destroyed step-bookmark of `to`")
+			log(ctx).Info("successfully released step-holds on or destroyed step-bookmark of `to`")
 		}
 
 	}()
 	go func() {
 		defer wg.Done()
+		ctx, endTask := trace.WithTask(ctx, "release-step-hold-from")
+		defer endTask()
+
 		if from == nil {
 			return
 		}
-		log.Debug("release step-hold of or step-bookmark on `from`")
+		log(ctx).Debug("release step-hold of or step-bookmark on `from`")
-		err := ReleaseStep(ctx, fs, from, p.jobId)
+		err := ReleaseStep(ctx, fs, *from, p.jobId)
 		if err != nil {
 			if dne, ok := err.(*zfs.DatasetDoesNotExist); ok {
 				// If bookmark cloning is not supported, `from` might be the old replication cursor
 				// and thus have already been destroyed by MoveReplicationCursor above
 				// In that case, nonexistence of `from` is not an error, otherwise it is.
-				fsp, err := zfs.NewDatasetPath(fs)
+				for _, c := range destroyedCursors {
-				if err != nil {
+					if c.GetFullPath() == dne.Path {
-					panic(err) // fs has been validated multiple times above
+						log(ctx).Info("`from` was a replication cursor and has already been destroyed")
-				}
-				for _, fsv := range destroyedCursors {
-					if fsv.ToAbsPath(fsp) == dne.Path {
-						log.Info("`from` was a replication cursor and has already been destroyed")
 						return
 					}
 				}
 				// fallthrough
 			}
-			log.WithError(err).Error("cannot release step-holds on or destroy step-bookmark of `from`")
+			log(ctx).WithError(err).Error("cannot release step-holds on or destroy step-bookmark of `from`")
 		} else {
-			log.Info("successfully released step-holds on or destroyed step-bookmark of `from`")
+			log(ctx).Info("successfully released step-holds on or destroyed step-bookmark of `from`")
 		}
 	}()
 	wg.Wait()
@@ -349,6 +458,8 @@ func (p *Sender) SendCompleted(ctx context.Context, r *pdu.SendCompletedReq) (*p
 }
 
 func (p *Sender) DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	dp, err := p.filterCheckFS(req.Filesystem)
 	if err != nil {
 		return nil, err
@@ -357,6 +468,8 @@ func (p *Sender) DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshots
 }
 
 func (p *Sender) Ping(ctx context.Context, req *pdu.PingReq) (*pdu.PingRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	res := pdu.PingRes{
 		Echo: req.GetMessage(),
 	}
@@ -364,14 +477,20 @@ func (p *Sender) Ping(ctx context.Context, req *pdu.PingReq) (*pdu.PingRes, erro
 }
 
 func (p *Sender) PingDataconn(ctx context.Context, req *pdu.PingReq) (*pdu.PingRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	return p.Ping(ctx, req)
 }
 
 func (p *Sender) WaitForConnectivity(ctx context.Context) error {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	return nil
 }
 
 func (p *Sender) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	dp, err := p.filterCheckFS(req.Filesystem)
 	if err != nil {
 		return nil, err
@@ -387,7 +506,7 @@ func (p *Sender) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCurs
 	return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Guid{Guid: cursor.Guid}}, nil
 }
 
-func (p *Sender) Receive(ctx context.Context, r *pdu.ReceiveReq, receive zfs.StreamCopier) (*pdu.ReceiveRes, error) {
+func (p *Sender) Receive(ctx context.Context, r *pdu.ReceiveReq, _ io.ReadCloser) (*pdu.ReceiveRes, error) {
 	return nil, fmt.Errorf("sender does not implement Receive()")
 }
 
@@ -502,6 +621,15 @@ func (f subroot) MapToLocal(fs string) (*zfs.DatasetPath, error) {
 }
 
 func (s *Receiver) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
+	// first make sure that root_fs is imported
+	if rphs, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, s.conf.RootWithoutClientComponent); err != nil {
+		return nil, errors.Wrap(err, "cannot determine whether root_fs exists")
+	} else if !rphs.FSExists {
+		return nil, errors.New("root_fs does not exist")
+	}
+
 	root := s.clientRootFromCtx(ctx)
 	filtered, err := zfs.ZFSListMapping(ctx, subroot{root})
 	if err != nil {
@@ -552,13 +680,16 @@ func (s *Receiver) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemR
 }
 
 func (s *Receiver) ListFilesystemVersions(ctx context.Context, req *pdu.ListFilesystemVersionsReq) (*pdu.ListFilesystemVersionsRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	root := s.clientRootFromCtx(ctx)
 	lp, err := subroot{root}.MapToLocal(req.GetFilesystem())
 	if err != nil {
 		return nil, err
 	}
+	// TODO share following code with sender
 
-	fsvs, err := zfs.ZFSListFilesystemVersions(lp, nil)
+	fsvs, err := zfs.ZFSListFilesystemVersions(ctx, lp, zfs.ListFilesystemVersionsOptions{})
 	if err != nil {
 		return nil, err
 	}
@@ -572,6 +703,8 @@ func (s *Receiver) ListFilesystemVersions(ctx context.Context, req *pdu.ListFile
 }
 
 func (s *Receiver) Ping(ctx context.Context, req *pdu.PingReq) (*pdu.PingRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	res := pdu.PingRes{
 		Echo: req.GetMessage(),
 	}
@@ -579,24 +712,30 @@ func (s *Receiver) Ping(ctx context.Context, req *pdu.PingReq) (*pdu.PingRes, er
 }
 
 func (s *Receiver) PingDataconn(ctx context.Context, req *pdu.PingReq) (*pdu.PingRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
 	return s.Ping(ctx, req)
 }
 
 func (s *Receiver) WaitForConnectivity(ctx context.Context) error {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
 	return nil
 }
 
-func (s *Receiver) ReplicationCursor(context.Context, *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
+func (s *Receiver) ReplicationCursor(ctx context.Context, _ *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
 	return nil, fmt.Errorf("ReplicationCursor not implemented for Receiver")
 }
 
-func (s *Receiver) Send(ctx context.Context, req *pdu.SendReq) (*pdu.SendRes, zfs.StreamCopier, error) {
+func (s *Receiver) Send(ctx context.Context, req *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
 	return nil, nil, fmt.Errorf("receiver does not implement Send()")
 }
 
 var maxConcurrentZFSRecvSemaphore = semaphore.New(envconst.Int64("ZREPL_ENDPOINT_MAX_CONCURRENT_RECV", 10))
 
-func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs.StreamCopier) (*pdu.ReceiveRes, error) {
+func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive io.ReadCloser) (*pdu.ReceiveRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	getLogger(ctx).Debug("incoming Receive")
 	defer receive.Close()
 
@@ -610,9 +749,6 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 	if to == nil {
 		return nil, errors.New("`To` must not be nil")
 	}
-	if err := to.ValidateInMemory(lp.ToString()); err != nil {
-		return nil, errors.Wrap(err, "`To` invalid")
-	}
 	if !to.IsSnapshot() {
 		return nil, errors.New("`To` must be a snapshot")
 	}
@@ -678,21 +814,30 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 		return nil, visitErr
 	}
 
+	log := getLogger(ctx).WithField("proto_fs", req.GetFilesystem()).WithField("local_fs", lp.ToString())
+
 	// determine whether we need to rollback the filesystem / change its placeholder state
 	var clearPlaceholderProperty bool
 	var recvOpts zfs.RecvOptions
 	ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, lp)
-	if err == nil && ph.FSExists && ph.IsPlaceholder {
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot get placeholder state")
+	}
+	log.WithField("placeholder_state", fmt.Sprintf("%#v", ph)).Debug("placeholder state")
+	if ph.FSExists && ph.IsPlaceholder {
 		recvOpts.RollbackAndForceRecv = true
 		clearPlaceholderProperty = true
 	}
+
 	if clearPlaceholderProperty {
+		log.Info("clearing placeholder property")
 		if err := zfs.ZFSSetPlaceholder(ctx, lp, false); err != nil {
 			return nil, fmt.Errorf("cannot clear placeholder property for forced receive: %s", err)
 		}
 	}
 
 	if req.ClearResumeToken && ph.FSExists {
+		log.Info("clearing resume token")
 		if err := zfs.ZFSRecvClearResumeToken(ctx, lp.ToString()); err != nil {
 			return nil, errors.Wrap(err, "cannot clear resume token")
 		}
@@ -703,7 +848,7 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 		return nil, errors.Wrap(err, "cannot determine whether we can use resumable send & recv")
 	}
 
-	getLogger(ctx).Debug("acquire concurrent recv semaphore")
+	log.Debug("acquire concurrent recv semaphore")
 	// TODO use try-acquire and fail with resource-exhaustion rpc status
 	// => would require handling on the client-side
 	// => this is a dataconn endpoint, doesn't have the status code semantics of gRPC
@@ -713,28 +858,104 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 	}
 	defer guard.Release()
 
-	getLogger(ctx).WithField("opts", fmt.Sprintf("%#v", recvOpts)).Debug("start receive command")
+	log.Info("peeking 1M ahead")
+	var peek bytes.Buffer
+	var MaxPeek = envconst.Int64("ZREPL_ENDPOINT_RECV_PEEK_SIZE", 1<<20)
+	if _, err := io.Copy(&peek, io.LimitReader(receive, MaxPeek)); err != nil {
+		log.WithError(err).Error("cannot read peek-buffer from send stream")
+	}
+	var peekCopy bytes.Buffer
+	if n, err := peekCopy.Write(peek.Bytes()); err != nil || n != peek.Len() {
+		panic(peek.Len())
+	}
+
+	log.WithField("opts", fmt.Sprintf("%#v", recvOpts)).Debug("start receive command")
 
 	snapFullPath := to.FullPath(lp.ToString())
-	if err := zfs.ZFSRecv(ctx, lp.ToString(), to, receive, recvOpts); err != nil {
+	if err := zfs.ZFSRecv(ctx, lp.ToString(), to, chainedio.NewChainedReader(&peek, receive), recvOpts); err != nil {
-		getLogger(ctx).
+
+		// best-effort rollback of placeholder state if the recv didn't start
+		_, resumableStatePresent := err.(*zfs.RecvFailedWithResumeTokenErr)
+		disablePlaceholderRestoration := envconst.Bool("ZREPL_ENDPOINT_DISABLE_PLACEHOLDER_RESTORATION", false)
+		placeholderRestored := !ph.IsPlaceholder
+		if !disablePlaceholderRestoration && !resumableStatePresent && recvOpts.RollbackAndForceRecv && ph.FSExists && ph.IsPlaceholder && clearPlaceholderProperty {
+			log.Info("restoring placeholder property")
+			if phErr := zfs.ZFSSetPlaceholder(ctx, lp, true); phErr != nil {
+				log.WithError(phErr).Error("cannot restore placeholder property after failed receive, subsequent replications will likely fail with a different error")
+				// fallthrough
+			} else {
+				placeholderRestored = true
+			}
+			// fallthrough
+		}
+
+		// deal with failing initial encrypted send & recv
+		if _, ok := err.(*zfs.RecvDestroyOrOverwriteEncryptedErr); ok && ph.IsPlaceholder && placeholderRestored {
+			msg := `cannot automatically replace placeholder filesystem with incoming send stream - please see receive-side log for details`
+			err := errors.New(msg)
+			log.Error(msg)
+
+			log.Error(`zrepl creates placeholder filesystems on the receiving side of a replication to match the sending side's dataset hierarchy`)
+			log.Error(`zrepl uses zfs receive -F to replace those placeholders with incoming full sends`)
+			log.Error(`OpenZFS native encryption prohibits zfs receive -F for encrypted filesystems`)
+			log.Error(`the current zrepl placeholder filesystem concept is thus incompatible with OpenZFS native encryption`)
+
+			tempStartFullRecvFS := lp.Copy().ToString() + ".zrepl.initial-recv"
+			tempStartFullRecvFSDP, dpErr := zfs.NewDatasetPath(tempStartFullRecvFS)
+			if dpErr != nil {
+				log.WithError(dpErr).Error("cannot determine temporary filesystem name for initial encrypted recv workaround")
+				return nil, err // yes, err, not dpErr
+			}
+
+			log := log.WithField("temp_recv_fs", tempStartFullRecvFS)
+			log.Error(`as a workaround, zrepl will now attempt to re-receive the beginning of the stream into a temporary filesystem temp_recv_fs`)
+			log.Error(`if that step succeeds: shut down zrepl and use 'zfs rename' to swap temp_recv_fs with local_fs, then restart zrepl`)
+			log.Error(`replication will then resume using resumable send+recv`)
+
+			tempPH, phErr := zfs.ZFSGetFilesystemPlaceholderState(ctx, tempStartFullRecvFSDP)
+			if phErr != nil {
+				log.WithError(phErr).Error("cannot determine placeholder state of temp_recv_fs")
+				return nil, err // yes, err, not dpErr
+			}
+			if tempPH.FSExists {
+				log.Error("temp_recv_fs already exists, assuming a (partial) initial recv to that filesystem has already been done")
+				return nil, err
+			}
+
+			recvOpts.RollbackAndForceRecv = false
+			recvOpts.SavePartialRecvState = true
+			rerecvErr := zfs.ZFSRecv(ctx, tempStartFullRecvFS, to, chainedio.NewChainedReader(&peekCopy), recvOpts)
+			if _, isResumable := rerecvErr.(*zfs.RecvFailedWithResumeTokenErr); rerecvErr == nil || isResumable {
+				log.Error("completed re-receive into temporary filesystem temp_recv_fs, now shut down zrepl and use zfs rename to swap temp_recv_fs with local_fs")
+			} else {
+				log.WithError(rerecvErr).Error("failed to receive the beginning of the stream into temporary filesystem temp_recv_fs")
+				log.Error("we advise you to collect the error log and current configuration, open an issue on GitHub, and revert to your previous configuration in the meantime")
+			}
+
+			log.Error(`if you would like to see improvements to this situation, please open an issue on GitHub`)
+			return nil, err
+		}
+
+		log.
 			WithError(err).
-			WithField("opts", recvOpts).
+			WithField("opts", fmt.Sprintf("%#v", recvOpts)).
 			Error("zfs receive failed")
+
 		return nil, err
 	}
 
 	// validate that we actually received what the sender claimed
-	if err := to.ValidateExists(ctx, lp.ToString()); err != nil {
+	toRecvd, err := to.ValidateExistsAndGetVersion(ctx, lp.ToString())
+	if err != nil {
 		msg := "receive request's `To` version does not match what we received in the stream"
-		getLogger(ctx).WithError(err).WithField("snap", snapFullPath).Error(msg)
+		log.WithError(err).WithField("snap", snapFullPath).Error(msg)
-		getLogger(ctx).Error("aborting recv request, but keeping received snapshot for inspection")
+		log.Error("aborting recv request, but keeping received snapshot for inspection")
 		return nil, errors.Wrap(err, msg)
 	}
 
 	if s.conf.UpdateLastReceivedHold {
-		getLogger(ctx).Debug("move last-received-hold")
+		log.Debug("move last-received-hold")
-		if err := MoveLastReceivedHold(ctx, lp.ToString(), *to, s.conf.JobID); err != nil {
+		if err := MoveLastReceivedHold(ctx, lp.ToString(), toRecvd, s.conf.JobID); err != nil {
 			return nil, errors.Wrap(err, "cannot move last-received-hold")
 		}
 	}
@@ -743,6 +964,8 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 }
 
 func (s *Receiver) DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	root := s.clientRootFromCtx(ctx)
 	lp, err := subroot{root}.MapToLocal(req.Filesystem)
 	if err != nil {
@@ -752,6 +975,7 @@ func (s *Receiver) DestroySnapshots(ctx context.Context, req *pdu.DestroySnapsho
 }
 
 func (p *Receiver) HintMostRecentCommonAncestor(ctx context.Context, r *pdu.HintMostRecentCommonAncestorReq) (*pdu.HintMostRecentCommonAncestorRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
 	// we don't move last-received-hold as part of this hint
 	// because that wouldn't give us any benefit wrt resumability.
 	//
@@ -760,7 +984,9 @@ func (p *Receiver) HintMostRecentCommonAncestor(ctx context.Context, r *pdu.Hint
 	return &pdu.HintMostRecentCommonAncestorRes{}, nil
 }
 
-func (p *Receiver) SendCompleted(context.Context, *pdu.SendCompletedReq) (*pdu.SendCompletedRes, error) {
+func (p *Receiver) SendCompleted(ctx context.Context, _ *pdu.SendCompletedReq) (*pdu.SendCompletedRes, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
+
 	return &pdu.SendCompletedRes{}, nil
 }
 
@@ -782,7 +1008,7 @@ func doDestroySnapshots(ctx context.Context, lp *zfs.DatasetPath, snaps []*pdu.F
 			ErrOut:     &errs[i],
 		}
 	}
-	zfs.ZFSDestroyFilesystemVersions(reqs)
+	zfs.ZFSDestroyFilesystemVersions(ctx, reqs)
 	for i := range reqs {
 		if errs[i] != nil {
 			if de, ok := errs[i].(*zfs.DestroySnapshotsError); ok && len(de.Reason) == 1 {

endpoint/endpoint_zfs.go

@@ -1,503 +0,0 @@
-package endpoint
-
-import (
-	"context"
-	"fmt"
-	"regexp"
-	"sort"
-
-	"github.com/kr/pretty"
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/zfs"
-)
-
-var stepHoldTagRE = regexp.MustCompile("^zrepl_STEP_J_(.+)")
-
-func StepHoldTag(jobid JobID) (string, error) {
-	return stepHoldTagImpl(jobid.String())
-}
-
-func stepHoldTagImpl(jobid string) (string, error) {
-	t := fmt.Sprintf("zrepl_STEP_J_%s", jobid)
-	if err := zfs.ValidHoldTag(t); err != nil {
-		return "", err
-	}
-	return t, nil
-}
-
-// err != nil always means that the bookmark is not a step bookmark
-func ParseStepHoldTag(tag string) (JobID, error) {
-	match := stepHoldTagRE.FindStringSubmatch(tag)
-	if match == nil {
-		return JobID{}, fmt.Errorf("parse hold tag: match regex %q", stepHoldTagRE)
-	}
-	jobID, err := MakeJobID(match[1])
-	if err != nil {
-		return JobID{}, errors.Wrap(err, "parse hold tag: invalid job id field")
-	}
-	return jobID, nil
-}
-
-const stepBookmarkNamePrefix = "zrepl_STEP"
-
-// v must be validated by caller
-func StepBookmarkName(fs string, guid uint64, id JobID) (string, error) {
-	return stepBookmarkNameImpl(fs, guid, id.String())
-}
-
-func stepBookmarkNameImpl(fs string, guid uint64, jobid string) (string, error) {
-	return makeJobAndGuidBookmarkName(stepBookmarkNamePrefix, fs, guid, jobid)
-}
-
-// name is the full bookmark name, including dataset path
-//
-// err != nil always means that the bookmark is not a step bookmark
-func ParseStepBookmarkName(fullname string) (guid uint64, jobID JobID, err error) {
-	guid, jobID, err = parseJobAndGuidBookmarkName(fullname, stepBookmarkNamePrefix)
-	if err != nil {
-		err = errors.Wrap(err, "parse step bookmark name") // no shadow!
-	}
-	return guid, jobID, err
-}
-
-const replicationCursorBookmarkNamePrefix = "zrepl_CURSOR"
-
-func ReplicationCursorBookmarkName(fs string, guid uint64, id JobID) (string, error) {
-	return replicationCursorBookmarkNameImpl(fs, guid, id.String())
-}
-
-func replicationCursorBookmarkNameImpl(fs string, guid uint64, jobid string) (string, error) {
-	return makeJobAndGuidBookmarkName(replicationCursorBookmarkNamePrefix, fs, guid, jobid)
-}
-
-var ErrV1ReplicationCursor = fmt.Errorf("bookmark name is a v1-replication cursor")
-
-//err != nil always means that the bookmark is not a valid replication bookmark
-//
-// Returns ErrV1ReplicationCursor as error if the bookmark is a v1 replication cursor
-func ParseReplicationCursorBookmarkName(fullname string) (uint64, JobID, error) {
-
-	// check for legacy cursors
-	{
-		if err := zfs.EntityNamecheck(fullname, zfs.EntityTypeBookmark); err != nil {
-			return 0, JobID{}, errors.Wrap(err, "parse replication cursor bookmark name")
-		}
-		_, _, name, err := zfs.DecomposeVersionString(fullname)
-		if err != nil {
-			return 0, JobID{}, errors.Wrap(err, "parse replication cursor bookmark name: decompose version string")
-		}
-		const V1ReplicationCursorBookmarkName = "zrepl_replication_cursor"
-		if name == V1ReplicationCursorBookmarkName {
-			return 0, JobID{}, ErrV1ReplicationCursor
-		}
-	}
-
-	guid, jobID, err := parseJobAndGuidBookmarkName(fullname, replicationCursorBookmarkNamePrefix)
-	if err != nil {
-		err = errors.Wrap(err, "parse replication cursor bookmark name") // no shadow
-	}
-	return guid, jobID, err
-}
-
-// may return nil for both values, indicating there is no cursor
-func GetMostRecentReplicationCursorOfJob(ctx context.Context, fs string, jobID JobID) (*zfs.FilesystemVersion, error) {
-	fsp, err := zfs.NewDatasetPath(fs)
-	if err != nil {
-		return nil, err
-	}
-	candidates, err := GetReplicationCursors(ctx, fsp, jobID)
-	if err != nil || len(candidates) == 0 {
-		return nil, err
-	}
-
-	sort.Slice(candidates, func(i, j int) bool {
-		return candidates[i].CreateTXG < candidates[j].CreateTXG
-	})
-
-	mostRecent := candidates[len(candidates)-1]
-	return &mostRecent, nil
-}
-
-func GetReplicationCursors(ctx context.Context, fs *zfs.DatasetPath, jobID JobID) ([]zfs.FilesystemVersion, error) {
-
-	listOut := &ListHoldsAndBookmarksOutput{}
-	if err := listZFSHoldsAndBookmarksImplFS(ctx, listOut, fs); err != nil {
-		return nil, errors.Wrap(err, "get replication cursor: list bookmarks and holds")
-	}
-
-	if len(listOut.V1ReplicationCursors) > 0 {
-		getLogger(ctx).WithField("bookmark", pretty.Sprint(listOut.V1ReplicationCursors)).
-			Warn("found v1-replication cursor bookmarks, consider running migration 'replication-cursor:v1-v2' after successful replication with this zrepl version")
-	}
-
-	candidates := make([]zfs.FilesystemVersion, 0)
-	for _, v := range listOut.ReplicationCursorBookmarks {
-		zv := zfs.ZFSSendArgVersion{
-			RelName: "#" + v.Name,
-			GUID:    v.Guid,
-		}
-		if err := zv.ValidateExists(ctx, v.FS); err != nil {
-			getLogger(ctx).WithError(err).WithField("bookmark", zv.FullPath(v.FS)).
-				Error("found invalid replication cursor bookmark")
-			continue
-		}
-		candidates = append(candidates, v.v)
-	}
-
-	return candidates, nil
-}
-
-// `target` is validated before replication cursor is set. if validation fails, the cursor is not moved.
-//
-// returns ErrBookmarkCloningNotSupported if version is a bookmark and bookmarking bookmarks is not supported by ZFS
-func MoveReplicationCursor(ctx context.Context, fs string, target *zfs.ZFSSendArgVersion, jobID JobID) (destroyedCursors []zfs.FilesystemVersion, err error) {
-
-	if !target.IsSnapshot() {
-		return nil, zfs.ErrBookmarkCloningNotSupported
-	}
-
-	snapProps, err := target.ValidateExistsAndGetCheckedProps(ctx, fs)
-	if err != nil {
-		return nil, errors.Wrapf(err, "invalid replication cursor target %q (guid=%v)", target.RelName, target.GUID)
-	}
-
-	bookmarkname, err := ReplicationCursorBookmarkName(fs, snapProps.Guid, jobID)
-	if err != nil {
-		return nil, errors.Wrap(err, "determine replication cursor name")
-	}
-
-	// idempotently create bookmark (guid is encoded in it, hence we'll most likely add a new one
-	// cleanup the old one afterwards
-
-	err = zfs.ZFSBookmark(ctx, fs, *target, bookmarkname)
-	if err != nil {
-		if err == zfs.ErrBookmarkCloningNotSupported {
-			return nil, err // TODO go1.13 use wrapping
-		}
-		return nil, errors.Wrapf(err, "cannot create bookmark")
-	}
-
-	destroyedCursors, err = DestroyObsoleteReplicationCursors(ctx, fs, target, jobID)
-	if err != nil {
-		return nil, errors.Wrap(err, "destroy obsolete replication cursors")
-	}
-
-	return destroyedCursors, nil
-}
-
-func DestroyObsoleteReplicationCursors(ctx context.Context, fs string, target *zfs.ZFSSendArgVersion, jobID JobID) (destroyed []zfs.FilesystemVersion, err error) {
-	return destroyBookmarksOlderThan(ctx, fs, target, jobID, func(shortname string) (accept bool) {
-		_, parsedID, err := ParseReplicationCursorBookmarkName(fs + "#" + shortname)
-		return err == nil && parsedID == jobID
-	})
-}
-
-// idempotently hold / step-bookmark `version`
-//
-// returns ErrBookmarkCloningNotSupported if version is a bookmark and bookmarking bookmarks is not supported by ZFS
-func HoldStep(ctx context.Context, fs string, v *zfs.ZFSSendArgVersion, jobID JobID) error {
-	if err := v.ValidateExists(ctx, fs); err != nil {
-		return err
-	}
-	if v.IsSnapshot() {
-
-		tag, err := StepHoldTag(jobID)
-		if err != nil {
-			return errors.Wrap(err, "step hold tag")
-		}
-
-		if err := zfs.ZFSHold(ctx, fs, *v, tag); err != nil {
-			return errors.Wrap(err, "step hold: zfs")
-		}
-
-		return nil
-	}
-
-	v.MustBeBookmark()
-
-	bmname, err := StepBookmarkName(fs, v.GUID, jobID)
-	if err != nil {
-		return errors.Wrap(err, "create step bookmark: determine bookmark name")
-	}
-	// idempotently create bookmark
-	err = zfs.ZFSBookmark(ctx, fs, *v, bmname)
-	if err != nil {
-		if err == zfs.ErrBookmarkCloningNotSupported {
-			// TODO we could actually try to find a local snapshot that has the requested GUID
-			// 		however, the replication algorithm prefers snapshots anyways, so this quest
-			// 		is most likely not going to be successful. Also, there's the possibility that
-			//      the caller might want to filter what snapshots are eligible, and this would
-			//      complicate things even further.
-			return err // TODO go1.13 use wrapping
-		}
-		return errors.Wrap(err, "create step bookmark: zfs")
-	}
-	return nil
-}
-
-// idempotently release the step-hold on v if v is a snapshot
-// or idempotently destroy the step-bookmark  of v if v is a bookmark
-//
-// note that this operation leaves v itself untouched, unless v is the step-bookmark itself, in which case v is destroyed
-//
-// returns an instance of *zfs.DatasetDoesNotExist if `v` does not exist
-func ReleaseStep(ctx context.Context, fs string, v *zfs.ZFSSendArgVersion, jobID JobID) error {
-
-	if err := v.ValidateExists(ctx, fs); err != nil {
-		return err
-	}
-
-	if v.IsSnapshot() {
-		tag, err := StepHoldTag(jobID)
-		if err != nil {
-			return errors.Wrap(err, "step release tag")
-		}
-
-		if err := zfs.ZFSRelease(ctx, tag, v.FullPath(fs)); err != nil {
-			return errors.Wrap(err, "step release: zfs")
-		}
-
-		return nil
-	}
-
-	v.MustBeBookmark()
-
-	bmname, err := StepBookmarkName(fs, v.GUID, jobID)
-	if err != nil {
-		return errors.Wrap(err, "step release: determine bookmark name")
-	}
-	// idempotently destroy bookmark
-
-	if err := zfs.ZFSDestroyIdempotent(ctx, bmname); err != nil {
-		return errors.Wrap(err, "step release: bookmark destroy: zfs")
-	}
-
-	return nil
-}
-
-// release {step holds, step bookmarks} earlier and including `mostRecent`
-func ReleaseStepAll(ctx context.Context, fs string, mostRecent *zfs.ZFSSendArgVersion, jobID JobID) error {
-
-	if err := mostRecent.ValidateInMemory(fs); err != nil {
-		return err
-	}
-
-	tag, err := StepHoldTag(jobID)
-	if err != nil {
-		return errors.Wrap(err, "step release all: tag")
-	}
-
-	err = zfs.ZFSReleaseAllOlderAndIncludingGUID(ctx, fs, mostRecent.GUID, tag)
-	if err != nil {
-		return errors.Wrapf(err, "step release all: release holds older and including %q", mostRecent.FullPath(fs))
-	}
-
-	_, err = destroyBookmarksOlderThan(ctx, fs, mostRecent, jobID, func(shortname string) bool {
-		_, parsedId, parseErr := ParseStepBookmarkName(fs + "#" + shortname)
-		return parseErr == nil && parsedId == jobID
-	})
-	if err != nil {
-		return errors.Wrapf(err, "step release all: destroy bookmarks older than %q", mostRecent.FullPath(fs))
-	}
-
-	return nil
-}
-
-var lastReceivedHoldTagRE = regexp.MustCompile("^zrepl_last_received_J_(.+)$")
-
-// err != nil always means that the bookmark is not a step bookmark
-func ParseLastReceivedHoldTag(tag string) (JobID, error) {
-	match := lastReceivedHoldTagRE.FindStringSubmatch(tag)
-	if match == nil {
-		return JobID{}, errors.Errorf("parse last-received-hold tag: does not match regex %s", lastReceivedHoldTagRE.String())
-	}
-	jobId, err := MakeJobID(match[1])
-	if err != nil {
-		return JobID{}, errors.Wrap(err, "parse last-received-hold tag: invalid job id field")
-	}
-	return jobId, nil
-}
-
-func LastReceivedHoldTag(jobID JobID) (string, error) {
-	return lastReceivedHoldImpl(jobID.String())
-}
-
-func lastReceivedHoldImpl(jobid string) (string, error) {
-	tag := fmt.Sprintf("zrepl_last_received_J_%s", jobid)
-	if err := zfs.ValidHoldTag(tag); err != nil {
-		return "", err
-	}
-	return tag, nil
-}
-
-func MoveLastReceivedHold(ctx context.Context, fs string, to zfs.ZFSSendArgVersion, jobID JobID) error {
-	if err := to.ValidateExists(ctx, fs); err != nil {
-		return err
-	}
-	if err := zfs.EntityNamecheck(to.FullPath(fs), zfs.EntityTypeSnapshot); err != nil {
-		return err
-	}
-
-	tag, err := LastReceivedHoldTag(jobID)
-	if err != nil {
-		return errors.Wrap(err, "last-received-hold: hold tag")
-	}
-
-	// we never want to be without a hold
-	// => hold new one before releasing old hold
-
-	err = zfs.ZFSHold(ctx, fs, to, tag)
-	if err != nil {
-		return errors.Wrap(err, "last-received-hold: hold newly received")
-	}
-
-	err = zfs.ZFSReleaseAllOlderThanGUID(ctx, fs, to.GUID, tag)
-	if err != nil {
-		return errors.Wrap(err, "last-received-hold: release older holds")
-	}
-
-	return nil
-}
-
-type ListHoldsAndBookmarksOutputBookmarkV1ReplicationCursor struct {
-	FS   string
-	Name string
-}
-
-type ListHoldsAndBookmarksOutput struct {
-	StepBookmarks []*ListHoldsAndBookmarksOutputBookmark
-	StepHolds     []*ListHoldsAndBookmarksOutputHold
-
-	ReplicationCursorBookmarks []*ListHoldsAndBookmarksOutputBookmark
-	V1ReplicationCursors       []*ListHoldsAndBookmarksOutputBookmarkV1ReplicationCursor
-	LastReceivedHolds          []*ListHoldsAndBookmarksOutputHold
-}
-
-type ListHoldsAndBookmarksOutputBookmark struct {
-	FS, Name string
-	Guid     uint64
-	JobID    JobID
-	v        zfs.FilesystemVersion
-}
-
-type ListHoldsAndBookmarksOutputHold struct {
-	FS            string
-	Snap          string
-	SnapGuid      uint64
-	SnapCreateTXG uint64
-	Tag           string
-	JobID         JobID
-}
-
-// List all holds and bookmarks managed by endpoint
-func ListZFSHoldsAndBookmarks(ctx context.Context, fsfilter zfs.DatasetFilter) (*ListHoldsAndBookmarksOutput, error) {
-
-	// initialize all fields so that JSON serialization of output looks pretty (see client/holds.go)
-	// however, listZFSHoldsAndBookmarksImplFS shouldn't rely on it
-	out := &ListHoldsAndBookmarksOutput{
-		StepBookmarks:              make([]*ListHoldsAndBookmarksOutputBookmark, 0),
-		StepHolds:                  make([]*ListHoldsAndBookmarksOutputHold, 0),
-		ReplicationCursorBookmarks: make([]*ListHoldsAndBookmarksOutputBookmark, 0),
-		V1ReplicationCursors:       make([]*ListHoldsAndBookmarksOutputBookmarkV1ReplicationCursor, 0),
-		LastReceivedHolds:          make([]*ListHoldsAndBookmarksOutputHold, 0),
-	}
-
-	fss, err := zfs.ZFSListMapping(ctx, fsfilter)
-	if err != nil {
-		return nil, errors.Wrap(err, "list filesystems")
-	}
-
-	for _, fs := range fss {
-		err := listZFSHoldsAndBookmarksImplFS(ctx, out, fs)
-		if err != nil {
-			return nil, errors.Wrapf(err, "list holds and bookmarks on %q", fs.ToString())
-		}
-	}
-	return out, nil
-}
-
-func listZFSHoldsAndBookmarksImplFS(ctx context.Context, out *ListHoldsAndBookmarksOutput, fs *zfs.DatasetPath) error {
-	fsvs, err := zfs.ZFSListFilesystemVersions(fs, nil)
-	if err != nil {
-		return errors.Wrapf(err, "list filesystem versions of %q", fs)
-	}
-	for _, v := range fsvs {
-		switch v.Type {
-		case zfs.Bookmark:
-			listZFSHoldsAndBookmarksImplTryParseBookmark(ctx, out, fs, v)
-		case zfs.Snapshot:
-			holds, err := zfs.ZFSHolds(ctx, fs.ToString(), v.Name)
-			if err != nil {
-				return errors.Wrapf(err, "get holds of %q", v.ToAbsPath(fs))
-			}
-			for _, tag := range holds {
-				listZFSHoldsAndBookmarksImplSnapshotTryParseHold(ctx, out, fs, v, tag)
-			}
-		default:
-			continue
-		}
-	}
-	return nil
-}
-
-// pure function, err != nil always indicates parsing error
-func listZFSHoldsAndBookmarksImplTryParseBookmark(ctx context.Context, out *ListHoldsAndBookmarksOutput, fs *zfs.DatasetPath, v zfs.FilesystemVersion) {
-	var err error
-
-	if v.Type != zfs.Bookmark {
-		panic("impl error")
-	}
-
-	fullname := v.ToAbsPath(fs)
-
-	bm := &ListHoldsAndBookmarksOutputBookmark{
-		FS: fs.ToString(), Name: v.Name, v: v,
-	}
-	bm.Guid, bm.JobID, err = ParseStepBookmarkName(fullname)
-	if err == nil {
-		out.StepBookmarks = append(out.StepBookmarks, bm)
-		return
-	}
-
-	bm.Guid, bm.JobID, err = ParseReplicationCursorBookmarkName(fullname)
-	if err == nil {
-		out.ReplicationCursorBookmarks = append(out.ReplicationCursorBookmarks, bm)
-		return
-	} else if err == ErrV1ReplicationCursor {
-		v1rc := &ListHoldsAndBookmarksOutputBookmarkV1ReplicationCursor{
-			FS: fs.ToString(), Name: v.Name,
-		}
-		out.V1ReplicationCursors = append(out.V1ReplicationCursors, v1rc)
-		return
-	}
-}
-
-// pure function, err != nil always indicates parsing error
-func listZFSHoldsAndBookmarksImplSnapshotTryParseHold(ctx context.Context, out *ListHoldsAndBookmarksOutput, fs *zfs.DatasetPath, v zfs.FilesystemVersion, holdTag string) {
-	var err error
-
-	if v.Type != zfs.Snapshot {
-		panic("impl error")
-	}
-
-	hold := &ListHoldsAndBookmarksOutputHold{
-		FS:            fs.ToString(),
-		Snap:          v.Name,
-		SnapGuid:      v.Guid,
-		SnapCreateTXG: v.CreateTXG,
-		Tag:           holdTag,
-	}
-	hold.JobID, err = ParseStepHoldTag(holdTag)
-	if err == nil {
-		out.StepHolds = append(out.StepHolds, hold)
-		return
-	}
-
-	hold.JobID, err = ParseLastReceivedHoldTag(holdTag)
-	if err == nil {
-		out.LastReceivedHolds = append(out.LastReceivedHolds, hold)
-		return
-	}
-
-}

endpoint/endpoint_zfs_abstraction.go

@@ -0,0 +1,840 @@
+package endpoint
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+	"github.com/zrepl/zrepl/util/envconst"
+	"github.com/zrepl/zrepl/util/semaphore"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type AbstractionType string
+
+// Implementation note:
+// There are a lot of exhaustive switches on AbstractionType in the code base.
+// When adding a new abstraction type, make sure to search and update them!
+const (
+	AbstractionStepBookmark                AbstractionType = "step-bookmark"
+	AbstractionStepHold                    AbstractionType = "step-hold"
+	AbstractionLastReceivedHold            AbstractionType = "last-received-hold"
+	AbstractionReplicationCursorBookmarkV1 AbstractionType = "replication-cursor-bookmark-v1"
+	AbstractionReplicationCursorBookmarkV2 AbstractionType = "replication-cursor-bookmark-v2"
+)
+
+var AbstractionTypesAll = map[AbstractionType]bool{
+	AbstractionStepBookmark:                true,
+	AbstractionStepHold:                    true,
+	AbstractionLastReceivedHold:            true,
+	AbstractionReplicationCursorBookmarkV1: true,
+	AbstractionReplicationCursorBookmarkV2: true,
+}
+
+// Implementation Note:
+// Whenever you add a new accessor, adjust AbstractionJSON.MarshalJSON accordingly
+type Abstraction interface {
+	GetType() AbstractionType
+	GetFS() string
+	GetName() string
+	GetFullPath() string
+	GetJobID() *JobID // may return nil if the abstraction does not have a JobID
+	GetCreateTXG() uint64
+	GetFilesystemVersion() zfs.FilesystemVersion
+	String() string
+	// destroy the abstraction: either releases the hold or destroys the bookmark
+	Destroy(context.Context) error
+	json.Marshaler
+}
+
+func (t AbstractionType) Validate() error {
+	switch t {
+	case AbstractionStepBookmark:
+		return nil
+	case AbstractionStepHold:
+		return nil
+	case AbstractionLastReceivedHold:
+		return nil
+	case AbstractionReplicationCursorBookmarkV1:
+		return nil
+	case AbstractionReplicationCursorBookmarkV2:
+		return nil
+	default:
+		return errors.Errorf("unknown abstraction type %q", t)
+	}
+}
+
+func (t AbstractionType) MustValidate() error {
+	if err := t.Validate(); err != nil {
+		panic(err)
+	}
+	return nil
+}
+
+type AbstractionJSON struct{ Abstraction }
+
+var _ json.Marshaler = (*AbstractionJSON)(nil)
+
+func (a AbstractionJSON) MarshalJSON() ([]byte, error) {
+	type S struct {
+		Type              AbstractionType
+		FS                string
+		Name              string
+		FullPath          string
+		JobID             *JobID // may return nil if the abstraction does not have a JobID
+		CreateTXG         uint64
+		FilesystemVersion zfs.FilesystemVersion
+		String            string
+	}
+	v := S{
+		Type:              a.Abstraction.GetType(),
+		FS:                a.Abstraction.GetFS(),
+		Name:              a.Abstraction.GetName(),
+		FullPath:          a.Abstraction.GetFullPath(),
+		JobID:             a.Abstraction.GetJobID(),
+		CreateTXG:         a.Abstraction.GetCreateTXG(),
+		FilesystemVersion: a.Abstraction.GetFilesystemVersion(),
+		String:            a.Abstraction.String(),
+	}
+	return json.Marshal(v)
+}
+
+type AbstractionTypeSet map[AbstractionType]bool
+
+func AbstractionTypeSetFromStrings(sts []string) (AbstractionTypeSet, error) {
+	ats := make(map[AbstractionType]bool, len(sts))
+	for i, t := range sts {
+		at := AbstractionType(t)
+		if err := at.Validate(); err != nil {
+			return nil, errors.Wrapf(err, "invalid abstraction type #%d %q", i+1, t)
+		}
+		ats[at] = true
+	}
+	return ats, nil
+}
+
+func (s AbstractionTypeSet) ContainsAll(q AbstractionTypeSet) bool {
+	for k := range q {
+		if _, ok := s[k]; !ok {
+			return false
+		}
+	}
+	return true
+}
+
+func (s AbstractionTypeSet) ContainsAnyOf(q AbstractionTypeSet) bool {
+	for k := range q {
+		if _, ok := s[k]; ok {
+			return true
+		}
+	}
+	return false
+}
+
+func (s AbstractionTypeSet) String() string {
+	sts := make([]string, 0, len(s))
+	for i := range s {
+		sts = append(sts, string(i))
+	}
+	sts = sort.StringSlice(sts)
+	return strings.Join(sts, ",")
+}
+
+func (s AbstractionTypeSet) Validate() error {
+	for k := range s {
+		if err := k.Validate(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type BookmarkExtractor func(fs *zfs.DatasetPath, v zfs.FilesystemVersion) Abstraction
+
+// returns nil if the abstraction type is not bookmark-based
+func (t AbstractionType) BookmarkExtractor() BookmarkExtractor {
+	switch t {
+	case AbstractionStepBookmark:
+		return StepBookmarkExtractor
+	case AbstractionReplicationCursorBookmarkV1:
+		return ReplicationCursorV1Extractor
+	case AbstractionReplicationCursorBookmarkV2:
+		return ReplicationCursorV2Extractor
+	case AbstractionStepHold:
+		return nil
+	case AbstractionLastReceivedHold:
+		return nil
+	default:
+		panic(fmt.Sprintf("unimpl: %q", t))
+	}
+}
+
+type HoldExtractor = func(fs *zfs.DatasetPath, v zfs.FilesystemVersion, tag string) Abstraction
+
+// returns nil if the abstraction type is not hold-based
+func (t AbstractionType) HoldExtractor() HoldExtractor {
+	switch t {
+	case AbstractionStepBookmark:
+		return nil
+	case AbstractionReplicationCursorBookmarkV1:
+		return nil
+	case AbstractionReplicationCursorBookmarkV2:
+		return nil
+	case AbstractionStepHold:
+		return StepHoldExtractor
+	case AbstractionLastReceivedHold:
+		return LastReceivedHoldExtractor
+	default:
+		panic(fmt.Sprintf("unimpl: %q", t))
+	}
+}
+
+type ListZFSHoldsAndBookmarksQuery struct {
+	FS ListZFSHoldsAndBookmarksQueryFilesystemFilter
+	// What abstraction types should match (any contained in the set)
+	What AbstractionTypeSet
+
+	// The output for the query must satisfy _all_ (AND) requirements of all fields in this query struct.
+
+	// if not nil: JobID of the hold or bookmark in question must be equal
+	// else: JobID of the hold or bookmark can be any value
+	JobID *JobID
+
+	// zero-value means any CreateTXG is acceptable
+	CreateTXG CreateTXGRange
+
+	// Number of concurrently queried filesystems. Must be >= 1
+	Concurrency int64
+}
+
+type CreateTXGRangeBound struct {
+	CreateTXG uint64
+	Inclusive *zfs.NilBool // must not be nil
+}
+
+// A non-empty range of CreateTXGs
+//
+// If both Since and Until are nil, any CreateTXG is acceptable
+type CreateTXGRange struct {
+	// if not nil: The hold's snapshot or the bookmark's createtxg must be greater than (or equal) Since
+	// else: CreateTXG of the hold or bookmark can be any value accepted by Until
+	Since *CreateTXGRangeBound
+	// if not nil: The hold's snapshot or the bookmark's createtxg must be less than (or equal) Until
+	// else: CreateTXG of the hold or bookmark can be any value accepted by Since
+	Until *CreateTXGRangeBound
+}
+
+// FS == nil XOR Filter == nil
+type ListZFSHoldsAndBookmarksQueryFilesystemFilter struct {
+	FS     *string
+	Filter zfs.DatasetFilter
+}
+
+func (q *ListZFSHoldsAndBookmarksQuery) Validate() error {
+	if err := q.FS.Validate(); err != nil {
+		return errors.Wrap(err, "FS")
+	}
+	if q.JobID != nil {
+		q.JobID.MustValidate() // FIXME
+	}
+	if err := q.CreateTXG.Validate(); err != nil {
+		return errors.Wrap(err, "CreateTXGRange")
+	}
+	if err := q.What.Validate(); err != nil {
+		return err
+	}
+	if q.Concurrency < 1 {
+		return errors.New("Concurrency must be >= 1")
+	}
+	return nil
+}
+
+var createTXGRangeBoundAllowCreateTXG0 = envconst.Bool("ZREPL_ENDPOINT_LIST_ABSTRACTIONS_QUERY_CREATETXG_RANGE_BOUND_ALLOW_0", false)
+
+func (i *CreateTXGRangeBound) Validate() error {
+	if err := i.Inclusive.Validate(); err != nil {
+		return errors.Wrap(err, "Inclusive")
+	}
+	if i.CreateTXG == 0 && !createTXGRangeBoundAllowCreateTXG0 {
+		return errors.New("CreateTXG must be non-zero")
+	}
+	return nil
+
+}
+
+func (f *ListZFSHoldsAndBookmarksQueryFilesystemFilter) Validate() error {
+	if f == nil {
+		return nil
+	}
+	fsSet := f.FS != nil
+	filterSet := f.Filter != nil
+	if fsSet && filterSet || !fsSet && !filterSet {
+		return fmt.Errorf("must set FS or Filter field, but fsIsSet=%v and filterIsSet=%v", fsSet, filterSet)
+	}
+	if fsSet {
+		if err := zfs.EntityNamecheck(*f.FS, zfs.EntityTypeFilesystem); err != nil {
+			return errors.Wrap(err, "FS invalid")
+		}
+	}
+	return nil
+}
+
+func (f *ListZFSHoldsAndBookmarksQueryFilesystemFilter) Filesystems(ctx context.Context) ([]string, error) {
+	if err := f.Validate(); err != nil {
+		panic(err)
+	}
+	if f.FS != nil {
+		return []string{*f.FS}, nil
+	}
+	if f.Filter != nil {
+		dps, err := zfs.ZFSListMapping(ctx, f.Filter)
+		if err != nil {
+			return nil, err
+		}
+		fss := make([]string, len(dps))
+		for i, dp := range dps {
+			fss[i] = dp.ToString()
+		}
+		return fss, nil
+	}
+	panic("unreachable")
+}
+
+func (r *CreateTXGRange) Validate() error {
+	if r.Since != nil {
+		if err := r.Since.Validate(); err != nil {
+			return errors.Wrap(err, "Since")
+		}
+	}
+	if r.Until != nil {
+		if err := r.Until.Validate(); err != nil {
+			return errors.Wrap(err, "Until")
+		}
+	}
+	if _, err := r.effectiveBounds(); err != nil {
+		return errors.Wrapf(err, "specified range %s is semantically invalid", r)
+	}
+	return nil
+}
+
+// inclusive-inclusive bounds
+type effectiveBounds struct {
+	sinceInclusive uint64
+	sinceUnbounded bool
+	untilInclusive uint64
+	untilUnbounded bool
+}
+
+// callers must have validated r.Since and r.Until before calling this method
+func (r *CreateTXGRange) effectiveBounds() (bounds effectiveBounds, err error) {
+
+	bounds.sinceUnbounded = r.Since == nil
+	bounds.untilUnbounded = r.Until == nil
+
+	if r.Since == nil && r.Until == nil {
+		return bounds, nil
+	}
+
+	if r.Since != nil {
+		bounds.sinceInclusive = r.Since.CreateTXG
+		if !r.Since.Inclusive.B {
+			if r.Since.CreateTXG == math.MaxUint64 {
+				return bounds, errors.Errorf("Since-exclusive (%v) must be less than math.MaxUint64 (%v)",
+					r.Since.CreateTXG, uint64(math.MaxUint64))
+			}
+			bounds.sinceInclusive++
+		}
+	}
+
+	if r.Until != nil {
+		bounds.untilInclusive = r.Until.CreateTXG
+		if !r.Until.Inclusive.B {
+			if r.Until.CreateTXG == 0 {
+				return bounds, errors.Errorf("Until-exclusive (%v) must be greater than 0", r.Until.CreateTXG)
+			}
+			bounds.untilInclusive--
+		}
+	}
+
+	if !bounds.sinceUnbounded && !bounds.untilUnbounded {
+		if bounds.sinceInclusive >= bounds.untilInclusive {
+			return bounds, errors.Errorf("effective range bounds are [%v,%v] which is empty or invalid", bounds.sinceInclusive, bounds.untilInclusive)
+		}
+		// fallthrough
+	}
+
+	return bounds, nil
+}
+
+func (r *CreateTXGRange) String() string {
+	var buf strings.Builder
+	if r.Since == nil {
+		fmt.Fprintf(&buf, "~")
+	} else {
+		if err := r.Since.Inclusive.Validate(); err != nil {
+			fmt.Fprintf(&buf, "?")
+		} else if r.Since.Inclusive.B {
+			fmt.Fprintf(&buf, "[")
+		} else {
+			fmt.Fprintf(&buf, "(")
+		}
+		fmt.Fprintf(&buf, "%d", r.Since.CreateTXG)
+	}
+
+	fmt.Fprintf(&buf, ",")
+
+	if r.Until == nil {
+		fmt.Fprintf(&buf, "~")
+	} else {
+		fmt.Fprintf(&buf, "%d", r.Until.CreateTXG)
+		if err := r.Until.Inclusive.Validate(); err != nil {
+			fmt.Fprintf(&buf, "?")
+		} else if r.Until.Inclusive.B {
+			fmt.Fprintf(&buf, "]")
+		} else {
+			fmt.Fprintf(&buf, ")")
+		}
+	}
+
+	return buf.String()
+}
+
+// panics if not .Validate()
+func (r *CreateTXGRange) IsUnbounded() bool {
+	if err := r.Validate(); err != nil {
+		panic(err)
+	}
+	bounds, err := r.effectiveBounds()
+	if err != nil {
+		panic(err)
+	}
+	return bounds.sinceUnbounded && bounds.untilUnbounded
+}
+
+// panics if not .Validate()
+func (r *CreateTXGRange) Contains(qCreateTxg uint64) bool {
+	if err := r.Validate(); err != nil {
+		panic(err)
+	}
+
+	bounds, err := r.effectiveBounds()
+	if err != nil {
+		panic(err)
+	}
+
+	sinceMatches := bounds.sinceUnbounded || bounds.sinceInclusive <= qCreateTxg
+	untilMatches := bounds.untilUnbounded || qCreateTxg <= bounds.untilInclusive
+
+	return sinceMatches && untilMatches
+}
+
+type ListAbstractionsError struct {
+	FS   string
+	Snap string
+	What string
+	Err  error
+}
+
+func (e ListAbstractionsError) Error() string {
+	if e.FS == "" {
+		return fmt.Sprintf("list endpoint abstractions: %s: %s", e.What, e.Err)
+	} else {
+		v := e.FS
+		if e.Snap != "" {
+			v = fmt.Sprintf("%s@%s", e.FS, e.Snap)
+		}
+		return fmt.Sprintf("list endpoint abstractions on %q: %s: %s", v, e.What, e.Err)
+	}
+}
+
+type putListAbstractionErr func(err error, fs string, what string)
+type putListAbstraction func(a Abstraction)
+
+type ListAbstractionsErrors []ListAbstractionsError
+
+func (e ListAbstractionsErrors) Error() string {
+	if len(e) == 0 {
+		panic(e)
+	}
+	if len(e) == 1 {
+		return fmt.Sprintf("list endpoint abstractions: %s", e[0])
+	}
+	msgs := make([]string, len(e))
+	for i := range e {
+		msgs[i] = e[i].Error()
+	}
+	return fmt.Sprintf("list endpoint abstractions: multiple errors:\n%s", strings.Join(msgs, "\n"))
+}
+
+func ListAbstractions(ctx context.Context, query ListZFSHoldsAndBookmarksQuery) (out []Abstraction, outErrs []ListAbstractionsError, err error) {
+	outChan, outErrsChan, err := ListAbstractionsStreamed(ctx, query)
+	if err != nil {
+		return nil, nil, err
+	}
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for a := range outChan {
+			out = append(out, a)
+		}
+	}()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for err := range outErrsChan {
+			outErrs = append(outErrs, err)
+		}
+	}()
+	wg.Wait()
+	return out, outErrs, nil
+}
+
+// if err != nil, the returned channels are both nil
+// if err == nil, both channels must be fully drained by the caller to avoid leaking goroutines
+func ListAbstractionsStreamed(ctx context.Context, query ListZFSHoldsAndBookmarksQuery) (<-chan Abstraction, <-chan ListAbstractionsError, error) {
+
+	// impl note: structure the query processing in such a way that
+	// a minimum amount of zfs shell-outs needs to be done
+
+	if err := query.Validate(); err != nil {
+		return nil, nil, errors.Wrap(err, "validate query")
+	}
+
+	fss, err := query.FS.Filesystems(ctx)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "list filesystems")
+	}
+
+	outErrs := make(chan ListAbstractionsError)
+	out := make(chan Abstraction)
+
+	errCb := func(err error, fs string, what string) {
+		outErrs <- ListAbstractionsError{Err: err, FS: fs, What: what}
+	}
+	emitAbstraction := func(a Abstraction) {
+		jobIdMatches := query.JobID == nil || a.GetJobID() == nil || *a.GetJobID() == *query.JobID
+
+		createTXGMatches := query.CreateTXG.Contains(a.GetCreateTXG())
+
+		if jobIdMatches && createTXGMatches {
+			out <- a
+		}
+	}
+
+	sem := semaphore.New(int64(query.Concurrency))
+	ctx, endTask := trace.WithTask(ctx, "list-abstractions-streamed-producer")
+	go func() {
+		defer endTask()
+		defer close(out)
+		defer close(outErrs)
+
+		_, add, wait := trace.WithTaskGroup(ctx, "list-abstractions-impl-fs")
+		defer wait()
+		for i := range fss {
+			add(func(ctx context.Context) {
+				g, err := sem.Acquire(ctx)
+				if err != nil {
+					errCb(err, fss[i], err.Error())
+					return
+				}
+				func() {
+					defer g.Release()
+					listAbstractionsImplFS(ctx, fss[i], &query, emitAbstraction, errCb)
+				}()
+			})
+		}
+	}()
+
+	return out, outErrs, nil
+}
+
+func listAbstractionsImplFS(ctx context.Context, fs string, query *ListZFSHoldsAndBookmarksQuery, emitCandidate putListAbstraction, errCb putListAbstractionErr) {
+	fsp, err := zfs.NewDatasetPath(fs)
+	if err != nil {
+		panic(err)
+	}
+
+	if len(query.What) == 0 {
+		return
+	}
+
+	whatTypes := zfs.VersionTypeSet{}
+	for what := range query.What {
+		if e := what.BookmarkExtractor(); e != nil {
+			whatTypes[zfs.Bookmark] = true
+		}
+		if e := what.HoldExtractor(); e != nil {
+			whatTypes[zfs.Snapshot] = true
+		}
+	}
+	fsvs, err := zfs.ZFSListFilesystemVersions(ctx, fsp, zfs.ListFilesystemVersionsOptions{
+		Types: whatTypes,
+	})
+	if err != nil {
+		errCb(err, fs, "list filesystem versions")
+		return
+	}
+
+	for at := range query.What {
+		bmE := at.BookmarkExtractor()
+		holdE := at.HoldExtractor()
+		if bmE == nil && holdE == nil || bmE != nil && holdE != nil {
+			panic("implementation error: extractors misconfigured for " + at)
+		}
+		for _, v := range fsvs {
+			var a Abstraction
+			if v.Type == zfs.Bookmark && bmE != nil {
+				a = bmE(fsp, v)
+			}
+			if v.Type == zfs.Snapshot && holdE != nil && query.CreateTXG.Contains(v.GetCreateTXG()) && (!v.UserRefs.Valid || v.UserRefs.Value > 0) {
+				holds, err := zfs.ZFSHolds(ctx, fsp.ToString(), v.Name)
+				if err != nil {
+					errCb(err, v.ToAbsPath(fsp), "get hold on snap")
+					continue
+				}
+				for _, tag := range holds {
+					a = holdE(fsp, v, tag)
+				}
+			}
+			if a != nil {
+				emitCandidate(a)
+			}
+		}
+	}
+}
+
+type BatchDestroyResult struct {
+	Abstraction
+	DestroyErr error
+}
+
+var _ json.Marshaler = (*BatchDestroyResult)(nil)
+
+func (r BatchDestroyResult) MarshalJSON() ([]byte, error) {
+	err := ""
+	if r.DestroyErr != nil {
+		err = r.DestroyErr.Error()
+	}
+	s := struct {
+		Abstraction AbstractionJSON
+		DestroyErr  string
+	}{
+		AbstractionJSON{r.Abstraction},
+		err,
+	}
+	return json.Marshal(s)
+}
+
+func BatchDestroy(ctx context.Context, abs []Abstraction) <-chan BatchDestroyResult {
+	// hold-based batching: per snapshot
+	// bookmark-based batching: none possible via CLI
+	// => not worth the trouble for now, will be worth it once we start using channel programs
+	// => TODO: actual batching using channel programs
+	res := make(chan BatchDestroyResult, len(abs))
+	go func() {
+		for _, a := range abs {
+			res <- BatchDestroyResult{
+				a,
+				a.Destroy(ctx),
+			}
+		}
+		close(res)
+	}()
+	return res
+}
+
+type StalenessInfo struct {
+	ConstructedWithQuery ListZFSHoldsAndBookmarksQuery
+	Live                 []Abstraction
+	Stale                []Abstraction
+}
+
+type fsAndJobId struct {
+	fs    string
+	jobId JobID
+}
+
+type ListStaleQueryError struct {
+	error
+}
+
+// returns *ListStaleQueryError if the given query cannot be used for determining staleness info
+func ListStale(ctx context.Context, q ListZFSHoldsAndBookmarksQuery) (*StalenessInfo, error) {
+	if !q.CreateTXG.IsUnbounded() {
+		// we must determine the most recent step per FS, can't allow that
+		return nil, &ListStaleQueryError{errors.New("ListStale cannot have Until != nil set on query")}
+	}
+
+	// if asking for step holds, must also as for step bookmarks (same kind of abstraction)
+	// as well as replication cursor bookmarks (for firstNotStale)
+	ifAnyThenAll := AbstractionTypeSet{
+		AbstractionStepHold:                    true,
+		AbstractionStepBookmark:                true,
+		AbstractionReplicationCursorBookmarkV2: true,
+	}
+	if q.What.ContainsAnyOf(ifAnyThenAll) && !q.What.ContainsAll(ifAnyThenAll) {
+		return nil, &ListStaleQueryError{errors.Errorf("ListStale requires query to ask for all of %s", ifAnyThenAll.String())}
+	}
+
+	// ----------------- done validating query for listStaleFiltering -----------------------
+
+	qAbs, absErr, err := ListAbstractions(ctx, q)
+	if err != nil {
+		return nil, err
+	}
+	if len(absErr) > 0 {
+		// can't go on here because we can't determine the most recent step
+		return nil, ListAbstractionsErrors(absErr)
+	}
+
+	si := listStaleFiltering(qAbs, q.CreateTXG.Since)
+	si.ConstructedWithQuery = q
+	return si, nil
+}
+
+type fsAjobAtype struct {
+	fsAndJobId
+	Type AbstractionType
+}
+
+// For step holds and bookmarks, only those older than the most recent replication cursor
+// of their (filesystem,job) is considered because younger ones cannot be stale by definition
+// (if we destroy them, we might actually lose the hold on the `To` for an ongoing incremental replication)
+//
+// For replication cursors and last-received-holds, only the most recent one is kept.
+//
+// the returned StalenessInfo.ConstructedWithQuery is not set
+func listStaleFiltering(abs []Abstraction, sinceBound *CreateTXGRangeBound) *StalenessInfo {
+
+	var noJobId []Abstraction
+	by := make(map[fsAjobAtype][]Abstraction)
+	for _, a := range abs {
+		if a.GetJobID() == nil {
+			noJobId = append(noJobId, a)
+			continue
+		}
+		faj := fsAjobAtype{fsAndJobId{a.GetFS(), *a.GetJobID()}, a.GetType()}
+		l := by[faj]
+		l = append(l, a)
+		by[faj] = l
+	}
+
+	type stepFirstNotStaleCandidate struct {
+		cursor *Abstraction
+		step   *Abstraction
+	}
+	stepFirstNotStaleCandidates := make(map[fsAndJobId]stepFirstNotStaleCandidate) // empty map => will always return nil
+	for _, a := range abs {
+		if a.GetJobID() == nil {
+			continue // already put those into always-live list noJobId in above loop
+		}
+		key := fsAndJobId{a.GetFS(), *a.GetJobID()}
+		c := stepFirstNotStaleCandidates[key]
+
+		switch a.GetType() {
+		// stepFirstNotStaleCandidate.cursor
+		case AbstractionReplicationCursorBookmarkV2:
+			if c.cursor == nil || (*c.cursor).GetCreateTXG() < a.GetCreateTXG() {
+				a := a
+				c.cursor = &a
+			}
+
+		// stepFirstNotStaleCandidate.step
+		case AbstractionStepBookmark:
+			fallthrough
+		case AbstractionStepHold:
+			if c.step == nil || (*c.step).GetCreateTXG() < a.GetCreateTXG() {
+				a := a
+				c.step = &a
+			}
+
+		// not interested in the others
+		default:
+			continue // not relevant
+
+		}
+
+		stepFirstNotStaleCandidates[key] = c
+	}
+
+	ret := &StalenessInfo{
+		Live:  noJobId,
+		Stale: []Abstraction{},
+	}
+
+	for k := range by {
+		l := by[k]
+
+		if k.Type == AbstractionStepHold || k.Type == AbstractionStepBookmark {
+			// all older than the most recent cursor are stale, others are always live
+
+			// if we don't have a replication cursor yet, use untilBound = nil
+			// to consider all steps stale (...at first)
+			var untilBound *CreateTXGRangeBound
+			{
+				sfnsc := stepFirstNotStaleCandidates[k.fsAndJobId]
+
+				// if there's a replication cursor, use it as a cutoff between live and stale
+				// if there's none, we are in initial replication and only need to keep
+				// the most recent step hold live, since that's what our initial replication strategy
+				// uses (both initially and on resume)
+				// (FIXME hardcoded replication strategy)
+				if sfnsc.cursor != nil {
+					untilBound = &CreateTXGRangeBound{
+						CreateTXG: (*sfnsc.cursor).GetCreateTXG(),
+						// if we have a cursor, can throw away step hold on both From and To
+						Inclusive: &zfs.NilBool{B: true},
+					}
+				} else if sfnsc.step != nil {
+					untilBound = &CreateTXGRangeBound{
+						CreateTXG: (*sfnsc.step).GetCreateTXG(),
+						// if we don't have a cursor, the step most recent step hold is our
+						// initial replication cursor and it's possibly still live (interrupted initial replication)
+						Inclusive: &zfs.NilBool{B: false},
+					}
+				} else {
+					untilBound = nil // consider everything stale
+				}
+			}
+			staleRange := CreateTXGRange{
+				Since: sinceBound,
+				Until: untilBound,
+			}
+
+			// partition by staleRange
+			for _, a := range l {
+				if staleRange.Contains(a.GetCreateTXG()) {
+					ret.Stale = append(ret.Stale, a)
+				} else {
+					ret.Live = append(ret.Live, a)
+				}
+			}
+
+		} else if k.Type == AbstractionReplicationCursorBookmarkV2 || k.Type == AbstractionLastReceivedHold {
+			// all but the most recent are stale by definition (we always _move_ them)
+			// NOTE: must not use firstNotStale in this branch, not computed for these types
+
+			// sort descending (highest createtxg first), then cut off
+			sort.Slice(l, func(i, j int) bool {
+				return l[i].GetCreateTXG() > l[j].GetCreateTXG()
+			})
+			if len(l) > 0 {
+				ret.Live = append(ret.Live, l[0])
+				ret.Stale = append(ret.Stale, l[1:]...)
+			}
+		} else {
+			ret.Live = append(ret.Live, l...)
+		}
+	}
+
+	return ret
+
+}

endpoint/endpoint_zfs_abstraction_cursor_and_last_received_hold.go

@@ -0,0 +1,386 @@
+package endpoint
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"sort"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/util/errorarray"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+const replicationCursorBookmarkNamePrefix = "zrepl_CURSOR"
+
+func ReplicationCursorBookmarkName(fs string, guid uint64, id JobID) (string, error) {
+	return replicationCursorBookmarkNameImpl(fs, guid, id.String())
+}
+
+func replicationCursorBookmarkNameImpl(fs string, guid uint64, jobid string) (string, error) {
+	return makeJobAndGuidBookmarkName(replicationCursorBookmarkNamePrefix, fs, guid, jobid)
+}
+
+var ErrV1ReplicationCursor = fmt.Errorf("bookmark name is a v1-replication cursor")
+
+//err != nil always means that the bookmark is not a valid replication bookmark
+//
+// Returns ErrV1ReplicationCursor as error if the bookmark is a v1 replication cursor
+func ParseReplicationCursorBookmarkName(fullname string) (uint64, JobID, error) {
+
+	// check for legacy cursors
+	{
+		if err := zfs.EntityNamecheck(fullname, zfs.EntityTypeBookmark); err != nil {
+			return 0, JobID{}, errors.Wrap(err, "parse replication cursor bookmark name")
+		}
+		_, _, name, err := zfs.DecomposeVersionString(fullname)
+		if err != nil {
+			return 0, JobID{}, errors.Wrap(err, "parse replication cursor bookmark name: decompose version string")
+		}
+		const V1ReplicationCursorBookmarkName = "zrepl_replication_cursor"
+		if name == V1ReplicationCursorBookmarkName {
+			return 0, JobID{}, ErrV1ReplicationCursor
+		}
+		// fallthrough to main parser
+	}
+
+	guid, jobID, err := parseJobAndGuidBookmarkName(fullname, replicationCursorBookmarkNamePrefix)
+	if err != nil {
+		err = errors.Wrap(err, "parse replication cursor bookmark name") // no shadow
+	}
+	return guid, jobID, err
+}
+
+// may return nil for both values, indicating there is no cursor
+func GetMostRecentReplicationCursorOfJob(ctx context.Context, fs string, jobID JobID) (*zfs.FilesystemVersion, error) {
+	fsp, err := zfs.NewDatasetPath(fs)
+	if err != nil {
+		return nil, err
+	}
+	candidates, err := GetReplicationCursors(ctx, fsp, jobID)
+	if err != nil || len(candidates) == 0 {
+		return nil, err
+	}
+
+	sort.Slice(candidates, func(i, j int) bool {
+		return candidates[i].CreateTXG < candidates[j].CreateTXG
+	})
+
+	mostRecent := candidates[len(candidates)-1]
+	return &mostRecent, nil
+}
+
+func GetReplicationCursors(ctx context.Context, dp *zfs.DatasetPath, jobID JobID) ([]zfs.FilesystemVersion, error) {
+
+	fs := dp.ToString()
+	q := ListZFSHoldsAndBookmarksQuery{
+		FS: ListZFSHoldsAndBookmarksQueryFilesystemFilter{FS: &fs},
+		What: map[AbstractionType]bool{
+			AbstractionReplicationCursorBookmarkV1: true,
+			AbstractionReplicationCursorBookmarkV2: true,
+		},
+		JobID:       &jobID,
+		CreateTXG:   CreateTXGRange{},
+		Concurrency: 1,
+	}
+	abs, absErr, err := ListAbstractions(ctx, q)
+	if err != nil {
+		return nil, errors.Wrap(err, "get replication cursor: list bookmarks and holds")
+	}
+	if len(absErr) > 0 {
+		return nil, ListAbstractionsErrors(absErr)
+	}
+
+	var v1, v2 []Abstraction
+	for _, a := range abs {
+		switch a.GetType() {
+		case AbstractionReplicationCursorBookmarkV1:
+			v1 = append(v1, a)
+		case AbstractionReplicationCursorBookmarkV2:
+			v2 = append(v2, a)
+		default:
+			panic("unexpected abstraction: " + a.GetType())
+		}
+	}
+
+	if len(v1) > 0 {
+		getLogger(ctx).WithField("bookmark", v1).
+			Warn("found v1-replication cursor bookmarks, consider running migration 'replication-cursor:v1-v2' after successful replication with this zrepl version")
+	}
+
+	candidates := make([]zfs.FilesystemVersion, 0)
+	for _, v := range v2 {
+		candidates = append(candidates, v.GetFilesystemVersion())
+	}
+
+	return candidates, nil
+}
+
+type ReplicationCursorTarget interface {
+	IsSnapshot() bool
+	GetGuid() uint64
+	GetCreateTXG() uint64
+	ToSendArgVersion() zfs.ZFSSendArgVersion
+}
+
+// `target` is validated before replication cursor is set. if validation fails, the cursor is not moved.
+//
+// returns ErrBookmarkCloningNotSupported if version is a bookmark and bookmarking bookmarks is not supported by ZFS
+func MoveReplicationCursor(ctx context.Context, fs string, target ReplicationCursorTarget, jobID JobID) (destroyedCursors []Abstraction, err error) {
+
+	if !target.IsSnapshot() {
+		return nil, zfs.ErrBookmarkCloningNotSupported
+	}
+
+	bookmarkname, err := ReplicationCursorBookmarkName(fs, target.GetGuid(), jobID)
+	if err != nil {
+		return nil, errors.Wrap(err, "determine replication cursor name")
+	}
+
+	// idempotently create bookmark (guid is encoded in it, hence we'll most likely add a new one
+	// cleanup the old one afterwards
+
+	err = zfs.ZFSBookmark(ctx, fs, target.ToSendArgVersion(), bookmarkname)
+	if err != nil {
+		if err == zfs.ErrBookmarkCloningNotSupported {
+			return nil, err // TODO go1.13 use wrapping
+		}
+		return nil, errors.Wrapf(err, "cannot create bookmark")
+	}
+
+	destroyedCursors, err = DestroyObsoleteReplicationCursors(ctx, fs, target, jobID)
+	if err != nil {
+		return nil, errors.Wrap(err, "destroy obsolete replication cursors")
+	}
+
+	return destroyedCursors, nil
+}
+
+type ReplicationCursor interface {
+	GetCreateTXG() uint64
+}
+
+func DestroyObsoleteReplicationCursors(ctx context.Context, fs string, current ReplicationCursor, jobID JobID) (_ []Abstraction, err error) {
+
+	q := ListZFSHoldsAndBookmarksQuery{
+		FS: ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &fs,
+		},
+		What: AbstractionTypeSet{
+			AbstractionReplicationCursorBookmarkV2: true,
+		},
+		JobID: &jobID,
+		CreateTXG: CreateTXGRange{
+			Since: nil,
+			Until: &CreateTXGRangeBound{
+				CreateTXG: current.GetCreateTXG(),
+				Inclusive: &zfs.NilBool{B: false},
+			},
+		},
+		Concurrency: 1,
+	}
+	abs, absErr, err := ListAbstractions(ctx, q)
+	if err != nil {
+		return nil, errors.Wrap(err, "list abstractions")
+	}
+	if len(absErr) > 0 {
+		return nil, errors.Wrap(ListAbstractionsErrors(absErr), "list abstractions")
+	}
+
+	var destroyed []Abstraction
+	var errs []error
+	for res := range BatchDestroy(ctx, abs) {
+		log := getLogger(ctx).
+			WithField("replication_cursor_bookmark", res.Abstraction)
+		if res.DestroyErr != nil {
+			errs = append(errs, res.DestroyErr)
+			log.WithError(err).
+				Error("cannot destroy obsolete replication cursor bookmark")
+		} else {
+			destroyed = append(destroyed, res.Abstraction)
+			log.Info("destroyed obsolete replication cursor bookmark")
+		}
+	}
+	if len(errs) == 0 {
+		return destroyed, nil
+	} else {
+		return destroyed, errorarray.Wrap(errs, "destroy obsolete replication cursor")
+	}
+}
+
+var lastReceivedHoldTagRE = regexp.MustCompile("^zrepl_last_received_J_(.+)$")
+
+// err != nil always means that the bookmark is not a step bookmark
+func ParseLastReceivedHoldTag(tag string) (JobID, error) {
+	match := lastReceivedHoldTagRE.FindStringSubmatch(tag)
+	if match == nil {
+		return JobID{}, errors.Errorf("parse last-received-hold tag: does not match regex %s", lastReceivedHoldTagRE.String())
+	}
+	jobId, err := MakeJobID(match[1])
+	if err != nil {
+		return JobID{}, errors.Wrap(err, "parse last-received-hold tag: invalid job id field")
+	}
+	return jobId, nil
+}
+
+func LastReceivedHoldTag(jobID JobID) (string, error) {
+	return lastReceivedHoldImpl(jobID.String())
+}
+
+func lastReceivedHoldImpl(jobid string) (string, error) {
+	tag := fmt.Sprintf("zrepl_last_received_J_%s", jobid)
+	if err := zfs.ValidHoldTag(tag); err != nil {
+		return "", err
+	}
+	return tag, nil
+}
+
+func MoveLastReceivedHold(ctx context.Context, fs string, to zfs.FilesystemVersion, jobID JobID) error {
+
+	if !to.IsSnapshot() {
+		return errors.Errorf("last-received-hold: target must be a snapshot: %s", to.FullPath(fs))
+	}
+
+	tag, err := LastReceivedHoldTag(jobID)
+	if err != nil {
+		return errors.Wrap(err, "last-received-hold: hold tag")
+	}
+
+	// we never want to be without a hold
+	// => hold new one before releasing old hold
+
+	err = zfs.ZFSHold(ctx, fs, to, tag)
+	if err != nil {
+		return errors.Wrap(err, "last-received-hold: hold newly received")
+	}
+
+	q := ListZFSHoldsAndBookmarksQuery{
+		What: AbstractionTypeSet{
+			AbstractionLastReceivedHold: true,
+		},
+		FS: ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &fs,
+		},
+		JobID: &jobID,
+		CreateTXG: CreateTXGRange{
+			Since: nil,
+			Until: &CreateTXGRangeBound{
+				CreateTXG: to.GetCreateTXG(),
+				Inclusive: &zfs.NilBool{B: false},
+			},
+		},
+		Concurrency: 1,
+	}
+	abs, absErrs, err := ListAbstractions(ctx, q)
+	if err != nil {
+		return errors.Wrap(err, "last-received-hold: list")
+	}
+	if len(absErrs) > 0 {
+		return errors.Wrap(ListAbstractionsErrors(absErrs), "last-received-hold: list")
+	}
+
+	getLogger(ctx).WithField("last-received-holds", fmt.Sprintf("%s", abs)).Debug("releasing last-received-holds")
+
+	var errs []error
+	for res := range BatchDestroy(ctx, abs) {
+		log := getLogger(ctx).
+			WithField("last-received-hold", res.Abstraction)
+		if res.DestroyErr != nil {
+			errs = append(errs, res.DestroyErr)
+			log.WithError(err).
+				Error("cannot release last-received-hold")
+		} else {
+			log.Info("released last-received-hold")
+		}
+	}
+	if len(errs) == 0 {
+		return nil
+	} else {
+		return errorarray.Wrap(errs, "last-received-hold: release")
+	}
+}
+
+func ReplicationCursorV2Extractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion) (_ Abstraction) {
+	if v.Type != zfs.Bookmark {
+		panic("impl error")
+	}
+	fullname := v.ToAbsPath(fs)
+	guid, jobid, err := ParseReplicationCursorBookmarkName(fullname)
+	if err == nil {
+		if guid != v.Guid {
+			// TODO log this possibly tinkered-with bookmark
+			return nil
+		}
+		return &bookmarkBasedAbstraction{
+			Type:              AbstractionReplicationCursorBookmarkV2,
+			FS:                fs.ToString(),
+			FilesystemVersion: v,
+			JobID:             jobid,
+		}
+	}
+	return nil
+}
+
+func ReplicationCursorV1Extractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion) (_ Abstraction) {
+	if v.Type != zfs.Bookmark {
+		panic("impl error")
+	}
+	fullname := v.ToAbsPath(fs)
+	_, _, err := ParseReplicationCursorBookmarkName(fullname)
+	if err == ErrV1ReplicationCursor {
+		return &ReplicationCursorV1{
+			Type:              AbstractionReplicationCursorBookmarkV1,
+			FS:                fs.ToString(),
+			FilesystemVersion: v,
+		}
+	}
+	return nil
+}
+
+var _ HoldExtractor = LastReceivedHoldExtractor
+
+func LastReceivedHoldExtractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion, holdTag string) Abstraction {
+	var err error
+
+	if v.Type != zfs.Snapshot {
+		panic("impl error")
+	}
+
+	jobID, err := ParseLastReceivedHoldTag(holdTag)
+	if err == nil {
+		return &holdBasedAbstraction{
+			Type:              AbstractionLastReceivedHold,
+			FS:                fs.ToString(),
+			FilesystemVersion: v,
+			Tag:               holdTag,
+			JobID:             jobID,
+		}
+	}
+	return nil
+}
+
+type ReplicationCursorV1 struct {
+	Type AbstractionType
+	FS   string
+	zfs.FilesystemVersion
+}
+
+func (c ReplicationCursorV1) GetType() AbstractionType                    { return c.Type }
+func (c ReplicationCursorV1) GetFS() string                               { return c.FS }
+func (c ReplicationCursorV1) GetFullPath() string                         { return fmt.Sprintf("%s#%s", c.FS, c.GetName()) }
+func (c ReplicationCursorV1) GetJobID() *JobID                            { return nil }
+func (c ReplicationCursorV1) GetFilesystemVersion() zfs.FilesystemVersion { return c.FilesystemVersion }
+func (c ReplicationCursorV1) MarshalJSON() ([]byte, error) {
+	return json.Marshal(AbstractionJSON{c})
+}
+func (c ReplicationCursorV1) String() string {
+	return fmt.Sprintf("%s %s", c.Type, c.GetFullPath())
+}
+func (c ReplicationCursorV1) Destroy(ctx context.Context) error {
+	if err := zfs.ZFSDestroyIdempotent(ctx, c.GetFullPath()); err != nil {
+		return errors.Wrapf(err, "destroy %s %s: zfs", c.Type, c.GetFullPath())
+	}
+	return nil
+}

endpoint/endpoint_zfs_abstraction_step.go

@@ -0,0 +1,286 @@
+package endpoint
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/util/errorarray"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var stepHoldTagRE = regexp.MustCompile("^zrepl_STEP_J_(.+)")
+
+func StepHoldTag(jobid JobID) (string, error) {
+	return stepHoldTagImpl(jobid.String())
+}
+
+func stepHoldTagImpl(jobid string) (string, error) {
+	t := fmt.Sprintf("zrepl_STEP_J_%s", jobid)
+	if err := zfs.ValidHoldTag(t); err != nil {
+		return "", err
+	}
+	return t, nil
+}
+
+// err != nil always means that the bookmark is not a step bookmark
+func ParseStepHoldTag(tag string) (JobID, error) {
+	match := stepHoldTagRE.FindStringSubmatch(tag)
+	if match == nil {
+		return JobID{}, fmt.Errorf("parse hold tag: match regex %q", stepHoldTagRE)
+	}
+	jobID, err := MakeJobID(match[1])
+	if err != nil {
+		return JobID{}, errors.Wrap(err, "parse hold tag: invalid job id field")
+	}
+	return jobID, nil
+}
+
+const stepBookmarkNamePrefix = "zrepl_STEP"
+
+// v must be validated by caller
+func StepBookmarkName(fs string, guid uint64, id JobID) (string, error) {
+	return stepBookmarkNameImpl(fs, guid, id.String())
+}
+
+func stepBookmarkNameImpl(fs string, guid uint64, jobid string) (string, error) {
+	return makeJobAndGuidBookmarkName(stepBookmarkNamePrefix, fs, guid, jobid)
+}
+
+// name is the full bookmark name, including dataset path
+//
+// err != nil always means that the bookmark is not a step bookmark
+func ParseStepBookmarkName(fullname string) (guid uint64, jobID JobID, err error) {
+	guid, jobID, err = parseJobAndGuidBookmarkName(fullname, stepBookmarkNamePrefix)
+	if err != nil {
+		err = errors.Wrap(err, "parse step bookmark name") // no shadow!
+	}
+	return guid, jobID, err
+}
+
+// idempotently hold / step-bookmark `version`
+//
+// returns ErrBookmarkCloningNotSupported if version is a bookmark and bookmarking bookmarks is not supported by ZFS
+func HoldStep(ctx context.Context, fs string, v zfs.FilesystemVersion, jobID JobID) (Abstraction, error) {
+	if v.IsSnapshot() {
+
+		tag, err := StepHoldTag(jobID)
+		if err != nil {
+			return nil, errors.Wrap(err, "step hold tag")
+		}
+
+		if err := zfs.ZFSHold(ctx, fs, v, tag); err != nil {
+			return nil, errors.Wrap(err, "step hold: zfs")
+		}
+
+		return &holdBasedAbstraction{
+			Type:              AbstractionStepHold,
+			FS:                fs,
+			Tag:               tag,
+			JobID:             jobID,
+			FilesystemVersion: v,
+		}, nil
+	}
+
+	if !v.IsBookmark() {
+		panic(fmt.Sprintf("version must bei either snapshot or bookmark, got %#v", v))
+	}
+
+	bmname, err := StepBookmarkName(fs, v.Guid, jobID)
+	if err != nil {
+		return nil, errors.Wrap(err, "create step bookmark: determine bookmark name")
+	}
+	// idempotently create bookmark
+	err = zfs.ZFSBookmark(ctx, fs, v.ToSendArgVersion(), bmname)
+	if err != nil {
+		if err == zfs.ErrBookmarkCloningNotSupported {
+			// TODO we could actually try to find a local snapshot that has the requested GUID
+			// 		however, the replication algorithm prefers snapshots anyways, so this quest
+			// 		is most likely not going to be successful. Also, there's the possibility that
+			//      the caller might want to filter what snapshots are eligibile, and this would
+			//      complicate things even further.
+			return nil, err // TODO go1.13 use wrapping
+		}
+		return nil, errors.Wrap(err, "create step bookmark: zfs")
+	}
+	return &bookmarkBasedAbstraction{
+		Type:              AbstractionStepBookmark,
+		FS:                fs,
+		FilesystemVersion: v,
+		JobID:             jobID,
+	}, nil
+}
+
+// idempotently release the step-hold on v if v is a snapshot
+// or idempotently destroy the step-bookmark  of v if v is a bookmark
+//
+// note that this operation leaves v itself untouched, unless v is the step-bookmark itself, in which case v is destroyed
+//
+// returns an instance of *zfs.DatasetDoesNotExist if `v` does not exist
+func ReleaseStep(ctx context.Context, fs string, v zfs.FilesystemVersion, jobID JobID) error {
+
+	if v.IsSnapshot() {
+		tag, err := StepHoldTag(jobID)
+		if err != nil {
+			return errors.Wrap(err, "step release tag")
+		}
+
+		if err := zfs.ZFSRelease(ctx, tag, v.FullPath(fs)); err != nil {
+			return errors.Wrap(err, "step release: zfs")
+		}
+
+		return nil
+	}
+	if !v.IsBookmark() {
+		panic(fmt.Sprintf("impl error: expecting version to be a bookmark, got %#v", v))
+	}
+
+	bmname, err := StepBookmarkName(fs, v.Guid, jobID)
+	if err != nil {
+		return errors.Wrap(err, "step release: determine bookmark name")
+	}
+	// idempotently destroy bookmark
+
+	if err := zfs.ZFSDestroyIdempotent(ctx, bmname); err != nil {
+		return errors.Wrap(err, "step release: bookmark destroy: zfs")
+	}
+
+	return nil
+}
+
+// release {step holds, step bookmarks} earlier and including `mostRecent`
+func ReleaseStepCummulativeInclusive(ctx context.Context, fs string, since *CreateTXGRangeBound, mostRecent zfs.FilesystemVersion, jobID JobID) error {
+	q := ListZFSHoldsAndBookmarksQuery{
+		What: AbstractionTypeSet{
+			AbstractionStepHold:     true,
+			AbstractionStepBookmark: true,
+		},
+		FS: ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &fs,
+		},
+		JobID: &jobID,
+		CreateTXG: CreateTXGRange{
+			Since: since,
+			Until: &CreateTXGRangeBound{
+				CreateTXG: mostRecent.CreateTXG,
+				Inclusive: &zfs.NilBool{B: true},
+			},
+		},
+		Concurrency: 1,
+	}
+	abs, absErrs, err := ListAbstractions(ctx, q)
+	if err != nil {
+		return errors.Wrap(err, "step release cummulative: list")
+	}
+	if len(absErrs) > 0 {
+		return errors.Wrap(ListAbstractionsErrors(absErrs), "step release cummulative: list")
+	}
+
+	getLogger(ctx).WithField("step_holds_and_bookmarks", fmt.Sprintf("%s", abs)).Debug("releasing step holds and bookmarks")
+
+	var errs []error
+	for res := range BatchDestroy(ctx, abs) {
+		log := getLogger(ctx).
+			WithField("step_hold_or_bookmark", res.Abstraction)
+		if res.DestroyErr != nil {
+			errs = append(errs, res.DestroyErr)
+			log.WithError(err).
+				Error("cannot release step hold or bookmark")
+		} else {
+			log.Info("released step hold or bookmark")
+		}
+	}
+	if len(errs) == 0 {
+		return nil
+	} else {
+		return errorarray.Wrap(errs, "step release cummulative: release")
+	}
+}
+
+func TryReleaseStepStaleFS(ctx context.Context, fs string, jobID JobID) {
+
+	q := ListZFSHoldsAndBookmarksQuery{
+		FS: ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &fs,
+		},
+		JobID: &jobID,
+		What: AbstractionTypeSet{
+			AbstractionStepHold:                    true,
+			AbstractionStepBookmark:                true,
+			AbstractionReplicationCursorBookmarkV2: true,
+		},
+		Concurrency: 1,
+	}
+	staleness, err := ListStale(ctx, q)
+	if _, ok := err.(*ListStaleQueryError); ok {
+		panic(err)
+	} else if err != nil {
+		getLogger(ctx).WithError(err).Error("cannot list stale step holds and bookmarks")
+		return
+	}
+	for _, s := range staleness.Stale {
+		getLogger(ctx).WithField("stale_step_hold_or_bookmark", s).Info("batch-destroying stale step hold or bookmark")
+	}
+	for res := range BatchDestroy(ctx, staleness.Stale) {
+		if res.DestroyErr != nil {
+			getLogger(ctx).
+				WithField("stale_step_hold_or_bookmark", res.Abstraction).
+				WithError(res.DestroyErr).
+				Error("cannot destroy stale step-hold or bookmark")
+		} else {
+			getLogger(ctx).
+				WithField("stale_step_hold_or_bookmark", res.Abstraction).
+				WithError(res.DestroyErr).
+				Info("destroyed stale step-hold or bookmark")
+		}
+	}
+
+}
+
+var _ BookmarkExtractor = StepBookmarkExtractor
+
+func StepBookmarkExtractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion) (_ Abstraction) {
+	if v.Type != zfs.Bookmark {
+		panic("impl error")
+	}
+
+	fullname := v.ToAbsPath(fs)
+
+	guid, jobid, err := ParseStepBookmarkName(fullname)
+	if guid != v.Guid {
+		// TODO log this possibly tinkered-with bookmark
+		return nil
+	}
+	if err == nil {
+		bm := &bookmarkBasedAbstraction{
+			Type:              AbstractionStepBookmark,
+			FS:                fs.ToString(),
+			FilesystemVersion: v,
+			JobID:             jobid,
+		}
+		return bm
+	}
+	return nil
+}
+
+var _ HoldExtractor = StepHoldExtractor
+
+func StepHoldExtractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion, holdTag string) Abstraction {
+	if v.Type != zfs.Snapshot {
+		panic("impl error")
+	}
+
+	jobID, err := ParseStepHoldTag(holdTag)
+	if err == nil {
+		return &holdBasedAbstraction{
+			Type:              AbstractionStepHold,
+			FS:                fs.ToString(),
+			Tag:               holdTag,
+			FilesystemVersion: v,
+			JobID:             jobID,
+		}
+	}
+	return nil
+}

endpoint/endpoint_zfs_abstraction_test.go

@@ -0,0 +1,234 @@
+package endpoint
+
+import (
+	"fmt"
+	"math"
+	"runtime/debug"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func TestCreateTXGRange(t *testing.T) {
+
+	type testCaseExpectation struct {
+		input  uint64
+		expect bool
+	}
+	type testCase struct {
+		name                     string
+		config                   *CreateTXGRange
+		configAllowZeroCreateTXG bool
+		expectInvalid            bool
+		expectString             string
+		expect                   []testCaseExpectation
+	}
+
+	tcs := []testCase{
+		{
+			name:          "unbounded",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Since: nil,
+				Until: nil,
+			},
+			expectString: "~,~",
+			expect: []testCaseExpectation{
+				{0, true},
+				{math.MaxUint64, true},
+				{1, true},
+				{math.MaxUint64 - 1, true},
+			},
+		},
+		{
+			name:          "wrong order obvious",
+			expectInvalid: true,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{23, &zfs.NilBool{B: true}},
+				Until: &CreateTXGRangeBound{20, &zfs.NilBool{B: true}},
+			},
+			expectString: "[23,20]",
+		},
+		{
+			name:          "wrong order edge-case could also be empty",
+			expectInvalid: true,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{23, &zfs.NilBool{B: false}},
+				Until: &CreateTXGRangeBound{22, &zfs.NilBool{B: true}},
+			},
+			expectString: "(23,22]",
+		},
+		{
+			name:          "empty",
+			expectInvalid: true,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
+				Until: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
+			},
+			expectString: "(2,2)",
+		},
+		{
+			name:          "inclusive-since-exclusive-until",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: true}},
+				Until: &CreateTXGRangeBound{5, &zfs.NilBool{B: false}},
+			},
+			expectString: "[2,5)",
+			expect: []testCaseExpectation{
+				{0, false},
+				{1, false},
+				{2, true},
+				{3, true},
+				{4, true},
+				{5, false},
+				{6, false},
+			},
+		},
+		{
+			name:          "exclusive-since-inclusive-until",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
+				Until: &CreateTXGRangeBound{5, &zfs.NilBool{B: true}},
+			},
+			expectString: "(2,5]",
+			expect: []testCaseExpectation{
+				{0, false},
+				{1, false},
+				{2, false},
+				{3, true},
+				{4, true},
+				{5, true},
+				{6, false},
+			},
+		},
+		{
+			name:          "zero-createtxg-not-allowed-because-likely-programmer-error",
+			expectInvalid: true,
+			config: &CreateTXGRange{
+				Since: nil,
+				Until: &CreateTXGRangeBound{0, &zfs.NilBool{B: true}},
+			},
+			expectString: "~,0]",
+		},
+		{
+			name:          "half-open-no-until",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
+				Until: nil,
+			},
+			expectString: "(2,~",
+			expect: []testCaseExpectation{
+				{0, false},
+				{1, false},
+				{2, false},
+				{3, true},
+				{4, true},
+				{5, true},
+				{6, true},
+			},
+		},
+		{
+			name:          "half-open-no-since",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Since: nil,
+				Until: &CreateTXGRangeBound{4, &zfs.NilBool{B: true}},
+			},
+			expectString: "~,4]",
+			expect: []testCaseExpectation{
+				{0, true},
+				{1, true},
+				{2, true},
+				{3, true},
+				{4, true},
+				{5, false},
+			},
+		},
+		{
+			name:          "edgeSince",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{math.MaxUint64, &zfs.NilBool{B: true}},
+				Until: nil,
+			},
+			expectString: "[18446744073709551615,~",
+			expect: []testCaseExpectation{
+				{math.MaxUint64, true},
+				{math.MaxUint64 - 1, false},
+				{0, false},
+				{1, false},
+			},
+		},
+		{
+			name:          "edgeSinceNegative",
+			expectInvalid: true,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{math.MaxUint64, &zfs.NilBool{B: false}},
+				Until: nil,
+			},
+			expectString: "(18446744073709551615,~",
+		},
+		{
+			name:          "edgeUntil",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Until: &CreateTXGRangeBound{0, &zfs.NilBool{B: true}},
+			},
+			configAllowZeroCreateTXG: true,
+			expectString:             "~,0]",
+			expect: []testCaseExpectation{
+				{0, true},
+				{math.MaxUint64, false},
+				{1, false},
+			},
+		},
+		{
+			name:                     "edgeUntilNegative",
+			expectInvalid:            true,
+			configAllowZeroCreateTXG: true,
+			config: &CreateTXGRange{
+				Until: &CreateTXGRangeBound{0, &zfs.NilBool{B: false}},
+			},
+			expectString: "~,0)",
+		},
+	}
+
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			require.True(t, tc.expectInvalid != (len(tc.expect) > 0), "invalid test config: must either expect invalid or have expectations: %s", tc.name)
+			require.NotEmpty(t, tc.expectString)
+			assert.Equal(t, tc.expectString, tc.config.String())
+
+			save := createTXGRangeBoundAllowCreateTXG0
+			createTXGRangeBoundAllowCreateTXG0 = tc.configAllowZeroCreateTXG
+			defer func() {
+				createTXGRangeBoundAllowCreateTXG0 = save
+			}()
+
+			if tc.expectInvalid {
+				t.Run(tc.name, func(t *testing.T) {
+					assert.Error(t, tc.config.Validate())
+				})
+			} else {
+				for i, e := range tc.expect {
+					t.Run(fmt.Sprint(i), func(t *testing.T) {
+						defer func() {
+							v := recover()
+							if v != nil {
+								t.Fatalf("should not panic: %T %v\n%s", v, v, debug.Stack())
+							}
+						}()
+						assert.Equal(t, e.expect, tc.config.Contains(e.input))
+					})
+				}
+			}
+		})
+	}
+
+}

endpoint/endpoint_zfs_helpers_repr.go

@@ -1,7 +1,6 @@
 package endpoint
 
 import (
-	"context"
 	"fmt"
 	"regexp"
 	"strconv"
@@ -57,53 +56,3 @@ func parseJobAndGuidBookmarkName(fullname string, prefix string) (guid uint64, j
 
 	return guid, jobID, nil
 }
-
-func destroyBookmarksOlderThan(ctx context.Context, fs string, mostRecent *zfs.ZFSSendArgVersion, jobID JobID, filter func(shortname string) (accept bool)) (destroyed []zfs.FilesystemVersion, err error) {
-	if filter == nil {
-		panic(filter)
-	}
-
-	fsp, err := zfs.NewDatasetPath(fs)
-	if err != nil {
-		return nil, errors.Wrap(err, "invalid filesystem path")
-	}
-
-	mostRecentProps, err := mostRecent.ValidateExistsAndGetCheckedProps(ctx, fs)
-	if err != nil {
-		return nil, errors.Wrap(err, "validate most recent version argument")
-	}
-
-	stepBookmarks, err := zfs.ZFSListFilesystemVersions(fsp, zfs.FilterFromClosure(
-		func(t zfs.VersionType, name string) (accept bool, err error) {
-			if t != zfs.Bookmark {
-				return false, nil
-			}
-			return filter(name), nil
-		}))
-	if err != nil {
-		return nil, errors.Wrap(err, "list bookmarks")
-	}
-
-	// cut off all bookmarks prior to mostRecent's CreateTXG
-	var destroy []zfs.FilesystemVersion
-	for _, v := range stepBookmarks {
-		if v.Type != zfs.Bookmark {
-			panic("implementation error")
-		}
-		if !filter(v.Name) {
-			panic("inconsistent filter result")
-		}
-		if v.CreateTXG < mostRecentProps.CreateTXG {
-			destroy = append(destroy, v)
-		}
-	}
-
-	// FIXME use batch destroy, must adopt code to handle bookmarks
-	for _, v := range destroy {
-		if err := zfs.ZFSDestroyIdempotent(ctx, v.ToAbsPath(fsp)); err != nil {
-			return nil, errors.Wrap(err, "destroy bookmark")
-		}
-	}
-
-	return destroy, nil
-}

endpoint/endpoint_zfs_helpers_types.go

@@ -0,0 +1,74 @@
+package endpoint
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type bookmarkBasedAbstraction struct {
+	Type AbstractionType
+	FS   string
+	zfs.FilesystemVersion
+	JobID JobID
+}
+
+func (b bookmarkBasedAbstraction) GetType() AbstractionType { return b.Type }
+func (b bookmarkBasedAbstraction) GetFS() string            { return b.FS }
+func (b bookmarkBasedAbstraction) GetJobID() *JobID         { return &b.JobID }
+func (b bookmarkBasedAbstraction) GetFullPath() string {
+	return fmt.Sprintf("%s#%s", b.FS, b.Name) // TODO use zfs.FilesystemVersion.ToAbsPath
+}
+func (b bookmarkBasedAbstraction) MarshalJSON() ([]byte, error) {
+	return json.Marshal(AbstractionJSON{b})
+}
+func (b bookmarkBasedAbstraction) String() string {
+	return fmt.Sprintf("%s %s", b.Type, b.GetFullPath())
+}
+
+func (b bookmarkBasedAbstraction) GetFilesystemVersion() zfs.FilesystemVersion {
+	return b.FilesystemVersion
+}
+
+func (b bookmarkBasedAbstraction) Destroy(ctx context.Context) error {
+	if err := zfs.ZFSDestroyIdempotent(ctx, b.GetFullPath()); err != nil {
+		return errors.Wrapf(err, "destroy %s: zfs", b)
+	}
+	return nil
+}
+
+type holdBasedAbstraction struct {
+	Type AbstractionType
+	FS   string
+	zfs.FilesystemVersion
+	Tag   string
+	JobID JobID
+}
+
+func (h holdBasedAbstraction) GetType() AbstractionType { return h.Type }
+func (h holdBasedAbstraction) GetFS() string            { return h.FS }
+func (h holdBasedAbstraction) GetJobID() *JobID         { return &h.JobID }
+func (h holdBasedAbstraction) GetFullPath() string {
+	return fmt.Sprintf("%s@%s", h.FS, h.GetName()) // TODO use zfs.FilesystemVersion.ToAbsPath
+}
+func (h holdBasedAbstraction) MarshalJSON() ([]byte, error) {
+	return json.Marshal(AbstractionJSON{h})
+}
+func (h holdBasedAbstraction) String() string {
+	return fmt.Sprintf("%s %q on %s", h.Type, h.Tag, h.GetFullPath())
+}
+
+func (h holdBasedAbstraction) GetFilesystemVersion() zfs.FilesystemVersion {
+	return h.FilesystemVersion
+}
+
+func (h holdBasedAbstraction) Destroy(ctx context.Context) error {
+	if err := zfs.ZFSRelease(ctx, h.Tag, h.GetFullPath()); err != nil {
+		return errors.Wrapf(err, "release %s: zfs", h)
+	}
+	return nil
+}

go.mod

@@ -5,6 +5,7 @@ go 1.12
 require (
 	github.com/fatih/color v1.7.0
 	github.com/gdamore/tcell v1.2.0
+	github.com/gitchander/permutation v0.0.0-20181107151852-9e56b92e9909
 	github.com/go-logfmt/logfmt v0.4.0
 	github.com/go-sql-driver/mysql v1.4.1-0.20190907122137-b2c03bcae3d4
 	github.com/golang/protobuf v1.3.2
@@ -23,15 +24,18 @@ require (
 	github.com/pkg/profile v1.2.1
 	github.com/problame/go-netssh v0.0.0-20191209123953-18d8aa6923c7
 	github.com/prometheus/client_golang v1.2.1
+	github.com/prometheus/common v0.7.0
 	github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 // go1.12 thinks it needs this
 	github.com/spf13/cobra v0.0.2
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.4.0
+	github.com/willf/bitset v1.1.10
 	github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d
 	github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // go1.12 thinks it needs this
 	github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd
 	golang.org/x/net v0.0.0-20190613194153-d28f0bde5980
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/sys v0.0.0-20191026070338-33540a1f6037
+	gonum.org/v1/gonum v0.7.0 // indirect
 	google.golang.org/grpc v1.17.0
 )

go.sum

@@ -5,9 +5,12 @@ github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q
 github.com/OpenPeeDeeP/depguard v0.0.0-20180806142446-a69c782687b2/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
 github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alvaroloes/enumer v1.1.1/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
@@ -28,6 +31,7 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ftrvxmtrx/fd v0.0.0-20150925145434-c6d800382fff h1:zk1wwii7uXmI0znwU+lqg+wFL9G5+vm5I+9rv2let60=
@@ -36,6 +40,8 @@ github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdk
 github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
 github.com/gdamore/tcell v1.2.0 h1:ikixzsxc8K8o3V2/CEmyoEW8mJZaNYQQ3NP3VIQdUe4=
 github.com/gdamore/tcell v1.2.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM=
+github.com/gitchander/permutation v0.0.0-20181107151852-9e56b92e9909 h1:9NC8seTx6/zRmMTAdsHj/uOMi0EGHGQtjyLafBjk77Q=
+github.com/gitchander/permutation v0.0.0-20181107151852-9e56b92e9909/go.mod h1:lP+DW8LR6Rw3ru9Vo2/y/3iiLaLWmofYql/va+7zJOk=
 github.com/go-critic/go-critic v0.3.4/go.mod h1:AHR42Lk/E/aOznsrYdMYeIQS5RH10HZHSqP+rD6AJrc=
 github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -70,6 +76,7 @@ github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
@@ -122,6 +129,7 @@ github.com/jinzhu/copier v0.0.0-20170922082739-db4671f3a9b8/go.mod h1:yL958EeXv8
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -238,6 +246,7 @@ github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOms
 github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
 github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
@@ -272,6 +281,8 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
 github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
 github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc=
+github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d h1:yJIizrfO599ot2kQ6Af1enICnwBD3XoxgX3MrMwot2M=
 github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
@@ -289,6 +300,11 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2 h1:y102fOLFqhV41b+4GPiJoa0k/x+pJcEi2/HB1Y5T6fU=
+golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -328,17 +344,24 @@ golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181205014116-22934f0fdb62/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
+gonum.org/v1/gonum v0.7.0 h1:Hdks0L0hgznZLG9nzXb8vZ0rRvqNvAcgAp84y7Mwkgw=
+gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM=
+gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
+gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
 google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
@@ -349,7 +372,9 @@ google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9M
 google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -368,5 +393,6 @@ mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIa
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
 mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
 mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe/go.mod h1:BnhuWBAqxH3+J5bDybdxgw5ZfS+DsVd4iylsKQePN8o=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
 sourcegraph.com/sqs/pbtypes v1.0.0/go.mod h1:3AciMUv4qUuRHRHhOG4TZOB+72GdPVz5k+c648qsFS4=

main.go

@@ -17,7 +17,7 @@ func init() {
 	cli.AddSubcommand(client.PprofCmd)
 	cli.AddSubcommand(client.TestCmd)
 	cli.AddSubcommand(client.MigrateCmd)
-	cli.AddSubcommand(client.HoldsCmd)
+	cli.AddSubcommand(client.ZFSAbstractionsCmd)
 }
 
 func main() {

platformtest/harness/harness.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/fatih/color"
 	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/logging"
@@ -68,7 +69,9 @@ func doMain() error {
 		logger.Error(err.Error())
 		panic(err)
 	}
-	ctx := platformtest.WithLogger(context.Background(), logger)
+	ctx := context.Background()
+	defer trace.WithTaskFromStackUpdateCtx(&ctx)()
+	ctx = platformtest.WithLogger(ctx, logger)
 	ex := platformtest.NewEx(logger)
 
 	type invocation struct {

platformtest/platformtest_ops.go

@@ -24,6 +24,7 @@ type Stmt interface {
 type Op string
 
 const (
+	Comment         Op = "#"
 	AssertExists    Op = "!E"
 	AssertNotExists Op = "!N"
 	Add             Op = "+"
@@ -102,6 +103,26 @@ func (o *SnapOp) Run(ctx context.Context, e Execer) error {
 	}
 }
 
+type BookmarkOp struct {
+	Op       Op
+	Existing string
+	Bookmark string
+}
+
+func (o *BookmarkOp) Run(ctx context.Context, e Execer) error {
+	switch o.Op {
+	case Add:
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", "bookmark", o.Existing, o.Bookmark)
+	case Del:
+		if o.Existing != "" {
+			panic("existing must be empty for destroy, got " + o.Existing)
+		}
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", "destroy", o.Bookmark)
+	default:
+		panic(o.Op)
+	}
+}
+
 type RunOp struct {
 	RootDS string
 	Script string
@@ -255,16 +276,26 @@ nextLine:
 			op = AssertExists
 		case string(AssertNotExists):
 			op = AssertNotExists
+		case string(Comment):
+			op = Comment
+			continue
 		default:
 			return nil, &LineError{scan.Text(), fmt.Sprintf("invalid op %q", comps.Text())}
 		}
 
-		// FS / SNAP
+		// FS / SNAP / BOOKMARK
 		if err := expectMoreTokens(); err != nil {
 			return nil, err
 		}
 		if strings.ContainsAny(comps.Text(), "@") {
 			stmts = append(stmts, &SnapOp{Op: op, Path: fmt.Sprintf("%s/%s", rootds, comps.Text())})
+		} else if strings.ContainsAny(comps.Text(), "#") {
+			bookmark := fmt.Sprintf("%s/%s", rootds, comps.Text())
+			if err := expectMoreTokens(); err != nil {
+				return nil, err
+			}
+			existing := fmt.Sprintf("%s/%s", rootds, comps.Text())
+			stmts = append(stmts, &BookmarkOp{Op: op, Existing: existing, Bookmark: bookmark})
 		} else {
 			// FS
 			fs := comps.Text()

platformtest/tests/batchDestroy.go

@@ -32,7 +32,7 @@ func BatchDestroy(ctx *platformtest.Context) {
 			Name:       "2",
 		},
 	}
-	zfs.ZFSDestroyFilesystemVersions(reqs)
+	zfs.ZFSDestroyFilesystemVersions(ctx, reqs)
 	if *reqs[0].ErrOut != nil {
 		panic("expecting no error")
 	}

platformtest/tests/batchRelease.go

@@ -1,154 +0,0 @@
-package tests
-
-import (
-	"fmt"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/zrepl/zrepl/platformtest"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type rollupReleaseExpectTags struct {
-	Snap  string
-	Holds map[string]bool
-}
-
-func rollupReleaseTest(ctx *platformtest.Context, cb func(fs string) []rollupReleaseExpectTags) {
-
-	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
-	DESTROYROOT
-	CREATEROOT
-	+  "foo bar"
-	+  "foo bar@1"
-	+  "foo bar@2"
-	+  "foo bar@3"
-	+  "foo bar@4"
-	+  "foo bar@5"
-	+  "foo bar@6"
-	R  zfs hold zrepl_platformtest   "${ROOTDS}/foo bar@1"
-	R  zfs hold zrepl_platformtest_2 "${ROOTDS}/foo bar@2"
-	R  zfs hold zrepl_platformtest   "${ROOTDS}/foo bar@3"
-	R  zfs hold zrepl_platformtest   "${ROOTDS}/foo bar@5"
-	R  zfs hold zrepl_platformtest   "${ROOTDS}/foo bar@6"
-	R  zfs bookmark "${ROOTDS}/foo bar@5" "${ROOTDS}/foo bar#5"
-`)
-
-	fs := fmt.Sprintf("%s/foo bar", ctx.RootDataset)
-
-	expTags := cb(fs)
-
-	for _, exp := range expTags {
-		holds, err := zfs.ZFSHolds(ctx, fs, exp.Snap)
-		if err != nil {
-			panic(err)
-		}
-		for _, h := range holds {
-			if e, ok := exp.Holds[h]; !ok || !e {
-				panic(fmt.Sprintf("tag %q on snap %q not expected", h, exp.Snap))
-			}
-		}
-	}
-
-}
-
-func RollupReleaseIncluding(ctx *platformtest.Context) {
-	rollupReleaseTest(ctx, func(fs string) []rollupReleaseExpectTags {
-		guid5, err := zfs.ZFSGetGUID(ctx, fs, "@5")
-		require.NoError(ctx, err)
-
-		err = zfs.ZFSReleaseAllOlderAndIncludingGUID(ctx, fs, guid5, "zrepl_platformtest")
-		require.NoError(ctx, err)
-
-		return []rollupReleaseExpectTags{
-			{"1", map[string]bool{}},
-			{"2", map[string]bool{"zrepl_platformtest_2": true}},
-			{"3", map[string]bool{}},
-			{"4", map[string]bool{}},
-			{"5", map[string]bool{}},
-			{"6", map[string]bool{"zrepl_platformtest": true}},
-		}
-	})
-}
-
-func RollupReleaseExcluding(ctx *platformtest.Context) {
-	rollupReleaseTest(ctx, func(fs string) []rollupReleaseExpectTags {
-		guid5, err := zfs.ZFSGetGUID(ctx, fs, "@5")
-		require.NoError(ctx, err)
-
-		err = zfs.ZFSReleaseAllOlderThanGUID(ctx, fs, guid5, "zrepl_platformtest")
-		require.NoError(ctx, err)
-
-		return []rollupReleaseExpectTags{
-			{"1", map[string]bool{}},
-			{"2", map[string]bool{"zrepl_platformtest_2": true}},
-			{"3", map[string]bool{}},
-			{"4", map[string]bool{}},
-			{"5", map[string]bool{"zrepl_platformtest": true}},
-			{"6", map[string]bool{"zrepl_platformtest": true}},
-		}
-	})
-}
-
-func RollupReleaseMostRecentIsBookmarkWithoutSnapshot(ctx *platformtest.Context) {
-	rollupReleaseTest(ctx, func(fs string) []rollupReleaseExpectTags {
-		guid5, err := zfs.ZFSGetGUID(ctx, fs, "#5")
-		require.NoError(ctx, err)
-
-		err = zfs.ZFSRelease(ctx, "zrepl_platformtest", fs+"@5")
-		require.NoError(ctx, err)
-
-		err = zfs.ZFSDestroy(ctx, fs+"@5")
-		require.NoError(ctx, err)
-
-		err = zfs.ZFSReleaseAllOlderAndIncludingGUID(ctx, fs, guid5, "zrepl_platformtest")
-		require.NoError(ctx, err)
-
-		return []rollupReleaseExpectTags{
-			{"1", map[string]bool{}},
-			{"2", map[string]bool{"zrepl_platformtest_2": true}},
-			{"3", map[string]bool{}},
-			{"4", map[string]bool{}},
-			// {"5", map[string]bool{}}, doesn't exist
-			{"6", map[string]bool{"zrepl_platformtest": true}},
-		}
-	})
-}
-
-func RollupReleaseMostRecentIsBookmarkAndSnapshotStillExists(ctx *platformtest.Context) {
-	rollupReleaseTest(ctx, func(fs string) []rollupReleaseExpectTags {
-		guid5, err := zfs.ZFSGetGUID(ctx, fs, "#5")
-		require.NoError(ctx, err)
-
-		err = zfs.ZFSReleaseAllOlderAndIncludingGUID(ctx, fs, guid5, "zrepl_platformtest")
-		require.NoError(ctx, err)
-
-		return []rollupReleaseExpectTags{
-			{"1", map[string]bool{}},
-			{"2", map[string]bool{"zrepl_platformtest_2": true}},
-			{"3", map[string]bool{}},
-			{"4", map[string]bool{}},
-			{"5", map[string]bool{}},
-			{"6", map[string]bool{"zrepl_platformtest": true}},
-		}
-	})
-}
-
-func RollupReleaseMostRecentDoesntExist(ctx *platformtest.Context) {
-	rollupReleaseTest(ctx, func(fs string) []rollupReleaseExpectTags {
-
-		const nonexistentGuid = 0 // let's take our chances...
-		err := zfs.ZFSReleaseAllOlderAndIncludingGUID(ctx, fs, nonexistentGuid, "zrepl_platformtest")
-		require.Error(ctx, err)
-		require.Contains(ctx, err.Error(), "cannot find snapshot or bookmark with guid 0")
-
-		return []rollupReleaseExpectTags{
-			{"1", map[string]bool{"zrepl_platformtest": true}},
-			{"2", map[string]bool{"zrepl_platformtest_2": true}},
-			{"3", map[string]bool{"zrepl_platformtest": true}},
-			{"4", map[string]bool{"zrepl_platformtest": true}},
-			{"5", map[string]bool{"zrepl_platformtest": true}},
-			{"6", map[string]bool{"zrepl_platformtest": true}},
-		}
-	})
-}

platformtest/tests/helpers.go

@@ -5,6 +5,7 @@ import (
 	"math/rand"
 	"os"
 	"path"
+	"sort"
 	"strings"
 
 	"github.com/stretchr/testify/require"
@@ -25,6 +26,14 @@ func sendArgVersion(ctx *platformtest.Context, fs, relName string) zfs.ZFSSendAr
 	}
 }
 
+func fsversion(ctx *platformtest.Context, fs, relname string) zfs.FilesystemVersion {
+	v, err := zfs.ZFSGetFilesystemVersion(ctx, fs+relname)
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
 func mustDatasetPath(fs string) *zfs.DatasetPath {
 	p, err := zfs.NewDatasetPath(fs)
 	if err != nil {
@@ -47,10 +56,10 @@ func mustSnapshot(ctx *platformtest.Context, snap string) {
 	}
 }
 
-func mustGetProps(ctx *platformtest.Context, entity string) zfs.ZFSPropCreateTxgAndGuidProps {
+func mustGetFilesystemVersion(ctx *platformtest.Context, snapOrBookmark string) zfs.FilesystemVersion {
-	props, err := zfs.ZFSGetCreateTXGAndGuid(ctx, entity)
+	v, err := zfs.ZFSGetFilesystemVersion(ctx, snapOrBookmark)
 	check(err)
-	return props
+	return v
 }
 
 func check(err error) {
@@ -78,7 +87,7 @@ type dummySnapshotSituation struct {
 }
 
 type resumeSituation struct {
-	sendArgs         zfs.ZFSSendArgs
+	sendArgs         zfs.ZFSSendArgsUnvalidated
 	recvOpts         zfs.RecvOptions
 	sendErr, recvErr error
 	recvErrDecoded   *zfs.RecvFailedWithResumeTokenErr
@@ -107,7 +116,7 @@ func makeDummyDataSnapshots(ctx *platformtest.Context, sendFS string) (situation
 	return situation
 }
 
-func makeResumeSituation(ctx *platformtest.Context, src dummySnapshotSituation, recvFS string, sendArgs zfs.ZFSSendArgs, recvOptions zfs.RecvOptions) *resumeSituation {
+func makeResumeSituation(ctx *platformtest.Context, src dummySnapshotSituation, recvFS string, sendArgs zfs.ZFSSendArgsUnvalidated, recvOptions zfs.RecvOptions) *resumeSituation {
 
 	situation := &resumeSituation{}
 
@@ -115,14 +124,19 @@ func makeResumeSituation(ctx *platformtest.Context, src dummySnapshotSituation,
 	situation.recvOpts = recvOptions
 	require.True(ctx, recvOptions.SavePartialRecvState, "this method would be pointless otherwise")
 	require.Equal(ctx, sendArgs.FS, src.sendFS)
-
+	sendArgsValidated, err := sendArgs.Validate(ctx)
-	copier, err := zfs.ZFSSend(ctx, sendArgs)
 	situation.sendErr = err
 	if err != nil {
 		return situation
 	}
 
-	limitedCopier := zfs.NewReadCloserCopier(limitio.ReadCloser(copier, src.dummyDataLen/2))
+	copier, err := zfs.ZFSSend(ctx, sendArgsValidated)
+	situation.sendErr = err
+	if err != nil {
+		return situation
+	}
+
+	limitedCopier := limitio.ReadCloser(copier, src.dummyDataLen/2)
 	defer limitedCopier.Close()
 
 	require.NotNil(ctx, sendArgs.To)
@@ -137,3 +151,26 @@ func makeResumeSituation(ctx *platformtest.Context, src dummySnapshotSituation,
 
 	return situation
 }
+
+func versionRelnamesSorted(versions []zfs.FilesystemVersion) []string {
+	var vstrs []string
+	for _, v := range versions {
+		vstrs = append(vstrs, v.RelName())
+	}
+	sort.Strings(vstrs)
+	return vstrs
+}
+
+func datasetToStringSortedTrimPrefix(prefix *zfs.DatasetPath, paths []*zfs.DatasetPath) []string {
+	var pstrs []string
+	for _, p := range paths {
+		trimmed := p.Copy()
+		trimmed.TrimPrefix(prefix)
+		if trimmed.Length() == 0 {
+			continue
+		}
+		pstrs = append(pstrs, trimmed.ToString())
+	}
+	sort.Strings(pstrs)
+	return pstrs
+}

platformtest/tests/idempotentHold.go

@@ -22,7 +22,7 @@ func IdempotentHold(ctx *platformtest.Context) {
 	`)
 
 	fs := fmt.Sprintf("%s/foo bar", ctx.RootDataset)
-	v1 := sendArgVersion(ctx, fs, "@1")
+	v1 := fsversion(ctx, fs, "@1")
 
 	tag := "zrepl_platformtest"
 	err := zfs.ZFSHold(ctx, fs, v1, tag)
@@ -34,14 +34,4 @@ func IdempotentHold(ctx *platformtest.Context) {
 	if err != nil {
 		panic(err)
 	}
-
-	vnonexistent := zfs.ZFSSendArgVersion{
-		RelName: "@nonexistent",
-		GUID:    0xbadf00d,
-	}
-	err = zfs.ZFSHold(ctx, fs, vnonexistent, tag)
-	if err == nil {
-		panic("still expecting error for nonexistent snapshot")
-	}
-
 }

platformtest/tests/listFilesystemVersions.go

@@ -0,0 +1,179 @@
+package tests
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func ListFilesystemVersionsTypeFilteringAndPrefix(t *platformtest.Context) {
+	platformtest.Run(t, platformtest.PanicErr, t.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@foo 1"
+		+  "foo bar#foo 1" "foo bar@foo 1"
+		+  "foo bar#bookfoo 1" "foo bar@foo 1"
+		+  "foo bar@foo 2"
+		+  "foo bar#foo 2" "foo bar@foo 2"
+		+  "foo bar#bookfoo 2" "foo bar@foo 2"
+		+  "foo bar@blup 1"
+		+  "foo bar#blup 1" "foo bar@blup 1"
+		+  "foo bar@ foo with leading whitespace"
+
+		# repeat the whole thing for a child dataset to make sure we disable recursion
+
+		+  "foo bar/child dataset"
+		+  "foo bar/child dataset@foo 1"
+		+  "foo bar/child dataset#foo 1" "foo bar/child dataset@foo 1"
+		+  "foo bar/child dataset#bookfoo 1" "foo bar/child dataset@foo 1"
+		+  "foo bar/child dataset@foo 2"
+		+  "foo bar/child dataset#foo 2" "foo bar/child dataset@foo 2"
+		+  "foo bar/child dataset#bookfoo 2" "foo bar/child dataset@foo 2"
+		+  "foo bar/child dataset@blup 1"
+		+  "foo bar/child dataset#blup 1" "foo bar/child dataset@blup 1"
+		+  "foo bar/child dataset@ foo with leading whitespace"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", t.RootDataset)
+
+	// no options := all types
+	vs, err := zfs.ZFSListFilesystemVersions(t, mustDatasetPath(fs), zfs.ListFilesystemVersionsOptions{})
+	require.NoError(t, err)
+	require.Equal(t, []string{
+		"#blup 1", "#bookfoo 1", "#bookfoo 2", "#foo 1", "#foo 2",
+		"@ foo with leading whitespace", "@blup 1", "@foo 1", "@foo 2",
+	}, versionRelnamesSorted(vs))
+
+	// just snapshots
+	vs, err = zfs.ZFSListFilesystemVersions(t, mustDatasetPath(fs), zfs.ListFilesystemVersionsOptions{
+		Types: zfs.Snapshots,
+	})
+	require.NoError(t, err)
+	require.Equal(t, []string{"@ foo with leading whitespace", "@blup 1", "@foo 1", "@foo 2"}, versionRelnamesSorted(vs))
+
+	// just bookmarks
+	vs, err = zfs.ZFSListFilesystemVersions(t, mustDatasetPath(fs), zfs.ListFilesystemVersionsOptions{
+		Types: zfs.Bookmarks,
+	})
+	require.NoError(t, err)
+	require.Equal(t, []string{"#blup 1", "#bookfoo 1", "#bookfoo 2", "#foo 1", "#foo 2"}, versionRelnamesSorted(vs))
+
+	// just with prefix foo
+	vs, err = zfs.ZFSListFilesystemVersions(t, mustDatasetPath(fs), zfs.ListFilesystemVersionsOptions{
+		ShortnamePrefix: "foo",
+	})
+	require.NoError(t, err)
+	require.Equal(t, []string{"#foo 1", "#foo 2", "@foo 1", "@foo 2"}, versionRelnamesSorted(vs))
+
+}
+
+func ListFilesystemVersionsZeroExistIsNotAnError(t *platformtest.Context) {
+	platformtest.Run(t, platformtest.PanicErr, t.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	+ "foo bar"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", t.RootDataset)
+
+	vs, err := zfs.ZFSListFilesystemVersions(t, mustDatasetPath(fs), zfs.ListFilesystemVersionsOptions{})
+	require.Empty(t, vs)
+	require.NoError(t, err)
+	dsne, ok := err.(*zfs.DatasetDoesNotExist)
+	require.True(t, ok)
+	require.Equal(t, fs, dsne.Path)
+}
+
+func ListFilesystemVersionsFilesystemNotExist(t *platformtest.Context) {
+	platformtest.Run(t, platformtest.PanicErr, t.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	`)
+
+	nonexistentFS := fmt.Sprintf("%s/not existent", t.RootDataset)
+
+	vs, err := zfs.ZFSListFilesystemVersions(t, mustDatasetPath(nonexistentFS), zfs.ListFilesystemVersionsOptions{})
+	require.Empty(t, vs)
+	require.Error(t, err)
+	t.Logf("err = %T\n%s", err, err)
+	dsne, ok := err.(*zfs.DatasetDoesNotExist)
+	require.True(t, ok)
+	require.Equal(t, nonexistentFS, dsne.Path)
+}
+
+func ListFilesystemVersionsUserrefs(t *platformtest.Context) {
+	platformtest.Run(t, platformtest.PanicErr, t.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	+ "foo bar"
+	+ "foo bar@snap 1"
+	+ "foo bar#snap 1" "foo bar@snap 1"
+	+ "foo bar@snap 2"
+	+ "foo bar#snap 2" "foo bar@snap 2"
+	R zfs hold zrepl_platformtest "${ROOTDS}/foo bar@snap 2"
+	+ "foo bar@snap 3"
+	+ "foo bar#snap 3" "foo bar@snap 3"
+	R zfs hold zrepl_platformtest "${ROOTDS}/foo bar@snap 3"
+	R zfs hold zrepl_platformtest_second_hold "${ROOTDS}/foo bar@snap 3"
+	+ "foo bar@snap 4"
+	+ "foo bar#snap 4" "foo bar@snap 4"
+
+
+	+ "foo bar/child datset"
+	+ "foo bar/child datset@snap 1"
+	+ "foo bar/child datset#snap 1" "foo bar/child datset@snap 1"
+	+ "foo bar/child datset@snap 2"
+	+ "foo bar/child datset#snap 2" "foo bar/child datset@snap 2"
+	R zfs hold zrepl_platformtest "${ROOTDS}/foo bar/child datset@snap 2"
+	+ "foo bar/child datset@snap 3"
+	+ "foo bar/child datset#snap 3" "foo bar/child datset@snap 3"
+	R zfs hold zrepl_platformtest "${ROOTDS}/foo bar/child datset@snap 3"
+	R zfs hold zrepl_platformtest_second_hold "${ROOTDS}/foo bar/child datset@snap 3"
+	+ "foo bar/child datset@snap 4"
+	+ "foo bar/child datset#snap 4" "foo bar/child datset@snap 4"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", t.RootDataset)
+
+	vs, err := zfs.ZFSListFilesystemVersions(t, mustDatasetPath(fs), zfs.ListFilesystemVersionsOptions{})
+	require.NoError(t, err)
+
+	type expectation struct {
+		relName  string
+		userrefs zfs.OptionUint64
+	}
+
+	expect := []expectation{
+		{"#snap 1", zfs.OptionUint64{Valid: false}},
+		{"#snap 2", zfs.OptionUint64{Valid: false}},
+		{"#snap 3", zfs.OptionUint64{Valid: false}},
+		{"#snap 4", zfs.OptionUint64{Valid: false}},
+		{"@snap 1", zfs.OptionUint64{Value: 0, Valid: true}},
+		{"@snap 2", zfs.OptionUint64{Value: 1, Valid: true}},
+		{"@snap 3", zfs.OptionUint64{Value: 2, Valid: true}},
+		{"@snap 4", zfs.OptionUint64{Value: 0, Valid: true}},
+	}
+
+	sort.Slice(vs, func(i, j int) bool {
+		return strings.Compare(vs[i].RelName(), vs[j].RelName()) < 0
+	})
+
+	var expectRelNames []string
+	for _, e := range expect {
+		expectRelNames = append(expectRelNames, e.relName)
+	}
+
+	require.Equal(t, expectRelNames, versionRelnamesSorted(vs))
+
+	for i, e := range expect {
+		require.Equal(t, e.relName, vs[i].RelName())
+		require.Equal(t, e.userrefs, vs[i].UserRefs)
+	}
+
+}

platformtest/tests/listFilesystems.go

@@ -0,0 +1,34 @@
+package tests
+
+import (
+	"strings"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func ListFilesystemsNoFilter(t *platformtest.Context) {
+	platformtest.Run(t, platformtest.PanicErr, t.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		R  zfs create -V 10M "${ROOTDS}/bar baz"
+		+  "foo bar"
+		+  "foo bar/bar blup"
+		+  "foo bar/blah"
+		R  zfs create -V 10M "${ROOTDS}/foo bar/blah/a volume"
+	`)
+
+	fss, err := zfs.ZFSListMapping(t, zfs.NoFilter())
+	require.NoError(t, err)
+	var onlyTestPool []*zfs.DatasetPath
+	for _, fs := range fss {
+		if strings.HasPrefix(fs.ToString(), t.RootDataset) {
+			onlyTestPool = append(onlyTestPool, fs)
+		}
+	}
+	onlyTestPoolStr := datasetToStringSortedTrimPrefix(mustDatasetPath(t.RootDataset), onlyTestPool)
+	require.Equal(t, []string{"bar baz", "foo bar", "foo bar/bar blup", "foo bar/blah", "foo bar/blah/a volume"}, onlyTestPoolStr)
+
+}

platformtest/tests/replicationCursor.go

@@ -32,7 +32,7 @@ func ReplicationCursor(ctx *platformtest.Context) {
 	}
 
 	fs := ds.ToString()
-	snap := sendArgVersion(ctx, fs, "@1 with space")
+	snap := fsversion(ctx, fs, "@1 with space")
 
 	destroyed, err := endpoint.MoveReplicationCursor(ctx, fs, &snap, jobid)
 	if err != nil {
@@ -40,7 +40,7 @@ func ReplicationCursor(ctx *platformtest.Context) {
 	}
 	assert.Empty(ctx, destroyed)
 
-	snapProps, err := zfs.ZFSGetCreateTXGAndGuid(ctx, snap.FullPath(fs))
+	snapProps, err := zfs.ZFSGetFilesystemVersion(ctx, snap.FullPath(fs))
 	if err != nil {
 		panic(err)
 	}
@@ -57,13 +57,13 @@ func ReplicationCursor(ctx *platformtest.Context) {
 	}
 
 	// try moving
-	cursor1BookmarkName, err := endpoint.ReplicationCursorBookmarkName(fs, snap.GUID, jobid)
+	cursor1BookmarkName, err := endpoint.ReplicationCursorBookmarkName(fs, snap.Guid, jobid)
 	require.NoError(ctx, err)
 
-	snap2 := sendArgVersion(ctx, fs, "@2 with space")
+	snap2 := fsversion(ctx, fs, "@2 with space")
 	destroyed, err = endpoint.MoveReplicationCursor(ctx, fs, &snap2, jobid)
 	require.NoError(ctx, err)
 	require.Equal(ctx, 1, len(destroyed))
-	require.Equal(ctx, zfs.Bookmark, destroyed[0].Type)
+	require.Equal(ctx, endpoint.AbstractionReplicationCursorBookmarkV2, destroyed[0].GetType())
-	require.Equal(ctx, cursor1BookmarkName, destroyed[0].Name)
+	require.Equal(ctx, cursor1BookmarkName, destroyed[0].GetName())
 }

platformtest/tests/resumableRecvAndTokenHandling.go

@@ -25,7 +25,7 @@ func ResumableRecvAndTokenHandling(ctx *platformtest.Context) {
 
 	src := makeDummyDataSnapshots(ctx, sendFS)
 
-	s := makeResumeSituation(ctx, src, recvFS, zfs.ZFSSendArgs{
+	s := makeResumeSituation(ctx, src, recvFS, zfs.ZFSSendArgsUnvalidated{
 		FS:          sendFS,
 		To:          src.snapA,
 		Encrypted:   &zfs.NilBool{B: false},

platformtest/tests/sendArgsValidation.go

@@ -23,9 +23,9 @@ func SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden(ctx *platformt
 	`)
 
 	fs := fmt.Sprintf("%s/send er", ctx.RootDataset)
-	props := mustGetProps(ctx, fs+"@a snap")
+	props := mustGetFilesystemVersion(ctx, fs+"@a snap")
 
-	sendArgs := zfs.ZFSSendArgs{
+	sendArgs, err := zfs.ZFSSendArgsUnvalidated{
 		FS: fs,
 		To: &zfs.ZFSSendArgVersion{
 			RelName: "@a snap",
@@ -33,10 +33,15 @@ func SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden(ctx *platformt
 		},
 		Encrypted:   &zfs.NilBool{B: true},
 		ResumeToken: "",
-	}
+	}.Validate(ctx)
-	stream, err := zfs.ZFSSend(ctx, sendArgs)
+
+	var stream *zfs.SendStream
 	if err == nil {
-		defer stream.Close()
+		stream, err = zfs.ZFSSend(ctx, sendArgs) // no shadow
+		if err == nil {
+			defer stream.Close()
+		}
+		// fallthrough
 	}
 
 	if expectNotSupportedErr {
@@ -76,7 +81,7 @@ func SendArgsValidationResumeTokenEncryptionMismatchForbidden(ctx *platformtest.
 
 	src := makeDummyDataSnapshots(ctx, sendFS)
 
-	unencS := makeResumeSituation(ctx, src, unencRecvFS, zfs.ZFSSendArgs{
+	unencS := makeResumeSituation(ctx, src, unencRecvFS, zfs.ZFSSendArgsUnvalidated{
 		FS:        sendFS,
 		To:        src.snapA,
 		Encrypted: &zfs.NilBool{B: false}, // !
@@ -85,7 +90,7 @@ func SendArgsValidationResumeTokenEncryptionMismatchForbidden(ctx *platformtest.
 		SavePartialRecvState: true,
 	})
 
-	encS := makeResumeSituation(ctx, src, encRecvFS, zfs.ZFSSendArgs{
+	encS := makeResumeSituation(ctx, src, encRecvFS, zfs.ZFSSendArgsUnvalidated{
 		FS:        sendFS,
 		To:        src.snapA,
 		Encrypted: &zfs.NilBool{B: true}, // !
@@ -97,16 +102,10 @@ func SendArgsValidationResumeTokenEncryptionMismatchForbidden(ctx *platformtest.
 	// threat model: use of a crafted resume token that requests an unencrypted send
 	//               but send args require encrypted send
 	{
-		var maliciousSend zfs.ZFSSendArgs = encS.sendArgs
+		var maliciousSend zfs.ZFSSendArgsUnvalidated = encS.sendArgs
 		maliciousSend.ResumeToken = unencS.recvErrDecoded.ResumeTokenRaw
 
-		stream, err := zfs.ZFSSend(ctx, maliciousSend)
+		_, err := maliciousSend.Validate(ctx)
-		if err == nil {
-			defer stream.Close()
-		}
-		require.Nil(ctx, stream)
-		require.Error(ctx, err)
-		ctx.Logf("send err: %T %s", err, err)
 		validationErr, ok := err.(*zfs.ZFSSendArgsValidationError)
 		require.True(ctx, ok)
 		require.Equal(ctx, validationErr.What, zfs.ZFSSendArgsResumeTokenMismatch)
@@ -120,14 +119,10 @@ func SendArgsValidationResumeTokenEncryptionMismatchForbidden(ctx *platformtest.
 	// threat model: use of a crafted resume token that requests an encrypted send
 	//               but send args require unencrypted send
 	{
-		var maliciousSend zfs.ZFSSendArgs = unencS.sendArgs
+		var maliciousSend zfs.ZFSSendArgsUnvalidated = unencS.sendArgs
 		maliciousSend.ResumeToken = encS.recvErrDecoded.ResumeTokenRaw
 
-		stream, err := zfs.ZFSSend(ctx, maliciousSend)
+		_, err := maliciousSend.Validate(ctx)
-		if err == nil {
-			defer stream.Close()
-		}
-		require.Nil(ctx, stream)
 		require.Error(ctx, err)
 		ctx.Logf("send err: %T %s", err, err)
 		validationErr, ok := err.(*zfs.ZFSSendArgsValidationError)
@@ -169,7 +164,7 @@ func SendArgsValidationResumeTokenDifferentFilesystemForbidden(ctx *platformtest
 	src1 := makeDummyDataSnapshots(ctx, sendFS1)
 	src2 := makeDummyDataSnapshots(ctx, sendFS2)
 
-	rs := makeResumeSituation(ctx, src1, recvFS, zfs.ZFSSendArgs{
+	rs := makeResumeSituation(ctx, src1, recvFS, zfs.ZFSSendArgsUnvalidated{
 		FS:        sendFS1,
 		To:        src1.snapA,
 		Encrypted: &zfs.NilBool{B: false},
@@ -180,7 +175,7 @@ func SendArgsValidationResumeTokenDifferentFilesystemForbidden(ctx *platformtest
 
 	// threat model: forged resume token tries to steal a full send of snapA on fs2 by
 	//               presenting a resume token for full send of snapA on fs1
-	var maliciousSend zfs.ZFSSendArgs = zfs.ZFSSendArgs{
+	var maliciousSend zfs.ZFSSendArgsUnvalidated = zfs.ZFSSendArgsUnvalidated{
 		FS: sendFS2,
 		To: &zfs.ZFSSendArgVersion{
 			RelName: src2.snapA.RelName,
@@ -189,12 +184,7 @@ func SendArgsValidationResumeTokenDifferentFilesystemForbidden(ctx *platformtest
 		Encrypted:   &zfs.NilBool{B: false},
 		ResumeToken: rs.recvErrDecoded.ResumeTokenRaw,
 	}
-
+	_, err = maliciousSend.Validate(ctx)
-	stream, err := zfs.ZFSSend(ctx, maliciousSend)
-	if err == nil {
-		defer stream.Close()
-	}
-	require.Nil(ctx, stream)
 	require.Error(ctx, err)
 	ctx.Logf("send err: %T %s", err, err)
 	validationErr, ok := err.(*zfs.ZFSSendArgsValidationError)

platformtest/tests/tests.go

@@ -18,11 +18,6 @@ var Cases = []Case{
 	UndestroyableSnapshotParsing,
 	GetNonexistent,
 	ReplicationCursor,
-	RollupReleaseIncluding,
-	RollupReleaseExcluding,
-	RollupReleaseMostRecentIsBookmarkWithoutSnapshot,
-	RollupReleaseMostRecentIsBookmarkAndSnapshotStillExists,
-	RollupReleaseMostRecentDoesntExist,
 	IdempotentHold,
 	IdempotentBookmark,
 	IdempotentDestroy,
@@ -31,4 +26,9 @@ var Cases = []Case{
 	SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden,
 	SendArgsValidationResumeTokenEncryptionMismatchForbidden,
 	SendArgsValidationResumeTokenDifferentFilesystemForbidden,
+	ListFilesystemVersionsTypeFilteringAndPrefix,
+	ListFilesystemVersionsFilesystemNotExist,
+	ListFilesystemVersionsFilesystemNotExist,
+	ListFilesystemVersionsUserrefs,
+	ListFilesystemsNoFilter,
 }

replication/design.md

@@ -18,7 +18,32 @@ Hence, when trying to map algorithm to implementation, use the code in package `
       * the recv-side fs doesn't get newer snapshots than `to` in the meantime
         * guaranteed because the zrepl model of the receiver assumes ownership of the filesystems it receives into
           * if that assumption is broken, future replication attempts will fail with a conflict
-* The [Algorithm for Planning and Executing Replication of an Filesystems](#zrepl-algo-filesystem) is a design draft and not used
+* The [Algorithm for Planning and Executing Replication of an Filesystems](#zrepl-algo-filesystem) is a design draft and not used.
+  However, there were some noteworthy lessons learned when implementing the algorithm for a single step:
+  * In order to avoid leaking `step-hold`s and `step-bookmarks`, if the replication planner is invoked a second time after a replication step (either initial or incremental) has been attempted but failed to completed, the replication planner must
+    * A) either guarantee that it will resume that replication step, and continue as if nothing happened or
+    * B) release the step holds and bookmarks and clear the partially received state on the sending side.
+  * Option A is what we want to do: we use the step algorithm to achieve resumability in the first place!
+  * Option B is not done by zrepl except if the sending side doesn't support resuming.
+    In that case however, we need not release any holds since the behavior is to re-start the send
+    from the beginning.
+  * However, there is one **edge-case to Option A for initial replication**:
+    * If initial replication "`full a`" fails without leaving resumable state, the step holds on the sending side are still present, which makes sense because
+      * a) we want resumability and
+      * b) **the sending side cannot immediately be informed post-failure whether the initial replication left any state that would mandate keeping the step hold**, because the network connection might have failed.
+    * Thus, the sending side must keep the step hold for "`full a`" until it knows more.
+    * In the current implementation, it knows more when the next replication attempt is made, the planner is invoked, the diffing algorithm run, and the `HintMostRecentCommonAncestor` RPC is sent by the active side, communicating the most recent common version shared betwen sender and receiver.
+    * At this point, the sender can safely throw away any step holds with CreateTXG's older than that version.
+  * **The `step-hold`, `step-bookmark`, `last-received-hold` and `replication-cursor` abstractions are currently local concepts of package `endpoint` and not part of the replication protocol**
+    * This is not necessarilty the best design decision and should be revisited some point:
+      * The (quite expensive) `HintMostRecentCommonAncestor` RPC impl on the sender would not be necessary if step holds were part of the replication protocol:
+        * We only need the `HintMostRecentCommonAncestor` info for the aforementioned edge-case during initial replication, where the receive is aborted without any partial received state being stored on the receiver (due to network failure, wrong zfs invocation, bad permissions, etc):
+        * **The replication planner does not know about the step holds, thus it cannot deterministically pick up where it left of (right at the start of the last failing initial replication).**
+        * Instead, it will seem like no prior invocation happened at all, and it will apply its policy for initial replication to pick a new `full b != full a`, **thereby leaking the step holds of `full a`**.
+        * In contrast, if the replication planner created the step holds and knew about them, it could use the step holds as an indicator where it left off and re-start from there (of course asserting that the thereby inferred step is compatible with the state of the receiving side).
+      * (What we do in zrepl right now is to hard-code the initial replication policy, and hard-code that assumption in `endpoint.ListStale` as well.)
+      * The cummulative cleanup done in `HintMostRecentCommonAncestor` provides a nice self-healing aspect, though.
+
 * We also have [Notes on Planning and Executing Replication of Multiple Filesystems](#zrepl-algo-multiple-filesystems-notes)
 
 ---

replication/driver/replication_driver.go

@@ -10,6 +10,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
@@ -146,6 +147,12 @@ type fs struct {
 
 	l *chainlock.L
 
+	// ordering relationship that must be maintained for initial replication
+	initialRepOrd struct {
+		parents, children []*fs
+		parentDidUpdate   chan struct{}
+	}
+
 	planning struct {
 		done bool
 		err  *timedError
@@ -281,6 +288,17 @@ func Do(ctx context.Context, planner Planner) (ReportFunc, WaitFunc) {
 }
 
 func (a *attempt) do(ctx context.Context, prev *attempt) {
+	prevs := a.doGlobalPlanning(ctx, prev)
+	if prevs == nil {
+		return
+	}
+	a.doFilesystems(ctx, prevs)
+}
+
+// if no error occurs, returns a map that maps this attempt's a.fss to `prev`'s a.fss
+func (a *attempt) doGlobalPlanning(ctx context.Context, prev *attempt) map[*fs]*fs {
+	ctx, endSpan := trace.WithSpan(ctx, "plan")
+	defer endSpan()
 	pfss, err := a.planner.Plan(ctx)
 	errTime := time.Now()
 	defer a.l.Lock().Unlock()
@@ -288,7 +306,7 @@ func (a *attempt) do(ctx context.Context, prev *attempt) {
 		a.planErr = newTimedError(err, errTime)
 		a.fss = nil
 		a.finishedAt = time.Now()
-		return
+		return nil
 	}
 
 	for _, pfs := range pfss {
@@ -296,6 +314,7 @@ func (a *attempt) do(ctx context.Context, prev *attempt) {
 			fs: pfs,
 			l:  a.l,
 		}
+		fs.initialRepOrd.parentDidUpdate = make(chan struct{}, 1)
 		a.fss = append(a.fss, fs)
 	}
 
@@ -344,7 +363,7 @@ func (a *attempt) do(ctx context.Context, prev *attempt) {
 			a.planErr = newTimedError(errors.New(msg.String()), now)
 			a.fss = nil
 			a.finishedAt = now
-			return
+			return nil
 		}
 		for cur, fss := range prevFSs {
 			if len(fss) > 0 {
@@ -354,13 +373,37 @@ func (a *attempt) do(ctx context.Context, prev *attempt) {
 	}
 	// invariant: prevs contains an entry for each unambiguous correspondence
 
+	// build up parent-child relationship (FIXME (O(n^2), but who's going to have that many filesystems...))
+	for _, f1 := range a.fss {
+		fs1 := f1.fs.ReportInfo().Name
+		for _, f2 := range a.fss {
+			fs2 := f2.fs.ReportInfo().Name
+			if strings.HasPrefix(fs1, fs2) && fs1 != fs2 {
+				f1.initialRepOrd.parents = append(f1.initialRepOrd.parents, f2)
+				f2.initialRepOrd.children = append(f2.initialRepOrd.children, f1)
+			}
+		}
+	}
+
+	return prevs
+}
+
+func (a *attempt) doFilesystems(ctx context.Context, prevs map[*fs]*fs) {
+	ctx, endSpan := trace.WithSpan(ctx, "do-repl")
+	defer endSpan()
+
+	defer a.l.Lock().Unlock()
+
 	stepQueue := newStepQueue()
-	defer stepQueue.Start(1)() // TODO parallel replication
+	defer stepQueue.Start(envconst.Int("ZREPL_REPLICATION_EXPERIMENTAL_REPLICATION_CONCURRENCY", 1))() // TODO parallel replication
 	var fssesDone sync.WaitGroup
 	for _, f := range a.fss {
 		fssesDone.Add(1)
 		go func(f *fs) {
 			defer fssesDone.Done()
+			// avoid explosion of tasks with name f.report().Info.Name
+			ctx, endTask := trace.WithTaskAndSpan(ctx, "repl-fs", f.report().Info.Name)
+			defer endTask()
 			f.do(ctx, stepQueue, prevs[f])
 		}(f)
 	}
@@ -370,54 +413,76 @@ func (a *attempt) do(ctx context.Context, prev *attempt) {
 	a.finishedAt = time.Now()
 }
 
-func (fs *fs) do(ctx context.Context, pq *stepQueue, prev *fs) {
+func (f *fs) debug(format string, args ...interface{}) {
+	debugPrefix("fs=%s", f.fs.ReportInfo().Name)(format, args...)
+}
 
-	defer fs.l.Lock().Unlock()
+// wake up children that watch for f.{planning.{err,done},planned.{step,stepErr}}
+func (f *fs) initialRepOrdWakeupChildren() {
+	var children []string
+	for _, c := range f.initialRepOrd.children {
+		// no locking required, c.fs does not change
+		children = append(children, c.fs.ReportInfo().Name)
+	}
+	f.debug("wakeup children %s", children)
+	for _, child := range f.initialRepOrd.children {
+		select {
+		// no locking required, child.initialRepOrd does not change
+		case child.initialRepOrd.parentDidUpdate <- struct{}{}:
+		default:
+		}
+	}
+}
+
+func (f *fs) do(ctx context.Context, pq *stepQueue, prev *fs) {
+
+	defer f.l.Lock().Unlock()
+	defer f.initialRepOrdWakeupChildren()
 
 	// get planned steps from replication logic
 	var psteps []Step
 	var errTime time.Time
 	var err error
-	fs.l.DropWhile(func() {
+	f.l.DropWhile(func() {
 		// TODO hacky
 		// choose target time that is earlier than any snapshot, so fs planning is always prioritized
 		targetDate := time.Unix(0, 0)
-		defer pq.WaitReady(fs, targetDate)()
+		defer pq.WaitReady(ctx, f, targetDate)()
-		psteps, err = fs.fs.PlanFS(ctx) // no shadow
+		psteps, err = f.fs.PlanFS(ctx) // no shadow
-		errTime = time.Now()            // no shadow
+		errTime = time.Now()           // no shadow
 	})
-	debug := debugPrefix("fs=%s", fs.fs.ReportInfo().Name)
-	fs.planning.done = true
 	if err != nil {
-		fs.planning.err = newTimedError(err, errTime)
+		f.planning.err = newTimedError(err, errTime)
 		return
 	}
 	for _, pstep := range psteps {
 		step := &step{
-			l:    fs.l,
+			l:    f.l,
 			step: pstep,
 		}
-		fs.planned.steps = append(fs.planned.steps, step)
+		f.planned.steps = append(f.planned.steps, step)
 	}
-	debug("initial len(fs.planned.steps) = %d", len(fs.planned.steps))
+	// we're not done planning yet, f.planned.steps might still be changed by next block
+	// => don't set f.planning.done just yet
+	f.debug("initial len(fs.planned.steps) = %d", len(f.planned.steps))
 
 	// for not-first attempts, only allow fs.planned.steps
 	// up to including the originally planned target snapshot
 	if prev != nil && prev.planning.done && prev.planning.err == nil {
 		prevUncompleted := prev.planned.steps[prev.planned.step:]
 		if len(prevUncompleted) == 0 {
-			debug("prevUncompleted is empty")
+			f.debug("prevUncompleted is empty")
 			return
 		}
-		if len(fs.planned.steps) == 0 {
+		if len(f.planned.steps) == 0 {
-			debug("fs.planned.steps is empty")
+			f.debug("fs.planned.steps is empty")
 			return
 		}
 		prevFailed := prevUncompleted[0]
-		curFirst := fs.planned.steps[0]
+		curFirst := f.planned.steps[0]
 		// we assume that PlanFS retries prevFailed (using curFirst)
 		if !prevFailed.step.TargetEquals(curFirst.step) {
-			debug("Targets don't match")
+			f.debug("Targets don't match")
 			// Two options:
 			// A: planning algorithm is broken
 			// B: manual user intervention inbetween
@@ -433,44 +498,132 @@ func (fs *fs) do(ctx context.Context, pq *stepQueue, prev *fs) {
 			}
 			msg := fmt.Sprintf("last attempt's uncompleted step %s does not correspond to this attempt's first planned step %s",
 				stepFmt(prevFailed), stepFmt(curFirst))
-			fs.planned.stepErr = newTimedError(errors.New(msg), time.Now())
+			f.planned.stepErr = newTimedError(errors.New(msg), time.Now())
 			return
 		}
 		// only allow until step targets diverge
 		min := len(prevUncompleted)
-		if min > len(fs.planned.steps) {
+		if min > len(f.planned.steps) {
-			min = len(fs.planned.steps)
+			min = len(f.planned.steps)
 		}
 		diverge := 0
 		for ; diverge < min; diverge++ {
-			debug("diverge compare iteration %d", diverge)
+			f.debug("diverge compare iteration %d", diverge)
-			if !fs.planned.steps[diverge].step.TargetEquals(prevUncompleted[diverge].step) {
+			if !f.planned.steps[diverge].step.TargetEquals(prevUncompleted[diverge].step) {
 				break
 			}
 		}
-		debug("diverge is %d", diverge)
+		f.debug("diverge is %d", diverge)
-		fs.planned.steps = fs.planned.steps[0:diverge]
+		f.planned.steps = f.planned.steps[0:diverge]
 	}
-	debug("post-prev-merge len(fs.planned.steps) = %d", len(fs.planned.steps))
+	f.debug("post-prev-merge len(fs.planned.steps) = %d", len(f.planned.steps))
 
-	for i, s := range fs.planned.steps {
+	// now we are done planning (f.planned.steps won't change from now on)
-		var (
+	f.planning.done = true
-			err     error
+
-			errTime time.Time
+	// wait for parents' initial replication
-		)
+	var parents []string
-		// lock must not be held while executing step in order for reporting to work
+	for _, p := range f.initialRepOrd.parents {
-		fs.l.DropWhile(func() {
+		parents = append(parents, p.fs.ReportInfo().Name)
-			targetDate := s.step.TargetDate()
+	}
-			defer pq.WaitReady(fs, targetDate)()
+	f.debug("wait for parents %s", parents)
-			err = s.step.Step(ctx) // no shadow
+	for {
-			errTime = time.Now()   // no shadow
+		var initialReplicatingParentsWithErrors []string
+		allParentsPresentOnReceiver := true
+		f.l.DropWhile(func() {
+			for _, p := range f.initialRepOrd.parents {
+				p.l.HoldWhile(func() {
+					// (get the preconditions that allow us to inspect p.planned)
+					parentHasPlanningDone := p.planning.done && p.planning.err == nil
+					if !parentHasPlanningDone {
+						// if the parent couldn't be planned, we cannot know whether it needs initial replication
+						// or incremental replication => be conservative and assume it was initial replication
+						allParentsPresentOnReceiver = false
+						if p.planning.err != nil {
+							initialReplicatingParentsWithErrors = append(initialReplicatingParentsWithErrors, p.fs.ReportInfo().Name)
+						}
+						return
+					}
+					// now allowed to inspect p.planned
+
+					// if there are no steps to be done, the filesystem must exist on the receiving side
+					// (otherwise we'd replicate it, and there would be a step for that)
+					// (FIXME hardcoded initial replication policy, assuming the policy will always do _some_ initial replication)
+					parentHasNoSteps := len(p.planned.steps) == 0
+
+					// OR if it has completed at least one step
+					// (remember that .step points to the next step to be done)
+					// (TODO technically, we could make this step ready in the moment the recv-side
+					//  dataset exists, i.e. after the first few megabytes of transferred data, but we'd have to ask the receiver for that -> poll ListFilesystems RPC)
+					parentHasTakenAtLeastOneSuccessfulStep := !parentHasNoSteps && p.planned.step >= 1
+
+					parentFirstStepIsIncremental := // no need to lock for .report() because step.l == it's fs.l
+						len(p.planned.steps) > 0 && p.planned.steps[0].report().IsIncremental()
+
+					f.debug("parentHasNoSteps=%v parentFirstStepIsIncremental=%v parentHasTakenAtLeastOneSuccessfulStep=%v",
+						parentHasNoSteps, parentFirstStepIsIncremental, parentHasTakenAtLeastOneSuccessfulStep)
+
+					parentPresentOnReceiver := parentHasNoSteps || parentFirstStepIsIncremental || parentHasTakenAtLeastOneSuccessfulStep
+
+					allParentsPresentOnReceiver = allParentsPresentOnReceiver && parentPresentOnReceiver // no shadow
+
+					if !parentPresentOnReceiver && p.planned.stepErr != nil {
+						initialReplicatingParentsWithErrors = append(initialReplicatingParentsWithErrors, p.fs.ReportInfo().Name)
+					}
+
+				})
+			}
 		})
+
+		if len(initialReplicatingParentsWithErrors) > 0 {
+			f.planned.stepErr = newTimedError(fmt.Errorf("parent(s) failed during initial replication: %s", initialReplicatingParentsWithErrors), time.Now())
+			return
+		}
+
+		if allParentsPresentOnReceiver {
+			break // good to go
+		}
+
+		// wait for wakeups from parents, then check again
+		// lock must not be held while waiting in order for reporting to work
+		f.l.DropWhile(func() {
+			select {
+			case <-ctx.Done():
+				f.planned.stepErr = newTimedError(ctx.Err(), time.Now())
+				return
+			case <-f.initialRepOrd.parentDidUpdate:
+				// loop
+			}
+		})
+		if f.planned.stepErr != nil {
+			return
+		}
+	}
+
+	f.debug("all parents ready, start replication %s", parents)
+
+	// do our steps
+	for i, s := range f.planned.steps {
+		// lock must not be held while executing step in order for reporting to work
+		f.l.DropWhile(func() {
+			// wait for parallel replication
+			targetDate := s.step.TargetDate()
+			defer pq.WaitReady(ctx, f, targetDate)()
+			// do the step
+			ctx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("%#v", s.step.ReportInfo()))
+			defer endSpan()
+			err, errTime = s.step.Step(ctx), time.Now() // no shadow
+		})
+
 		if err != nil {
-			fs.planned.stepErr = newTimedError(err, errTime)
+			f.planned.stepErr = newTimedError(err, errTime)
 			break
 		}
-		fs.planned.step = i + 1 // fs.planned.step must be == len(fs.planned.steps) if all went OK
+		f.planned.step = i + 1 // fs.planned.step must be == len(fs.planned.steps) if all went OK
+
+		f.initialRepOrdWakeupChildren()
 	}
+
 }
 
 // caller must hold lock l

replication/driver/replication_driver_debug.go

@@ -26,6 +26,6 @@ type debugFunc func(format string, args ...interface{})
 func debugPrefix(prefixFormat string, prefixFormatArgs ...interface{}) debugFunc {
 	prefix := fmt.Sprintf(prefixFormat, prefixFormatArgs...)
 	return func(format string, args ...interface{}) {
-		debug("%s: %s", prefix, fmt.Sprintf(format, args))
+		debug("%s: %s", prefix, fmt.Sprintf(format, args...))
 	}
 }

replication/driver/replication_driver_logging.go

@@ -3,23 +3,10 @@ package driver
 import (
 	"context"
 
+	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
 )
 
-type Logger = logger.Logger
+func getLog(ctx context.Context) logger.Logger {
-
+	return logging.GetLogger(ctx, logging.SubsysReplication)
-type contextKey int
-
-const contextKeyLogger contextKey = iota + 1
-
-func getLog(ctx context.Context) Logger {
-	l, ok := ctx.Value(contextKeyLogger).(Logger)
-	if !ok {
-		l = logger.NewNullLogger()
-	}
-	return l
-}
-
-func WithLogger(ctx context.Context, log Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLogger, log)
 }

replication/driver/replication_driver_test.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/replication/report"
 
@@ -149,6 +150,7 @@ func (f *mockStep) ReportInfo() *report.StepInfo {
 func TestReplication(t *testing.T) {
 
 	ctx := context.Background()
+	defer trace.WithTaskFromStackUpdateCtx(&ctx)()
 
 	mp := &mockPlanner{}
 	getReport, wait := Do(ctx, mp)

replication/driver/replication_stepqueue.go

@@ -2,8 +2,10 @@ package driver
 
 import (
 	"container/heap"
+	"context"
 	"time"
 
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 	"github.com/zrepl/zrepl/util/chainlock"
 )
 
@@ -155,7 +157,8 @@ func (q *stepQueue) sendAndWaitForWakeup(ident interface{}, targetDate time.Time
 }
 
 // Wait for the ident with targetDate to be selected to run.
-func (q *stepQueue) WaitReady(ident interface{}, targetDate time.Time) StepCompletedFunc {
+func (q *stepQueue) WaitReady(ctx context.Context, ident interface{}, targetDate time.Time) StepCompletedFunc {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
 	if targetDate.IsZero() {
 		panic("targetDate of zero is reserved for marking Done")
 	}

replication/driver/replication_stepqueue_test.go

@@ -1,6 +1,7 @@
 package driver
 
 import (
+	"context"
 	"fmt"
 	"math"
 	"sort"
@@ -11,18 +12,23 @@ import (
 
 	"github.com/montanaflynn/stats"
 	"github.com/stretchr/testify/assert"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 )
 
 // FIXME: this test relies on timing and is thus rather flaky
 // (relies on scheduler responsiveness of < 500ms)
 func TestPqNotconcurrent(t *testing.T) {
+	ctx, end := trace.WithTaskFromStack(context.Background())
+	defer end()
 	var ctr uint32
 	q := newStepQueue()
 	var wg sync.WaitGroup
 	wg.Add(4)
 	go func() {
+		ctx, end := trace.WithTaskFromStack(ctx)
+		defer end()
 		defer wg.Done()
-		defer q.WaitReady("1", time.Unix(9999, 0))()
+		defer q.WaitReady(ctx, "1", time.Unix(9999, 0))()
 		ret := atomic.AddUint32(&ctr, 1)
 		assert.Equal(t, uint32(1), ret)
 		time.Sleep(1 * time.Second)
@@ -34,20 +40,26 @@ func TestPqNotconcurrent(t *testing.T) {
 
 	// while "1" is still running, queue in "2", "3" and "4"
 	go func() {
+		ctx, end := trace.WithTaskFromStack(ctx)
+		defer end()
 		defer wg.Done()
-		defer q.WaitReady("2", time.Unix(2, 0))()
+		defer q.WaitReady(ctx, "2", time.Unix(2, 0))()
 		ret := atomic.AddUint32(&ctr, 1)
 		assert.Equal(t, uint32(2), ret)
 	}()
 	go func() {
+		ctx, end := trace.WithTaskFromStack(ctx)
+		defer end()
 		defer wg.Done()
-		defer q.WaitReady("3", time.Unix(3, 0))()
+		defer q.WaitReady(ctx, "3", time.Unix(3, 0))()
 		ret := atomic.AddUint32(&ctr, 1)
 		assert.Equal(t, uint32(3), ret)
 	}()
 	go func() {
+		ctx, end := trace.WithTaskFromStack(ctx)
+		defer end()
 		defer wg.Done()
-		defer q.WaitReady("4", time.Unix(4, 0))()
+		defer q.WaitReady(ctx, "4", time.Unix(4, 0))()
 		ret := atomic.AddUint32(&ctr, 1)
 		assert.Equal(t, uint32(4), ret)
 	}()
@@ -77,6 +89,8 @@ func (r record) String() string {
 // Hence, perform some statistics on the wakeup times and assert that the mean wakeup
 // times for each step are close together.
 func TestPqConcurrent(t *testing.T) {
+	ctx, end := trace.WithTaskFromStack(context.Background())
+	defer end()
 
 	q := newStepQueue()
 	var wg sync.WaitGroup
@@ -90,12 +104,14 @@ func TestPqConcurrent(t *testing.T) {
 	records := make(chan []record, filesystems)
 	for fs := 0; fs < filesystems; fs++ {
 		go func(fs int) {
+			ctx, end := trace.WithTaskFromStack(ctx)
+			defer end()
 			defer wg.Done()
 			recs := make([]record, 0)
 			for step := 0; step < stepsPerFS; step++ {
 				pos := atomic.AddUint32(&globalCtr, 1)
 				t := time.Unix(int64(step), 0)
-				done := q.WaitReady(fs, t)
+				done := q.WaitReady(ctx, fs, t)
 				wakeAt := time.Since(begin)
 				time.Sleep(sleepTimePerStep)
 				done()

replication/logic/pdu/pdu.pb.go

@@ -46,7 +46,7 @@ func (x Tri) String() string {
 	return proto.EnumName(Tri_name, int32(x))
 }
 func (Tri) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{0}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{0}
 }
 
 type FilesystemVersion_VersionType int32
@@ -69,7 +69,7 @@ func (x FilesystemVersion_VersionType) String() string {
 	return proto.EnumName(FilesystemVersion_VersionType_name, int32(x))
 }
 func (FilesystemVersion_VersionType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{5, 0}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{5, 0}
 }
 
 type ListFilesystemReq struct {
@@ -82,7 +82,7 @@ func (m *ListFilesystemReq) Reset()         { *m = ListFilesystemReq{} }
 func (m *ListFilesystemReq) String() string { return proto.CompactTextString(m) }
 func (*ListFilesystemReq) ProtoMessage()    {}
 func (*ListFilesystemReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{0}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{0}
 }
 func (m *ListFilesystemReq) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ListFilesystemReq.Unmarshal(m, b)
@@ -113,7 +113,7 @@ func (m *ListFilesystemRes) Reset()         { *m = ListFilesystemRes{} }
 func (m *ListFilesystemRes) String() string { return proto.CompactTextString(m) }
 func (*ListFilesystemRes) ProtoMessage()    {}
 func (*ListFilesystemRes) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{1}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{1}
 }
 func (m *ListFilesystemRes) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ListFilesystemRes.Unmarshal(m, b)
@@ -154,7 +154,7 @@ func (m *Filesystem) Reset()         { *m = Filesystem{} }
 func (m *Filesystem) String() string { return proto.CompactTextString(m) }
 func (*Filesystem) ProtoMessage()    {}
 func (*Filesystem) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{2}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{2}
 }
 func (m *Filesystem) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_Filesystem.Unmarshal(m, b)
@@ -213,7 +213,7 @@ func (m *ListFilesystemVersionsReq) Reset()         { *m = ListFilesystemVersion
 func (m *ListFilesystemVersionsReq) String() string { return proto.CompactTextString(m) }
 func (*ListFilesystemVersionsReq) ProtoMessage()    {}
 func (*ListFilesystemVersionsReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{3}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{3}
 }
 func (m *ListFilesystemVersionsReq) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ListFilesystemVersionsReq.Unmarshal(m, b)
@@ -251,7 +251,7 @@ func (m *ListFilesystemVersionsRes) Reset()         { *m = ListFilesystemVersion
 func (m *ListFilesystemVersionsRes) String() string { return proto.CompactTextString(m) }
 func (*ListFilesystemVersionsRes) ProtoMessage()    {}
 func (*ListFilesystemVersionsRes) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{4}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{4}
 }
 func (m *ListFilesystemVersionsRes) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ListFilesystemVersionsRes.Unmarshal(m, b)
@@ -293,7 +293,7 @@ func (m *FilesystemVersion) Reset()         { *m = FilesystemVersion{} }
 func (m *FilesystemVersion) String() string { return proto.CompactTextString(m) }
 func (*FilesystemVersion) ProtoMessage()    {}
 func (*FilesystemVersion) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{5}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{5}
 }
 func (m *FilesystemVersion) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_FilesystemVersion.Unmarshal(m, b)
@@ -371,7 +371,7 @@ func (m *SendReq) Reset()         { *m = SendReq{} }
 func (m *SendReq) String() string { return proto.CompactTextString(m) }
 func (*SendReq) ProtoMessage()    {}
 func (*SendReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{6}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{6}
 }
 func (m *SendReq) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_SendReq.Unmarshal(m, b)
@@ -445,7 +445,7 @@ func (m *Property) Reset()         { *m = Property{} }
 func (m *Property) String() string { return proto.CompactTextString(m) }
 func (*Property) ProtoMessage()    {}
 func (*Property) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{7}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{7}
 }
 func (m *Property) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_Property.Unmarshal(m, b)
@@ -496,7 +496,7 @@ func (m *SendRes) Reset()         { *m = SendRes{} }
 func (m *SendRes) String() string { return proto.CompactTextString(m) }
 func (*SendRes) ProtoMessage()    {}
 func (*SendRes) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{8}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{8}
 }
 func (m *SendRes) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_SendRes.Unmarshal(m, b)
@@ -548,7 +548,7 @@ func (m *SendCompletedReq) Reset()         { *m = SendCompletedReq{} }
 func (m *SendCompletedReq) String() string { return proto.CompactTextString(m) }
 func (*SendCompletedReq) ProtoMessage()    {}
 func (*SendCompletedReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{9}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{9}
 }
 func (m *SendCompletedReq) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_SendCompletedReq.Unmarshal(m, b)
@@ -585,7 +585,7 @@ func (m *SendCompletedRes) Reset()         { *m = SendCompletedRes{} }
 func (m *SendCompletedRes) String() string { return proto.CompactTextString(m) }
 func (*SendCompletedRes) ProtoMessage()    {}
 func (*SendCompletedRes) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{10}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{10}
 }
 func (m *SendCompletedRes) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_SendCompletedRes.Unmarshal(m, b)
@@ -608,7 +608,7 @@ var xxx_messageInfo_SendCompletedRes proto.InternalMessageInfo
 type ReceiveReq struct {
 	Filesystem string             `protobuf:"bytes,1,opt,name=Filesystem,proto3" json:"Filesystem,omitempty"`
 	To         *FilesystemVersion `protobuf:"bytes,2,opt,name=To,proto3" json:"To,omitempty"`
-	// If true, the receiver should clear the resume token before perfoming the
+	// If true, the receiver should clear the resume token before performing the
 	// zfs recv of the stream in the request
 	ClearResumeToken     bool     `protobuf:"varint,3,opt,name=ClearResumeToken,proto3" json:"ClearResumeToken,omitempty"`
 	XXX_NoUnkeyedLiteral struct{} `json:"-"`
@@ -620,7 +620,7 @@ func (m *ReceiveReq) Reset()         { *m = ReceiveReq{} }
 func (m *ReceiveReq) String() string { return proto.CompactTextString(m) }
 func (*ReceiveReq) ProtoMessage()    {}
 func (*ReceiveReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{11}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{11}
 }
 func (m *ReceiveReq) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ReceiveReq.Unmarshal(m, b)
@@ -671,7 +671,7 @@ func (m *ReceiveRes) Reset()         { *m = ReceiveRes{} }
 func (m *ReceiveRes) String() string { return proto.CompactTextString(m) }
 func (*ReceiveRes) ProtoMessage()    {}
 func (*ReceiveRes) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{12}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{12}
 }
 func (m *ReceiveRes) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ReceiveRes.Unmarshal(m, b)
@@ -704,7 +704,7 @@ func (m *DestroySnapshotsReq) Reset()         { *m = DestroySnapshotsReq{} }
 func (m *DestroySnapshotsReq) String() string { return proto.CompactTextString(m) }
 func (*DestroySnapshotsReq) ProtoMessage()    {}
 func (*DestroySnapshotsReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{13}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{13}
 }
 func (m *DestroySnapshotsReq) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_DestroySnapshotsReq.Unmarshal(m, b)
@@ -750,7 +750,7 @@ func (m *DestroySnapshotRes) Reset()         { *m = DestroySnapshotRes{} }
 func (m *DestroySnapshotRes) String() string { return proto.CompactTextString(m) }
 func (*DestroySnapshotRes) ProtoMessage()    {}
 func (*DestroySnapshotRes) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{14}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{14}
 }
 func (m *DestroySnapshotRes) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_DestroySnapshotRes.Unmarshal(m, b)
@@ -795,7 +795,7 @@ func (m *DestroySnapshotsRes) Reset()         { *m = DestroySnapshotsRes{} }
 func (m *DestroySnapshotsRes) String() string { return proto.CompactTextString(m) }
 func (*DestroySnapshotsRes) ProtoMessage()    {}
 func (*DestroySnapshotsRes) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{15}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{15}
 }
 func (m *DestroySnapshotsRes) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_DestroySnapshotsRes.Unmarshal(m, b)
@@ -833,7 +833,7 @@ func (m *ReplicationCursorReq) Reset()         { *m = ReplicationCursorReq{} }
 func (m *ReplicationCursorReq) String() string { return proto.CompactTextString(m) }
 func (*ReplicationCursorReq) ProtoMessage()    {}
 func (*ReplicationCursorReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{16}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{16}
 }
 func (m *ReplicationCursorReq) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ReplicationCursorReq.Unmarshal(m, b)
@@ -874,7 +874,7 @@ func (m *ReplicationCursorRes) Reset()         { *m = ReplicationCursorRes{} }
 func (m *ReplicationCursorRes) String() string { return proto.CompactTextString(m) }
 func (*ReplicationCursorRes) ProtoMessage()    {}
 func (*ReplicationCursorRes) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{17}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{17}
 }
 func (m *ReplicationCursorRes) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ReplicationCursorRes.Unmarshal(m, b)
@@ -1010,7 +1010,7 @@ func (m *PingReq) Reset()         { *m = PingReq{} }
 func (m *PingReq) String() string { return proto.CompactTextString(m) }
 func (*PingReq) ProtoMessage()    {}
 func (*PingReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{18}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{18}
 }
 func (m *PingReq) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_PingReq.Unmarshal(m, b)
@@ -1049,7 +1049,7 @@ func (m *PingRes) Reset()         { *m = PingRes{} }
 func (m *PingRes) String() string { return proto.CompactTextString(m) }
 func (*PingRes) ProtoMessage()    {}
 func (*PingRes) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{19}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{19}
 }
 func (m *PingRes) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_PingRes.Unmarshal(m, b)
@@ -1077,7 +1077,16 @@ func (m *PingRes) GetEcho() string {
 }
 
 type HintMostRecentCommonAncestorReq struct {
-	Filesystem           string             `protobuf:"bytes,1,opt,name=Filesystem,proto3" json:"Filesystem,omitempty"`
+	Filesystem string `protobuf:"bytes,1,opt,name=Filesystem,proto3" json:"Filesystem,omitempty"`
+	// A copy of the FilesystemVersion on the sending side that the replication
+	// algorithm identified as a shared most recent common version between sending
+	// and receiving side.
+	//
+	// If nil, this is an indication that the replication algorithm could not
+	// find a common ancestor between the two sides.
+	// NOTE: nilness does not mean that replication never happened - there could
+	// as well be a replication conflict. thus, dont' jump to conclusions too
+	// rapidly here.
 	SenderVersion        *FilesystemVersion `protobuf:"bytes,2,opt,name=SenderVersion,proto3" json:"SenderVersion,omitempty"`
 	XXX_NoUnkeyedLiteral struct{}           `json:"-"`
 	XXX_unrecognized     []byte             `json:"-"`
@@ -1088,7 +1097,7 @@ func (m *HintMostRecentCommonAncestorReq) Reset()         { *m = HintMostRecentC
 func (m *HintMostRecentCommonAncestorReq) String() string { return proto.CompactTextString(m) }
 func (*HintMostRecentCommonAncestorReq) ProtoMessage()    {}
 func (*HintMostRecentCommonAncestorReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{20}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{20}
 }
 func (m *HintMostRecentCommonAncestorReq) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_HintMostRecentCommonAncestorReq.Unmarshal(m, b)
@@ -1132,7 +1141,7 @@ func (m *HintMostRecentCommonAncestorRes) Reset()         { *m = HintMostRecentC
 func (m *HintMostRecentCommonAncestorRes) String() string { return proto.CompactTextString(m) }
 func (*HintMostRecentCommonAncestorRes) ProtoMessage()    {}
 func (*HintMostRecentCommonAncestorRes) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_0f43b713cd3bf056, []int{21}
+	return fileDescriptor_pdu_e59763dc61674a79, []int{21}
 }
 func (m *HintMostRecentCommonAncestorRes) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_HintMostRecentCommonAncestorRes.Unmarshal(m, b)
@@ -1449,9 +1458,9 @@ var _Replication_serviceDesc = grpc.ServiceDesc{
 	Metadata: "pdu.proto",
 }
 
-func init() { proto.RegisterFile("pdu.proto", fileDescriptor_pdu_0f43b713cd3bf056) }
+func init() { proto.RegisterFile("pdu.proto", fileDescriptor_pdu_e59763dc61674a79) }
 
-var fileDescriptor_pdu_0f43b713cd3bf056 = []byte{
+var fileDescriptor_pdu_e59763dc61674a79 = []byte{
 	// 892 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x56, 0xdf, 0x6f, 0xdb, 0x36,
 	0x10, 0x8e, 0x6c, 0x39, 0x91, 0xcf, 0xe9, 0xea, 0x5c, 0xb2, 0x42, 0x13, 0xba, 0xce, 0xe3, 0x86,

replication/logic/pdu/pdu.proto

@@ -128,7 +128,17 @@ message PingRes {
 }
 
 message HintMostRecentCommonAncestorReq {
-    string Filesystem = 1;
+  string Filesystem = 1;
-    FilesystemVersion SenderVersion = 2;
+
+  // A copy of the FilesystemVersion on the sending side that the replication
+  // algorithm identified as a shared most recent common version between sending
+  // and receiving side.
+  //
+  // If nil, this is an indication that the replication algorithm could not
+  // find a common ancestor between the two sides.
+  // NOTE: nilness does not mean that replication never happened - there could
+  // as well be a replication conflict. thus, dont' jump to conclusions too
+  // rapidly here.
+  FilesystemVersion SenderVersion = 2;
 }
 message HintMostRecentCommonAncestorRes {}

replication/logic/replication_logic.go

@@ -4,11 +4,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"sync"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 
+	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/replication/driver"
 	. "github.com/zrepl/zrepl/replication/logic/diff"
 	"github.com/zrepl/zrepl/replication/logic/pdu"
@@ -38,7 +41,7 @@ type Sender interface {
 	// If a non-nil io.ReadCloser is returned, it is guaranteed to be closed before
 	// any next call to the parent github.com/zrepl/zrepl/replication.Endpoint.
 	// If the send request is for dry run the io.ReadCloser will be nil
-	Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, zfs.StreamCopier, error)
+	Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error)
 	SendCompleted(ctx context.Context, r *pdu.SendCompletedReq) (*pdu.SendCompletedRes, error)
 	ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error)
 }
@@ -47,7 +50,7 @@ type Receiver interface {
 	Endpoint
 	// Receive sends r and sendStream (the latter containing a ZFS send stream)
 	// to the parent github.com/zrepl/zrepl/replication.Endpoint.
-	Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs.StreamCopier) (*pdu.ReceiveRes, error)
+	Receive(ctx context.Context, req *pdu.ReceiveReq, receive io.ReadCloser) (*pdu.ReceiveRes, error)
 }
 
 type PlannerPolicy struct {
@@ -79,6 +82,8 @@ func (p *Planner) WaitForConnectivity(ctx context.Context) error {
 	var wg sync.WaitGroup
 	doPing := func(endpoint Endpoint, errOut *error) {
 		defer wg.Done()
+		ctx, endTask := trace.WithTaskFromStack(ctx)
+		defer endTask()
 		err := endpoint.WaitForConnectivity(ctx)
 		if err != nil {
 			*errOut = err
@@ -162,7 +167,7 @@ type Step struct {
 
 	// byteCounter is nil initially, and set later in Step.doReplication
 	// => concurrent read of that pointer from Step.ReportInfo must be protected
-	byteCounter    bytecounter.StreamCopier
+	byteCounter    bytecounter.ReadCloser
 	byteCounterMtx chainlock.L
 }
 
@@ -234,6 +239,7 @@ func resolveConflict(conflict error) (path []*pdu.FilesystemVersion, msg string)
 	if noCommonAncestor, ok := conflict.(*ConflictNoCommonAncestor); ok {
 		if len(noCommonAncestor.SortedReceiverVersions) == 0 {
 			// TODO this is hard-coded replication policy: most recent snapshot as source
+			// NOTE: Keep in sync with listStaleFiltering, it depends on this hard-coded assumption
 			var mostRecentSnap *pdu.FilesystemVersion
 			for n := len(noCommonAncestor.SortedSenderVersions) - 1; n >= 0; n-- {
 				if noCommonAncestor.SortedSenderVersions[n].Type == pdu.FilesystemVersion_Snapshot {
@@ -301,9 +307,11 @@ func (p *Planner) doPlanning(ctx context.Context) ([]*Filesystem, error) {
 
 func (fs *Filesystem) doPlanning(ctx context.Context) ([]*Step, error) {
 
-	log := getLogger(ctx).WithField("filesystem", fs.Path)
+	log := func(ctx context.Context) logger.Logger {
+		return getLogger(ctx).WithField("filesystem", fs.Path)
+	}
 
-	log.Debug("assessing filesystem")
+	log(ctx).Debug("assessing filesystem")
 
 	if fs.policy.EncryptedSend == True && !fs.senderFS.GetIsEncrypted() {
 		return nil, fmt.Errorf("sender filesystem is not encrypted but policy mandates encrypted send")
@@ -311,14 +319,14 @@ func (fs *Filesystem) doPlanning(ctx context.Context) ([]*Step, error) {
 
 	sfsvsres, err := fs.sender.ListFilesystemVersions(ctx, &pdu.ListFilesystemVersionsReq{Filesystem: fs.Path})
 	if err != nil {
-		log.WithError(err).Error("cannot get remote filesystem versions")
+		log(ctx).WithError(err).Error("cannot get remote filesystem versions")
 		return nil, err
 	}
 	sfsvs := sfsvsres.GetVersions()
 
 	if len(sfsvs) < 1 {
 		err := errors.New("sender does not have any versions")
-		log.Error(err.Error())
+		log(ctx).Error(err.Error())
 		return nil, err
 	}
 
@@ -326,7 +334,7 @@ func (fs *Filesystem) doPlanning(ctx context.Context) ([]*Step, error) {
 	if fs.receiverFS != nil && !fs.receiverFS.GetIsPlaceholder() {
 		rfsvsres, err := fs.receiver.ListFilesystemVersions(ctx, &pdu.ListFilesystemVersionsReq{Filesystem: fs.Path})
 		if err != nil {
-			log.WithError(err).Error("receiver error")
+			log(ctx).WithError(err).Error("receiver error")
 			return nil, err
 		}
 		rfsvs = rfsvsres.GetVersions()
@@ -338,35 +346,39 @@ func (fs *Filesystem) doPlanning(ctx context.Context) ([]*Step, error) {
 	var resumeTokenRaw string
 	if fs.receiverFS != nil && fs.receiverFS.ResumeToken != "" {
 		resumeTokenRaw = fs.receiverFS.ResumeToken // shadow
-		log.WithField("receiverFS.ResumeToken", resumeTokenRaw).Debug("decode receiver fs resume token")
+		log(ctx).WithField("receiverFS.ResumeToken", resumeTokenRaw).Debug("decode receiver fs resume token")
 		resumeToken, err = zfs.ParseResumeToken(ctx, resumeTokenRaw) // shadow
 		if err != nil {
 			// TODO in theory, we could do replication without resume token, but that would mean that
 			// we need to discard the resumable state on the receiver's side.
 			// Would be easy by setting UsedResumeToken=false in the RecvReq ...
 			// FIXME / CHECK semantics UsedResumeToken if SendReq.ResumeToken == ""
-			log.WithError(err).Error("cannot decode resume token, aborting")
+			log(ctx).WithError(err).Error("cannot decode resume token, aborting")
 			return nil, err
 		}
-		log.WithField("token", resumeToken).Debug("decode resume token")
+		log(ctx).WithField("token", resumeToken).Debug("decode resume token")
 	}
 
-	// give both sides a hint about how far the replication got
+	// give both sides a hint about how far prior replication attempts got
-	// This serves as a cumulative variant of SendCompleted and can be useful
+	// This serves as a cummulative variant of SendCompleted and can be useful
 	// for example to release stale holds from an earlier (interrupted) replication.
 	// TODO FIXME: enqueue this as a replication step instead of doing it here during planning
 	//             then again, the step should run regardless of planning success
 	//             so maybe a separate phase before PLANNING, then?
 	path, conflict := IncrementalPath(rfsvs, sfsvs)
-	var sender_mrca *pdu.FilesystemVersion // from sfsvs
+	var sender_mrca *pdu.FilesystemVersion
 	if conflict == nil && len(path) > 0 {
 		sender_mrca = path[0] // shadow
 	}
-	if sender_mrca != nil {
+	// yes, sender_mrca may be nil, indicating that we do not have an mrca
+	{
 		var wg sync.WaitGroup
 		doHint := func(ep Endpoint, name string) {
 			defer wg.Done()
-			log := log.WithField("to_side", name).
+			ctx, endTask := trace.WithTask(ctx, "hint-mrca-"+name)
+			defer endTask()
+
+			log := log(ctx).WithField("to_side", name).
 				WithField("sender_mrca", sender_mrca.String())
 			log.Debug("hint most recent common ancestor")
 			hint := &pdu.HintMostRecentCommonAncestorReq{
@@ -382,8 +394,6 @@ func (fs *Filesystem) doPlanning(ctx context.Context) ([]*Step, error) {
 		go doHint(fs.sender, "sender")
 		go doHint(fs.receiver, "receiver")
 		wg.Wait()
-	} else {
-		log.Debug("cannot identify most recent common ancestor, skipping hint")
 	}
 
 	var steps []*Step
@@ -427,7 +437,7 @@ func (fs *Filesystem) doPlanning(ctx context.Context) ([]*Step, error) {
 			encryptionMatches = true
 		}
 
-		log.WithField("fromVersion", fromVersion).
+		log(ctx).WithField("fromVersion", fromVersion).
 			WithField("toVersion", toVersion).
 			WithField("encryptionMatches", encryptionMatches).
 			Debug("result of resume-token-matching to sender's versions")
@@ -483,11 +493,11 @@ func (fs *Filesystem) doPlanning(ctx context.Context) ([]*Step, error) {
 			var msg string
 			path, msg = resolveConflict(conflict) // no shadowing allowed!
 			if path != nil {
-				log.WithField("conflict", conflict).Info("conflict")
+				log(ctx).WithField("conflict", conflict).Info("conflict")
-				log.WithField("resolution", msg).Info("automatically resolved")
+				log(ctx).WithField("resolution", msg).Info("automatically resolved")
 			} else {
-				log.WithField("conflict", conflict).Error("conflict")
+				log(ctx).WithField("conflict", conflict).Error("conflict")
-				log.WithField("problem", msg).Error("cannot resolve conflict")
+				log(ctx).WithField("problem", msg).Error("cannot resolve conflict")
 			}
 		}
 		if len(path) == 0 {
@@ -521,37 +531,35 @@ func (fs *Filesystem) doPlanning(ctx context.Context) ([]*Step, error) {
 	}
 
 	if len(steps) == 0 {
-		log.Info("planning determined that no replication steps are required")
+		log(ctx).Info("planning determined that no replication steps are required")
 	}
 
-	log.Debug("compute send size estimate")
+	log(ctx).Debug("compute send size estimate")
 	errs := make(chan error, len(steps))
-	var wg sync.WaitGroup
 	fanOutCtx, fanOutCancel := context.WithCancel(ctx)
+	_, fanOutAdd, fanOutWait := trace.WithTaskGroup(fanOutCtx, "compute-size-estimate")
 	defer fanOutCancel()
 	for _, step := range steps {
-		wg.Add(1)
+		step := step // local copy that is moved into the closure
-		go func(step *Step) {
+		fanOutAdd(func(ctx context.Context) {
-			defer wg.Done()
-
 			// TODO instead of the semaphore, rely on resource-exhaustion signaled by the remote endpoint to limit size-estimate requests
 			// Send is handled over rpc/dataconn ATM, which doesn't support the resource exhaustion status codes that gRPC defines
-			guard, err := fs.sizeEstimateRequestSem.Acquire(fanOutCtx)
+			guard, err := fs.sizeEstimateRequestSem.Acquire(ctx)
 			if err != nil {
 				fanOutCancel()
 				return
 			}
 			defer guard.Release()
 
-			err = step.updateSizeEstimate(fanOutCtx)
+			err = step.updateSizeEstimate(ctx)
 			if err != nil {
-				log.WithError(err).WithField("step", step).Error("error computing size estimate")
+				log(ctx).WithError(err).WithField("step", step).Error("error computing size estimate")
 				fanOutCancel()
 			}
 			errs <- err
-		}(step)
+		})
 	}
-	wg.Wait()
+	fanOutWait()
 	close(errs)
 	var significantErr error = nil
 	for err := range errs {
@@ -565,7 +573,7 @@ func (fs *Filesystem) doPlanning(ctx context.Context) ([]*Step, error) {
 		return nil, significantErr
 	}
 
-	log.Debug("filesystem planning finished")
+	log(ctx).Debug("filesystem planning finished")
 	return steps, nil
 }
 
@@ -602,23 +610,23 @@ func (s *Step) doReplication(ctx context.Context) error {
 
 	fs := s.parent.Path
 
-	log := getLogger(ctx)
+	log := getLogger(ctx).WithField("filesystem", fs)
 	sr := s.buildSendRequest(false)
 
 	log.Debug("initiate send request")
-	sres, sstreamCopier, err := s.sender.Send(ctx, sr)
+	sres, stream, err := s.sender.Send(ctx, sr)
 	if err != nil {
 		log.WithError(err).Error("send request failed")
 		return err
 	}
-	if sstreamCopier == nil {
+	if stream == nil {
 		err := errors.New("send request did not return a stream, broken endpoint implementation")
 		return err
 	}
-	defer sstreamCopier.Close()
+	defer stream.Close()
 
 	// Install a byte counter to track progress + for status report
-	byteCountingStream := bytecounter.NewStreamCopier(sstreamCopier)
+	byteCountingStream := bytecounter.NewReadCloser(stream)
 	s.byteCounterMtx.Lock()
 	s.byteCounter = byteCountingStream
 	s.byteCounterMtx.Unlock()

replication/logic/replication_logic_context.go

@@ -3,26 +3,10 @@ package logic
 import (
 	"context"
 
+	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
 )
 
-type contextKey int
+func getLogger(ctx context.Context) logger.Logger {
-
+	return logging.GetLogger(ctx, logging.SubsysReplication)
-const (
-	contextKeyLog contextKey = iota
-)
-
-type Logger = logger.Logger
-
-func WithLogger(ctx context.Context, l Logger) context.Context {
-	ctx = context.WithValue(ctx, contextKeyLog, l)
-	return ctx
-}
-
-func getLogger(ctx context.Context) Logger {
-	l, ok := ctx.Value(contextKeyLog).(Logger)
-	if !ok {
-		l = logger.NewNullLogger()
-	}
-	return l
 }

rpc/dataconn/dataconn_client.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"strings"
 
 	"github.com/golang/protobuf/proto"
@@ -11,7 +12,6 @@ import (
 	"github.com/zrepl/zrepl/replication/logic/pdu"
 	"github.com/zrepl/zrepl/rpc/dataconn/stream"
 	"github.com/zrepl/zrepl/transport"
-	"github.com/zrepl/zrepl/zfs"
 )
 
 type Client struct {
@@ -26,7 +26,7 @@ func NewClient(connecter transport.Connecter, log Logger) *Client {
 	}
 }
 
-func (c *Client) send(ctx context.Context, conn *stream.Conn, endpoint string, req proto.Message, streamCopier zfs.StreamCopier) error {
+func (c *Client) send(ctx context.Context, conn *stream.Conn, endpoint string, req proto.Message, stream io.ReadCloser) error {
 
 	var buf bytes.Buffer
 	_, memErr := buf.WriteString(endpoint)
@@ -46,8 +46,8 @@ func (c *Client) send(ctx context.Context, conn *stream.Conn, endpoint string, r
 		return err
 	}
 
-	if streamCopier != nil {
+	if stream != nil {
-		return conn.SendStream(ctx, streamCopier, ZFSStream)
+		return conn.SendStream(ctx, stream, ZFSStream)
 	} else {
 		return nil
 	}
@@ -109,7 +109,7 @@ func (c *Client) putWire(conn *stream.Conn) {
 	}
 }
 
-func (c *Client) ReqSend(ctx context.Context, req *pdu.SendReq) (*pdu.SendRes, zfs.StreamCopier, error) {
+func (c *Client) ReqSend(ctx context.Context, req *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) {
 	conn, err := c.getWire(ctx)
 	if err != nil {
 		return nil, nil, err
@@ -130,17 +130,19 @@ func (c *Client) ReqSend(ctx context.Context, req *pdu.SendReq) (*pdu.SendRes, z
 		return nil, nil, err
 	}
 
-	var copier zfs.StreamCopier = nil
+	var stream io.ReadCloser
 	if !req.DryRun {
 		putWireOnReturn = false
-		copier = &streamCopier{streamConn: conn, closeStreamOnClose: true}
+		stream, err = conn.ReadStream(ZFSStream, true) // no shadow
+		if err != nil {
+			return nil, nil, err
+		}
 	}
 
-	return &res, copier, nil
+	return &res, stream, nil
 }
 
-func (c *Client) ReqRecv(ctx context.Context, req *pdu.ReceiveReq, streamCopier zfs.StreamCopier) (*pdu.ReceiveRes, error) {
+func (c *Client) ReqRecv(ctx context.Context, req *pdu.ReceiveReq, stream io.ReadCloser) (*pdu.ReceiveRes, error) {
-
 	defer c.log.Debug("ReqRecv returns")
 	conn, err := c.getWire(ctx)
 	if err != nil {
@@ -166,7 +168,7 @@ func (c *Client) ReqRecv(ctx context.Context, req *pdu.ReceiveReq, streamCopier
 
 	sendErrChan := make(chan error)
 	go func() {
-		if err := c.send(ctx, conn, EndpointRecv, req, streamCopier); err != nil {
+		if err := c.send(ctx, conn, EndpointRecv, req, stream); err != nil {
 			sendErrChan <- err
 		} else {
 			sendErrChan <- nil

rpc/dataconn/dataconn_server.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
+	"sync"
 
 	"github.com/golang/protobuf/proto"
 
@@ -11,7 +13,6 @@ import (
 	"github.com/zrepl/zrepl/replication/logic/pdu"
 	"github.com/zrepl/zrepl/rpc/dataconn/stream"
 	"github.com/zrepl/zrepl/transport"
-	"github.com/zrepl/zrepl/zfs"
 )
 
 // WireInterceptor has a chance to exchange the context and connection on each client connection.
@@ -21,27 +22,44 @@ type WireInterceptor func(ctx context.Context, rawConn *transport.AuthConn) (con
 type Handler interface {
 	// Send handles a SendRequest.
 	// The returned io.ReadCloser is allowed to be nil, for example if the requested Send is a dry-run.
-	Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, zfs.StreamCopier, error)
+	Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error)
 	// Receive handles a ReceiveRequest.
 	// It is guaranteed that Server calls Receive with a stream that holds the IdleConnTimeout
 	// configured in ServerConfig.Shared.IdleConnTimeout.
-	Receive(ctx context.Context, r *pdu.ReceiveReq, receive zfs.StreamCopier) (*pdu.ReceiveRes, error)
+	Receive(ctx context.Context, r *pdu.ReceiveReq, receive io.ReadCloser) (*pdu.ReceiveRes, error)
 	// PingDataconn handles a PingReq
 	PingDataconn(ctx context.Context, r *pdu.PingReq) (*pdu.PingRes, error)
 }
 
 type Logger = logger.Logger
 
+type ContextInterceptorData interface {
+	FullMethod() string
+	ClientIdentity() string
+}
+
+type ContextInterceptor = func(ctx context.Context, data ContextInterceptorData, handler func(ctx context.Context))
+
 type Server struct {
 	h   Handler
 	wi  WireInterceptor
+	ci  ContextInterceptor
 	log Logger
 }
 
-func NewServer(wi WireInterceptor, logger Logger, handler Handler) *Server {
+var noopContextInteceptor = func(ctx context.Context, _ ContextInterceptorData, handler func(context.Context)) {
+	handler(ctx)
+}
+
+// wi and ci may be nil
+func NewServer(wi WireInterceptor, ci ContextInterceptor, logger Logger, handler Handler) *Server {
+	if ci == nil {
+		ci = noopContextInteceptor
+	}
 	return &Server{
 		h:   handler,
 		wi:  wi,
+		ci:  ci,
 		log: logger,
 	}
 }
@@ -50,16 +68,26 @@ func NewServer(wi WireInterceptor, logger Logger, handler Handler) *Server {
 // No accept errors are returned: they are logged to the Logger passed
 // to the constructor.
 func (s *Server) Serve(ctx context.Context, l transport.AuthenticatedListener) {
+	var wg sync.WaitGroup
+	defer wg.Wait()
 
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
 		<-ctx.Done()
-		s.log.Debug("context done")
+		s.log.Debug("context done, closing listener")
 		if err := l.Close(); err != nil {
 			s.log.WithError(err).Error("cannot close listener")
 		}
 	}()
 	conns := make(chan *transport.AuthConn)
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
+		defer close(conns)
 		for {
 			conn, err := l.Accept(ctx)
 			if err != nil {
@@ -74,10 +102,22 @@ func (s *Server) Serve(ctx context.Context, l transport.AuthenticatedListener) {
 		}
 	}()
 	for conn := range conns {
-		go s.serveConn(conn)
+		wg.Add(1)
+		go func(conn *transport.AuthConn) {
+			defer wg.Done()
+			s.serveConn(conn)
+		}(conn)
 	}
 }
 
+type contextInterceptorData struct {
+	fullMethod     string
+	clientIdentity string
+}
+
+func (d contextInterceptorData) FullMethod() string     { return d.fullMethod }
+func (d contextInterceptorData) ClientIdentity() string { return d.clientIdentity }
+
 func (s *Server) serveConn(nc *transport.AuthConn) {
 	s.log.Debug("serveConn begin")
 	defer s.log.Debug("serveConn done")
@@ -102,6 +142,17 @@ func (s *Server) serveConn(nc *transport.AuthConn) {
 	}
 	endpoint := string(header)
 
+	data := contextInterceptorData{
+		fullMethod:     endpoint,
+		clientIdentity: nc.ClientIdentity(),
+	}
+	s.ci(ctx, data, func(ctx context.Context) {
+		s.serveConnRequest(ctx, endpoint, c)
+	})
+}
+
+func (s *Server) serveConnRequest(ctx context.Context, endpoint string, c *stream.Conn) {
+
 	reqStructured, err := c.ReadStreamedMessage(ctx, RequestStructuredMaxSize, ReqStructured)
 	if err != nil {
 		s.log.WithError(err).Error("error reading structured part")
@@ -111,7 +162,7 @@ func (s *Server) serveConn(nc *transport.AuthConn) {
 	s.log.WithField("endpoint", endpoint).Debug("calling handler")
 
 	var res proto.Message
-	var sendStream zfs.StreamCopier
+	var sendStream io.ReadCloser
 	var handlerErr error
 	switch endpoint {
 	case EndpointSend:
@@ -127,7 +178,12 @@ func (s *Server) serveConn(nc *transport.AuthConn) {
 			s.log.WithError(err).Error("cannot unmarshal receive request")
 			return
 		}
-		res, handlerErr = s.h.Receive(ctx, &req, &streamCopier{streamConn: c, closeStreamOnClose: false}) // SHADOWING
+		stream, err := c.ReadStream(ZFSStream, false)
+		if err != nil {
+			s.log.WithError(err).Error("cannot open stream in receive request")
+			return
+		}
+		res, handlerErr = s.h.Receive(ctx, &req, stream) // SHADOWING
 	case EndpointPing:
 		var req pdu.PingReq
 		if err := proto.Unmarshal(reqStructured, &req); err != nil {

rpc/dataconn/dataconn_shared.go

@@ -1,12 +1,7 @@
 package dataconn
 
 import (
-	"io"
-	"sync"
 	"time"
-
-	"github.com/zrepl/zrepl/rpc/dataconn/stream"
-	"github.com/zrepl/zrepl/zfs"
 )
 
 const (
@@ -39,33 +34,3 @@ const (
 	responseHeaderHandlerOk          = "HANDLER OK\n"
 	responseHeaderHandlerErrorPrefix = "HANDLER ERROR:\n"
 )
-
-type streamCopier struct {
-	mtx                sync.Mutex
-	used               bool
-	streamConn         *stream.Conn
-	closeStreamOnClose bool
-}
-
-// WriteStreamTo implements zfs.StreamCopier
-func (s *streamCopier) WriteStreamTo(w io.Writer) zfs.StreamCopierError {
-	s.mtx.Lock()
-	defer s.mtx.Unlock()
-	if s.used {
-		panic("streamCopier used multiple times")
-	}
-	s.used = true
-	return s.streamConn.ReadStreamInto(w, ZFSStream)
-}
-
-// Close implements zfs.StreamCopier
-func (s *streamCopier) Close() error {
-	// only record the close here, what we do actually depends on whether
-	// the streamCopier is instantiated server-side or client-side
-	s.mtx.Lock()
-	defer s.mtx.Unlock()
-	if s.closeStreamOnClose {
-		return s.streamConn.Close()
-	}
-	return nil
-}

rpc/dataconn/microbenchmark/microbenchmark.go

@@ -29,7 +29,6 @@ import (
 	"github.com/zrepl/zrepl/rpc/dataconn/timeoutconn"
 	"github.com/zrepl/zrepl/transport"
 	"github.com/zrepl/zrepl/util/devnoop"
-	"github.com/zrepl/zrepl/zfs"
 )
 
 func orDie(err error) {
@@ -42,23 +41,9 @@ type readerStreamCopier struct{ io.Reader }
 
 func (readerStreamCopier) Close() error { return nil }
 
-type readerStreamCopierErr struct {
-	error
-}
-
-func (readerStreamCopierErr) IsReadError() bool  { return false }
-func (readerStreamCopierErr) IsWriteError() bool { return true }
-
-func (c readerStreamCopier) WriteStreamTo(w io.Writer) zfs.StreamCopierError {
-	var buf [1 << 21]byte
-	_, err := io.CopyBuffer(w, c.Reader, buf[:])
-	// always assume write error
-	return readerStreamCopierErr{err}
-}
-
 type devNullHandler struct{}
 
-func (devNullHandler) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, zfs.StreamCopier, error) {
+func (devNullHandler) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) {
 	var res pdu.SendRes
 	if args.devnoopReader {
 		return &res, readerStreamCopier{devnoop.Get()}, nil
@@ -67,12 +52,12 @@ func (devNullHandler) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, z
 	}
 }
 
-func (devNullHandler) Receive(ctx context.Context, r *pdu.ReceiveReq, stream zfs.StreamCopier) (*pdu.ReceiveRes, error) {
+func (devNullHandler) Receive(ctx context.Context, r *pdu.ReceiveReq, stream io.ReadCloser) (*pdu.ReceiveRes, error) {
 	var out io.Writer = os.Stdout
 	if args.devnoopWriter {
 		out = devnoop.Get()
 	}
-	err := stream.WriteStreamTo(out)
+	_, err := io.Copy(out, stream)
 	var res pdu.ReceiveRes
 	return &res, err
 }
@@ -127,7 +112,7 @@ func server() {
 	orDie(err)
 	l := tcpListener{nl.(*net.TCPListener), "fakeclientidentity"}
 
-	srv := dataconn.NewServer(nil, logger.NewStderrDebugLogger(), devNullHandler{})
+	srv := dataconn.NewServer(nil, nil, logger.NewStderrDebugLogger(), devNullHandler{})
 
 	ctx := context.Background()
 
@@ -172,7 +157,7 @@ func client() {
 		req := pdu.SendReq{}
 		_, stream, err := client.ReqSend(ctx, &req)
 		orDie(err)
-		err = stream.WriteStreamTo(os.Stdout)
+		_, err = io.Copy(os.Stdout, stream)
 		orDie(err)
 	case "recv":
 		var r io.Reader = os.Stdin

rpc/dataconn/stream/stream.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"net"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"unicode/utf8"
 
@@ -14,7 +15,6 @@ import (
 	"github.com/zrepl/zrepl/rpc/dataconn/base2bufpool"
 	"github.com/zrepl/zrepl/rpc/dataconn/frameconn"
 	"github.com/zrepl/zrepl/rpc/dataconn/heartbeatconn"
-	"github.com/zrepl/zrepl/zfs"
 )
 
 type Logger = logger.Logger
@@ -81,9 +81,13 @@ func doWriteStream(ctx context.Context, c *heartbeatconn.Conn, stream io.Reader,
 		err error
 	}
 
+	var wg sync.WaitGroup
+	defer wg.Wait()
 	reads := make(chan read, 5)
 	var stopReading uint32
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
 		defer close(reads)
 		for atomic.LoadUint32(&stopReading) == 0 {
 			buffer := bufpool.Get(1 << FramePayloadShift)
@@ -198,7 +202,7 @@ func (e ReadStreamError) Temporary() bool {
 	return false
 }
 
-var _ zfs.StreamCopierError = &ReadStreamError{}
+var _ net.Error = &ReadStreamError{}
 
 func (e ReadStreamError) IsReadError() bool {
 	return e.Kind != ReadStreamErrorKindWrite

rpc/dataconn/stream/stream_conn.go

@@ -14,7 +14,6 @@ import (
 
 	"github.com/zrepl/zrepl/rpc/dataconn/heartbeatconn"
 	"github.com/zrepl/zrepl/rpc/dataconn/timeoutconn"
-	"github.com/zrepl/zrepl/zfs"
 )
 
 type Conn struct {
@@ -40,15 +39,7 @@ type Conn struct {
 
 var readMessageSentinel = fmt.Errorf("read stream complete")
 
-type writeStreamToErrorUnknownState struct{}
+var errWriteStreamToErrorUnknownState = fmt.Errorf("dataconn read stream: connection is in unknown state")
-
-func (e writeStreamToErrorUnknownState) Error() string {
-	return "dataconn read stream: connection is in unknown state"
-}
-
-func (e writeStreamToErrorUnknownState) IsReadError() bool { return true }
-
-func (e writeStreamToErrorUnknownState) IsWriteError() bool { return false }
 
 func Wrap(nc timeoutconn.Wire, sendHeartbeatInterval, peerTimeout time.Duration) *Conn {
 	hc := heartbeatconn.Wrap(nc, sendHeartbeatInterval, peerTimeout)
@@ -123,14 +114,28 @@ func (c *Conn) ReadStreamedMessage(ctx context.Context, maxSize uint32, frameTyp
 	}
 }
 
+type StreamReader struct {
+	*io.PipeReader
+	conn             *Conn
+	closeConnOnClose bool
+}
+
+func (r *StreamReader) Close() error {
+	err := r.PipeReader.Close()
+	if r.closeConnOnClose {
+		r.conn.Close() // TODO error logging
+	}
+	return err
+}
+
 // WriteStreamTo reads a stream from Conn and writes it to w.
-func (c *Conn) ReadStreamInto(w io.Writer, frameType uint32) (err zfs.StreamCopierError) {
+func (c *Conn) ReadStream(frameType uint32, closeConnOnClose bool) (_ *StreamReader, err error) {
 
 	// if we are closed while writing, return that as an error
 	if closeGuard, cse := c.closeState.RWEntry(); cse != nil {
-		return cse
+		return nil, cse
 	} else {
-		defer func(err *zfs.StreamCopierError) {
+		defer func(err *error) {
 			if closed := closeGuard.RWExit(); closed != nil {
 				*err = closed
 			}
@@ -138,18 +143,23 @@ func (c *Conn) ReadStreamInto(w io.Writer, frameType uint32) (err zfs.StreamCopi
 	}
 
 	c.readMtx.Lock()
-	defer c.readMtx.Unlock()
 	if !c.readClean {
-		return writeStreamToErrorUnknownState{}
+		return nil, errWriteStreamToErrorUnknownState
 	}
-	var rse *ReadStreamError = readStream(c.frameReads, c.hc, w, frameType)
-	c.readClean = isConnCleanAfterRead(rse)
 
-	// https://golang.org/doc/faq#nil_error
+	r, w := io.Pipe()
-	if rse == nil {
+	go func() {
-		return nil
+		defer c.readMtx.Unlock()
-	}
+		var err *ReadStreamError = readStream(c.frameReads, c.hc, w, frameType)
-	return rse
+		if err != nil {
+			_ = w.CloseWithError(err) // doc guarantees that error will always be nil
+		} else {
+			w.Close()
+		}
+		c.readClean = isConnCleanAfterRead(err)
+	}()
+
+	return &StreamReader{PipeReader: r, conn: c, closeConnOnClose: closeConnOnClose}, nil
 }
 
 func (c *Conn) WriteStreamedMessage(ctx context.Context, buf io.Reader, frameType uint32) (err error) {
@@ -178,7 +188,7 @@ func (c *Conn) WriteStreamedMessage(ctx context.Context, buf io.Reader, frameTyp
 	return errConn
 }
 
-func (c *Conn) SendStream(ctx context.Context, src zfs.StreamCopier, frameType uint32) (err error) {
+func (c *Conn) SendStream(ctx context.Context, stream io.ReadCloser, frameType uint32) (err error) {
 
 	// if we are closed while reading, return that as an error
 	if closeGuard, cse := c.closeState.RWEntry(); cse != nil {
@@ -197,49 +207,17 @@ func (c *Conn) SendStream(ctx context.Context, src zfs.StreamCopier, frameType u
 		return fmt.Errorf("dataconn send stream: connection is in unknown state")
 	}
 
-	// avoid io.Pipe if zfs.StreamCopier is an io.Reader
+	errStream, errConn := writeStream(ctx, c.hc, stream, frameType)
-	var r io.Reader
-	var w *io.PipeWriter
-	streamCopierErrChan := make(chan zfs.StreamCopierError, 1)
-	if reader, ok := src.(io.Reader); ok {
-		r = reader
-		streamCopierErrChan <- nil
-		close(streamCopierErrChan)
-	} else {
-		r, w = io.Pipe()
-		go func() {
-			streamCopierErrChan <- src.WriteStreamTo(w)
-			w.Close()
-		}()
-	}
 
-	type writeStreamRes struct {
+	c.writeClean = isConnCleanAfterWrite(errConn) // TODO correct?
-		errStream, errConn error
-	}
-	writeStreamErrChan := make(chan writeStreamRes, 1)
-	go func() {
-		var res writeStreamRes
-		res.errStream, res.errConn = writeStream(ctx, c.hc, r, frameType)
-		if w != nil {
-			_ = w.CloseWithError(res.errStream) // always returns nil
-		}
-		writeStreamErrChan <- res
-	}()
 
-	writeRes := <-writeStreamErrChan
+	if errStream != nil {
-	streamCopierErr := <-streamCopierErrChan
+		return errStream
-	c.writeClean = isConnCleanAfterWrite(writeRes.errConn) // TODO correct?
+	} else if errConn != nil {
-	if streamCopierErr != nil && streamCopierErr.IsReadError() {
+		return errConn
-		return streamCopierErr // something on our side is bad
-	} else {
-		if writeRes.errStream != nil {
-			return writeRes.errStream
-		} else if writeRes.errConn != nil {
-			return writeRes.errConn
-		}
-		// TODO combined error?
-		return streamCopierErr
 	}
+	// TODO combined error?
+	return nil
 }
 
 type closeState struct {
@@ -248,17 +226,13 @@ type closeState struct {
 
 type closeStateErrConnectionClosed struct{}
 
-var _ zfs.StreamCopierError = (*closeStateErrConnectionClosed)(nil)
-var _ error = (*closeStateErrConnectionClosed)(nil)
 var _ net.Error = (*closeStateErrConnectionClosed)(nil)
 
 func (e *closeStateErrConnectionClosed) Error() string {
 	return "connection closed"
 }
-func (e *closeStateErrConnectionClosed) IsReadError() bool  { return true }
+func (e *closeStateErrConnectionClosed) Timeout() bool   { return false }
-func (e *closeStateErrConnectionClosed) IsWriteError() bool { return true }
+func (e *closeStateErrConnectionClosed) Temporary() bool { return false }
-func (e *closeStateErrConnectionClosed) Timeout() bool      { return false }
-func (e *closeStateErrConnectionClosed) Temporary() bool    { return false }
 
 func (s *closeState) CloseEntry() error {
 	firstCloser := atomic.AddUint32(&s.closeCount, 1) == 1
@@ -273,7 +247,7 @@ type closeStateEntry struct {
 	entryCount uint32
 }
 
-func (s *closeState) RWEntry() (e *closeStateEntry, err zfs.StreamCopierError) {
+func (s *closeState) RWEntry() (e *closeStateEntry, err net.Error) {
 	entry := &closeStateEntry{s, atomic.LoadUint32(&s.closeCount)}
 	if entry.entryCount > 0 {
 		return nil, &closeStateErrConnectionClosed{}
@@ -281,7 +255,7 @@ func (s *closeState) RWEntry() (e *closeStateEntry, err zfs.StreamCopierError) {
 	return entry, nil
 }
 
-func (e *closeStateEntry) RWExit() zfs.StreamCopierError {
+func (e *closeStateEntry) RWExit() net.Error {
 	if atomic.LoadUint32(&e.entryCount) == e.entryCount {
 		// no calls to Close() while running rw operation
 		return nil

rpc/grpcclientidentity/authlistener_grpc_adaptor.go

@@ -99,10 +99,23 @@ func (*transportCredentials) OverrideServerName(string) error {
 	panic("not implemented")
 }
 
-type ContextInterceptor = func(ctx context.Context) context.Context
+type ContextInterceptorData interface {
+	FullMethod() string
+	ClientIdentity() string
+}
 
-func NewInterceptors(logger Logger, clientIdentityKey interface{}, ctxInterceptor ContextInterceptor) (unary grpc.UnaryServerInterceptor, stream grpc.StreamServerInterceptor) {
+type contextInterceptorData struct {
-	unary = func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
+	fullMethod     string
+	clientIdentity string
+}
+
+func (d contextInterceptorData) FullMethod() string     { return d.fullMethod }
+func (d contextInterceptorData) ClientIdentity() string { return d.clientIdentity }
+
+type Interceptor = func(ctx context.Context, data ContextInterceptorData, handler func(ctx context.Context))
+
+func NewInterceptors(logger Logger, clientIdentityKey interface{}, interceptor Interceptor) (unary grpc.UnaryServerInterceptor, stream grpc.StreamServerInterceptor) {
+	unary = func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
 		logger.WithField("fullMethod", info.FullMethod).Debug("request")
 		p, ok := peer.FromContext(ctx)
 		if !ok {
@@ -115,10 +128,18 @@ func NewInterceptors(logger Logger, clientIdentityKey interface{}, ctxIntercepto
 		}
 		logger.WithField("peer_client_identity", a.clientIdentity).Debug("peer client identity")
 		ctx = context.WithValue(ctx, clientIdentityKey, a.clientIdentity)
-		if ctxInterceptor != nil {
+		data := contextInterceptorData{
-			ctx = ctxInterceptor(ctx)
+			fullMethod:     info.FullMethod,
+			clientIdentity: a.clientIdentity,
 		}
-		return handler(ctx, req)
+		var (
+			resp interface{}
+			err  error
+		)
+		interceptor(ctx, data, func(ctx context.Context) {
+			resp, err = handler(ctx, req) // no-shadow
+		})
+		return resp, err
 	}
 	stream = func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
 		panic("unimplemented")

rpc/grpcclientidentity/grpchelper/authlistener_grpc_adaptor_wrapper.go

@@ -50,7 +50,7 @@ func ClientConn(cn transport.Connecter, log Logger) *grpc.ClientConn {
 }
 
 // NewServer is a convenience interface around the TransportCredentials and Interceptors interface.
-func NewServer(authListener transport.AuthenticatedListener, clientIdentityKey interface{}, logger grpcclientidentity.Logger, ctxInterceptor grpcclientidentity.ContextInterceptor) (srv *grpc.Server, serve func() error) {
+func NewServer(authListener transport.AuthenticatedListener, clientIdentityKey interface{}, logger grpcclientidentity.Logger, ctxInterceptor grpcclientidentity.Interceptor) (srv *grpc.Server, serve func() error) {
 	ka := grpc.KeepaliveParams(keepalive.ServerParameters{
 		Time:    StartKeepalivesAfterInactivityDuration,
 		Timeout: KeepalivePeerTimeout,

rpc/netadaptor/authlistener_netlistener_adaptor.go

@@ -33,8 +33,12 @@ import (
 
 type Logger = logger.Logger
 
+type acceptRes struct {
+	conn *transport.AuthConn
+	err  error
+}
 type acceptReq struct {
-	callback chan net.Conn
+	callback chan acceptRes
 }
 
 type Listener struct {
@@ -64,10 +68,19 @@ func New(authListener transport.AuthenticatedListener, l Logger) *Listener {
 // The returned net.Conn is guaranteed to be *transport.AuthConn, i.e., the type of connection
 // returned by the wrapped transport.AuthenticatedListener.
 func (a Listener) Accept() (net.Conn, error) {
-	req := acceptReq{make(chan net.Conn, 1)}
+	req := acceptReq{make(chan acceptRes, 1)}
-	a.accepts <- req
+
-	conn := <-req.callback
+	select {
-	return conn, nil
+	case a.accepts <- req:
+	case <-a.stop:
+		return nil, fmt.Errorf("already closed") // TODO net.Error
+	}
+
+	res, ok := <-req.callback
+	if !ok {
+		return nil, fmt.Errorf("already closed") // TODO net.Error
+	}
+	return res.conn, res.err
 }
 
 func (a Listener) handleAccept() {
@@ -77,18 +90,9 @@ func (a Listener) handleAccept() {
 			a.logger.Debug("handleAccept stop accepting")
 			return
 		case req := <-a.accepts:
-			for {
+			a.logger.Debug("accept authListener")
-				a.logger.Debug("accept authListener")
+			authConn, err := a.al.Accept(context.Background())
-				authConn, err := a.al.Accept(context.Background())
+			req.callback <- acceptRes{authConn, err}
-				if err != nil {
-					a.logger.WithError(err).Error("accept error")
-					continue
-				}
-				a.logger.WithField("type", fmt.Sprintf("%T", authConn)).
-					Debug("accept complete")
-				req.callback <- authConn
-				break
-			}
 		}
 	}
 }

rpc/rpc_client.go

@@ -4,11 +4,13 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"net"
 	"sync"
 	"sync/atomic"
 	"time"
 
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 	"google.golang.org/grpc"
 
 	"github.com/google/uuid"
@@ -20,7 +22,6 @@ import (
 	"github.com/zrepl/zrepl/rpc/versionhandshake"
 	"github.com/zrepl/zrepl/transport"
 	"github.com/zrepl/zrepl/util/envconst"
-	"github.com/zrepl/zrepl/zfs"
 )
 
 // Client implements the active side of a replication setup.
@@ -82,49 +83,76 @@ func (c *Client) Close() {
 
 // callers must ensure that the returned io.ReadCloser is closed
 // TODO expose dataClient interface to the outside world
-func (c *Client) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, zfs.StreamCopier, error) {
+func (c *Client) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) {
+	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.Send")
+	defer endSpan()
+
 	// TODO the returned sendStream may return a read error created by the remote side
-	res, streamCopier, err := c.dataClient.ReqSend(ctx, r)
+	res, stream, err := c.dataClient.ReqSend(ctx, r)
 	if err != nil {
 		return nil, nil, err
 	}
-	if streamCopier == nil {
+	if stream == nil {
 		return res, nil, nil
 	}
 
-	return res, streamCopier, nil
+	return res, stream, nil
 
 }
 
-func (c *Client) Receive(ctx context.Context, req *pdu.ReceiveReq, streamCopier zfs.StreamCopier) (*pdu.ReceiveRes, error) {
+func (c *Client) Receive(ctx context.Context, req *pdu.ReceiveReq, stream io.ReadCloser) (*pdu.ReceiveRes, error) {
-	return c.dataClient.ReqRecv(ctx, req, streamCopier)
+	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.Receive")
+	defer endSpan()
+
+	return c.dataClient.ReqRecv(ctx, req, stream)
 }
 
 func (c *Client) ListFilesystems(ctx context.Context, in *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error) {
+	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.ListFilesystems")
+	defer endSpan()
+
 	return c.controlClient.ListFilesystems(ctx, in)
 }
 
 func (c *Client) ListFilesystemVersions(ctx context.Context, in *pdu.ListFilesystemVersionsReq) (*pdu.ListFilesystemVersionsRes, error) {
+	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.ListFilesystemVersions")
+	defer endSpan()
+
 	return c.controlClient.ListFilesystemVersions(ctx, in)
 }
 
 func (c *Client) DestroySnapshots(ctx context.Context, in *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error) {
+	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.DestroySnapshots")
+	defer endSpan()
+
 	return c.controlClient.DestroySnapshots(ctx, in)
 }
 
 func (c *Client) ReplicationCursor(ctx context.Context, in *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
+	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.ReplicationCursor")
+	defer endSpan()
+
 	return c.controlClient.ReplicationCursor(ctx, in)
 }
 
 func (c *Client) SendCompleted(ctx context.Context, in *pdu.SendCompletedReq) (*pdu.SendCompletedRes, error) {
+	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.SendCompleted")
+	defer endSpan()
+
 	return c.controlClient.SendCompleted(ctx, in)
 }
 
 func (c *Client) HintMostRecentCommonAncestor(ctx context.Context, in *pdu.HintMostRecentCommonAncestorReq) (*pdu.HintMostRecentCommonAncestorRes, error) {
+	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.HintMostRecentCommonAncestor")
+	defer endSpan()
+
 	return c.controlClient.HintMostRecentCommonAncestor(ctx, in)
 }
 
 func (c *Client) WaitForConnectivity(ctx context.Context) error {
+	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.WaitForConnectivity")
+	defer endSpan()
+
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 	msg := uuid.New().String()

rpc/rpc_logging.go

@@ -3,17 +3,12 @@ package rpc
 import (
 	"context"
 
+	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
 )
 
 type Logger = logger.Logger
 
-type contextKey int
-
-const (
-	contextKeyLoggers contextKey = iota
-)
-
 /// All fields must be non-nil
 type Loggers struct {
 	General Logger
@@ -21,11 +16,10 @@ type Loggers struct {
 	Data    Logger
 }
 
-func WithLoggers(ctx context.Context, loggers Loggers) context.Context {
-	ctx = context.WithValue(ctx, contextKeyLoggers, loggers)
-	return ctx
-}
-
 func GetLoggersOrPanic(ctx context.Context) Loggers {
-	return ctx.Value(contextKeyLoggers).(Loggers)
+	return Loggers{
+		General: logging.GetLogger(ctx, logging.SubsysRPC),
+		Control: logging.GetLogger(ctx, logging.SubsysRPCControl),
+		Data:    logging.GetLogger(ctx, logging.SubsysRPCData),
+	}
 }

rpc/rpc_server.go

@@ -7,6 +7,7 @@ import (
 	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/replication/logic/pdu"
 	"github.com/zrepl/zrepl/rpc/dataconn"
+	"github.com/zrepl/zrepl/rpc/grpcclientidentity"
 	"github.com/zrepl/zrepl/rpc/grpcclientidentity/grpchelper"
 	"github.com/zrepl/zrepl/rpc/versionhandshake"
 	"github.com/zrepl/zrepl/transport"
@@ -30,7 +31,20 @@ type Server struct {
 	dataServerServe    serveFunc
 }
 
-type HandlerContextInterceptor func(ctx context.Context) context.Context
+type HandlerContextInterceptorData interface {
+	FullMethod() string
+	ClientIdentity() string
+}
+
+type interceptorData struct {
+	prefixMethod string
+	wrapped      HandlerContextInterceptorData
+}
+
+func (d interceptorData) ClientIdentity() string { return d.wrapped.ClientIdentity() }
+func (d interceptorData) FullMethod() string     { return d.prefixMethod + d.wrapped.FullMethod() }
+
+type HandlerContextInterceptor func(ctx context.Context, data HandlerContextInterceptorData, handler func(ctx context.Context))
 
 // config must be valid (use its Validate function).
 func NewServer(handler Handler, loggers Loggers, ctxInterceptor HandlerContextInterceptor) *Server {
@@ -38,7 +52,10 @@ func NewServer(handler Handler, loggers Loggers, ctxInterceptor HandlerContextIn
 	// setup control server
 	controlServerServe := func(ctx context.Context, controlListener transport.AuthenticatedListener, errOut chan<- error) {
 
-		controlServer, serve := grpchelper.NewServer(controlListener, endpoint.ClientIdentityKey, loggers.Control, ctxInterceptor)
+		var controlCtxInterceptor grpcclientidentity.Interceptor = func(ctx context.Context, data grpcclientidentity.ContextInterceptorData, handler func(ctx context.Context)) {
+			ctxInterceptor(ctx, interceptorData{"control://", data}, handler)
+		}
+		controlServer, serve := grpchelper.NewServer(controlListener, endpoint.ClientIdentityKey, loggers.Control, controlCtxInterceptor)
 		pdu.RegisterReplicationServer(controlServer, handler)
 
 		// give time for graceful stop until deadline expires, then hard stop
@@ -47,8 +64,9 @@ func NewServer(handler Handler, loggers Loggers, ctxInterceptor HandlerContextIn
 			if dl, ok := ctx.Deadline(); ok {
 				go time.AfterFunc(dl.Sub(dl), controlServer.Stop)
 			}
-			loggers.Control.Debug("shutting down control server")
+			loggers.Control.Debug("gracefully shutting down control server")
 			controlServer.GracefulStop()
+			loggers.Control.Debug("gracdeful shut down of control server complete")
 		}()
 
 		errOut <- serve()
@@ -58,12 +76,12 @@ func NewServer(handler Handler, loggers Loggers, ctxInterceptor HandlerContextIn
 	dataServerClientIdentitySetter := func(ctx context.Context, wire *transport.AuthConn) (context.Context, *transport.AuthConn) {
 		ci := wire.ClientIdentity()
 		ctx = context.WithValue(ctx, endpoint.ClientIdentityKey, ci)
-		if ctxInterceptor != nil {
-			ctx = ctxInterceptor(ctx) // SHADOWING
-		}
 		return ctx, wire
 	}
-	dataServer := dataconn.NewServer(dataServerClientIdentitySetter, loggers.Data, handler)
+	var dataCtxInterceptor dataconn.ContextInterceptor = func(ctx context.Context, data dataconn.ContextInterceptorData, handler func(ctx context.Context)) {
+		ctxInterceptor(ctx, interceptorData{"data://", data}, handler)
+	}
+	dataServer := dataconn.NewServer(dataServerClientIdentitySetter, dataCtxInterceptor, loggers.Data, handler)
 	dataServerServe := func(ctx context.Context, dataListener transport.AuthenticatedListener, errOut chan<- error) {
 		dataServer.Serve(ctx, dataListener)
 		errOut <- nil // TODO bad design of dataServer?
@@ -84,6 +102,8 @@ func NewServer(handler Handler, loggers Loggers, ctxInterceptor HandlerContextIn
 // Serve never returns an error, it logs them to the Server's logger.
 func (s *Server) Serve(ctx context.Context, l transport.AuthenticatedListener) {
 	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	defer s.logger.Debug("rpc.(*Server).Serve done")
 
 	l = versionhandshake.Listener(l, envconst.Duration("ZREPL_RPC_SERVER_VERSIONHANDSHAKE_TIMEOUT", 10*time.Second))
 

rpc/transportmux/transportmux.go

@@ -7,33 +7,23 @@ package transportmux
 
 import (
 	"context"
+	"sync/atomic"
+	"syscall"
 
 	"fmt"
 	"io"
 	"net"
 	"time"
 
+	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/transport"
 )
 
-type contextKey int
-
-const (
-	contextKeyLog contextKey = 1 + iota
-)
-
 type Logger = logger.Logger
 
-func WithLogger(ctx context.Context, log Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLog, log)
-}
-
 func getLog(ctx context.Context) Logger {
-	if l, ok := ctx.Value(contextKeyLog).(Logger); ok {
+	return logging.GetLogger(ctx, logging.SubsysTransportMux)
-		return l
-	}
-	return logger.NewNullLogger()
 }
 
 type acceptRes struct {
@@ -42,12 +32,31 @@ type acceptRes struct {
 }
 
 type demuxListener struct {
-	conns chan acceptRes
+	closed int32
+	conns  chan acceptRes
+}
+
+var ErrClosed = &net.OpError{
+	Op:     "accept",
+	Net:    "demux",
+	Source: nil,
+	Addr:   nil,
+	Err:    syscall.EINVAL,
 }
 
 func (l *demuxListener) Accept(ctx context.Context) (*transport.AuthConn, error) {
-	res := <-l.conns
+	if atomic.LoadInt32(&l.closed) != 0 {
-	return res.conn, res.err
+		return nil, ErrClosed
+	}
+	select {
+	case r, ok := <-l.conns:
+		if !ok {
+			return nil, ErrClosed
+		}
+		return r.conn, r.err
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
 }
 
 type demuxAddr struct{}
@@ -59,7 +68,10 @@ func (l *demuxListener) Addr() net.Addr {
 	return demuxAddr{}
 }
 
-func (l *demuxListener) Close() error { return nil } // TODO
+func (l *demuxListener) Close() error {
+	atomic.StoreInt32(&l.closed, 1)
+	return nil
+}
 
 // Exact length of a label in bytes (0-byte padded if it is shorter).
 // This is a protocol constant, changing it breaks the wire protocol.
@@ -90,7 +102,10 @@ func Demux(ctx context.Context, rawListener transport.AuthenticatedListener, lab
 		if _, ok := padded[labelPadded]; ok {
 			return nil, fmt.Errorf("duplicate label %q", label)
 		}
-		dl := &demuxListener{make(chan acceptRes)}
+		dl := &demuxListener{
+			closed: 0,
+			conns:  make(chan acceptRes, 1),
+		}
 		padded[labelPadded] = dl
 		ret[label] = dl
 	}
@@ -103,10 +118,37 @@ func Demux(ctx context.Context, rawListener transport.AuthenticatedListener, lab
 		if err := rawListener.Close(); err != nil {
 			getLog(ctx).WithError(err).Error("error closing listener")
 		}
+
+		drainConns := func(ch chan acceptRes) {
+			for c := range ch {
+				if c.conn != nil {
+					if err := c.conn.Close(); err != nil {
+						getLog(ctx).WithError(err).Error("error closing connection while draining after listener was closed")
+					}
+				}
+			}
+		}
+		for _, dl := range ret {
+			atomic.StoreInt32(&dl.(*demuxListener).closed, 1)
+			drainConns(dl.(*demuxListener).conns)
+		}
 	}()
 
 	go func() {
+		defer func() {
+			for _, dl := range ret {
+				close(dl.(*demuxListener).conns)
+			}
+		}()
+
 		for {
+			select {
+			case <-ctx.Done():
+				getLog(ctx).WithError(ctx.Err()).Info("stop accepting new connections after context done")
+				return
+			default:
+			}
+
 			rawConn, err := rawListener.Accept(ctx)
 			if err != nil {
 				if ctx.Err() != nil {
@@ -147,7 +189,6 @@ func Demux(ctx context.Context, rawListener transport.AuthenticatedListener, lab
 			if err != nil {
 				getLog(ctx).WithError(err).Error("cannot reset deadline")
 			}
-			// blocking is intentional
 			demuxListener.conns <- acceptRes{conn: rawConn, err: nil}
 		}
 	}()

rpc/versionhandshake/versionhandshake.go

@@ -152,7 +152,7 @@ func (m *HandshakeMessage) DecodeReader(r io.Reader, maxLen int) error {
 
 func DoHandshakeCurrentVersion(conn net.Conn, deadline time.Time) *HandshakeError {
 	// current protocol version is hardcoded here
-	return DoHandshakeVersion(conn, deadline, 2)
+	return DoHandshakeVersion(conn, deadline, 3)
 }
 
 const HandshakeMessageMaxLen = 16 * 4096

transport/tcp/serve_tcp.go

@@ -65,6 +65,12 @@ type TCPAuthListener struct {
 }
 
 func (f *TCPAuthListener) Accept(ctx context.Context) (*transport.AuthConn, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	go func() {
+		<-ctx.Done()
+		cancel()
+	}()
 	nc, err := f.TCPListener.AcceptTCP()
 	if err != nil {
 		return nil, err

transport/transport.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"syscall"
 
+	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/rpc/dataconn/timeoutconn"
 	"github.com/zrepl/zrepl/zfs"
@@ -66,19 +67,8 @@ func ValidateClientIdentity(in string) (err error) {
 	return nil
 }
 
-type contextKey int
-
-const contextKeyLog contextKey = 0
-
 type Logger = logger.Logger
 
-func WithLogger(ctx context.Context, log Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLog, log)
-}
-
 func GetLogger(ctx context.Context) Logger {
-	if log, ok := ctx.Value(contextKeyLog).(Logger); ok {
+	return logging.GetLogger(ctx, logging.SubsysTransport)
-		return log
-	}
-	return logger.NewNullLogger()
 }

util/bytecounter/bytecounter_readcloser.go

@@ -0,0 +1,39 @@
+package bytecounter
+
+import (
+	"io"
+	"sync/atomic"
+)
+
+// ReadCloser wraps an io.ReadCloser, reimplementing
+// its interface and counting the bytes written to during copying.
+type ReadCloser interface {
+	io.ReadCloser
+	Count() int64
+}
+
+// NewReadCloser wraps rc.
+func NewReadCloser(rc io.ReadCloser) ReadCloser {
+	return &readCloser{rc, 0}
+}
+
+type readCloser struct {
+	rc    io.ReadCloser
+	count int64
+}
+
+func (r *readCloser) Count() int64 {
+	return atomic.LoadInt64(&r.count)
+}
+
+var _ io.ReadCloser = &readCloser{}
+
+func (r *readCloser) Close() error {
+	return r.rc.Close()
+}
+
+func (r *readCloser) Read(p []byte) (int, error) {
+	n, err := r.rc.Read(p)
+	atomic.AddInt64(&r.count, int64(n))
+	return n, err
+}