move implementation to internal/ directory (#828)

internal/cli/cli.go

@@ -0,0 +1,155 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/internal/daemon/logging/trace"
+
+	"github.com/zrepl/zrepl/internal/config"
+)
+
+var rootArgs struct {
+	configPath string
+}
+
+var rootCmd = &cobra.Command{
+	Use:   "zrepl",
+	Short: "One-stop ZFS replication solution",
+}
+
+func init() {
+	rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
+}
+
+var genCompletionCmd = &cobra.Command{
+	Use:   "gencompletion",
+	Short: "generate shell auto-completions",
+}
+
+type completionCmdInfo struct {
+	genFunc func(outpath string) error
+	help    string
+}
+
+var completionCmdMap = map[string]completionCmdInfo{
+	"zsh": {
+		rootCmd.GenZshCompletionFile,
+		"  save to file `_zrepl` in your zsh's $fpath",
+	},
+	"bash": {
+		rootCmd.GenBashCompletionFile,
+		"  save to a path and source that path in your .bashrc",
+	},
+}
+
+func init() {
+	for sh, info := range completionCmdMap {
+		sh, info := sh, info
+		genCompletionCmd.AddCommand(&cobra.Command{
+			Use:     fmt.Sprintf("%s path/to/out/file", sh),
+			Short:   fmt.Sprintf("generate %s completions", sh),
+			Example: info.help,
+			Run: func(cmd *cobra.Command, args []string) {
+				if len(args) != 1 {
+					fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
+					err := cmd.Usage()
+					if err != nil {
+						panic(err)
+					}
+					os.Exit(1)
+				}
+				if err := info.genFunc(args[0]); err != nil {
+					fmt.Fprintf(os.Stderr, "error generating %s completion: %s", sh, err)
+					os.Exit(1)
+				}
+			},
+		})
+	}
+	rootCmd.AddCommand(genCompletionCmd)
+}
+
+type Subcommand struct {
+	Use              string
+	Short            string
+	Example          string
+	NoRequireConfig  bool
+	Run              func(ctx context.Context, subcommand *Subcommand, args []string) error
+	SetupFlags       func(f *pflag.FlagSet)
+	SetupSubcommands func() []*Subcommand
+
+	config    *config.Config
+	configErr error
+}
+
+func (s *Subcommand) ConfigParsingError() error {
+	return s.configErr
+}
+
+func (s *Subcommand) Config() *config.Config {
+	if !s.NoRequireConfig && s.config == nil {
+		panic("command that requires config is running and has no config set")
+	}
+	return s.config
+}
+
+func (s *Subcommand) run(cmd *cobra.Command, args []string) {
+	s.tryParseConfig()
+	ctx := context.Background()
+	endTask := trace.WithTaskFromStackUpdateCtx(&ctx)
+	defer endTask()
+	err := s.Run(ctx, s, args)
+	endTask()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+		os.Exit(1)
+	}
+}
+
+func (s *Subcommand) tryParseConfig() {
+	config, err := config.ParseConfig(rootArgs.configPath)
+	s.configErr = err
+	if err != nil {
+		if s.NoRequireConfig {
+			// doesn't matter
+			return
+		} else {
+			fmt.Fprintf(os.Stderr, "could not parse config: %s\n", err)
+			os.Exit(1)
+		}
+	}
+	s.config = config
+}
+
+func AddSubcommand(s *Subcommand) {
+	addSubcommandToCobraCmd(rootCmd, s)
+}
+
+func addSubcommandToCobraCmd(c *cobra.Command, s *Subcommand) {
+	cmd := cobra.Command{
+		Use:     s.Use,
+		Short:   s.Short,
+		Example: s.Example,
+	}
+	if s.SetupSubcommands == nil {
+		cmd.Run = s.run
+	} else {
+		for _, sub := range s.SetupSubcommands() {
+			addSubcommandToCobraCmd(&cmd, sub)
+		}
+	}
+	if s.SetupFlags != nil {
+		s.SetupFlags(cmd.Flags())
+	}
+	c.AddCommand(&cmd)
+}
+
+func Run() {
+	if err := rootCmd.Execute(); err != nil {
+		os.Exit(1)
+	}
+}

internal/client/configcheck.go

@@ -0,0 +1,131 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/kr/pretty"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+	"github.com/zrepl/yaml-config"
+
+	"github.com/zrepl/zrepl/internal/cli"
+	"github.com/zrepl/zrepl/internal/config"
+	"github.com/zrepl/zrepl/internal/daemon/job"
+	"github.com/zrepl/zrepl/internal/daemon/logging"
+	"github.com/zrepl/zrepl/internal/logger"
+)
+
+var configcheckArgs struct {
+	format        string
+	what          string
+	skipCertCheck bool
+}
+
+var ConfigcheckCmd = &cli.Subcommand{
+	Use:   "configcheck",
+	Short: "check if config can be parsed without errors",
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]")
+		f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]")
+		f.BoolVar(&configcheckArgs.skipCertCheck, "skip-cert-check", false, "skip checking cert files")
+	},
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		formatMap := map[string]func(interface{}){
+			"": func(i interface{}) {},
+			"pretty": func(i interface{}) {
+				if _, err := pretty.Println(i); err != nil {
+					panic(err)
+				}
+			},
+			"json": func(i interface{}) {
+				if err := json.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
+					panic(err)
+				}
+			},
+			"yaml": func(i interface{}) {
+				if err := yaml.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
+					panic(err)
+				}
+			},
+		}
+
+		formatter, ok := formatMap[configcheckArgs.format]
+		if !ok {
+			return fmt.Errorf("unsupported --format %q", configcheckArgs.format)
+		}
+
+		var hadErr bool
+
+		parseFlags := config.ParseFlagsNone
+
+		if configcheckArgs.skipCertCheck {
+			parseFlags |= config.ParseFlagsNoCertCheck
+		}
+
+		// further: try to build jobs
+		confJobs, err := job.JobsFromConfig(subcommand.Config(), parseFlags)
+
+		if err != nil {
+			err := errors.Wrap(err, "cannot build jobs from config")
+			if configcheckArgs.what == "jobs" {
+				return err
+			} else {
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				confJobs = nil
+				hadErr = true
+			}
+		}
+
+		// further: try to build logging outlets
+		outlets, err := logging.OutletsFromConfig(*subcommand.Config().Global.Logging)
+		if err != nil {
+			err := errors.Wrap(err, "cannot build logging from config")
+			if configcheckArgs.what == "logging" {
+				return err
+			} else {
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				outlets = nil
+				hadErr = true
+			}
+		}
+
+		whatMap := map[string]func(){
+			"all": func() {
+				o := struct {
+					config  *config.Config
+					jobs    []job.Job
+					logging *logger.Outlets
+				}{
+					subcommand.Config(),
+					confJobs,
+					outlets,
+				}
+				formatter(o)
+			},
+			"config": func() {
+				formatter(subcommand.Config())
+			},
+			"jobs": func() {
+				formatter(confJobs)
+			},
+			"logging": func() {
+				formatter(outlets)
+			},
+		}
+
+		wf, ok := whatMap[configcheckArgs.what]
+		if !ok {
+			return fmt.Errorf("unsupported --format %q", configcheckArgs.what)
+		}
+		wf()
+
+		if hadErr {
+			return fmt.Errorf("config parsing failed")
+		} else {
+			return nil
+		}
+	},
+}

internal/client/jsonclient.go

@@ -0,0 +1,49 @@
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net"
+	"net/http"
+
+	"github.com/pkg/errors"
+)
+
+func controlHttpClient(sockpath string) (client http.Client, err error) {
+	return http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", sockpath)
+			},
+		},
+	}, nil
+}
+
+func jsonRequestResponse(c http.Client, endpoint string, req interface{}, res interface{}) error {
+	var buf bytes.Buffer
+	encodeErr := json.NewEncoder(&buf).Encode(req)
+	if encodeErr != nil {
+		return encodeErr
+	}
+
+	resp, err := c.Post("http://unix"+endpoint, "application/json", &buf)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		var msg bytes.Buffer
+		_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
+		return errors.Errorf("%s", msg.String())
+	}
+
+	decodeError := json.NewDecoder(resp.Body).Decode(&res)
+	if decodeError != nil {
+		return decodeError
+	}
+
+	return nil
+}

internal/client/migrate.go

@@ -0,0 +1,268 @@
+package client
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/fatih/color"
+	"github.com/kr/pretty"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/internal/daemon/job"
+	"github.com/zrepl/zrepl/internal/endpoint"
+	"github.com/zrepl/zrepl/internal/zfs"
+
+	"github.com/zrepl/zrepl/internal/cli"
+	"github.com/zrepl/zrepl/internal/config"
+)
+
+var (
+	MigrateCmd = &cli.Subcommand{
+		Use:   "migrate",
+		Short: "perform migration of the on-disk / zfs properties",
+		SetupSubcommands: func() []*cli.Subcommand {
+			return migrations
+		},
+	}
+)
+
+var migrations = []*cli.Subcommand{
+	&cli.Subcommand{
+		Use: "0.0.X:0.1:placeholder",
+		Run: doMigratePlaceholder0_1,
+		SetupFlags: func(f *pflag.FlagSet) {
+			f.BoolVar(&migratePlaceholder0_1Args.dryRun, "dry-run", false, "dry run")
+		},
+	},
+	&cli.Subcommand{
+		Use: "replication-cursor:v1-v2",
+		Run: doMigrateReplicationCursor,
+		SetupFlags: func(f *pflag.FlagSet) {
+			f.BoolVar(&migrateReplicationCursorArgs.dryRun, "dry-run", false, "dry run")
+		},
+	},
+}
+
+var migratePlaceholder0_1Args struct {
+	dryRun bool
+}
+
+func doMigratePlaceholder0_1(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	if len(args) != 0 {
+		return fmt.Errorf("migration does not take arguments, got %v", args)
+	}
+
+	cfg := sc.Config()
+
+	allFSS, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
+	if err != nil {
+		return errors.Wrap(err, "cannot list filesystems")
+	}
+
+	type workItem struct {
+		jobName string
+		rootFS  *zfs.DatasetPath
+		fss     []*zfs.DatasetPath
+	}
+	var wis []workItem
+	for i, j := range cfg.Jobs {
+		var rfsS string
+		switch job := j.Ret.(type) {
+		case *config.SinkJob:
+			rfsS = job.RootFS
+		case *config.PullJob:
+			rfsS = job.RootFS
+		default:
+			fmt.Printf("ignoring job %q (%d/%d, type %T)\n", j.Name(), i, len(cfg.Jobs), j.Ret)
+			continue
+		}
+		rfs, err := zfs.NewDatasetPath(rfsS)
+		if err != nil {
+			return errors.Wrapf(err, "root fs for job %q is not a valid dataset path", j.Name())
+		}
+		var fss []*zfs.DatasetPath
+		for _, fs := range allFSS {
+			if fs.HasPrefix(rfs) {
+				fss = append(fss, fs)
+			}
+		}
+		wis = append(wis, workItem{j.Name(), rfs, fss})
+	}
+
+	for _, wi := range wis {
+		fmt.Printf("job %q => migrate filesystems below root_fs %q\n", wi.jobName, wi.rootFS.ToString())
+		if len(wi.fss) == 0 {
+			fmt.Printf("\tno filesystems\n")
+			continue
+		}
+		for _, fs := range wi.fss {
+			fmt.Printf("\t%q ... ", fs.ToString())
+			r, err := zfs.ZFSMigrateHashBasedPlaceholderToCurrent(ctx, fs, migratePlaceholder0_1Args.dryRun)
+			if err != nil {
+				fmt.Printf("error: %s\n", err)
+			} else if !r.NeedsModification {
+				fmt.Printf("unchanged (placeholder=%v)\n", r.OriginalState.IsPlaceholder)
+			} else {
+				fmt.Printf("migrate (placeholder=%v) (old value = %q)\n",
+					r.OriginalState.IsPlaceholder, r.OriginalState.RawLocalPropertyValue)
+			}
+		}
+	}
+
+	return nil
+}
+
+var migrateReplicationCursorArgs struct {
+	dryRun bool
+}
+
+var bold = color.New(color.Bold)
+var succ = color.New(color.FgGreen)
+var fail = color.New(color.FgRed)
+
+var migrateReplicationCursorSkipSentinel = fmt.Errorf("skipping this filesystem")
+
+func doMigrateReplicationCursor(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	if len(args) != 0 {
+		return fmt.Errorf("migration does not take arguments, got %v", args)
+	}
+
+	cfg := sc.Config()
+	jobs, err := job.JobsFromConfig(cfg, config.ParseFlagsNone)
+	if err != nil {
+		fmt.Printf("cannot parse config:\n%s\n\n", err)
+		fmt.Printf("NOTE: this migration was released together with a change in job name requirements.\n")
+		return fmt.Errorf("exiting migration after error")
+	}
+
+	v1cursorJobs := make([]job.Job, 0, len(cfg.Jobs))
+	for i, j := range cfg.Jobs {
+		if jobs[i].Name() != j.Name() {
+			panic("implementation error")
+		}
+		switch j.Ret.(type) {
+		case *config.PushJob:
+			v1cursorJobs = append(v1cursorJobs, jobs[i])
+		case *config.SourceJob:
+			v1cursorJobs = append(v1cursorJobs, jobs[i])
+		default:
+			fmt.Printf("ignoring job %q (%d/%d, type %T), not supposed to create v1 replication cursors\n", j.Name(), i, len(cfg.Jobs), j.Ret)
+			continue
+		}
+	}
+
+	// scan all filesystems for v1 replication cursors
+
+	fss, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
+	if err != nil {
+		return errors.Wrap(err, "list filesystems")
+	}
+
+	var hadError bool
+	for _, fs := range fss {
+
+		bold.Printf("INSPECT FILESYSTEM %q\n", fs.ToString())
+
+		err := doMigrateReplicationCursorFS(ctx, v1cursorJobs, fs)
+		if err == migrateReplicationCursorSkipSentinel {
+			bold.Printf("FILESYSTEM SKIPPED\n")
+		} else if err != nil {
+			hadError = true
+			fail.Printf("MIGRATION FAILED: %s\n", err)
+		} else {
+			succ.Printf("FILESYSTEM %q COMPLETE\n", fs.ToString())
+		}
+	}
+
+	if hadError {
+		fail.Printf("\n\none or more filesystems could not be migrated, please inspect output and or re-run migration")
+		return errors.Errorf("")
+	}
+	return nil
+}
+
+func doMigrateReplicationCursorFS(ctx context.Context, v1CursorJobs []job.Job, fs *zfs.DatasetPath) error {
+
+	var owningJob job.Job = nil
+	for _, job := range v1CursorJobs {
+		conf := job.SenderConfig()
+		if conf == nil {
+			continue
+		}
+		pass, err := conf.FSF.Filter(fs)
+		if err != nil {
+			return errors.Wrapf(err, "filesystem filter error in job %q for fs %q", job.Name(), fs.ToString())
+		}
+		if !pass {
+			continue
+		}
+		if owningJob != nil {
+			return errors.Errorf("jobs %q and %q both match %q\ncannot attribute replication cursor to either one", owningJob.Name(), job.Name(), fs)
+		}
+		owningJob = job
+	}
+	if owningJob == nil {
+		fmt.Printf("no job's Filesystems filter matches\n")
+		return migrateReplicationCursorSkipSentinel
+	}
+	fmt.Printf("identified owning job %q\n", owningJob.Name())
+
+	bookmarks, err := zfs.ZFSListFilesystemVersions(ctx, fs, zfs.ListFilesystemVersionsOptions{
+		Types: zfs.Bookmarks,
+	})
+	if err != nil {
+		return errors.Wrapf(err, "list filesystem versions of %q", fs.ToString())
+	}
+
+	var oldCursor *zfs.FilesystemVersion
+	for i, fsv := range bookmarks {
+		_, _, err := endpoint.ParseReplicationCursorBookmarkName(fsv.ToAbsPath(fs))
+		if err != endpoint.ErrV1ReplicationCursor {
+			continue
+		}
+
+		if oldCursor != nil {
+			fmt.Printf("unexpected v1 replication cursor candidate: %q", fsv.ToAbsPath(fs))
+			return errors.Wrap(err, "multiple filesystem versions identified as v1 replication cursors")
+		}
+
+		oldCursor = &bookmarks[i]
+
+	}
+
+	if oldCursor == nil {
+		bold.Printf("no v1 replication cursor found for filesystem %q\n", fs.ToString())
+		return migrateReplicationCursorSkipSentinel
+	}
+
+	fmt.Printf("found v1 replication cursor:\n%s\n", pretty.Sprint(oldCursor))
+
+	mostRecentNew, err := endpoint.GetMostRecentReplicationCursorOfJob(ctx, fs.ToString(), owningJob.SenderConfig().JobID)
+	if err != nil {
+		return errors.Wrapf(err, "get most recent v2 replication cursor")
+	}
+
+	if mostRecentNew == nil {
+		return errors.Errorf("no v2 replication cursor found for job %q on filesystem %q", owningJob.SenderConfig().JobID, fs.ToString())
+	}
+
+	fmt.Printf("most recent v2 replication cursor:\n%#v", oldCursor)
+
+	if !(mostRecentNew.CreateTXG >= oldCursor.CreateTXG) {
+		return errors.Errorf("v1 replication cursor createtxg is higher than v2 cursor's, skipping this filesystem")
+	}
+
+	fmt.Printf("determined that v2 cursor is bookmark of same or newer version than v1 cursor\n")
+	fmt.Printf("destroying v1 cursor %q\n", oldCursor.ToAbsPath(fs))
+
+	if migrateReplicationCursorArgs.dryRun {
+		succ.Printf("DRY RUN\n")
+	} else {
+		if err := zfs.ZFSDestroyFilesystemVersion(ctx, fs, oldCursor); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

internal/client/migrate_test.go

@@ -0,0 +1,16 @@
+package client
+
+import "testing"
+
+func TestMigrationsUnambiguousNames(t *testing.T) {
+	names := make(map[string]bool)
+	for _, mig := range migrations {
+		if _, ok := names[mig.Use]; ok {
+			t.Errorf("duplicate migration name %q", mig.Use)
+			t.FailNow()
+			return
+		} else {
+			names[mig.Use] = true
+		}
+	}
+}

internal/client/pprof.go

@@ -0,0 +1,10 @@
+package client
+
+import "github.com/zrepl/zrepl/internal/cli"
+
+var PprofCmd = &cli.Subcommand{
+	Use: "pprof",
+	SetupSubcommands: func() []*cli.Subcommand {
+		return []*cli.Subcommand{PprofListenCmd, pprofActivityTraceCmd}
+	},
+}

internal/client/pprof_activity_trace.go

@@ -0,0 +1,44 @@
+package client
+
+import (
+	"context"
+	"io"
+	"log"
+	"os"
+
+	"golang.org/x/net/websocket"
+
+	"github.com/zrepl/zrepl/internal/cli"
+)
+
+var pprofActivityTraceCmd = &cli.Subcommand{
+	Use:   "activity-trace ZREPL_PPROF_HOST:ZREPL_PPROF_PORT",
+	Short: "attach to zrepl daemon with activated pprof listener and dump an activity-trace to stdout",
+	Run:   runPProfActivityTrace,
+}
+
+func runPProfActivityTrace(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+	log := log.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Printf("exiting after error")
+		os.Exit(1)
+	}
+
+	if len(args) != 1 {
+		log.Printf("exactly one positional argument is required")
+		die()
+	}
+
+	url := "ws://" + args[0] + "/debug/zrepl/activity-trace" // FIXME dont' repeat that
+
+	log.Printf("attaching to activity trace stream %s", url)
+	ws, err := websocket.Dial(url, "", url)
+	if err != nil {
+		log.Printf("error: %s", err)
+		die()
+	}
+
+	_, err = io.Copy(os.Stdout, ws)
+	return err
+}

internal/client/pprof_listen.go

@@ -0,0 +1,68 @@
+package client
+
+import (
+	"context"
+	"errors"
+	"log"
+	"os"
+
+	"github.com/zrepl/zrepl/internal/cli"
+	"github.com/zrepl/zrepl/internal/config"
+	"github.com/zrepl/zrepl/internal/daemon"
+)
+
+var pprofListenCmd struct {
+	daemon.PprofServerControlMsg
+}
+
+var PprofListenCmd = &cli.Subcommand{
+	Use:   "listen off | [on TCP_LISTEN_ADDRESS]",
+	Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		if len(args) < 1 {
+			goto enargs
+		}
+		switch args[0] {
+		case "on":
+			pprofListenCmd.Run = true
+			if len(args) != 2 {
+				return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
+			}
+			pprofListenCmd.HttpListenAddress = args[1]
+		case "off":
+			if len(args) != 1 {
+				goto enargs
+			}
+			pprofListenCmd.Run = false
+		}
+
+		RunPProf(subcommand.Config())
+		return nil
+	enargs:
+		return errors.New("invalid number of positional arguments")
+
+	},
+}
+
+func RunPProf(conf *config.Config) {
+	log := log.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Printf("exiting after error")
+		os.Exit(1)
+	}
+
+	log.Printf("connecting to zrepl daemon")
+
+	httpc, err := controlHttpClient(conf.Global.Control.SockPath)
+	if err != nil {
+		log.Printf("error creating http client: %s", err)
+		die()
+	}
+	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofListenCmd.PprofServerControlMsg, struct{}{})
+	if err != nil {
+		log.Printf("error sending control message: %s", err)
+		die()
+	}
+	log.Printf("finished")
+}

internal/client/signal.go

@@ -0,0 +1,42 @@
+package client
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/internal/cli"
+	"github.com/zrepl/zrepl/internal/config"
+	"github.com/zrepl/zrepl/internal/daemon"
+)
+
+var SignalCmd = &cli.Subcommand{
+	Use:   "signal [wakeup|reset] JOB",
+	Short: "wake up a job from wait state or abort its current invocation",
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		return runSignalCmd(subcommand.Config(), args)
+	},
+}
+
+func runSignalCmd(config *config.Config, args []string) error {
+	if len(args) != 2 {
+		return errors.Errorf("Expected 2 arguments: [wakeup|reset] JOB")
+	}
+
+	httpc, err := controlHttpClient(config.Global.Control.SockPath)
+	if err != nil {
+		return err
+	}
+
+	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointSignal,
+		struct {
+			Name string
+			Op   string
+		}{
+			Name: args[1],
+			Op:   args[0],
+		},
+		struct{}{},
+	)
+	return err
+}

internal/client/status/client/client.go

@@ -0,0 +1,111 @@
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net"
+	"net/http"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/internal/daemon"
+)
+
+type Client struct {
+	h *http.Client
+}
+
+func New(network, addr string) (*Client, error) {
+	httpc, err := makeControlHttpClient(func(_ context.Context) (net.Conn, error) { return net.Dial(network, addr) })
+	if err != nil {
+		return nil, err
+	}
+	return &Client{httpc}, nil
+}
+
+func (c *Client) Status() (s daemon.Status, _ error) {
+	err := jsonRequestResponse(c.h, daemon.ControlJobEndpointStatus,
+		struct{}{},
+		&s,
+	)
+	return s, err
+}
+
+func (c *Client) StatusRaw() ([]byte, error) {
+	var r json.RawMessage
+	err := jsonRequestResponse(c.h, daemon.ControlJobEndpointStatus, struct{}{}, &r)
+	if err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+
+func (c *Client) signal(job, sig string) error {
+	return jsonRequestResponse(c.h, daemon.ControlJobEndpointSignal,
+		struct {
+			Name string
+			Op   string
+		}{
+			Name: job,
+			Op:   sig,
+		},
+		struct{}{},
+	)
+}
+
+func (c *Client) SignalWakeup(job string) error {
+	return c.signal(job, "wakeup")
+}
+
+func (c *Client) SignalReset(job string) error {
+	return c.signal(job, "reset")
+}
+
+func makeControlHttpClient(dialfunc func(context.Context) (net.Conn, error)) (client *http.Client, err error) {
+	return &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+				return dialfunc(ctx)
+			},
+		},
+	}, nil
+}
+
+func jsonRequestResponse(c *http.Client, endpoint string, req interface{}, res interface{}) error {
+	var buf bytes.Buffer
+	encodeErr := json.NewEncoder(&buf).Encode(req)
+	if encodeErr != nil {
+		return encodeErr
+	}
+
+	hreq, err := http.NewRequest("POST", "http://unix"+endpoint, &buf)
+	if err != nil {
+		return err
+	}
+	hreq.Header.Set("Content-Type", "application/json")
+	// Prevent EOF errors when client request frequency and server keepalive close are at the same time.
+	// Found this by watching http.Server.ConnState changes, then found
+	// https://stackoverflow.com/questions/17714494/golang-http-request-results-in-eof-errors-when-making-multiple-requests-successi
+	// Note: The issue seems even more prounounced with local TCP sockets than unix domain sockets. So, I used that for debugging.
+	hreq.Close = true
+	resp, err := c.Do(hreq)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		var msg bytes.Buffer
+		_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
+		return errors.Errorf("%s", msg.String())
+	}
+
+	decodeError := json.NewDecoder(resp.Body).Decode(&res)
+	if decodeError != nil {
+		return decodeError
+	}
+
+	return nil
+}

internal/client/status/status.go

@@ -0,0 +1,97 @@
+package status
+
+import (
+	"context"
+	"os"
+	"time"
+
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/internal/cli"
+	"github.com/zrepl/zrepl/internal/client/status/client"
+	"github.com/zrepl/zrepl/internal/config"
+	"github.com/zrepl/zrepl/internal/daemon"
+	"github.com/zrepl/zrepl/internal/util/choices"
+)
+
+type Client interface {
+	Status() (daemon.Status, error)
+	StatusRaw() ([]byte, error)
+	SignalWakeup(job string) error
+	SignalReset(job string) error
+}
+
+type statusFlags struct {
+	Mode  choices.Choices
+	Job   string
+	Delay time.Duration
+}
+
+var statusv2Flags statusFlags
+
+type statusv2Mode int
+
+const (
+	StatusV2ModeInteractive statusv2Mode = 1 + iota
+	StatusV2ModeDump
+	StatusV2ModeRaw
+	StatusV2ModeLegacy
+)
+
+var Subcommand = &cli.Subcommand{
+	Use:   "status",
+	Short: "retrieve & display daemon status information",
+	SetupFlags: func(f *pflag.FlagSet) {
+		statusv2Flags.Mode.Init(
+			"interactive", StatusV2ModeInteractive,
+			"dump", StatusV2ModeDump,
+			"raw", StatusV2ModeRaw,
+			"legacy", StatusV2ModeLegacy,
+		)
+		statusv2Flags.Mode.SetTypeString("mode")
+		statusv2Flags.Mode.SetDefaultValue(StatusV2ModeInteractive)
+		f.Var(&statusv2Flags.Mode, "mode", statusv2Flags.Mode.Usage())
+		f.StringVar(&statusv2Flags.Job, "job", "", "only show specified job (works in \"dump\" and \"interactive\" mode)")
+		f.DurationVarP(&statusv2Flags.Delay, "delay", "d", 1*time.Second, "use -d 3s for 3 seconds delay (minimum delay is 1s)")
+	},
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		return runStatusV2Command(ctx, subcommand.Config(), args)
+	},
+}
+
+func runStatusV2Command(ctx context.Context, config *config.Config, args []string) error {
+
+	c, err := client.New("unix", config.Global.Control.SockPath)
+	if err != nil {
+		return errors.Wrapf(err, "connect to daemon socket at %q", config.Global.Control.SockPath)
+	}
+
+	mode := statusv2Flags.Mode.Value().(statusv2Mode)
+
+	if !isatty.IsTerminal(os.Stdout.Fd()) && mode != StatusV2ModeDump && mode != StatusV2ModeRaw {
+		dumpmode, err := statusv2Flags.Mode.InputForChoice(StatusV2ModeDump)
+		if err != nil {
+			panic(err)
+		}
+		rawmode, err := statusv2Flags.Mode.InputForChoice(StatusV2ModeRaw)
+		if err != nil {
+			panic(err)
+		}
+		return errors.Errorf("error: stdout is not a tty, please use --mode %s or --mode %s", dumpmode, rawmode)
+	}
+
+	switch mode {
+	case StatusV2ModeInteractive:
+		return interactive(c, statusv2Flags)
+	case StatusV2ModeDump:
+		return dump(c, statusv2Flags.Job)
+	case StatusV2ModeRaw:
+		return raw(c)
+	case StatusV2ModeLegacy:
+		return legacy(c, statusv2Flags)
+	default:
+		panic("unreachable")
+	}
+}

internal/client/status/status_dump.go

@@ -0,0 +1,70 @@
+package status
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/gdamore/tcell/v2"
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/internal/client/status/viewmodel"
+)
+
+func dump(c Client, job string) error {
+	s, err := c.Status()
+	if err != nil {
+		return err
+	}
+
+	if job != "" {
+		if _, ok := s.Jobs[job]; !ok {
+			return errors.Errorf("job %q not found", job)
+		}
+	}
+
+	width := (1 << 31) - 1
+	wrap := false
+	hline := strings.Repeat("-", 80)
+	if isatty.IsTerminal(os.Stdout.Fd()) {
+		wrap = true
+		screen, err := tcell.NewScreen()
+		if err != nil {
+			return errors.Wrap(err, "get terminal dimensions")
+		}
+		if err := screen.Init(); err != nil {
+			return errors.Wrap(err, "init screen")
+		}
+		width, _ = screen.Size()
+		screen.Fini()
+		hline = strings.Repeat("-", width)
+	}
+
+	m := viewmodel.New()
+	params := viewmodel.Params{
+		Report:                  s.Jobs,
+		ReportFetchError:        nil,
+		SelectedJob:             nil,
+		FSFilter:                func(s string) bool { return true },
+		DetailViewWidth:         width,
+		DetailViewWrap:          wrap,
+		ShortKeybindingOverview: "",
+	}
+	m.Update(params)
+	for _, j := range m.Jobs() {
+		if job != "" && j.Name() != job {
+			continue
+		}
+		params.SelectedJob = j
+		m.Update(params)
+		fmt.Println(m.SelectedJob().FullDescription())
+		if job != "" {
+			return nil
+		} else {
+			fmt.Println(hline)
+		}
+	}
+
+	return nil
+}

internal/client/status/status_interactive.go

@@ -0,0 +1,347 @@
+package status
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	tview "code.rocketnine.space/tslocum/cview"
+	"github.com/gdamore/tcell/v2"
+
+	"github.com/zrepl/zrepl/internal/client/status/viewmodel"
+)
+
+func interactive(c Client, flag statusFlags) error {
+
+	// Set this so we don't overwrite the default terminal colors
+	// See https://github.com/rivo/tview/blob/master/styles.go
+	tview.Styles.PrimitiveBackgroundColor = tcell.ColorDefault
+	tview.Styles.ContrastBackgroundColor = tcell.ColorDefault
+	tview.Styles.PrimaryTextColor = tcell.ColorDefault
+	tview.Styles.BorderColor = tcell.ColorDefault
+	app := tview.NewApplication()
+
+	jobDetailSplit := tview.NewFlex()
+	jobMenu := tview.NewTreeView()
+	jobMenuRoot := tview.NewTreeNode("jobs")
+	jobMenuRoot.SetSelectable(true)
+	jobMenu.SetRoot(jobMenuRoot)
+	jobMenu.SetCurrentNode(jobMenuRoot)
+	jobMenu.SetSelectedTextColor(tcell.ColorGreen)
+	jobTextDetail := tview.NewTextView()
+	jobTextDetail.SetWrap(false)
+
+	jobMenu.SetBorder(true)
+	jobTextDetail.SetBorder(true)
+
+	toolbarSplit := tview.NewFlex()
+	toolbarSplit.SetDirection(tview.FlexRow)
+	inputBarContainer := tview.NewFlex()
+	fsFilterInput := tview.NewInputField()
+	fsFilterInput.SetBorder(false)
+	fsFilterInput.SetFieldBackgroundColor(tcell.ColorDefault)
+	inputBarLabel := tview.NewTextView()
+	inputBarLabel.SetText("[::b]FILTER ")
+	inputBarLabel.SetDynamicColors(true)
+	inputBarContainer.AddItem(inputBarLabel, 7, 1, false)
+	inputBarContainer.AddItem(fsFilterInput, 0, 10, false)
+	toolbarSplit.AddItem(inputBarContainer, 1, 0, false)
+	toolbarSplit.AddItem(jobDetailSplit, 0, 10, false)
+
+	bottombar := tview.NewFlex()
+	bottombar.SetDirection(tview.FlexColumn)
+	bottombarDateView := tview.NewTextView()
+	bottombar.AddItem(bottombarDateView, len(time.Now().String()), 0, false)
+	bottomBarStatus := tview.NewTextView()
+	bottomBarStatus.SetDynamicColors(true)
+	bottomBarStatus.SetTextAlign(tview.AlignRight)
+	bottombar.AddItem(bottomBarStatus, 0, 10, false)
+	toolbarSplit.AddItem(bottombar, 1, 0, false)
+
+	tabbableWithJobMenu := []tview.Primitive{jobMenu, jobTextDetail, fsFilterInput}
+	tabbableWithoutJobMenu := []tview.Primitive{jobTextDetail, fsFilterInput}
+	var tabbable []tview.Primitive
+	tabbableActiveIndex := 0
+	tabbableRedraw := func() {
+		if len(tabbable) == 0 {
+			app.SetFocus(nil)
+			return
+		}
+		if tabbableActiveIndex >= len(tabbable) {
+			app.SetFocus(tabbable[0])
+			return
+		}
+		app.SetFocus(tabbable[tabbableActiveIndex])
+	}
+	tabbableCycle := func() {
+		if len(tabbable) == 0 {
+			return
+		}
+		tabbableActiveIndex = (tabbableActiveIndex + 1) % len(tabbable)
+		app.SetFocus(tabbable[tabbableActiveIndex])
+		tabbableRedraw()
+	}
+
+	jobMenuVisisble := false
+	reconfigureJobDetailSplit := func(setJobMenuVisible bool) {
+		if jobMenuVisisble == setJobMenuVisible {
+			return
+		}
+		jobMenuVisisble = setJobMenuVisible
+		if setJobMenuVisible {
+			jobDetailSplit.RemoveItem(jobTextDetail)
+			jobDetailSplit.AddItem(jobMenu, 0, 1, true)
+			jobDetailSplit.AddItem(jobTextDetail, 0, 5, false)
+			tabbable = tabbableWithJobMenu
+		} else {
+			jobDetailSplit.RemoveItem(jobMenu)
+			tabbable = tabbableWithoutJobMenu
+		}
+		tabbableRedraw()
+	}
+
+	showModal := func(m *tview.Modal, modalDoneFunc func(idx int, label string)) {
+		preModalFocus := app.GetFocus()
+		m.SetDoneFunc(func(idx int, label string) {
+			if modalDoneFunc != nil {
+				modalDoneFunc(idx, label)
+			}
+			app.SetRoot(toolbarSplit, true)
+			app.SetFocus(preModalFocus)
+			app.Draw()
+		})
+		app.SetRoot(m, true)
+		app.Draw()
+	}
+
+	app.SetRoot(toolbarSplit, true)
+	// initial focus
+	tabbableActiveIndex = len(tabbable)
+	tabbableCycle()
+	reconfigureJobDetailSplit(true)
+
+	m := viewmodel.New()
+	params := &viewmodel.Params{
+		Report:                  nil,
+		SelectedJob:             nil,
+		FSFilter:                func(_ string) bool { return true },
+		DetailViewWidth:         100,
+		DetailViewWrap:          false,
+		ShortKeybindingOverview: "[::b]Q[::-] quit  [::b]<TAB>[::-] switch panes [::b]W[::-] wrap lines  [::b]Shift+M[::-] toggle navbar  [::b]Shift+S[::-] signal job [::b]</>[::-] filter filesystems",
+	}
+	paramsMtx := &sync.Mutex{}
+	var redraw func()
+	viewmodelupdate := func(cb func(*viewmodel.Params)) {
+		paramsMtx.Lock()
+		defer paramsMtx.Unlock()
+		cb(params)
+		m.Update(*params)
+	}
+	redraw = func() {
+		jobs := m.Jobs()
+		if flag.Job != "" {
+			job_found := false
+			for _, job := range jobs {
+				if strings.Compare(flag.Job, job.Name()) == 0 {
+					jobs = []*viewmodel.Job{job}
+					job_found = true
+					break
+				}
+			}
+			if !job_found {
+				jobs = nil
+			}
+		}
+		redrawJobsList := false
+		var selectedJobN *tview.TreeNode
+		if len(jobMenuRoot.GetChildren()) == len(jobs) {
+			for i, jobN := range jobMenuRoot.GetChildren() {
+				if jobN.GetReference().(*viewmodel.Job) != jobs[i] {
+					redrawJobsList = true
+					break
+				}
+				if jobN.GetReference().(*viewmodel.Job) == m.SelectedJob() {
+					selectedJobN = jobN
+				}
+			}
+		} else {
+			redrawJobsList = true
+		}
+		if redrawJobsList {
+			selectedJobN = nil
+			children := make([]*tview.TreeNode, len(jobs))
+			for i := range jobs {
+				jobN := tview.NewTreeNode(jobs[i].JobTreeTitle())
+				jobN.SetReference(jobs[i])
+				jobN.SetSelectable(true)
+				children[i] = jobN
+				jobN.SetSelectedFunc(func() {
+					viewmodelupdate(func(p *viewmodel.Params) {
+						p.SelectedJob = jobN.GetReference().(*viewmodel.Job)
+					})
+				})
+				if jobs[i] == m.SelectedJob() {
+					selectedJobN = jobN
+				}
+			}
+			jobMenuRoot.SetChildren(children)
+		}
+
+		if selectedJobN != nil && jobMenu.GetCurrentNode() != selectedJobN {
+			jobMenu.SetCurrentNode(selectedJobN)
+		} else if selectedJobN == nil {
+			// select something, otherwise selection breaks (likely bug in tview)
+			jobMenu.SetCurrentNode(jobMenuRoot)
+		}
+
+		if selJ := m.SelectedJob(); selJ != nil {
+			jobTextDetail.SetText(selJ.FullDescription())
+		} else {
+			jobTextDetail.SetText("please select a job")
+		}
+
+		bottombardatestring := m.DateString()
+		bottombarDateView.SetText(bottombardatestring)
+		bottombar.ResizeItem(bottombarDateView, len(bottombardatestring), 0)
+
+		bottomBarStatus.SetText(m.BottomBarStatus())
+
+		app.Draw()
+
+	}
+
+	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				app.Suspend(func() {
+					panic(err)
+				})
+			}
+		}()
+		for {
+			st, err := c.Status()
+			viewmodelupdate(func(p *viewmodel.Params) {
+				p.Report = st.Jobs
+				p.ReportFetchError = err
+			})
+			app.QueueUpdateDraw(redraw)
+
+			time.Sleep(flag.Delay)
+		}
+	}()
+
+	jobMenu.SetChangedFunc(func(jobN *tview.TreeNode) {
+		viewmodelupdate(func(p *viewmodel.Params) {
+			p.SelectedJob, _ = jobN.GetReference().(*viewmodel.Job)
+		})
+		redraw()
+		jobTextDetail.ScrollToBeginning()
+	})
+	jobMenu.SetSelectedFunc(func(jobN *tview.TreeNode) {
+		app.SetFocus(jobTextDetail)
+	})
+
+	app.SetBeforeDrawFunc(func(screen tcell.Screen) bool {
+		viewmodelupdate(func(p *viewmodel.Params) {
+			_, _, p.DetailViewWidth, _ = jobTextDetail.GetInnerRect()
+		})
+		return false
+	})
+
+	app.SetInputCapture(func(e *tcell.EventKey) *tcell.EventKey {
+		if e.Key() == tcell.KeyTab {
+			tabbableCycle()
+			return nil
+		}
+
+		if e.Key() == tcell.KeyRune && app.GetFocus() == fsFilterInput {
+			return e
+		}
+
+		if e.Key() == tcell.KeyRune && e.Rune() == '/' {
+			if app.GetFocus() != fsFilterInput {
+				app.SetFocus(fsFilterInput)
+			}
+			return e
+		}
+
+		if e.Key() == tcell.KeyRune && e.Rune() == 'M' {
+			reconfigureJobDetailSplit(!jobMenuVisisble)
+			return nil
+		}
+
+		if e.Key() == tcell.KeyRune && e.Rune() == 'q' {
+			app.Stop()
+		}
+
+		if e.Key() == tcell.KeyRune && e.Rune() == 'S' {
+			job, ok := jobMenu.GetCurrentNode().GetReference().(*viewmodel.Job)
+			if !ok {
+				return nil
+			}
+			signals := []string{"wakeup", "reset"}
+			clientFuncs := []func(job string) error{c.SignalWakeup, c.SignalReset}
+			sigMod := tview.NewModal()
+			sigMod.SetBackgroundColor(tcell.ColorDefault)
+			sigMod.SetBorder(true)
+			sigMod.GetForm().SetButtonTextColorFocused(tcell.ColorGreen)
+			sigMod.AddButtons(signals)
+			sigMod.SetText(fmt.Sprintf("Send a signal to job %q", job.Name()))
+			showModal(sigMod, func(idx int, _ string) {
+				go func() {
+					if idx == -1 {
+						return
+					}
+					err := clientFuncs[idx](job.Name())
+					if err != nil {
+						app.QueueUpdate(func() {
+							me := tview.NewModal()
+							me.SetText(fmt.Sprintf("signal error: %s", err))
+							me.AddButtons([]string{"Close"})
+							showModal(me, nil)
+						})
+					}
+				}()
+			})
+		}
+
+		return e
+	})
+
+	fsFilterInput.SetChangedFunc(func(searchterm string) {
+		viewmodelupdate(func(p *viewmodel.Params) {
+			p.FSFilter = func(fs string) bool {
+				r, err := regexp.Compile(searchterm)
+				if err != nil {
+					return true
+				}
+				return r.MatchString(fs)
+			}
+		})
+		redraw()
+		jobTextDetail.ScrollToBeginning()
+	})
+	fsFilterInput.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+		if event.Key() == tcell.KeyEnter {
+			app.SetFocus(jobTextDetail)
+			return nil
+		}
+		return event
+	})
+
+	jobTextDetail.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+		if event.Key() == tcell.KeyRune && event.Rune() == 'w' {
+			// toggle wrapping
+			viewmodelupdate(func(p *viewmodel.Params) {
+				p.DetailViewWrap = !p.DetailViewWrap
+			})
+			redraw()
+			return nil
+		}
+		return event
+	})
+
+	return app.Run()
+}

internal/client/status/status_legacy.go

@@ -0,0 +1,128 @@
+package status
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	tview "code.rocketnine.space/tslocum/cview"
+	"github.com/gdamore/tcell/v2"
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/internal/client/status/viewmodel"
+)
+
+func legacy(c Client, flag statusFlags) error {
+
+	// Set this so we don't overwrite the default terminal colors
+	// See https://github.com/rivo/tview/blob/master/styles.go
+	tview.Styles.PrimitiveBackgroundColor = tcell.ColorDefault
+	tview.Styles.ContrastBackgroundColor = tcell.ColorDefault
+	tview.Styles.PrimaryTextColor = tcell.ColorDefault
+	tview.Styles.BorderColor = tcell.ColorDefault
+	app := tview.NewApplication()
+
+	textView := tview.NewTextView()
+	textView.SetWrap(true)
+	textView.SetScrollable(true) // so that it allows us to set scroll position
+	textView.SetScrollBarVisibility(tview.ScrollBarNever)
+
+	app.SetRoot(textView, true)
+
+	width := (1 << 31) - 1
+	wrap := false
+	if isatty.IsTerminal(os.Stdout.Fd()) {
+		wrap = true
+		screen, err := tcell.NewScreen()
+		if err != nil {
+			return errors.Wrap(err, "get terminal dimensions")
+		}
+		if err := screen.Init(); err != nil {
+			return errors.Wrap(err, "init screen")
+		}
+		width, _ = screen.Size()
+		screen.Fini()
+	}
+
+	paramsMtx := &sync.Mutex{}
+	params := viewmodel.Params{
+		Report:                  nil,
+		ReportFetchError:        nil,
+		SelectedJob:             nil,
+		FSFilter:                func(s string) bool { return true },
+		DetailViewWidth:         width,
+		DetailViewWrap:          wrap,
+		ShortKeybindingOverview: "",
+	}
+
+	redraw := func() {
+		textView.Clear()
+
+		paramsMtx.Lock()
+		defer paramsMtx.Unlock()
+
+		if params.ReportFetchError != nil {
+			fmt.Fprintln(textView, params.ReportFetchError.Error())
+		} else if params.Report != nil {
+			m := viewmodel.New()
+			m.Update(params)
+			for _, j := range m.Jobs() {
+				if flag.Job != "" && j.Name() != flag.Job {
+					continue
+				}
+				params.SelectedJob = j
+				m.Update(params)
+				fmt.Fprintln(textView, m.SelectedJob().FullDescription())
+				if flag.Job != "" {
+					break
+				} else {
+					hline := strings.Repeat("-", params.DetailViewWidth)
+					fmt.Fprintln(textView, hline)
+				}
+			}
+		} else {
+			fmt.Fprintln(textView, "waiting for request results")
+		}
+		textView.ScrollToBeginning()
+	}
+
+	app.SetBeforeDrawFunc(func(screen tcell.Screen) bool {
+		// sync resizes to `params`
+		paramsMtx.Lock()
+		_, _, newWidth, _ := textView.GetInnerRect()
+		if newWidth != params.DetailViewWidth {
+			params.DetailViewWidth = newWidth
+			app.QueueUpdateDraw(redraw)
+		}
+		paramsMtx.Unlock()
+
+		textView.ScrollToBeginning() // has the effect of inhibiting user scrolls
+		return false
+	})
+
+	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				app.Suspend(func() {
+					panic(err)
+				})
+			}
+		}()
+		for {
+			st, err := c.Status()
+			paramsMtx.Lock()
+			params.Report = st.Jobs
+			params.ReportFetchError = err
+			paramsMtx.Unlock()
+
+			app.QueueUpdateDraw(redraw)
+
+			time.Sleep(flag.Delay)
+		}
+	}()
+
+	return app.Run()
+}

internal/client/status/status_raw.go

@@ -0,0 +1,18 @@
+package status
+
+import (
+	"bytes"
+	"io"
+	"os"
+)
+
+func raw(c Client) error {
+	b, err := c.StatusRaw()
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(os.Stdout, bytes.NewReader(b)); err != nil {
+		return err
+	}
+	return nil
+}

internal/client/status/viewmodel/bytecountbinary.go

@@ -0,0 +1,26 @@
+package viewmodel
+
+import (
+	"fmt"
+	"math"
+)
+
+func ByteCountBinaryUint(b uint64) string {
+	if b > math.MaxInt64 {
+		panic(b)
+	}
+	return ByteCountBinary(int64(b))
+}
+
+func ByteCountBinary(b int64) string {
+	const unit = 1024
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := unit, 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
+}

internal/client/status/viewmodel/bytesprogresshistory.go

@@ -0,0 +1,49 @@
+package viewmodel
+
+import "time"
+
+type byteProgressMeasurement struct {
+	time time.Time
+	val  uint64
+}
+
+type bytesProgressHistory struct {
+	last        *byteProgressMeasurement // pointer as poor man's optional
+	changeCount int
+	lastChange  time.Time
+}
+
+func (p *bytesProgressHistory) Update(currentVal uint64) (bytesPerSecondAvg int64, changeCount int) {
+
+	if p.last == nil {
+		p.last = &byteProgressMeasurement{
+			time: time.Now(),
+			val:  currentVal,
+		}
+		return 0, 0
+	}
+
+	if p.last.val != currentVal {
+		p.changeCount++
+		p.lastChange = time.Now()
+	}
+
+	if time.Since(p.lastChange) > 3*time.Second {
+		p.last = nil
+		return 0, 0
+	}
+
+	var deltaV int64
+	if currentVal >= p.last.val {
+		deltaV = int64(currentVal - p.last.val)
+	} else {
+		deltaV = -int64(p.last.val - currentVal)
+	}
+	deltaT := time.Since(p.last.time)
+	rate := float64(deltaV) / deltaT.Seconds()
+
+	p.last.time = time.Now()
+	p.last.val = currentVal
+
+	return int64(rate), p.changeCount
+}

internal/client/status/viewmodel/render.go

@@ -0,0 +1,660 @@
+package viewmodel
+
+import (
+	"fmt"
+	"math"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/go-playground/validator/v10"
+	yaml "github.com/zrepl/yaml-config"
+
+	"github.com/zrepl/zrepl/internal/client/status/viewmodel/stringbuilder"
+	"github.com/zrepl/zrepl/internal/daemon"
+	"github.com/zrepl/zrepl/internal/daemon/job"
+	"github.com/zrepl/zrepl/internal/daemon/pruner"
+	"github.com/zrepl/zrepl/internal/daemon/snapper"
+	"github.com/zrepl/zrepl/internal/replication/report"
+)
+
+type M struct {
+	jobs            map[string]*Job
+	jobsList        []*Job
+	selectedJob     *Job
+	dateString      string
+	bottomBarStatus string
+}
+
+type Job struct {
+	// long-lived
+	name         string
+	byteProgress *bytesProgressHistory
+
+	lastStatus      *job.Status
+	fulldescription string
+}
+
+func New() *M {
+	return &M{
+		jobs:        make(map[string]*Job),
+		jobsList:    make([]*Job, 0),
+		selectedJob: nil,
+	}
+}
+
+type FilterFunc func(string) bool
+
+type Params struct {
+	Report                  map[string]*job.Status
+	ReportFetchError        error
+	SelectedJob             *Job
+	FSFilter                FilterFunc `validate:"required"`
+	DetailViewWidth         int        `validate:"gte=1"`
+	DetailViewWrap          bool
+	ShortKeybindingOverview string
+}
+
+var validate = validator.New()
+
+func (m *M) Update(p Params) {
+
+	if err := validate.Struct(p); err != nil {
+		panic(err)
+	}
+
+	if p.ReportFetchError != nil {
+		m.bottomBarStatus = fmt.Sprintf("[red::]status fetch: %s", p.ReportFetchError)
+	} else {
+		m.bottomBarStatus = p.ShortKeybindingOverview
+		for jobname, st := range p.Report {
+			// TODO handle job renames & deletions
+			j, ok := m.jobs[jobname]
+			if !ok {
+				j = &Job{
+					name:         jobname,
+					byteProgress: &bytesProgressHistory{},
+				}
+				m.jobs[jobname] = j
+				m.jobsList = append(m.jobsList, j)
+			}
+			j.lastStatus = st
+		}
+	}
+
+	// filter out internal jobs
+	var jobsList []*Job
+	for _, j := range m.jobsList {
+		if daemon.IsInternalJobName(j.name) {
+			continue
+		}
+		jobsList = append(jobsList, j)
+	}
+	m.jobsList = jobsList
+
+	// determinism!
+	sort.Slice(m.jobsList, func(i, j int) bool {
+		return strings.Compare(m.jobsList[i].name, m.jobsList[j].name) < 0
+	})
+
+	// try to not lose the selected job
+	m.selectedJob = nil
+	for _, j := range m.jobsList {
+		j.updateFullDescription(p)
+		if j == p.SelectedJob {
+			m.selectedJob = j
+		}
+	}
+
+	m.dateString = time.Now().Format(time.RFC3339)
+
+}
+
+func (m *M) BottomBarStatus() string { return m.bottomBarStatus }
+
+func (m *M) Jobs() []*Job { return m.jobsList }
+
+// may be nil
+func (m *M) SelectedJob() *Job { return m.selectedJob }
+
+func (m *M) DateString() string { return m.dateString }
+
+func (j *Job) updateFullDescription(p Params) {
+	width := p.DetailViewWidth
+	if !p.DetailViewWrap {
+		width = 10000000 // FIXME
+	}
+	b := stringbuilder.New(stringbuilder.Config{
+		IndentMultiplier: 3,
+		Width:            width,
+	})
+	drawJob(b, j.name, j.lastStatus, j.byteProgress, p.FSFilter)
+	j.fulldescription = b.String()
+}
+
+func (j *Job) JobTreeTitle() string {
+	return j.name
+}
+
+func (j *Job) FullDescription() string {
+	return j.fulldescription
+}
+
+func (j *Job) Name() string {
+	return j.name
+}
+
+func drawJob(t *stringbuilder.B, name string, v *job.Status, history *bytesProgressHistory, fsfilter FilterFunc) {
+
+	t.Printf("Job: %s\n", name)
+	t.Printf("Type: %s\n\n", v.Type)
+
+	if v.Type == job.TypePush || v.Type == job.TypePull {
+		activeStatus, ok := v.JobSpecific.(*job.ActiveSideStatus)
+		if !ok || activeStatus == nil {
+			t.Printf("ActiveSideStatus is null")
+			t.Newline()
+			return
+		}
+
+		t.Printf("Replication:")
+		t.AddIndentAndNewline(1)
+		renderReplicationReport(t, activeStatus.Replication, history, fsfilter)
+		t.AddIndentAndNewline(-1)
+
+		t.Printf("Pruning Sender:")
+		t.AddIndentAndNewline(1)
+		renderPrunerReport(t, activeStatus.PruningSender, fsfilter)
+		t.AddIndentAndNewline(-1)
+
+		t.Printf("Pruning Receiver:")
+		t.AddIndentAndNewline(1)
+		renderPrunerReport(t, activeStatus.PruningReceiver, fsfilter)
+		t.AddIndentAndNewline(-1)
+
+		if v.Type == job.TypePush {
+			t.Printf("Snapshotting:")
+			t.AddIndentAndNewline(1)
+			renderSnapperReport(t, activeStatus.Snapshotting, fsfilter)
+			t.AddIndentAndNewline(-1)
+		}
+
+	} else if v.Type == job.TypeSnap {
+		snapStatus, ok := v.JobSpecific.(*job.SnapJobStatus)
+		if !ok || snapStatus == nil {
+			t.Printf("SnapJobStatus is null")
+			t.Newline()
+			return
+		}
+		t.Printf("Pruning snapshots:")
+		t.AddIndentAndNewline(1)
+		renderPrunerReport(t, snapStatus.Pruning, fsfilter)
+		t.AddIndentAndNewline(-1)
+		t.Printf("Snapshotting:")
+		t.AddIndentAndNewline(1)
+		renderSnapperReport(t, snapStatus.Snapshotting, fsfilter)
+		t.AddIndentAndNewline(-1)
+	} else if v.Type == job.TypeSource {
+
+		st := v.JobSpecific.(*job.PassiveStatus)
+		t.Printf("Snapshotting:\n")
+		t.AddIndent(1)
+		renderSnapperReport(t, st.Snapper, fsfilter)
+		t.AddIndentAndNewline(-1)
+
+	} else {
+		t.Printf("No status representation for job type '%s', dumping as YAML", v.Type)
+		t.Newline()
+		asYaml, err := yaml.Marshal(v.JobSpecific)
+		if err != nil {
+			t.Printf("Error marshaling status to YAML: %s", err)
+			t.Newline()
+			return
+		}
+		t.Write(string(asYaml))
+		t.Newline()
+	}
+}
+
+func printFilesystemStatus(t *stringbuilder.B, rep *report.FilesystemReport, maxFS int) {
+
+	expected, replicated, containsInvalidSizeEstimates := rep.BytesSum()
+	sizeEstimationImpreciseNotice := ""
+	if containsInvalidSizeEstimates {
+		sizeEstimationImpreciseNotice = " (some steps lack size estimation)"
+	}
+	if rep.CurrentStep < len(rep.Steps) && rep.Steps[rep.CurrentStep].Info.BytesExpected == 0 {
+		sizeEstimationImpreciseNotice = " (step lacks size estimation)"
+	}
+
+	userVisisbleCurrentStep, userVisibleTotalSteps := rep.CurrentStep, len(rep.Steps)
+	// `.CurrentStep` is == len(rep.Steps) if all steps are done.
+	// Until then, it's an index into .Steps that starts at 0.
+	// For the user, we want it to start at 1.
+	if rep.CurrentStep >= len(rep.Steps) {
+		// rep.CurrentStep is what we want to show.
+		// We check for >= and not == for robustness.
+	} else {
+		// We're not done yet, so, make step count start at 1
+		// (The `.State` is included in the output, indicating we're not done yet)
+		userVisisbleCurrentStep = rep.CurrentStep + 1
+	}
+	status := fmt.Sprintf("%s (step %d/%d, %s/%s)%s",
+		strings.ToUpper(string(rep.State)),
+		userVisisbleCurrentStep, userVisibleTotalSteps,
+		ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected),
+		sizeEstimationImpreciseNotice,
+	)
+
+	activeIndicator := " "
+	if rep.BlockedOn == report.FsBlockedOnNothing &&
+		(rep.State == report.FilesystemPlanning || rep.State == report.FilesystemStepping) {
+		activeIndicator = "*"
+	}
+	t.AddIndent(1)
+	t.Printf("%s %s %s ",
+		activeIndicator,
+		stringbuilder.RightPad(rep.Info.Name, maxFS, " "),
+		status)
+
+	next := ""
+	if err := rep.Error(); err != nil {
+		next = err.Err
+	} else if rep.State != report.FilesystemDone {
+		if nextStep := rep.NextStep(); nextStep != nil {
+			if nextStep.IsIncremental() {
+				next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
+			} else {
+				next = fmt.Sprintf("next: full send %s", nextStep.Info.To)
+			}
+			attribs := []string{}
+
+			if nextStep.Info.Resumed {
+				attribs = append(attribs, "resumed")
+			}
+
+			if len(attribs) > 0 {
+				next += fmt.Sprintf(" (%s)", strings.Join(attribs, ", "))
+			}
+		} else {
+			next = "" // individual FSes may still be in planning state
+		}
+
+	}
+	t.Printf("%s", next)
+
+	t.AddIndent(-1)
+	t.Newline()
+}
+
+func renderReplicationReport(t *stringbuilder.B, rep *report.Report, history *bytesProgressHistory, fsfilter FilterFunc) {
+	if rep == nil {
+		t.Printf("...\n")
+		return
+	}
+
+	if rep.WaitReconnectError != nil {
+		t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: %s", rep.WaitReconnectError)
+		t.Newline()
+	}
+	if !rep.WaitReconnectSince.IsZero() {
+		delta := time.Until(rep.WaitReconnectUntil).Round(time.Second)
+		if rep.WaitReconnectUntil.IsZero() || delta > 0 {
+			var until string
+			if rep.WaitReconnectUntil.IsZero() {
+				until = "waiting indefinitely"
+			} else {
+				until = fmt.Sprintf("hard fail in %s @ %s", delta, rep.WaitReconnectUntil)
+			}
+			t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnecting with exponential backoff (since %s) (%s)",
+				rep.WaitReconnectSince, until)
+		} else {
+			t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnects reached hard-fail timeout @ %s", rep.WaitReconnectUntil)
+		}
+		t.Newline()
+	}
+
+	// TODO visualize more than the latest attempt by folding all attempts into one
+	if len(rep.Attempts) == 0 {
+		t.Printf("no attempts made yet")
+		return
+	} else {
+		t.Printf("Attempt #%d", len(rep.Attempts))
+		if len(rep.Attempts) > 1 {
+			t.Printf(". Previous attempts failed with the following statuses:")
+			t.AddIndentAndNewline(1)
+			for i, a := range rep.Attempts[:len(rep.Attempts)-1] {
+				t.PrintfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)\n", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt))
+			}
+			t.AddIndentAndNewline(-1)
+		} else {
+			t.Newline()
+		}
+	}
+
+	latest := rep.Attempts[len(rep.Attempts)-1]
+	sort.Slice(latest.Filesystems, func(i, j int) bool {
+		return latest.Filesystems[i].Info.Name < latest.Filesystems[j].Info.Name
+	})
+
+	// apply filter
+	filtered := make([]*report.FilesystemReport, 0, len(latest.Filesystems))
+	for _, fs := range latest.Filesystems {
+		if !fsfilter(fs.Info.Name) {
+			continue
+		}
+		filtered = append(filtered, fs)
+	}
+	latest.Filesystems = filtered
+
+	t.Printf("Status: %s", latest.State)
+	t.Newline()
+	if !latest.FinishAt.IsZero() {
+		t.Printf("Last Run: %s (lasted %s)\n", latest.FinishAt.Round(time.Second), latest.FinishAt.Sub(latest.StartAt).Round(time.Second))
+	} else {
+		t.Printf("Started: %s (lasting %s)\n", latest.StartAt.Round(time.Second), time.Since(latest.StartAt).Round(time.Second))
+	}
+
+	if latest.State == report.AttemptPlanningError {
+		t.Printf("Problem: ")
+		t.PrintfDrawIndentedAndWrappedIfMultiline("%s", latest.PlanError)
+		t.Newline()
+	} else if latest.State == report.AttemptFanOutError {
+		t.Printf("Problem: one or more of the filesystems encountered errors")
+		t.Newline()
+	}
+
+	if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
+		// Draw global progress bar
+		// Progress: [---------------]
+		expected, replicated, containsInvalidSizeEstimates := latest.BytesSum()
+		rate, changeCount := history.Update(replicated)
+		eta := time.Duration(0)
+		if rate > 0 {
+			eta = time.Duration((float64(expected)-float64(replicated))/float64(rate)) * time.Second
+		}
+
+		if !latest.State.IsTerminal() {
+			t.Write("Progress: ")
+			t.DrawBar(50, replicated, expected, changeCount)
+			t.Write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected), ByteCountBinary(rate)))
+			if eta != 0 {
+				t.Write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
+			}
+			t.Newline()
+		}
+		if containsInvalidSizeEstimates {
+			t.Write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
+			t.Newline()
+		}
+
+		if len(latest.Filesystems) == 0 {
+			t.Write("NOTE: no filesystems were considered for replication!")
+			t.Newline()
+		}
+
+		var maxFSLen int
+		for _, fs := range latest.Filesystems {
+			if len(fs.Info.Name) > maxFSLen {
+				maxFSLen = len(fs.Info.Name)
+			}
+		}
+		for _, fs := range latest.Filesystems {
+			printFilesystemStatus(t, fs, maxFSLen)
+		}
+
+	}
+}
+
+func humanizeDuration(duration time.Duration) string {
+	days := int64(duration.Hours() / 24)
+	hours := int64(math.Mod(duration.Hours(), 24))
+	minutes := int64(math.Mod(duration.Minutes(), 60))
+	seconds := int64(math.Mod(duration.Seconds(), 60))
+
+	var parts []string
+
+	force := false
+	chunks := []int64{days, hours, minutes, seconds}
+	for i, chunk := range chunks {
+		if force || chunk > 0 {
+			padding := 0
+			if force {
+				padding = 2
+			}
+			parts = append(parts, fmt.Sprintf("%*d%c", padding, chunk, "dhms"[i]))
+			force = true
+		}
+	}
+
+	return strings.Join(parts, " ")
+}
+
+func renderPrunerReport(t *stringbuilder.B, r *pruner.Report, fsfilter FilterFunc) {
+	if r == nil {
+		t.Printf("...\n")
+		return
+	}
+
+	state, err := pruner.StateString(r.State)
+	if err != nil {
+		t.Printf("Status: %q (parse error: %q)\n", r.State, err)
+		return
+	}
+
+	t.Printf("Status: %s", state)
+	t.Newline()
+
+	if r.Error != "" {
+		t.Printf("Error: %s\n", r.Error)
+	}
+
+	type commonFS struct {
+		*pruner.FSReport
+		completed bool
+	}
+	all := make([]commonFS, 0, len(r.Pending)+len(r.Completed))
+	for i := range r.Pending {
+		all = append(all, commonFS{&r.Pending[i], false})
+	}
+	for i := range r.Completed {
+		all = append(all, commonFS{&r.Completed[i], true})
+	}
+
+	// filter all
+	filtered := make([]commonFS, 0, len(all))
+	for _, fs := range all {
+		if fsfilter(fs.FSReport.Filesystem) {
+			filtered = append(filtered, fs)
+		}
+	}
+	all = filtered
+
+	switch state {
+	case pruner.Plan:
+		fallthrough
+	case pruner.PlanErr:
+		return
+	}
+
+	if len(all) == 0 {
+		t.Printf("nothing to do\n")
+		return
+	}
+
+	var totalDestroyCount, completedDestroyCount int
+	var maxFSname int
+	for _, fs := range all {
+		totalDestroyCount += len(fs.DestroyList)
+		if fs.completed {
+			completedDestroyCount += len(fs.DestroyList)
+		}
+		if maxFSname < len(fs.Filesystem) {
+			maxFSname = len(fs.Filesystem)
+		}
+	}
+
+	// global progress bar
+	if !state.IsTerminal() {
+		progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
+		t.Write("Progress: ")
+		t.Write("[")
+		t.Write(stringbuilder.Times("=", progress))
+		t.Write(">")
+		t.Write(stringbuilder.Times("-", 80-progress))
+		t.Write("]")
+		t.Printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
+		t.Newline()
+	}
+
+	sort.SliceStable(all, func(i, j int) bool {
+		return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
+	})
+
+	// Draw a table-like representation of 'all'
+	for _, fs := range all {
+		t.Write(stringbuilder.RightPad(fs.Filesystem, maxFSname, " "))
+		t.Write(" ")
+		if !fs.SkipReason.NotSkipped() {
+			t.Printf("skipped: %s\n", fs.SkipReason)
+			continue
+		}
+		if fs.LastError != "" {
+			if strings.ContainsAny(fs.LastError, "\r\n") {
+				t.Printf("ERROR:")
+				t.PrintfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError)
+			} else {
+				t.PrintfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError)
+			}
+			t.Newline()
+			continue
+		}
+
+		pruneRuleActionStr := fmt.Sprintf("(destroy %d of %d snapshots)",
+			len(fs.DestroyList), len(fs.SnapshotList))
+
+		if fs.completed {
+			t.Printf("Completed  %s\n", pruneRuleActionStr)
+			continue
+		}
+
+		t.Write("Pending    ") // whitespace is padding 10
+		if len(fs.DestroyList) == 1 {
+			t.Write(fs.DestroyList[0].Name)
+		} else {
+			t.Write(pruneRuleActionStr)
+		}
+		t.Newline()
+	}
+
+}
+
+func renderSnapperReport(t *stringbuilder.B, r *snapper.Report, fsfilter FilterFunc) {
+	if r == nil {
+		t.Printf("<no snapshotting report available>\n")
+		return
+	}
+	t.Printf("Type: %s\n", r.Type)
+	if r.Periodic != nil {
+		renderSnapperReportPeriodic(t, r.Periodic, fsfilter)
+	} else if r.Cron != nil {
+		renderSnapperReportCron(t, r.Cron, fsfilter)
+	} else {
+		t.Printf("<no details available>")
+	}
+}
+
+func renderSnapperReportPeriodic(t *stringbuilder.B, r *snapper.PeriodicReport, fsfilter FilterFunc) {
+	t.Printf("Status: %s", r.State)
+	t.Newline()
+
+	if r.Error != "" {
+		t.Printf("Error: %s\n", r.Error)
+	}
+	if !r.SleepUntil.IsZero() {
+		t.Printf("Sleep until: %s\n", r.SleepUntil)
+	}
+
+	renderSnapperPlanReportFilesystem(t, r.Progress, fsfilter)
+}
+
+func renderSnapperReportCron(t *stringbuilder.B, r *snapper.CronReport, fsfilter FilterFunc) {
+	t.Printf("State: %s\n", r.State)
+
+	now := time.Now()
+	if r.WakeupTime.After(now) {
+		t.Printf("Sleep until: %s (%s remaining)\n", r.WakeupTime, r.WakeupTime.Sub(now).Round(time.Second))
+	} else {
+		t.Printf("Started: %s (lasting %s)\n", r.WakeupTime, now.Sub(r.WakeupTime).Round(time.Second))
+	}
+
+	renderSnapperPlanReportFilesystem(t, r.Progress, fsfilter)
+}
+
+func renderSnapperPlanReportFilesystem(t *stringbuilder.B, fss []*snapper.ReportFilesystem, fsfilter FilterFunc) {
+	sort.Slice(fss, func(i, j int) bool {
+		return strings.Compare(fss[i].Path, fss[j].Path) == -1
+	})
+
+	dur := func(d time.Duration) string {
+		return d.Round(100 * time.Millisecond).String()
+	}
+
+	type row struct {
+		path, state, duration, remainder, hookReport string
+	}
+	var widths struct {
+		path, state, duration int
+	}
+	rows := make([]*row, 0, len(fss))
+	for _, fs := range fss {
+		if !fsfilter(fs.Path) {
+			continue
+		}
+		r := &row{
+			path:  fs.Path,
+			state: fs.State.String(),
+		}
+		if fs.HooksHadError {
+			r.hookReport = fs.Hooks // FIXME render here, not in daemon
+		}
+		switch fs.State {
+		case snapper.SnapPending:
+			r.duration = "..."
+			r.remainder = ""
+		case snapper.SnapStarted:
+			r.duration = dur(time.Since(fs.StartAt))
+			r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
+		case snapper.SnapDone:
+			fallthrough
+		case snapper.SnapError:
+			r.duration = dur(fs.DoneAt.Sub(fs.StartAt))
+			r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
+		}
+		rows = append(rows, r)
+		if len(r.path) > widths.path {
+			widths.path = len(r.path)
+		}
+		if len(r.state) > widths.state {
+			widths.state = len(r.state)
+		}
+		if len(r.duration) > widths.duration {
+			widths.duration = len(r.duration)
+		}
+	}
+
+	for _, r := range rows {
+		path := stringbuilder.RightPad(r.path, widths.path, " ")
+		state := stringbuilder.RightPad(r.state, widths.state, " ")
+		duration := stringbuilder.RightPad(r.duration, widths.duration, " ")
+		t.Printf("%s %s %s", path, state, duration)
+		t.PrintfDrawIndentedAndWrappedIfMultiline(" %s", r.remainder)
+		if r.hookReport != "" {
+			t.AddIndent(1)
+			t.Newline()
+			t.Printf("%s", r.hookReport)
+			t.AddIndent(-1)
+		}
+		t.Newline()
+	}
+}

internal/client/status/viewmodel/stringbuilder/stringbuilder.go

@@ -0,0 +1,119 @@
+package stringbuilder
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/go-playground/validator/v10"
+)
+
+type B struct {
+	// const
+	indentMultiplier int
+
+	// mut
+	sb     *strings.Builder
+	indent int
+	width  int
+	x, y   int
+}
+
+type Config struct {
+	IndentMultiplier int `validate:"gte=1"`
+	Width            int `validate:"gte=1"`
+}
+
+var validate = validator.New()
+
+func New(config Config) *B {
+
+	if err := validate.Struct(config); err != nil {
+		panic(err)
+	}
+
+	return &B{sb: &strings.Builder{}, width: config.Width, indentMultiplier: config.IndentMultiplier}
+}
+
+func (b *B) String() string { return b.sb.String() }
+
+func (w *B) Newline() {
+	w.Write("\n")
+}
+
+func (w *B) PrintfDrawIndentedAndWrappedIfMultiline(format string, args ...interface{}) {
+	whole := fmt.Sprintf(format, args...)
+	if strings.ContainsAny(whole, "\n\r") {
+		w.AddIndent(1)
+		defer w.AddIndent(-1)
+	}
+	w.Write(whole)
+}
+
+func (w *B) Printf(format string, args ...interface{}) {
+	whole := fmt.Sprintf(format, args...)
+	w.Write(whole)
+}
+
+func (t *B) AddIndent(delta int) {
+	t.indent += delta * t.indentMultiplier
+}
+
+func (t *B) AddIndentAndNewline(delta int) {
+	t.indent += delta * t.indentMultiplier
+	t.Write("\n")
+}
+
+func (w *B) Write(s string) {
+	for _, c := range s {
+		if c == '\n' {
+			fmt.Fprint(w.sb, "\n")
+			w.x = 0
+			fmt.Fprint(w.sb, Times(" ", w.indent-w.x))
+			w.x = w.indent
+			w.y++
+			continue
+		}
+		if w.x >= w.width {
+			fmt.Fprint(w.sb, "\n")
+			w.x = 0
+			fmt.Fprint(w.sb, Times(" ", w.indent-w.x))
+			w.x = w.indent
+		}
+		fmt.Fprintf(w.sb, "%c", c)
+		w.x++
+	}
+}
+
+func Times(str string, n int) (out string) {
+	for i := 0; i < n; i++ {
+		out += str
+	}
+	return
+}
+
+func RightPad(str string, length int, pad string) string {
+	if len(str) > length {
+		return str[:length]
+	}
+	return str + strings.Repeat(pad, length-len(str))
+}
+
+// changeCount = 0 indicates stall / no progress
+func (w *B) DrawBar(length int, bytes, totalBytes uint64, changeCount int) {
+	const arrowPositions = `>\|/`
+	var completedLength int
+	if totalBytes > 0 {
+		completedLength = int(uint64(length) * bytes / totalBytes)
+		if completedLength > length {
+			completedLength = length
+		}
+	} else if totalBytes == bytes {
+		completedLength = length
+	}
+
+	w.Write("[")
+	w.Write(Times("=", completedLength))
+	w.Write(string(arrowPositions[changeCount%len(arrowPositions)]))
+	w.Write(Times("-", length-completedLength))
+	w.Write("]")
+}

internal/client/stdinserver.go

@@ -0,0 +1,51 @@
+package client
+
+import (
+	"os"
+
+	"github.com/problame/go-netssh"
+
+	"github.com/zrepl/zrepl/internal/cli"
+	"github.com/zrepl/zrepl/internal/config"
+
+	"context"
+	"errors"
+	"log"
+	"path"
+)
+
+var StdinserverCmd = &cli.Subcommand{
+	Use:   "stdinserver CLIENT_IDENTITY",
+	Short: "stdinserver transport mode (started from authorized_keys file as forced command)",
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		return runStdinserver(subcommand.Config(), args)
+	},
+}
+
+func runStdinserver(config *config.Config, args []string) error {
+
+	// NOTE: the netssh proxying protocol requires exiting with non-zero status if anything goes wrong
+	defer os.Exit(1)
+
+	log := log.New(os.Stderr, "", log.LUTC|log.Ldate|log.Ltime)
+
+	if len(args) != 1 || args[0] == "" {
+		err := errors.New("must specify client_identity as positional argument")
+		return err
+	}
+
+	identity := args[0]
+	unixaddr := path.Join(config.Global.Serve.StdinServer.SockDir, identity)
+
+	log.Printf("proxying client identity '%s' to zrepl daemon '%s'", identity, unixaddr)
+
+	ctx := netssh.ContextWithLog(context.TODO(), log)
+
+	err := netssh.Proxy(ctx, unixaddr)
+	if err == nil {
+		log.Print("proxying finished successfully, exiting with status 0")
+		os.Exit(0)
+	}
+	log.Printf("error proxying: %s", err)
+	return nil
+}

internal/client/testcmd.go

@@ -0,0 +1,220 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/internal/cli"
+	"github.com/zrepl/zrepl/internal/config"
+	"github.com/zrepl/zrepl/internal/daemon/filters"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+var TestCmd = &cli.Subcommand{
+	Use: "test",
+	SetupSubcommands: func() []*cli.Subcommand {
+		return []*cli.Subcommand{testFilter, testPlaceholder, testDecodeResumeToken}
+	},
+}
+
+var testFilterArgs struct {
+	job   string
+	all   bool
+	input string
+}
+
+var testFilter = &cli.Subcommand{
+	Use:   "filesystems --job JOB [--all | --input INPUT]",
+	Short: "test filesystems filter specified in push or source job",
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&testFilterArgs.job, "job", "", "the name of the push or source job")
+		f.StringVar(&testFilterArgs.input, "input", "", "a filesystem name to test against the job's filters")
+		f.BoolVar(&testFilterArgs.all, "all", false, "test all local filesystems")
+	},
+	Run: runTestFilterCmd,
+}
+
+func runTestFilterCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+
+	if testFilterArgs.job == "" {
+		return fmt.Errorf("must specify --job flag")
+	}
+	if !(testFilterArgs.all != (testFilterArgs.input != "")) { // xor
+		return fmt.Errorf("must set one: --all or --input")
+	}
+
+	conf := subcommand.Config()
+
+	var confFilter config.FilesystemsFilter
+	job, err := conf.Job(testFilterArgs.job)
+	if err != nil {
+		return err
+	}
+	switch j := job.Ret.(type) {
+	case *config.SourceJob:
+		confFilter = j.Filesystems
+	case *config.PushJob:
+		confFilter = j.Filesystems
+	case *config.SnapJob:
+		confFilter = j.Filesystems
+	default:
+		return fmt.Errorf("job type %T does not have filesystems filter", j)
+	}
+
+	f, err := filters.DatasetMapFilterFromConfig(confFilter)
+	if err != nil {
+		return fmt.Errorf("filter invalid: %s", err)
+	}
+
+	var fsnames []string
+	if testFilterArgs.input != "" {
+		fsnames = []string{testFilterArgs.input}
+	} else {
+		out, err := zfs.ZFSList(ctx, []string{"name"})
+		if err != nil {
+			return fmt.Errorf("could not list ZFS filesystems: %s", err)
+		}
+		for _, row := range out {
+
+			fsnames = append(fsnames, row[0])
+		}
+	}
+
+	fspaths := make([]*zfs.DatasetPath, len(fsnames))
+	for i, fsname := range fsnames {
+		path, err := zfs.NewDatasetPath(fsname)
+		if err != nil {
+			return err
+		}
+		fspaths[i] = path
+	}
+
+	hadFilterErr := false
+	for _, in := range fspaths {
+		var res string
+		var errStr string
+		pass, err := f.Filter(in)
+		if err != nil {
+			res = "ERROR"
+			errStr = err.Error()
+			hadFilterErr = true
+		} else if pass {
+			res = "ACCEPT"
+		} else {
+			res = "REJECT"
+		}
+		fmt.Printf("%s\t%s\t%s\n", res, in.ToString(), errStr)
+	}
+
+	if hadFilterErr {
+		return fmt.Errorf("filter errors occurred")
+	}
+	return nil
+}
+
+var testPlaceholderArgs struct {
+	ds  string
+	all bool
+}
+
+var testPlaceholder = &cli.Subcommand{
+	Use:   "placeholder [--all | --dataset DATASET]",
+	Short: fmt.Sprintf("list received placeholder filesystems (zfs property %q)", zfs.PlaceholderPropertyName),
+	Example: `
+	placeholder --all
+	placeholder --dataset path/to/sink/clientident/fs`,
+	NoRequireConfig: true,
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&testPlaceholderArgs.ds, "dataset", "", "dataset path (not required to exist)")
+		f.BoolVar(&testPlaceholderArgs.all, "all", false, "list tab-separated placeholder status of all filesystems")
+	},
+	Run: runTestPlaceholder,
+}
+
+func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+
+	var checkDPs []*zfs.DatasetPath
+	var datasetWasExplicitArgument bool
+
+	// all actions first
+	if testPlaceholderArgs.all {
+		datasetWasExplicitArgument = false
+		out, err := zfs.ZFSList(ctx, []string{"name"})
+		if err != nil {
+			return errors.Wrap(err, "could not list ZFS filesystems")
+		}
+		for _, row := range out {
+			dp, err := zfs.NewDatasetPath(row[0])
+			if err != nil {
+				panic(err)
+			}
+			checkDPs = append(checkDPs, dp)
+		}
+	} else {
+		datasetWasExplicitArgument = true
+		dp, err := zfs.NewDatasetPath(testPlaceholderArgs.ds)
+		if err != nil {
+			return err
+		}
+		if dp.Empty() {
+			return fmt.Errorf("must specify --dataset DATASET or --all")
+		}
+		checkDPs = append(checkDPs, dp)
+	}
+
+	fmt.Printf("IS_PLACEHOLDER\tDATASET\tzrepl:placeholder\n")
+	for _, dp := range checkDPs {
+		ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, dp)
+		if err != nil {
+			return errors.Wrap(err, "cannot get placeholder state")
+		}
+		if !ph.FSExists {
+			if datasetWasExplicitArgument {
+				return errors.Errorf("filesystem %q does not exist", ph.FS)
+			} else {
+				// got deleted between ZFSList and ZFSGetFilesystemPlaceholderState
+				continue
+			}
+		}
+		is := "yes"
+		if !ph.IsPlaceholder {
+			is = "no"
+		}
+		fmt.Printf("%s\t%s\t%s\n", is, dp.ToString(), ph.RawLocalPropertyValue)
+	}
+	return nil
+}
+
+var testDecodeResumeTokenArgs struct {
+	token string
+}
+
+var testDecodeResumeToken = &cli.Subcommand{
+	Use:   "decoderesumetoken --token TOKEN",
+	Short: "decode resume token",
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&testDecodeResumeTokenArgs.token, "token", "", "the resume token obtained from the receive_resume_token property")
+	},
+	Run: runTestDecodeResumeTokenCmd,
+}
+
+func runTestDecodeResumeTokenCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+	if testDecodeResumeTokenArgs.token == "" {
+		return fmt.Errorf("token argument must be specified")
+	}
+	token, err := zfs.ParseResumeToken(ctx, testDecodeResumeTokenArgs.token)
+	if err != nil {
+		return err
+	}
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", " ")
+	if err := enc.Encode(&token); err != nil {
+		panic(err)
+	}
+	return nil
+}

internal/client/version.go

@@ -0,0 +1,75 @@
+package client
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/internal/cli"
+	"github.com/zrepl/zrepl/internal/config"
+	"github.com/zrepl/zrepl/internal/daemon"
+	"github.com/zrepl/zrepl/internal/version"
+)
+
+var versionArgs struct {
+	Show      string
+	Config    *config.Config
+	ConfigErr error
+}
+
+var VersionCmd = &cli.Subcommand{
+	Use:             "version",
+	Short:           "print version of zrepl binary and running daemon",
+	NoRequireConfig: true,
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)")
+	},
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		versionArgs.Config = subcommand.Config()
+		versionArgs.ConfigErr = subcommand.ConfigParsingError()
+		return runVersionCmd()
+	},
+}
+
+func runVersionCmd() error {
+	args := versionArgs
+
+	if args.Show != "daemon" && args.Show != "client" && args.Show != "" {
+		return fmt.Errorf("show flag must be 'client' or 'server' or be left empty")
+	}
+
+	var clientVersion, daemonVersion *version.ZreplVersionInformation
+	if args.Show == "client" || args.Show == "" {
+		clientVersion = version.NewZreplVersionInformation()
+		fmt.Printf("client: %s\n", clientVersion.String())
+	}
+	if args.Show == "daemon" || args.Show == "" {
+
+		if args.ConfigErr != nil {
+			return fmt.Errorf("config parsing error: %s", args.ConfigErr)
+		}
+
+		httpc, err := controlHttpClient(args.Config.Global.Control.SockPath)
+		if err != nil {
+			return fmt.Errorf("server: error: %s\n", err)
+		}
+
+		var info version.ZreplVersionInformation
+		err = jsonRequestResponse(httpc, daemon.ControlJobEndpointVersion, "", &info)
+		if err != nil {
+			return fmt.Errorf("server: error: %s\n", err)
+		}
+		daemonVersion = &info
+		fmt.Printf("server: %s\n", daemonVersion.String())
+	}
+
+	if args.Show == "" {
+		if clientVersion.Version != daemonVersion.Version {
+			fmt.Fprintf(os.Stderr, "WARNING: client version != daemon version, restart zrepl daemon\n")
+		}
+	}
+
+	return nil
+}

internal/client/zfsabstractions.go

@@ -0,0 +1,147 @@
+package client
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/internal/cli"
+	"github.com/zrepl/zrepl/internal/daemon/filters"
+	"github.com/zrepl/zrepl/internal/endpoint"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+var (
+	ZFSAbstractionsCmd = &cli.Subcommand{
+		Use:   "zfs-abstraction",
+		Short: "manage abstractions that zrepl builds on top of ZFS",
+		SetupSubcommands: func() []*cli.Subcommand {
+			return []*cli.Subcommand{
+				zabsCmdList,
+				zabsCmdReleaseAll,
+				zabsCmdReleaseStale,
+				zabsCmdCreate,
+			}
+		},
+	}
+)
+
+// a common set of CLI flags that map to the fields of an
+// endpoint.ListZFSHoldsAndBookmarksQuery
+type zabsFilterFlags struct {
+	Filesystems FilesystemsFilterFlag
+	Job         JobIDFlag
+	Types       AbstractionTypesFlag
+	Concurrency int64
+}
+
+// produce a query from the CLI flags
+func (f zabsFilterFlags) Query() (endpoint.ListZFSHoldsAndBookmarksQuery, error) {
+	q := endpoint.ListZFSHoldsAndBookmarksQuery{
+		FS:          f.Filesystems.FlagValue(),
+		What:        f.Types.FlagValue(),
+		JobID:       f.Job.FlagValue(),
+		Concurrency: f.Concurrency,
+	}
+	return q, q.Validate()
+}
+
+func (f *zabsFilterFlags) registerZabsFilterFlags(s *pflag.FlagSet, verb string) {
+	// Note: the default value is defined in the .FlagValue methods
+	s.Var(&f.Filesystems, "fs", fmt.Sprintf("only %s holds on the specified filesystem [default: all filesystems] [comma-separated list of <dataset-pattern>:<ok|!> pairs]", verb))
+	s.Var(&f.Job, "job", fmt.Sprintf("only %s holds created by the specified job [default: any job]", verb))
+
+	variants := make([]string, 0, len(endpoint.AbstractionTypesAll))
+	for v := range endpoint.AbstractionTypesAll {
+		variants = append(variants, string(v))
+	}
+	sort.Strings(variants)
+	variantsJoined := strings.Join(variants, "|")
+	s.Var(&f.Types, "type", fmt.Sprintf("only %s holds of the specified type [default: all] [comma-separated list of %s]", verb, variantsJoined))
+
+	s.Int64VarP(&f.Concurrency, "concurrency", "p", 1, "number of concurrently queried filesystems")
+}
+
+type JobIDFlag struct{ J *endpoint.JobID }
+
+func (f *JobIDFlag) Set(s string) error {
+	if len(s) == 0 {
+		*f = JobIDFlag{J: nil}
+		return nil
+	}
+
+	jobID, err := endpoint.MakeJobID(s)
+	if err != nil {
+		return err
+	}
+	*f = JobIDFlag{J: &jobID}
+	return nil
+}
+func (f JobIDFlag) Type() string               { return "job-ID" }
+func (f JobIDFlag) String() string             { return fmt.Sprint(f.J) }
+func (f JobIDFlag) FlagValue() *endpoint.JobID { return f.J }
+
+type AbstractionTypesFlag map[endpoint.AbstractionType]bool
+
+func (f *AbstractionTypesFlag) Set(s string) error {
+	ats, err := endpoint.AbstractionTypeSetFromStrings(strings.Split(s, ","))
+	if err != nil {
+		return err
+	}
+	*f = AbstractionTypesFlag(ats)
+	return nil
+}
+func (f AbstractionTypesFlag) Type() string { return "abstraction-type" }
+func (f AbstractionTypesFlag) String() string {
+	return endpoint.AbstractionTypeSet(f).String()
+}
+func (f AbstractionTypesFlag) FlagValue() map[endpoint.AbstractionType]bool {
+	if len(f) > 0 {
+		return f
+	}
+	return endpoint.AbstractionTypesAll
+}
+
+type FilesystemsFilterFlag struct {
+	F endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter
+}
+
+func (flag *FilesystemsFilterFlag) Set(s string) error {
+	mappings := strings.Split(s, ",")
+	if len(mappings) == 1 && !strings.Contains(mappings[0], ":") {
+		flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &mappings[0],
+		}
+		return nil
+	}
+
+	f := filters.NewDatasetMapFilter(len(mappings), true)
+	for _, m := range mappings {
+		thisMappingErr := fmt.Errorf("expecting comma-separated list of <dataset-pattern>:<ok|!> pairs, got %q", m)
+		lhsrhs := strings.SplitN(m, ":", 2)
+		if len(lhsrhs) != 2 {
+			return thisMappingErr
+		}
+		err := f.Add(lhsrhs[0], lhsrhs[1])
+		if err != nil {
+			return fmt.Errorf("%s: %s", thisMappingErr, err)
+		}
+	}
+	flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+		Filter: f,
+	}
+	return nil
+}
+func (flag FilesystemsFilterFlag) Type() string { return "filesystem filter spec" }
+func (flag FilesystemsFilterFlag) String() string {
+	return fmt.Sprintf("%v", flag.F)
+}
+func (flag FilesystemsFilterFlag) FlagValue() endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter {
+	var z FilesystemsFilterFlag
+	if flag == z {
+		return endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{Filter: zfs.NoFilter()}
+	}
+	return flag.F
+}

internal/client/zfsabstractions_create.go

@@ -0,0 +1,14 @@
+package client
+
+import "github.com/zrepl/zrepl/internal/cli"
+
+var zabsCmdCreate = &cli.Subcommand{
+	Use:             "create",
+	NoRequireConfig: true,
+	Short:           `create zrepl ZFS abstractions (mostly useful for debugging & development, users should not need to use this command)`,
+	SetupSubcommands: func() []*cli.Subcommand {
+		return []*cli.Subcommand{
+			zabsCmdCreateStepHold,
+		}
+	},
+}

internal/client/zfsabstractions_create_step_hold.go

@@ -0,0 +1,58 @@
+package client
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/internal/cli"
+	"github.com/zrepl/zrepl/internal/endpoint"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+var zabsCreateStepHoldFlags struct {
+	target string
+	jobid  JobIDFlag
+}
+
+var zabsCmdCreateStepHold = &cli.Subcommand{
+	Use:             "step",
+	Run:             doZabsCreateStep,
+	NoRequireConfig: true,
+	Short:           `create a step hold or bookmark`,
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVarP(&zabsCreateStepHoldFlags.target, "target", "t", "", "snapshot to be held / bookmark to be held")
+		f.VarP(&zabsCreateStepHoldFlags.jobid, "jobid", "j", "jobid for which the hold is installed")
+	},
+}
+
+func doZabsCreateStep(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	if len(args) > 0 {
+		return errors.New("subcommand takes no arguments")
+	}
+
+	f := &zabsCreateStepHoldFlags
+
+	fs, _, _, err := zfs.DecomposeVersionString(f.target)
+	if err != nil {
+		return errors.Wrapf(err, "%q invalid target", f.target)
+	}
+
+	if f.jobid.FlagValue() == nil {
+		return errors.Errorf("jobid must be set")
+	}
+
+	v, err := zfs.ZFSGetFilesystemVersion(ctx, f.target)
+	if err != nil {
+		return errors.Wrapf(err, "get info about target %q", f.target)
+	}
+
+	step, err := endpoint.HoldStep(ctx, fs, v, *f.jobid.FlagValue())
+	if err != nil {
+		return errors.Wrap(err, "create step hold")
+	}
+	fmt.Println(step.String())
+	return nil
+}

internal/client/zfsabstractions_list.go

@@ -0,0 +1,100 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/internal/cli"
+	"github.com/zrepl/zrepl/internal/endpoint"
+	"github.com/zrepl/zrepl/internal/util/chainlock"
+)
+
+var zabsListFlags struct {
+	Filter zabsFilterFlags
+	Json   bool
+}
+
+var zabsCmdList = &cli.Subcommand{
+	Use:             "list",
+	Short:           `list zrepl ZFS abstractions`,
+	Run:             doZabsList,
+	NoRequireConfig: true,
+	SetupFlags: func(f *pflag.FlagSet) {
+		zabsListFlags.Filter.registerZabsFilterFlags(f, "list")
+		f.BoolVar(&zabsListFlags.Json, "json", false, "emit JSON")
+	},
+}
+
+func doZabsList(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsListFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	abstractions, errors, drainDone, err := endpoint.ListAbstractionsStreamed(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+	defer drainDone()
+
+	var line chainlock.L
+	var wg sync.WaitGroup
+	defer wg.Wait()
+
+	// print results
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		enc := json.NewEncoder(os.Stdout)
+		for a := range abstractions {
+			func() {
+				defer line.Lock().Unlock()
+				if zabsListFlags.Json {
+					enc.SetIndent("", "  ")
+					if err := enc.Encode(a); err != nil {
+						panic(err)
+					}
+					fmt.Println()
+				} else {
+					fmt.Println(a)
+				}
+			}()
+		}
+	}()
+
+	// print errors to stderr
+	errorColor := color.New(color.FgRed)
+	var errorsSlice []endpoint.ListAbstractionsError
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for err := range errors {
+			func() {
+				defer line.Lock().Unlock()
+				errorsSlice = append(errorsSlice, err)
+				errorColor.Fprintf(os.Stderr, "%s\n", err)
+			}()
+		}
+	}()
+	wg.Wait()
+	if len(errorsSlice) > 0 {
+		errorColor.Add(color.Bold).Fprintf(os.Stderr, "there were errors in listing the abstractions")
+		return fmt.Errorf("")
+	} else {
+		return nil
+	}
+
+}

internal/client/zfsabstractions_release.go

@@ -0,0 +1,143 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/internal/cli"
+	"github.com/zrepl/zrepl/internal/endpoint"
+)
+
+// shared between release-all and release-step
+var zabsReleaseFlags struct {
+	Filter zabsFilterFlags
+	Json   bool
+	DryRun bool
+}
+
+func registerZabsReleaseFlags(s *pflag.FlagSet) {
+	zabsReleaseFlags.Filter.registerZabsFilterFlags(s, "release")
+	s.BoolVar(&zabsReleaseFlags.Json, "json", false, "emit json instead of pretty-printed")
+	s.BoolVar(&zabsReleaseFlags.DryRun, "dry-run", false, "do a dry-run")
+}
+
+var zabsCmdReleaseAll = &cli.Subcommand{
+	Use:             "release-all",
+	Run:             doZabsReleaseAll,
+	NoRequireConfig: true,
+	Short:           `(DANGEROUS) release ALL zrepl ZFS abstractions (mostly useful for uninstalling zrepl completely or for "de-zrepl-ing" a filesystem)`,
+	SetupFlags:      registerZabsReleaseFlags,
+}
+
+var zabsCmdReleaseStale = &cli.Subcommand{
+	Use:             "release-stale",
+	Run:             doZabsReleaseStale,
+	NoRequireConfig: true,
+	Short:           `release stale zrepl ZFS abstractions (useful if zrepl has a bug and does not do it by itself)`,
+	SetupFlags:      registerZabsReleaseFlags,
+}
+
+func doZabsReleaseAll(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsReleaseFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	abstractions, listErrors, err := endpoint.ListAbstractions(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+	if len(listErrors) > 0 {
+		color.New(color.FgRed).Fprintf(os.Stderr, "there were errors in listing the abstractions:\n%s\n", listErrors)
+		// proceed anyways with rest of abstractions
+	}
+
+	return doZabsRelease_Common(ctx, abstractions)
+}
+
+func doZabsReleaseStale(ctx context.Context, sc *cli.Subcommand, args []string) error {
+
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsReleaseFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	stalenessInfo, err := endpoint.ListStale(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+
+	return doZabsRelease_Common(ctx, stalenessInfo.Stale)
+}
+
+func doZabsRelease_Common(ctx context.Context, destroy []endpoint.Abstraction) error {
+
+	if zabsReleaseFlags.DryRun {
+		if zabsReleaseFlags.Json {
+			m, err := json.MarshalIndent(destroy, "", "  ")
+			if err != nil {
+				panic(err)
+			}
+			if _, err := os.Stdout.Write(m); err != nil {
+				panic(err)
+			}
+			fmt.Println()
+		} else {
+			for _, a := range destroy {
+				fmt.Printf("would destroy %s\n", a)
+			}
+		}
+		return nil
+	}
+
+	outcome := endpoint.BatchDestroy(ctx, destroy)
+	hadErr := false
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	colorErr := color.New(color.FgRed)
+	printfSuccess := color.New(color.FgGreen).FprintfFunc()
+	printfSection := color.New(color.Bold).FprintfFunc()
+
+	for res := range outcome {
+		hadErr = hadErr || res.DestroyErr != nil
+		if zabsReleaseFlags.Json {
+			err := enc.Encode(res)
+			if err != nil {
+				colorErr.Fprintf(os.Stderr, "cannot marshal there were errors in destroying the abstractions")
+			}
+		} else {
+			printfSection(os.Stdout, "destroy %s ...", res.Abstraction)
+			if res.DestroyErr != nil {
+				colorErr.Fprintf(os.Stdout, " failed:\n%s\n", res.DestroyErr)
+			} else {
+				printfSuccess(os.Stdout, " OK\n")
+			}
+		}
+	}
+
+	if hadErr {
+		colorErr.Add(color.Bold).Fprintf(os.Stderr, "there were errors in destroying the abstractions")
+		return fmt.Errorf("")
+	} else {
+		return nil
+	}
+}

internal/config/config.go

@@ -0,0 +1,702 @@
+package config
+
+import (
+	"fmt"
+	"log/syslog"
+	"os"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/robfig/cron/v3"
+	"github.com/zrepl/yaml-config"
+
+	"github.com/zrepl/zrepl/internal/util/datasizeunit"
+	zfsprop "github.com/zrepl/zrepl/internal/zfs/property"
+)
+
+type ParseFlags uint
+
+const (
+	ParseFlagsNone        ParseFlags = 0
+	ParseFlagsNoCertCheck ParseFlags = 1 << iota
+)
+
+type Config struct {
+	Jobs   []JobEnum `yaml:"jobs,optional"`
+	Global *Global   `yaml:"global,optional,fromdefaults"`
+}
+
+func (c *Config) Job(name string) (*JobEnum, error) {
+	for _, j := range c.Jobs {
+		if j.Name() == name {
+			return &j, nil
+		}
+	}
+	return nil, fmt.Errorf("job %q not defined in config", name)
+}
+
+type JobEnum struct {
+	Ret interface{}
+}
+
+func (j JobEnum) Name() string {
+	var name string
+	switch v := j.Ret.(type) {
+	case *SnapJob:
+		name = v.Name
+	case *PushJob:
+		name = v.Name
+	case *SinkJob:
+		name = v.Name
+	case *PullJob:
+		name = v.Name
+	case *SourceJob:
+		name = v.Name
+	default:
+		panic(fmt.Sprintf("unknown job type %T", v))
+	}
+	return name
+}
+
+type ActiveJob struct {
+	Type               string                `yaml:"type"`
+	Name               string                `yaml:"name"`
+	Connect            ConnectEnum           `yaml:"connect"`
+	Pruning            PruningSenderReceiver `yaml:"pruning"`
+	Replication        *Replication          `yaml:"replication,optional,fromdefaults"`
+	ConflictResolution *ConflictResolution   `yaml:"conflict_resolution,optional,fromdefaults"`
+}
+
+type ConflictResolution struct {
+	InitialReplication string `yaml:"initial_replication,optional,default=most_recent"`
+}
+
+type PassiveJob struct {
+	Type  string    `yaml:"type"`
+	Name  string    `yaml:"name"`
+	Serve ServeEnum `yaml:"serve"`
+}
+
+type SnapJob struct {
+	Type         string            `yaml:"type"`
+	Name         string            `yaml:"name"`
+	Pruning      PruningLocal      `yaml:"pruning"`
+	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
+	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+}
+
+type SendOptions struct {
+	Encrypted        bool `yaml:"encrypted,optional,default=false"`
+	Raw              bool `yaml:"raw,optional,default=false"`
+	SendProperties   bool `yaml:"send_properties,optional,default=false"`
+	BackupProperties bool `yaml:"backup_properties,optional,default=false"`
+	LargeBlocks      bool `yaml:"large_blocks,optional,default=false"`
+	Compressed       bool `yaml:"compressed,optional,default=false"`
+	EmbeddedData     bool `yaml:"embedded_data,optional,default=false"`
+	Saved            bool `yaml:"saved,optional,default=false"`
+
+	BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
+}
+
+type RecvOptions struct {
+	// Note: we cannot enforce encrypted recv as the ZFS cli doesn't provide a mechanism for it
+	// Encrypted bool `yaml:"may_encrypted"`
+	// Future:
+	// Reencrypt bool `yaml:"reencrypt"`
+
+	Properties *PropertyRecvOptions `yaml:"properties,fromdefaults"`
+
+	BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
+
+	Placeholder *PlaceholderRecvOptions `yaml:"placeholder,fromdefaults"`
+}
+
+var _ yaml.Unmarshaler = &datasizeunit.Bits{}
+
+type BandwidthLimit struct {
+	Max            datasizeunit.Bits `yaml:"max,default=-1 B"`
+	BucketCapacity datasizeunit.Bits `yaml:"bucket_capacity,default=128 KiB"`
+}
+
+type Replication struct {
+	Protection  *ReplicationOptionsProtection  `yaml:"protection,optional,fromdefaults"`
+	Concurrency *ReplicationOptionsConcurrency `yaml:"concurrency,optional,fromdefaults"`
+}
+
+type ReplicationOptionsProtection struct {
+	Initial     string `yaml:"initial,optional,default=guarantee_resumability"`
+	Incremental string `yaml:"incremental,optional,default=guarantee_resumability"`
+}
+
+type ReplicationOptionsConcurrency struct {
+	Steps         int `yaml:"steps,optional,default=1"`
+	SizeEstimates int `yaml:"size_estimates,optional,default=4"`
+}
+
+type PropertyRecvOptions struct {
+	Inherit  []zfsprop.Property          `yaml:"inherit,optional"`
+	Override map[zfsprop.Property]string `yaml:"override,optional"`
+}
+
+type PlaceholderRecvOptions struct {
+	Encryption string `yaml:"encryption,default=unspecified"`
+}
+
+type PushJob struct {
+	ActiveJob    `yaml:",inline"`
+	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
+	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+	Send         *SendOptions      `yaml:"send,fromdefaults,optional"`
+}
+
+func (j *PushJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
+func (j *PushJob) GetSendOptions() *SendOptions      { return j.Send }
+
+type PullJob struct {
+	ActiveJob `yaml:",inline"`
+	RootFS    string                   `yaml:"root_fs"`
+	Interval  PositiveDurationOrManual `yaml:"interval"`
+	Recv      *RecvOptions             `yaml:"recv,fromdefaults,optional"`
+}
+
+func (j *PullJob) GetRootFS() string             { return j.RootFS }
+func (j *PullJob) GetAppendClientIdentity() bool { return false }
+func (j *PullJob) GetRecvOptions() *RecvOptions  { return j.Recv }
+
+type PositiveDurationOrManual struct {
+	Interval time.Duration
+	Manual   bool
+}
+
+var _ yaml.Unmarshaler = (*PositiveDurationOrManual)(nil)
+
+func (i *PositiveDurationOrManual) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	var s string
+	if err := u(&s, true); err != nil {
+		return err
+	}
+	switch s {
+	case "manual":
+		i.Manual = true
+		i.Interval = 0
+	case "":
+		return fmt.Errorf("value must not be empty")
+	default:
+		i.Manual = false
+		i.Interval, err = parsePositiveDuration(s)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type SinkJob struct {
+	PassiveJob `yaml:",inline"`
+	RootFS     string       `yaml:"root_fs"`
+	Recv       *RecvOptions `yaml:"recv,optional,fromdefaults"`
+}
+
+func (j *SinkJob) GetRootFS() string             { return j.RootFS }
+func (j *SinkJob) GetAppendClientIdentity() bool { return true }
+func (j *SinkJob) GetRecvOptions() *RecvOptions  { return j.Recv }
+
+type SourceJob struct {
+	PassiveJob   `yaml:",inline"`
+	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
+	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+	Send         *SendOptions      `yaml:"send,optional,fromdefaults"`
+}
+
+func (j *SourceJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
+func (j *SourceJob) GetSendOptions() *SendOptions      { return j.Send }
+
+type FilesystemsFilter map[string]bool
+
+type SnapshottingEnum struct {
+	Ret interface{}
+}
+
+type SnapshottingPeriodic struct {
+	Type                    string            `yaml:"type"`
+	Prefix                  string            `yaml:"prefix"`
+	Interval                *PositiveDuration `yaml:"interval"`
+	Hooks                   HookList          `yaml:"hooks,optional"`
+	TimestampFormattingSpec `yaml:",inline"`
+}
+
+type TimestampFormattingSpec struct {
+	TimestampFormat   string `yaml:"timestamp_format,optional,default=dense"`
+	TimestampLocation string `yaml:"timestamp_location,optional,default=UTC"`
+}
+
+type CronSpec struct {
+	Schedule cron.Schedule
+}
+
+var _ yaml.Unmarshaler = &CronSpec{}
+
+func (s *CronSpec) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
+	var specString string
+	if err := unmarshal(&specString, false); err != nil {
+		return err
+	}
+
+	// Use standard cron format.
+	// Disable the various "descriptors" (@daily, etc)
+	// They are just aliases to "top of hour", "midnight", etc.
+	parser := cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.SecondOptional)
+
+	sched, err := parser.Parse(specString)
+	if err != nil {
+		return errors.Wrap(err, "cron syntax invalid")
+	}
+	s.Schedule = sched
+	return nil
+}
+
+type SnapshottingCron struct {
+	Type                    string   `yaml:"type"`
+	Prefix                  string   `yaml:"prefix"`
+	Cron                    CronSpec `yaml:"cron"`
+	Hooks                   HookList `yaml:"hooks,optional"`
+	TimestampFormattingSpec `yaml:",inline"`
+}
+
+type SnapshottingManual struct {
+	Type string `yaml:"type"`
+}
+
+type PruningSenderReceiver struct {
+	KeepSender   []PruningEnum `yaml:"keep_sender"`
+	KeepReceiver []PruningEnum `yaml:"keep_receiver"`
+}
+
+type PruningLocal struct {
+	Keep []PruningEnum `yaml:"keep"`
+}
+
+type LoggingOutletEnumList []LoggingOutletEnum
+
+func (l *LoggingOutletEnumList) SetDefault() {
+	def := `
+type: "stdout"
+time: true
+level: "warn"
+format: "human"
+`
+	s := &StdoutLoggingOutlet{}
+	err := yaml.UnmarshalStrict([]byte(def), &s)
+	if err != nil {
+		panic(err)
+	}
+	*l = []LoggingOutletEnum{LoggingOutletEnum{Ret: s}}
+}
+
+var _ yaml.Defaulter = &LoggingOutletEnumList{}
+
+type Global struct {
+	Logging    *LoggingOutletEnumList `yaml:"logging,optional,fromdefaults"`
+	Monitoring []MonitoringEnum       `yaml:"monitoring,optional"`
+	Control    *GlobalControl         `yaml:"control,optional,fromdefaults"`
+	Serve      *GlobalServe           `yaml:"serve,optional,fromdefaults"`
+}
+
+type ConnectEnum struct {
+	Ret interface{}
+}
+
+type ConnectCommon struct {
+	Type string `yaml:"type"`
+}
+
+type TCPConnect struct {
+	ConnectCommon `yaml:",inline"`
+	Address       string        `yaml:"address,hostport"`
+	DialTimeout   time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
+}
+
+type TLSConnect struct {
+	ConnectCommon `yaml:",inline"`
+	Address       string        `yaml:"address,hostport"`
+	Ca            string        `yaml:"ca"`
+	Cert          string        `yaml:"cert"`
+	Key           string        `yaml:"key"`
+	ServerCN      string        `yaml:"server_cn"`
+	DialTimeout   time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
+}
+
+type SSHStdinserverConnect struct {
+	ConnectCommon        `yaml:",inline"`
+	Host                 string        `yaml:"host"`
+	User                 string        `yaml:"user"`
+	Port                 uint16        `yaml:"port"`
+	IdentityFile         string        `yaml:"identity_file"`
+	TransportOpenCommand []string      `yaml:"transport_open_command,optional"` //TODO unused
+	SSHCommand           string        `yaml:"ssh_command,optional"`            //TODO unused
+	Options              []string      `yaml:"options,optional"`
+	DialTimeout          time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
+}
+
+type LocalConnect struct {
+	ConnectCommon  `yaml:",inline"`
+	ListenerName   string        `yaml:"listener_name"`
+	ClientIdentity string        `yaml:"client_identity"`
+	DialTimeout    time.Duration `yaml:"dial_timeout,zeropositive,default=2s"`
+}
+
+type ServeEnum struct {
+	Ret interface{}
+}
+
+type ServeCommon struct {
+	Type string `yaml:"type"`
+}
+
+type TCPServe struct {
+	ServeCommon    `yaml:",inline"`
+	Listen         string            `yaml:"listen,hostport"`
+	ListenFreeBind bool              `yaml:"listen_freebind,default=false"`
+	Clients        map[string]string `yaml:"clients"`
+}
+
+type TLSServe struct {
+	ServeCommon      `yaml:",inline"`
+	Listen           string        `yaml:"listen,hostport"`
+	ListenFreeBind   bool          `yaml:"listen_freebind,default=false"`
+	Ca               string        `yaml:"ca"`
+	Cert             string        `yaml:"cert"`
+	Key              string        `yaml:"key"`
+	ClientCNs        []string      `yaml:"client_cns"`
+	HandshakeTimeout time.Duration `yaml:"handshake_timeout,zeropositive,default=10s"`
+}
+
+type StdinserverServer struct {
+	ServeCommon      `yaml:",inline"`
+	ClientIdentities []string `yaml:"client_identities"`
+}
+
+type LocalServe struct {
+	ServeCommon  `yaml:",inline"`
+	ListenerName string `yaml:"listener_name"`
+}
+
+type PruningEnum struct {
+	Ret interface{}
+}
+
+type PruneKeepNotReplicated struct {
+	Type                 string `yaml:"type"`
+	KeepSnapshotAtCursor bool   `yaml:"keep_snapshot_at_cursor,optional,default=true"`
+}
+
+type PruneKeepLastN struct {
+	Type  string `yaml:"type"`
+	Count int    `yaml:"count"`
+	Regex string `yaml:"regex,optional"`
+}
+
+type PruneKeepRegex struct { // FIXME rename to KeepRegex
+	Type   string `yaml:"type"`
+	Regex  string `yaml:"regex"`
+	Negate bool   `yaml:"negate,optional,default=false"`
+}
+
+type LoggingOutletEnum struct {
+	Ret interface{}
+}
+
+type LoggingOutletCommon struct {
+	Type   string `yaml:"type"`
+	Level  string `yaml:"level"`
+	Format string `yaml:"format"`
+}
+
+type StdoutLoggingOutlet struct {
+	LoggingOutletCommon `yaml:",inline"`
+	Time                bool `yaml:"time,default=true"`
+	Color               bool `yaml:"color,default=true"`
+}
+
+type SyslogLoggingOutlet struct {
+	LoggingOutletCommon `yaml:",inline"`
+	Facility            *SyslogFacility `yaml:"facility,optional,fromdefaults"`
+	RetryInterval       time.Duration   `yaml:"retry_interval,positive,default=10s"`
+}
+
+type TCPLoggingOutlet struct {
+	LoggingOutletCommon `yaml:",inline"`
+	Address             string               `yaml:"address,hostport"`
+	Net                 string               `yaml:"net,default=tcp"`
+	RetryInterval       time.Duration        `yaml:"retry_interval,positive,default=10s"`
+	TLS                 *TCPLoggingOutletTLS `yaml:"tls,optional"`
+}
+
+type TCPLoggingOutletTLS struct {
+	CA   string `yaml:"ca"`
+	Cert string `yaml:"cert"`
+	Key  string `yaml:"key"`
+}
+
+type MonitoringEnum struct {
+	Ret interface{}
+}
+
+type PrometheusMonitoring struct {
+	Type           string `yaml:"type"`
+	Listen         string `yaml:"listen,hostport"`
+	ListenFreeBind bool   `yaml:"listen_freebind,default=false"`
+}
+
+type SyslogFacility syslog.Priority
+
+func (f *SyslogFacility) SetDefault() {
+	*f = SyslogFacility(syslog.LOG_LOCAL0)
+}
+
+var _ yaml.Defaulter = (*SyslogFacility)(nil)
+
+type GlobalControl struct {
+	SockPath string `yaml:"sockpath,default=/var/run/zrepl/control"`
+}
+
+type GlobalServe struct {
+	StdinServer *GlobalStdinServer `yaml:"stdinserver,optional,fromdefaults"`
+}
+
+type GlobalStdinServer struct {
+	SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
+}
+
+type HookList []HookEnum
+
+type HookEnum struct {
+	Ret interface{}
+}
+
+type HookCommand struct {
+	Path               string            `yaml:"path"`
+	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
+	Filesystems        FilesystemsFilter `yaml:"filesystems,optional,default={'<': true}"`
+	HookSettingsCommon `yaml:",inline"`
+}
+
+type HookPostgresCheckpoint struct {
+	HookSettingsCommon `yaml:",inline"`
+	DSN                string            `yaml:"dsn"`
+	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
+	Filesystems        FilesystemsFilter `yaml:"filesystems"` // required, user should not CHECKPOINT for every FS
+}
+
+type HookMySQLLockTables struct {
+	HookSettingsCommon `yaml:",inline"`
+	DSN                string            `yaml:"dsn"`
+	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
+	Filesystems        FilesystemsFilter `yaml:"filesystems"`
+}
+
+type HookSettingsCommon struct {
+	Type       string `yaml:"type"`
+	ErrIsFatal bool   `yaml:"err_is_fatal,optional,default=false"`
+}
+
+func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) {
+	var in struct {
+		Type string
+	}
+	if err := u(&in, true); err != nil {
+		return nil, err
+	}
+	if in.Type == "" {
+		return nil, &yaml.TypeError{Errors: []string{"must specify type"}}
+	}
+
+	v, ok := types[in.Type]
+	if !ok {
+		return nil, &yaml.TypeError{Errors: []string{fmt.Sprintf("invalid type name %q", in.Type)}}
+	}
+	if err := u(v, false); err != nil {
+		return nil, err
+	}
+	return v, nil
+}
+
+func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"snap":   &SnapJob{},
+		"push":   &PushJob{},
+		"sink":   &SinkJob{},
+		"pull":   &PullJob{},
+		"source": &SourceJob{},
+	})
+	return
+}
+
+func (t *ConnectEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"tcp":             &TCPConnect{},
+		"tls":             &TLSConnect{},
+		"ssh+stdinserver": &SSHStdinserverConnect{},
+		"local":           &LocalConnect{},
+	})
+	return
+}
+
+func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"tcp":         &TCPServe{},
+		"tls":         &TLSServe{},
+		"stdinserver": &StdinserverServer{},
+		"local":       &LocalServe{},
+	})
+	return
+}
+
+func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"not_replicated": &PruneKeepNotReplicated{},
+		"last_n":         &PruneKeepLastN{},
+		"grid":           &PruneGrid{},
+		"regex":          &PruneKeepRegex{},
+	})
+	return
+}
+
+func (t *SnapshottingEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"periodic": &SnapshottingPeriodic{},
+		"manual":   &SnapshottingManual{},
+		"cron":     &SnapshottingCron{},
+	})
+	return
+}
+
+func (t *LoggingOutletEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"stdout": &StdoutLoggingOutlet{},
+		"syslog": &SyslogLoggingOutlet{},
+		"tcp":    &TCPLoggingOutlet{},
+	})
+	return
+}
+
+func (t *MonitoringEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"prometheus": &PrometheusMonitoring{},
+	})
+	return
+}
+
+func (t *SyslogFacility) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	var s string
+	if err := u(&s, true); err != nil {
+		return err
+	}
+	var level syslog.Priority
+	switch s {
+	case "kern":
+		level = syslog.LOG_KERN
+	case "user":
+		level = syslog.LOG_USER
+	case "mail":
+		level = syslog.LOG_MAIL
+	case "daemon":
+		level = syslog.LOG_DAEMON
+	case "auth":
+		level = syslog.LOG_AUTH
+	case "syslog":
+		level = syslog.LOG_SYSLOG
+	case "lpr":
+		level = syslog.LOG_LPR
+	case "news":
+		level = syslog.LOG_NEWS
+	case "uucp":
+		level = syslog.LOG_UUCP
+	case "cron":
+		level = syslog.LOG_CRON
+	case "authpriv":
+		level = syslog.LOG_AUTHPRIV
+	case "ftp":
+		level = syslog.LOG_FTP
+	case "local0":
+		level = syslog.LOG_LOCAL0
+	case "local1":
+		level = syslog.LOG_LOCAL1
+	case "local2":
+		level = syslog.LOG_LOCAL2
+	case "local3":
+		level = syslog.LOG_LOCAL3
+	case "local4":
+		level = syslog.LOG_LOCAL4
+	case "local5":
+		level = syslog.LOG_LOCAL5
+	case "local6":
+		level = syslog.LOG_LOCAL6
+	case "local7":
+		level = syslog.LOG_LOCAL7
+	default:
+		return fmt.Errorf("invalid syslog level: %q", s)
+	}
+	*t = SyslogFacility(level)
+	return nil
+}
+
+func (t *HookEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"command":             &HookCommand{},
+		"postgres-checkpoint": &HookPostgresCheckpoint{},
+		"mysql-lock-tables":   &HookMySQLLockTables{},
+	})
+	return
+}
+
+var ConfigFileDefaultLocations = []string{
+	"/etc/zrepl/zrepl.yml",
+	"/usr/local/etc/zrepl/zrepl.yml",
+}
+
+func ParseConfig(path string) (i *Config, err error) {
+
+	if path == "" {
+		// Try default locations
+		for _, l := range ConfigFileDefaultLocations {
+			stat, statErr := os.Stat(l)
+			if statErr != nil {
+				continue
+			}
+			if !stat.Mode().IsRegular() {
+				err = errors.Errorf("file at default location is not a regular file: %s", l)
+				return
+			}
+			path = l
+			break
+		}
+	}
+
+	var bytes []byte
+
+	if bytes, err = os.ReadFile(path); err != nil {
+		return
+	}
+
+	return ParseConfigBytes(bytes)
+}
+
+func ParseConfigBytes(bytes []byte) (*Config, error) {
+	var c *Config
+	if err := yaml.UnmarshalStrict(bytes, &c); err != nil {
+		return nil, err
+	}
+	if c != nil {
+		return c, nil
+	}
+	// There was no yaml document in the file, deserialize from default.
+	// => See TestFromdefaultsEmptyDoc in yaml-config package.
+	if err := yaml.UnmarshalStrict([]byte("{}"), &c); err != nil {
+		return nil, err
+	}
+	if c == nil {
+		panic("the fallback to deserialize from `{}` should work")
+	}
+	return c, nil
+}

internal/config/config_dep_test.go

@@ -0,0 +1,90 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/zrepl/yaml-config"
+)
+
+type A struct {
+	B  *B     `yaml:"b,optional,fromdefaults"`
+	A1 string `yaml:"a1,optional"`
+}
+
+type B struct {
+	C *C     `yaml:"c,optional,fromdefaults"`
+	D string `yaml:"d,default=ddd"`
+	E string `yaml:"e,optional"`
+}
+
+type C struct {
+	Q string `yaml:"q,optional"`
+	R string `yaml:"r,default=r"`
+}
+
+func TestDepFromDefaults(t *testing.T) {
+
+	type testcase struct {
+		name   string
+		yaml   string
+		expect *A
+	}
+
+	tcs := []testcase{
+		{
+			name: "empty",
+			yaml: `{}`,
+			expect: &A{
+				B: &B{
+					C: &C{
+						R: "r",
+					},
+					D: "ddd",
+				},
+			},
+		},
+		{
+			name: "a1 set",
+			yaml: `{"a1":"blah"}`,
+			expect: &A{
+				A1: "blah",
+				B: &B{
+					C: &C{
+						R: "r",
+					},
+					D: "ddd",
+				},
+			},
+		},
+		{
+			name: "D set",
+			yaml: `
+b:
+  d: 4d
+`,
+			expect: &A{
+				B: &B{
+					D: "4d",
+					C: &C{
+						R: "r",
+					},
+				},
+			},
+		},
+	}
+
+	for tci := range tcs {
+		t.Run(fmt.Sprintf("%d-%s", tci, tcs[tci].name), func(t *testing.T) {
+			tc := tcs[tci]
+
+			var a A
+			err := yaml.UnmarshalStrict([]byte(tc.yaml), &a)
+			require.NoError(t, err)
+
+			require.Equal(t, tc.expect, &a)
+		})
+	}
+
+}

internal/config/config_duration.go

@@ -0,0 +1,106 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
+	"time"
+
+	"github.com/kr/pretty"
+	"github.com/zrepl/yaml-config"
+)
+
+type Duration struct{ d time.Duration }
+
+func (d Duration) Duration() time.Duration { return d.d }
+
+var _ yaml.Unmarshaler = &Duration{}
+
+func (d *Duration) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
+	var s string
+	err := unmarshal(&s, false)
+	if err != nil {
+		return err
+	}
+	d.d, err = parseDuration(s)
+	if err != nil {
+		d.d = 0
+		return &yaml.TypeError{Errors: []string{fmt.Sprintf("cannot parse value %q: %s", s, err)}}
+	}
+	return nil
+}
+
+type PositiveDuration struct{ d Duration }
+
+var _ yaml.Unmarshaler = &PositiveDuration{}
+
+func (d PositiveDuration) Duration() time.Duration { return d.d.Duration() }
+
+func (d *PositiveDuration) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
+	err := d.d.UnmarshalYAML(unmarshal)
+	if err != nil {
+		return err
+	}
+	if d.d.Duration() <= 0 {
+		return fmt.Errorf("duration must be positive, got %s", d.d.Duration())
+	}
+	return nil
+}
+
+func parsePositiveDuration(e string) (time.Duration, error) {
+	d, err := parseDuration(e)
+	if err != nil {
+		return d, err
+	}
+	if d <= 0 {
+		return 0, errors.New("duration must be positive integer")
+	}
+	return d, err
+}
+
+var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*([\+-]?\d+)\s*(|s|m|h|d|w)\s*$`)
+
+func parseDuration(e string) (d time.Duration, err error) {
+	comps := durationStringRegex.FindStringSubmatch(e)
+	if comps == nil {
+		err = fmt.Errorf("must match %s", durationStringRegex)
+		return
+	}
+	if len(comps) != 3 {
+		panic(pretty.Sprint(comps))
+	}
+
+	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+
+	var durationUnit time.Duration
+	switch comps[2] {
+	case "":
+		if durationFactor != 0 {
+			err = fmt.Errorf("missing time unit")
+			return
+		} else {
+			// It's the case where user specified '0'.
+			// We want to allow this, just like time.ParseDuration.
+		}
+	case "s":
+		durationUnit = time.Second
+	case "m":
+		durationUnit = time.Minute
+	case "h":
+		durationUnit = time.Hour
+	case "d":
+		durationUnit = 24 * time.Hour
+	case "w":
+		durationUnit = 24 * 7 * time.Hour
+	default:
+		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
+		return
+	}
+
+	d = time.Duration(durationFactor) * durationUnit
+	return
+}

internal/config/config_global_test.go

@@ -0,0 +1,115 @@
+package config
+
+import (
+	"fmt"
+	"log/syslog"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/zrepl/yaml-config"
+)
+
+func testValidGlobalSection(t *testing.T, s string) *Config {
+	jobdef := `
+jobs:
+- name: dummyjob
+  type: sink
+  serve:
+    type: tcp
+    listen: ":2342"
+    clients: {
+      "10.0.0.1":"foo"
+    }
+  root_fs: zroot/foo
+`
+	_, err := ParseConfigBytes([]byte(jobdef))
+	require.NoError(t, err)
+	return testValidConfig(t, s+jobdef)
+}
+
+func TestOutletTypes(t *testing.T) {
+	conf := testValidGlobalSection(t, `
+global:
+  logging:
+  - type: stdout
+    level: debug
+    format: human
+  - type: syslog
+    level: info
+    retry_interval: 20s
+    format: human
+  - type: tcp
+    level: debug
+    format: json
+    address: logserver.example.com:1234
+  - type: tcp
+    level: debug
+    format: json
+    address: encryptedlogserver.example.com:1234
+    retry_interval: 20s 
+    tls:
+      ca: /etc/zrepl/log/ca.crt
+      cert: /etc/zrepl/log/key.pem
+      key: /etc/zrepl/log/cert.pem
+`)
+	assert.Equal(t, 4, len(*conf.Global.Logging))
+	assert.NotNil(t, (*conf.Global.Logging)[3].Ret.(*TCPLoggingOutlet).TLS)
+}
+
+func TestDefaultLoggingOutlet(t *testing.T) {
+	conf := testValidGlobalSection(t, "")
+	assert.Equal(t, 1, len(*conf.Global.Logging))
+	o := (*conf.Global.Logging)[0].Ret.(*StdoutLoggingOutlet)
+	assert.Equal(t, "warn", o.Level)
+	assert.Equal(t, "human", o.Format)
+}
+
+func TestPrometheusMonitoring(t *testing.T) {
+	conf := testValidGlobalSection(t, `
+global:
+  monitoring:
+    - type: prometheus
+      listen: ':9811'
+`)
+	assert.Equal(t, ":9811", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
+}
+
+func TestSyslogLoggingOutletFacility(t *testing.T) {
+	type SyslogFacilityPriority struct {
+		Facility string
+		Priority syslog.Priority
+	}
+	syslogFacilitiesPriorities := []SyslogFacilityPriority{
+		{"", syslog.LOG_LOCAL0}, // default
+		{"kern", syslog.LOG_KERN}, {"daemon", syslog.LOG_DAEMON}, {"auth", syslog.LOG_AUTH},
+		{"syslog", syslog.LOG_SYSLOG}, {"lpr", syslog.LOG_LPR}, {"news", syslog.LOG_NEWS},
+		{"uucp", syslog.LOG_UUCP}, {"cron", syslog.LOG_CRON}, {"authpriv", syslog.LOG_AUTHPRIV},
+		{"ftp", syslog.LOG_FTP}, {"local0", syslog.LOG_LOCAL0}, {"local1", syslog.LOG_LOCAL1},
+		{"local2", syslog.LOG_LOCAL2}, {"local3", syslog.LOG_LOCAL3}, {"local4", syslog.LOG_LOCAL4},
+		{"local5", syslog.LOG_LOCAL5}, {"local6", syslog.LOG_LOCAL6}, {"local7", syslog.LOG_LOCAL7},
+	}
+
+	for _, sFP := range syslogFacilitiesPriorities {
+		logcfg := fmt.Sprintf(`
+global:
+  logging:
+  - type: syslog
+    level: info
+    format: human
+    facility: %s
+`, sFP.Facility)
+		conf := testValidGlobalSection(t, logcfg)
+		assert.Equal(t, 1, len(*conf.Global.Logging))
+		assert.True(t, SyslogFacility(sFP.Priority) == *(*conf.Global.Logging)[0].Ret.(*SyslogLoggingOutlet).Facility)
+	}
+}
+
+func TestLoggingOutletEnumList_SetDefaults(t *testing.T) {
+	e := &LoggingOutletEnumList{}
+	var i yaml.Defaulter = e
+	require.NotPanics(t, func() {
+		i.SetDefault()
+		assert.Equal(t, "warn", (*e)[0].Ret.(*StdoutLoggingOutlet).Level)
+	})
+}

internal/config/config_jobid_test.go

@@ -0,0 +1 @@
+package config

internal/config/config_minimal_test.go

@@ -0,0 +1,32 @@
+package config
+
+import (
+	"testing"
+)
+
+func TestJobsOnlyWorks(t *testing.T) {
+	testValidConfig(t, `
+jobs:
+- name: push
+  type: push
+  # snapshot the filesystems matched by the left-hand-side of the mapping
+  # every 10m with zrepl_ as prefix
+  connect:
+    type: tcp
+    address: localhost:2342
+  filesystems: {
+    "pool1/var/db<": true,
+    "pool1/usr/home<": true,
+    "pool1/usr/home/paranoid": false, #don't backup paranoid user
+    "pool1/poudriere/ports<": false #don't backup the ports trees
+  }
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    keep_receiver:
+    - type: last_n
+      count: 1
+`)
+}

internal/config/config_positiveintervalormanual_test.go

@@ -0,0 +1,41 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/zrepl/yaml-config"
+)
+
+func TestPositiveDurationOrManual(t *testing.T) {
+	cases := []struct {
+		Comment, Input string
+		Result         *PositiveDurationOrManual
+	}{
+		{"empty is error", "", nil},
+		{"negative is error", "-1s", nil},
+		{"zero seconds is error", "0s", nil},
+		{"zero is error", "0", nil},
+		{"non-manual is error", "something", nil},
+		{"positive seconds works", "1s", &PositiveDurationOrManual{Manual: false, Interval: 1 * time.Second}},
+		{"manual works", "manual", &PositiveDurationOrManual{Manual: true, Interval: 0}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.Comment, func(t *testing.T) {
+			var out struct {
+				FieldName PositiveDurationOrManual `yaml:"fieldname"`
+			}
+			input := fmt.Sprintf("\nfieldname: %s\n", tc.Input)
+			err := yaml.UnmarshalStrict([]byte(input), &out)
+			if tc.Result == nil {
+				assert.Error(t, err)
+				t.Logf("%#v", out)
+			} else {
+				assert.Equal(t, *tc.Result, out.FieldName)
+			}
+		})
+	}
+
+}

internal/config/config_recv_test.go

@@ -0,0 +1,134 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	zfsprop "github.com/zrepl/zrepl/internal/zfs/property"
+)
+
+func TestRecvOptions(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: pull
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  root_fs: "zreplplatformtest"
+  %s
+  interval: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+
+`
+
+	recv_properties_empty := `
+  recv:
+    properties:
+`
+
+	recv_inherit_empty := `
+  recv:
+    properties:
+      inherit:
+`
+
+	recv_inherit := `
+  recv:
+    properties:
+      inherit:
+        - testprop
+`
+
+	recv_override_empty := `
+  recv:
+    properties:
+      override:
+`
+
+	recv_override := `
+  recv:
+    properties:
+      override:
+        testprop2: "test123"
+`
+
+	recv_override_and_inherit := `
+  recv:
+    properties:
+      inherit:
+        - testprop
+      override:
+        testprop2: "test123"
+`
+
+	recv_empty := `
+  recv: {}
+`
+
+	recv_not_specified := `
+`
+
+	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
+
+	t.Run("recv_inherit_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_inherit_empty))
+		assert.NotNil(t, c)
+	})
+
+	t.Run("recv_inherit", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_inherit))
+		inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
+		assert.NotEmpty(t, inherit)
+		assert.Contains(t, inherit, zfsprop.Property("testprop"))
+	})
+
+	t.Run("recv_override_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_override_empty))
+		assert.NotNil(t, c)
+	})
+
+	t.Run("recv_override", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_override))
+		override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
+		require.Len(t, override, 1)
+		require.Equal(t, "test123", override["testprop2"])
+
+	})
+
+	t.Run("recv_override_and_inherit", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_override_and_inherit))
+		inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
+		override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
+		assert.NotEmpty(t, inherit)
+		assert.Contains(t, inherit, zfsprop.Property("testprop"))
+		assert.NotEmpty(t, override)
+		assert.Equal(t, "test123", override["testprop2"])
+	})
+
+	t.Run("recv_properties_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_properties_empty))
+		assert.NotNil(t, c)
+	})
+
+	t.Run("recv_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_empty))
+		assert.NotNil(t, c)
+	})
+
+	t.Run("send_not_specified", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_not_specified))
+		assert.NotNil(t, c)
+	})
+
+}

internal/config/config_send_test.go

@@ -0,0 +1,146 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSendOptions(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  %s
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+	encrypted_false := `
+  send:
+    encrypted: false
+`
+
+	encrypted_true := `
+  send:
+    encrypted: true
+`
+
+	raw_true := `
+  send:
+    raw: true
+
+`
+
+	raw_false := `
+  send:
+    raw: false
+
+`
+
+	raw_and_encrypted := `
+  send:
+    encrypted: true
+    raw: true
+`
+
+	properties_and_encrypted := `
+  send:
+    encrypted: true
+    send_properties: true
+`
+
+	properties_true := `
+  send:
+    send_properties: true
+`
+
+	properties_false := `
+  send:
+    send_properties: false
+`
+
+	send_empty := `
+  send: {}
+`
+
+	send_not_specified := `
+`
+
+	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
+
+	t.Run("encrypted_false", func(t *testing.T) {
+		c := testValidConfig(t, fill(encrypted_false))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, false, encrypted)
+	})
+	t.Run("encrypted_true", func(t *testing.T) {
+		c := testValidConfig(t, fill(encrypted_true))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, true, encrypted)
+	})
+
+	t.Run("send_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(send_empty))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, false, encrypted)
+	})
+
+	t.Run("properties_and_encrypted", func(t *testing.T) {
+		c := testValidConfig(t, fill(properties_and_encrypted))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
+		assert.Equal(t, true, encrypted)
+		assert.Equal(t, true, properties)
+	})
+
+	t.Run("properties_false", func(t *testing.T) {
+		c := testValidConfig(t, fill(properties_false))
+		properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
+		assert.Equal(t, false, properties)
+	})
+
+	t.Run("properties_true", func(t *testing.T) {
+		c := testValidConfig(t, fill(properties_true))
+		properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
+		assert.Equal(t, true, properties)
+	})
+
+	t.Run("raw_true", func(t *testing.T) {
+		c := testValidConfig(t, fill(raw_true))
+		raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
+		assert.Equal(t, true, raw)
+	})
+
+	t.Run("raw_false", func(t *testing.T) {
+		c := testValidConfig(t, fill(raw_false))
+		raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
+		assert.Equal(t, false, raw)
+	})
+
+	t.Run("raw_and_encrypted", func(t *testing.T) {
+		c := testValidConfig(t, fill(raw_and_encrypted))
+		raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, true, raw)
+		assert.Equal(t, true, encrypted)
+	})
+
+	t.Run("send_not_specified", func(t *testing.T) {
+		c := testValidConfig(t, fill(send_not_specified))
+		assert.NotNil(t, c)
+	})
+
+}

internal/config/config_snapshotting_test.go

@@ -0,0 +1,178 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSnapshotting(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  %s
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+	manual := `
+  snapshotting:
+    type: manual
+`
+	periodic := `
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    timestamp_format: dense
+    interval: 10m
+`
+	cron := `
+  snapshotting:
+    type: cron
+    prefix: zrepl_
+    timestamp_format: human
+    cron: "10 * * * *"
+`
+
+	periodicDaily := `
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 1d
+`
+
+	hooks := `
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 10m
+    hooks:
+    - type: command
+      path: /tmp/path/to/command
+    - type: command
+      path: /tmp/path/to/command
+      filesystems: { "zroot<": true, "<": false }
+    - type: postgres-checkpoint
+      dsn: "host=localhost port=5432 user=postgres sslmode=disable"
+      filesystems: {
+          "tank/postgres/data11": true
+      }
+    - type: mysql-lock-tables
+      dsn: "root@tcp(localhost)/"
+      filesystems: {
+        "tank/mysql": true
+      }
+`
+
+	fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
+	var c *Config
+
+	t.Run("manual", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(manual))
+		snm := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingManual)
+		assert.Equal(t, "manual", snm.Type)
+	})
+
+	t.Run("periodic", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(periodic))
+		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
+		assert.Equal(t, "periodic", snp.Type)
+		assert.Equal(t, 10*time.Minute, snp.Interval.Duration())
+		assert.Equal(t, "zrepl_", snp.Prefix)
+	})
+
+	t.Run("periodicDaily", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(periodicDaily))
+		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
+		assert.Equal(t, "periodic", snp.Type)
+		assert.Equal(t, 24*time.Hour, snp.Interval.Duration())
+		assert.Equal(t, "zrepl_", snp.Prefix)
+		assert.Equal(t, "dense", snp.TimestampFormat)
+	})
+
+	t.Run("cron", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(cron))
+		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingCron)
+		assert.Equal(t, "cron", snp.Type)
+		assert.Equal(t, "zrepl_", snp.Prefix)
+		assert.Equal(t, "human", snp.TimestampFormat)
+	})
+
+	t.Run("hooks", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(hooks))
+		hs := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic).Hooks
+		assert.Equal(t, hs[0].Ret.(*HookCommand).Filesystems["<"], true)
+		assert.Equal(t, hs[1].Ret.(*HookCommand).Filesystems["zroot<"], true)
+		assert.Equal(t, hs[2].Ret.(*HookPostgresCheckpoint).Filesystems["tank/postgres/data11"], true)
+		assert.Equal(t, hs[3].Ret.(*HookMySQLLockTables).Filesystems["tank/mysql"], true)
+	})
+
+}
+
+func TestSnapshottingTimestampDefaults(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  %s
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+
+	periodic := `
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 10m
+`
+	cron := `
+  snapshotting:
+    type: cron
+    prefix: zrepl_
+    cron: "10 * * * *"
+`
+
+	fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
+	var c *Config
+
+	t.Run("periodic", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(periodic))
+		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
+		assert.Equal(t, "periodic", snp.Type)
+		assert.Equal(t, 10*time.Minute, snp.Interval.Duration())
+		assert.Equal(t, "zrepl_", snp.Prefix)
+		assert.Equal(t, snp.TimestampFormat, "dense")
+		assert.Equal(t, snp.TimestampLocation, "UTC")
+	})
+
+	t.Run("cron", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(cron))
+		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingCron)
+		assert.Equal(t, "cron", snp.Type)
+		assert.Equal(t, "zrepl_", snp.Prefix)
+		assert.Equal(t, snp.TimestampFormat, "dense")
+		assert.Equal(t, snp.TimestampLocation, "UTC")
+	})
+}

internal/config/config_test.go

@@ -0,0 +1,157 @@
+package config
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+	"text/template"
+
+	"github.com/kr/pretty"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/zrepl/yaml-config"
+)
+
+func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
+	paths, err := filepath.Glob("./samples/*")
+	if err != nil {
+		t.Errorf("glob failed: %+v", err)
+	}
+
+	paths = append(paths, "../../packaging/systemd-default-zrepl.yml")
+
+	for _, p := range paths {
+
+		if path.Ext(p) != ".yml" {
+			t.Logf("skipping file %s", p)
+			continue
+		}
+
+		t.Run(p, func(t *testing.T) {
+			c, err := ParseConfig(p)
+			if err != nil {
+				t.Errorf("error parsing %s:\n%+v", p, err)
+			}
+
+			t.Logf("file: %s", p)
+			t.Log(pretty.Sprint(c))
+		})
+
+	}
+
+}
+
+// template must be a template/text template with a single '{{ . }}' as placeholder for val
+//
+//nolint:deadcode,unused
+func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
+	tmp, err := template.New("master").Parse(tmpl)
+	if err != nil {
+		panic(err)
+	}
+	var buf bytes.Buffer
+	err = tmp.Execute(&buf, val)
+	if err != nil {
+		panic(err)
+	}
+	return testValidConfig(t, buf.String())
+}
+
+func testValidConfig(t *testing.T, input string) *Config {
+	t.Helper()
+	conf, err := testConfig(t, input)
+	require.NoError(t, err)
+	require.NotNil(t, conf)
+	return conf
+}
+
+func testConfig(t *testing.T, input string) (*Config, error) {
+	t.Helper()
+	return ParseConfigBytes([]byte(input))
+}
+
+func trimSpaceEachLineAndPad(s, pad string) string {
+	var out strings.Builder
+	scan := bufio.NewScanner(strings.NewReader(s))
+	for scan.Scan() {
+		fmt.Fprintf(&out, "%s%s\n", pad, bytes.TrimSpace(scan.Bytes()))
+	}
+	return out.String()
+}
+
+func TestTrimSpaceEachLineAndPad(t *testing.T) {
+	foo := `
+	foo
+	bar baz
+	`
+	assert.Equal(t, "  \n  foo\n  bar baz\n  \n", trimSpaceEachLineAndPad(foo, "  "))
+}
+
+func TestCronSpec(t *testing.T) {
+
+	expectAccept := []string{
+		`"* * * * *"`,
+		`"0-10 * * * *"`,
+		`"* 0-5,8,12 * * *"`,
+	}
+
+	expectFail := []string{
+		`* * * *`,
+		``,
+		`23`,
+		`"@reboot"`,
+		`"@every 1h30m"`,
+		`"@daily"`,
+		`* * * * * *`,
+	}
+
+	for _, input := range expectAccept {
+		t.Run(input, func(t *testing.T) {
+			s := fmt.Sprintf("spec: %s\n", input)
+			var v struct {
+				Spec CronSpec
+			}
+			v.Spec.Schedule = nil
+			t.Logf("input:\n%s", s)
+			err := yaml.UnmarshalStrict([]byte(s), &v)
+			t.Logf("error: %T %s", err, err)
+			require.NoError(t, err)
+			require.NotNil(t, v.Spec.Schedule)
+		})
+	}
+
+	for _, input := range expectFail {
+		t.Run(input, func(t *testing.T) {
+			s := fmt.Sprintf("spec: %s\n", input)
+			var v struct {
+				Spec CronSpec
+			}
+			v.Spec.Schedule = nil
+			t.Logf("input: %q", s)
+			err := yaml.UnmarshalStrict([]byte(s), &v)
+			t.Logf("error: %T %s", err, err)
+			require.Error(t, err)
+			require.Nil(t, v.Spec.Schedule)
+		})
+	}
+
+}
+
+func TestEmptyConfig(t *testing.T) {
+	cases := []string{
+		"",
+		"\n",
+		"---",
+		"---\n",
+	}
+	for _, input := range cases {
+		config := testValidConfig(t, input)
+		require.NotNil(t, config)
+		require.NotNil(t, config.Global)
+		require.Empty(t, config.Jobs)
+	}
+}

internal/config/config_transport_test.go

@@ -0,0 +1,99 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestTransportConnect(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+%s
+  filesystems: {"<": true}
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+
+	mconf := func(s string) string { return fmt.Sprintf(tmpl, s) }
+
+	type test struct {
+		Name        string
+		ExpectError bool
+		Connect     string
+	}
+
+	testTable := []test{
+		{
+			Name:        "tcp_with_address_and_port",
+			ExpectError: false,
+			Connect: `
+			type: tcp
+			address: 10.0.0.23:42
+			`,
+		},
+		{
+			Name:        "tls_with_host_and_port",
+			ExpectError: false,
+			Connect: `
+			type: tls
+			address: "server1.foo.bar:8888"
+			ca:   /etc/zrepl/ca.crt
+			cert: /etc/zrepl/backupserver.fullchain
+			key:  /etc/zrepl/backupserver.key
+			server_cn: "server1"
+			`,
+		},
+		{
+			Name:        "tcp_without_port",
+			ExpectError: true,
+			Connect: `
+			type: tcp
+			address: 10.0.0.23
+			`,
+		},
+		{
+			Name:        "tls_without_port",
+			ExpectError: true,
+			Connect: `
+			type: tls
+			address: 10.0.0.23
+			ca:   /etc/zrepl/ca.crt
+			cert: /etc/zrepl/backupserver.fullchain
+			key:  /etc/zrepl/backupserver.key
+			server_cn: "server1"
+			`,
+		},
+	}
+
+	for _, tc := range testTable {
+		t.Run(tc.Name, func(t *testing.T) {
+			require.NotEmpty(t, tc.Connect)
+			connect := trimSpaceEachLineAndPad(tc.Connect, "    ")
+			conf := mconf(connect)
+			config, err := testConfig(t, conf)
+			if tc.ExpectError && err == nil {
+				t.Errorf("expected test failure, but got valid config %v", config)
+				return
+			}
+			if !tc.ExpectError && err != nil {
+				t.Errorf("not expecint test failure but got error: %s", err)
+				return
+			}
+			t.Logf("error=%v", err)
+			t.Logf("config=%v", config)
+		})
+	}
+
+}

internal/config/retentiongrid.go

@@ -0,0 +1,119 @@
+package config
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type RetentionIntervalList []RetentionInterval
+
+type PruneGrid struct {
+	Type  string                `yaml:"type"`
+	Grid  RetentionIntervalList `yaml:"grid"`
+	Regex string                `yaml:"regex"`
+}
+
+type RetentionInterval struct {
+	length    time.Duration
+	keepCount int
+}
+
+func (i *RetentionInterval) Length() time.Duration {
+	return i.length
+}
+
+func (i *RetentionInterval) KeepCount() int {
+	return i.keepCount
+}
+
+const RetentionGridKeepCountAll int = -1
+
+func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	var in string
+	if err := u(&in, true); err != nil {
+		return err
+	}
+
+	intervals, err := ParseRetentionIntervalSpec(in)
+	if err != nil {
+		return err
+	}
+
+	*t = intervals
+
+	return nil
+}
+
+var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
+
+func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {
+
+	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
+	if comps == nil {
+		err = fmt.Errorf("retention string does not match expected format")
+		return
+	}
+
+	times, err := strconv.Atoi(comps[1])
+	if err != nil {
+		return nil, err
+	} else if times <= 0 {
+		return nil, fmt.Errorf("contains factor <= 0")
+	}
+
+	duration, err := parsePositiveDuration(comps[2])
+	if err != nil {
+		return nil, err
+	}
+
+	keepCount := 1
+	if comps[3] != "" {
+		// Decompose key=value, comma separated
+		// For now, only keep_count is supported
+		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
+		res := re.FindStringSubmatch(comps[4])
+		if res == nil || len(res) != 2 {
+			err = fmt.Errorf("interval parameter contains unknown parameters")
+			return
+		}
+		if res[1] == "all" {
+			keepCount = RetentionGridKeepCountAll
+		} else {
+			keepCount, err = strconv.Atoi(res[1])
+			if err != nil {
+				err = fmt.Errorf("cannot parse keep_count value")
+				return
+			}
+		}
+	}
+
+	intervals = make([]RetentionInterval, times)
+	for i := range intervals {
+		intervals[i] = RetentionInterval{
+			length:    duration,
+			keepCount: keepCount,
+		}
+	}
+
+	return
+
+}
+
+func ParseRetentionIntervalSpec(s string) (intervals []RetentionInterval, err error) {
+
+	ges := strings.Split(s, "|")
+	intervals = make([]RetentionInterval, 0, 7*len(ges))
+
+	for intervalIdx, e := range ges {
+		parsed, err := parseRetentionGridIntervalString(e)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
+		}
+		intervals = append(intervals, parsed...)
+	}
+
+	return
+}

internal/config/samples/bandwidth_limit.yml

@@ -0,0 +1,41 @@
+
+jobs:
+  - type: sink
+    name: "limited_sink"
+    root_fs: "fs0"
+    recv:
+      bandwidth_limit:
+        max: 12345 B
+    serve:
+      type: local
+      listener_name: localsink
+
+  - type: push
+    name: "limited_push"
+    connect:
+      type: local
+      listener_name: localsink
+      client_identity: local_backup
+    filesystems: {
+      "root<": true,
+    }
+    send:
+      bandwidth_limit:
+        max: 54321 B
+        bucket_capacity: 1024 B
+    snapshotting:
+      type: manual
+    pruning:
+      keep_sender:
+        - type: last_n
+          count: 1
+      keep_receiver:
+        - type: last_n
+          count: 1
+
+  - type: sink
+    name: "nolimit_sink"
+    root_fs: "fs1"
+    serve:
+      type: local
+      listener_name: localsink

internal/config/samples/hooks/template.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -e
+
+[ "$ZREPL_DRYRUN" = "true" ] && DRYRUN="echo DRYRUN: "
+
+pre_snapshot() {
+    $DRYRUN date
+}
+
+post_snapshot() {
+    $DRYRUN date
+}
+
+case "$ZREPL_HOOKTYPE" in
+    pre_snapshot|post_snapshot)
+        "$ZREPL_HOOKTYPE"
+        ;;
+    *)
+        printf 'Unrecognized hook type: %s\n' "$ZREPL_HOOKTYPE"
+        exit 255
+        ;;
+esac

internal/config/samples/local.yml

@@ -0,0 +1,31 @@
+
+jobs:
+  - type: sink
+    name: "local_sink"
+    root_fs: "storage/zrepl/sink"
+    serve:
+      type: local
+      listener_name: localsink
+
+  - type: push
+    name: "backup_system"
+    connect:
+      type: local
+      listener_name: localsink
+      client_identity: local_backup
+    filesystems: {
+      "system<": true,
+    }
+    snapshotting:
+      type: periodic
+      interval: 10m
+      prefix: zrepl_
+    pruning:
+      keep_sender:
+      - type: not_replicated
+      - type: last_n
+        count: 10
+      keep_receiver:
+      - type: grid
+        grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+        regex: "zrepl_.*"

internal/config/samples/pull.yml

@@ -0,0 +1,24 @@
+jobs:
+- name: pull_servers
+  type: pull
+  connect:
+    type: tls
+    address: "server1.foo.bar:8888"
+    ca: "/certs/ca.crt"
+    cert: "/certs/cert.crt"
+    key: "/certs/key.pem"
+    server_cn: "server1"
+  root_fs: "pool2/backup_servers"
+  interval: 10m
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    - type: last_n
+      count: 10
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 14x1d
+      regex: "zrepl_.*"
+    keep_receiver:
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+      regex: "zrepl_.*"

internal/config/samples/pull_ssh.yml

@@ -0,0 +1,28 @@
+jobs:
+
+- name: pull_servers
+  type: pull
+  connect:
+    type: ssh+stdinserver
+    host: app-srv.example.com
+    user: root
+    port: 22
+    identity_file: /etc/zrepl/ssh/identity
+    options: # optional, default [], `-o` arguments passed to ssh
+    - "Compression=yes"
+  root_fs: "pool2/backup_servers"
+  interval: 10m
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    - type: last_n
+      count: 10
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 14x1d
+      regex: "^zrepl_.*"
+    keep_receiver:
+    - type: regex
+      regex: keep_
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+      regex: "^zrepl_.*"

internal/config/samples/push.yml

@@ -0,0 +1,26 @@
+jobs:
+  - type: push
+    name: "push"
+    filesystems: {
+      "<": true,
+      "tmp": false
+    }
+    connect:
+      type: tcp
+      address: "backup-server.foo.bar:8888"
+    snapshotting:
+      type: manual
+    send:
+      encrypted: false
+    pruning:
+      keep_sender:
+        - type: not_replicated
+        - type: last_n
+          count: 10
+        - type: grid
+          grid: 1x1h(keep=all) | 24x1h | 14x1d
+          regex: "^zrepl_.*"
+      keep_receiver:
+        - type: grid
+          grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+          regex: "^zrepl_.*"

internal/config/samples/quickstart_backup_to_external_disk.yml

@@ -0,0 +1,88 @@
+# This config serves as an example for a local zrepl installation that
+# backups the entire zpool `system` to `backuppool/zrepl/sink`
+#
+# The requirements covered by this setup are described in the zrepl documentation's
+# quick start section which inlines this example.
+#
+# CUSTOMIZATIONS YOU WILL LIKELY WANT TO APPLY:
+# - adjust the name of the production pool `system` in the `filesystems` filter of jobs `snapjob` and `push_to_drive`
+# - adjust the name of the backup pool `backuppool` in the `backuppool_sink` job
+# - adjust the occurences of `myhostname` to the name of the system you are backing up (cannot be easily changed once you start replicating)
+# - make sure the `zrepl_` prefix is not being used by any other zfs tools you might have installed (it likely isn't)
+
+jobs:
+
+# this job takes care of snapshot creation + pruning
+- name: snapjob
+  type: snap
+  filesystems: {
+      "system<": true,
+  }
+  # create snapshots with prefix `zrepl_` every 15 minutes
+  snapshotting:
+    type: periodic
+    interval: 15m
+    prefix: zrepl_
+  pruning:
+    keep:
+    # fade-out scheme for snapshots starting with `zrepl_`
+    # - keep all created in the last hour
+    # - then destroy snapshots such that we keep 24 each 1 hour apart
+    # - then destroy snapshots such that we keep 14 each 1 day apart
+    # - then destroy all older snapshots
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 14x1d
+      regex: "^zrepl_.*"
+    # keep all snapshots that don't have the `zrepl_` prefix
+    - type: regex
+      negate: true
+      regex: "^zrepl_.*"
+
+# This job pushes to the local sink defined in job `backuppool_sink`.
+# We trigger replication manually from the command line / udev rules using
+#  `zrepl signal wakeup push_to_drive`
+- type: push
+  name: push_to_drive
+  connect:
+    type: local
+    listener_name: backuppool_sink
+    client_identity: myhostname
+  filesystems: {
+      "system<": true
+  }
+  send:
+    encrypted: true
+  replication:
+    protection:
+      initial: guarantee_resumability
+      # Downgrade protection to guarantee_incremental which uses zfs bookmarks instead of zfs holds.
+      # Thus, when we yank out the backup drive during replication
+      # - we might not be able to resume the interrupted replication step because the partially received `to` snapshot of a `from`->`to` step may be pruned any time
+      # - but in exchange we get back the disk space allocated by `to` when we prune it
+      # - and because we still have the bookmarks created by `guarantee_incremental`, we can still do incremental replication of `from`->`to2` in the future
+      incremental: guarantee_incremental
+  snapshotting:
+    type: manual
+  pruning:
+    # no-op prune rule on sender (keep all snapshots), job `snapshot` takes care of this
+    keep_sender:
+    - type: regex
+      regex: ".*"
+    # retain 
+    keep_receiver:
+    # longer retention on the backup drive, we have more space there
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 360x1d
+      regex: "^zrepl_.*"
+    # retain all non-zrepl snapshots on the backup drive
+    - type: regex
+      negate: true
+      regex: "^zrepl_.*"
+
+# This job receives from job `push_to_drive` into `backuppool/zrepl/sink/myhostname`
+- type: sink
+  name: backuppool_sink
+  root_fs: "backuppool/zrepl/sink"
+  serve:
+    type: local
+    listener_name: backuppool_sink

internal/config/samples/quickstart_continuous_server_backup_receiver.yml

@@ -0,0 +1,12 @@
+jobs:
+- name: sink
+  type: sink
+  serve:
+      type: tls
+      listen: ":8888"
+      ca: "/etc/zrepl/prod.crt"
+      cert: "/etc/zrepl/backups.crt"
+      key: "/etc/zrepl/backups.key"
+      client_cns:
+        - "prod"
+  root_fs: "storage/zrepl/sink"

internal/config/samples/quickstart_continuous_server_backup_sender.yml

@@ -0,0 +1,28 @@
+jobs:
+- name: prod_to_backups
+  type: push
+  connect:
+    type: tls
+    address: "backups.example.com:8888"
+    ca: /etc/zrepl/backups.crt
+    cert: /etc/zrepl/prod.crt
+    key:  /etc/zrepl/prod.key
+    server_cn: "backups"
+  filesystems: {
+    "zroot<": true,
+    "zroot/var/tmp<": false,
+    "zroot/usr/home/paranoid": false
+  }
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 10m
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 30x1d | 6x30d
+      regex: "^zrepl_"

internal/config/samples/quickstart_fan_out_replication_source.yml

@@ -0,0 +1,59 @@
+jobs:
+# Separate job for snapshots and pruning
+- name: snapshots
+  type: snap
+  filesystems:
+    'tank<': true # all filesystems
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 10m
+  pruning:
+    keep:
+      # Keep non-zrepl snapshots
+      - type: regex
+        negate: true
+        regex: '^zrepl_'
+      # Time-based snapshot retention
+      - type: grid
+        grid: 1x1h(keep=all) | 24x1h | 30x1d | 12x30d
+        regex: '^zrepl_'
+
+# Source job for target B
+- name: target_b
+  type: source
+  serve:
+    type: tls
+    listen: :8888
+    ca: /etc/zrepl/b.example.com.crt
+    cert: /etc/zrepl/a.example.com.crt
+    key: /etc/zrepl/a.example.com.key
+    client_cns:
+      - b.example.com
+  filesystems:
+    'tank<': true # all filesystems
+  # Snapshots are handled by the separate snap job
+  snapshotting:
+    type: manual
+
+# Source job for target C
+- name: target_c
+  type: source
+  serve:
+    type: tls
+    listen: :8889
+    ca: /etc/zrepl/c.example.com.crt
+    cert: /etc/zrepl/a.example.com.crt
+    key: /etc/zrepl/a.example.com.key
+    client_cns:
+      - c.example.com
+  filesystems:
+    'tank<': true # all filesystems
+  # Snapshots are handled by the separate snap job
+  snapshotting:
+    type: manual
+
+# Source jobs for remaining targets. Each one should listen on a different port
+# and reference the correct certificate and client CN.
+# - name: target_c
+#   ...

internal/config/samples/quickstart_fan_out_replication_target.yml

@@ -0,0 +1,30 @@
+jobs:
+# Pull from source server A
+- name: source_a
+  type: pull
+  connect:
+    type: tls
+    # Use the correct port for this specific client (eg. B is 8888, C is 8889, etc.)
+    address: a.example.com:8888
+    ca: /etc/zrepl/a.example.com.crt
+    # Use the correct key pair for this specific client
+    cert: /etc/zrepl/b.example.com.crt
+    key: /etc/zrepl/b.example.com.key
+    server_cn: a.example.com
+  root_fs: pool0/backup
+  interval: 10m
+  pruning:
+    keep_sender:
+      # Source does the pruning in its snap job
+      - type: regex
+        regex: '.*'
+    # Receiver-side pruning can be configured as desired on each target server
+    keep_receiver:
+      # Keep non-zrepl snapshots
+      - type: regex
+        negate: true
+        regex: '^zrepl_'
+      # Time-based snapshot retention
+      - type: grid
+        grid: 1x1h(keep=all) | 24x1h | 30x1d | 12x30d
+        regex: '^zrepl_'

internal/config/samples/sink.yml

@@ -0,0 +1,13 @@
+jobs:
+  - type: sink
+    name: "laptop_sink"
+    root_fs: "pool2/backup_laptops"
+    serve:
+      type: tls
+      listen: "192.168.122.189:8888"
+      ca: "ca.pem"
+      cert: "cert.pem"
+      key: "key.pem"
+      client_cns:
+        - "laptop1"
+        - "homeserver"

internal/config/samples/snap.yml

@@ -0,0 +1,14 @@
+jobs:
+- name: snapjob
+  type: snap
+  filesystems: {
+    "tank/frequently_changed<": true,
+  }
+  snapshotting:
+    type: periodic
+    interval: 2m
+    prefix: zrepl_snapjob_
+  pruning:
+    keep:
+      - type: last_n
+        count: 60

internal/config/samples/snap_cron.yml

@@ -0,0 +1,14 @@
+jobs:
+- name: snapjob
+  type: snap
+  filesystems: {
+    "tank<": true,
+  }
+  snapshotting:
+    type: cron
+    prefix: zrepl_snapjob_
+    cron: "*/5 * * * *"
+  pruning:
+    keep:
+      - type: last_n
+        count: 60

internal/config/samples/source.yml

@@ -0,0 +1,21 @@
+jobs:
+- name: pull_source
+  type: source
+  serve:
+    type: tcp
+    listen: "0.0.0.0:8888"
+    clients: {
+      "192.168.122.123" :               "mysql01",
+      "192.168.122.42" :                "mx01",
+      "2001:0db8:85a3::8a2e:0370:7334": "gateway",
+      "10.23.42.0/24":       "cluster-*",
+      "fde4:8dba:82e1::/64": "san-*",
+    }
+  filesystems: {
+    "<": true,
+    "secret": false
+  }
+  snapshotting:
+    type: periodic
+    interval: 10m
+    prefix: zrepl_

internal/config/samples/source_ssh.yml

@@ -0,0 +1,17 @@
+jobs:
+- name: pull_source
+  type: source
+  serve:
+    type: stdinserver
+    client_identities:
+      - "client1"
+      - "client2"
+  filesystems: {
+    "<": true,
+    "secret": false
+  }
+  snapshotting:
+    type: periodic
+    interval: 10m
+    prefix: zrepl_
+

internal/daemon/control.go

@@ -0,0 +1,293 @@
+package daemon
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/zrepl/zrepl/internal/daemon/job"
+	"github.com/zrepl/zrepl/internal/daemon/nethelpers"
+	"github.com/zrepl/zrepl/internal/endpoint"
+	"github.com/zrepl/zrepl/internal/logger"
+	"github.com/zrepl/zrepl/internal/util/envconst"
+	"github.com/zrepl/zrepl/internal/version"
+	"github.com/zrepl/zrepl/internal/zfs"
+	"github.com/zrepl/zrepl/internal/zfs/zfscmd"
+)
+
+type controlJob struct {
+	sockaddr *net.UnixAddr
+	jobs     *jobs
+}
+
+func newControlJob(sockpath string, jobs *jobs) (j *controlJob, err error) {
+	j = &controlJob{jobs: jobs}
+
+	j.sockaddr, err = net.ResolveUnixAddr("unix", sockpath)
+	if err != nil {
+		err = errors.Wrap(err, "cannot resolve unix address")
+		return
+	}
+
+	return
+}
+
+func (j *controlJob) Name() string { return jobNameControl }
+
+func (j *controlJob) Status() *job.Status { return &job.Status{Type: job.TypeInternal} }
+
+func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
+
+func (j *controlJob) SenderConfig() *endpoint.SenderConfig { return nil }
+
+var promControl struct {
+	requestBegin    *prometheus.CounterVec
+	requestFinished *prometheus.HistogramVec
+}
+
+func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) {
+	promControl.requestBegin = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "zrepl",
+		Subsystem: "control",
+		Name:      "request_begin",
+		Help:      "number of request we started to handle",
+	}, []string{"endpoint"})
+
+	promControl.requestFinished = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "zrepl",
+		Subsystem: "control",
+		Name:      "request_finished",
+		Help:      "time it took a request to finish",
+		Buckets:   []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3, 10e-3, 100e-3, 200e-3, 400e-3, 800e-3, 1, 10, 20},
+	}, []string{"endpoint"})
+	registerer.MustRegister(promControl.requestBegin)
+	registerer.MustRegister(promControl.requestFinished)
+}
+
+const (
+	ControlJobEndpointPProf   string = "/debug/pprof"
+	ControlJobEndpointVersion string = "/version"
+	ControlJobEndpointStatus  string = "/status"
+	ControlJobEndpointSignal  string = "/signal"
+)
+
+func (j *controlJob) Run(ctx context.Context) {
+
+	log := job.GetLogger(ctx)
+	defer log.Info("control job finished")
+
+	l, err := nethelpers.ListenUnixPrivate(j.sockaddr)
+	if err != nil {
+		log.WithError(err).Error("error listening")
+		return
+	}
+
+	pprofServer := NewPProfServer(ctx)
+	if listen := envconst.String("ZREPL_DAEMON_AUTOSTART_PPROF_SERVER", ""); listen != "" {
+		pprofServer.Control(PprofServerControlMsg{
+			Run:               true,
+			HttpListenAddress: listen,
+		})
+	}
+
+	mux := http.NewServeMux()
+	mux.Handle(ControlJobEndpointPProf,
+		requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) {
+			var msg PprofServerControlMsg
+			err := decoder(&msg)
+			if err != nil {
+				return nil, errors.Errorf("decode failed")
+			}
+			pprofServer.Control(msg)
+			return struct{}{}, nil
+		}}})
+
+	mux.Handle(ControlJobEndpointVersion,
+		requestLogger{log: log, handler: jsonResponder{log, func() (interface{}, error) {
+			return version.NewZreplVersionInformation(), nil
+		}}})
+
+	mux.Handle(ControlJobEndpointStatus,
+		// don't log requests to status endpoint, too spammy
+		jsonResponder{log, func() (interface{}, error) {
+			jobs := j.jobs.status()
+			globalZFS := zfscmd.GetReport()
+			envconstReport := envconst.GetReport()
+			s := Status{
+				Jobs: jobs,
+				Global: GlobalStatus{
+					ZFSCmds:   globalZFS,
+					Envconst:  envconstReport,
+					OsEnviron: os.Environ(),
+				}}
+			return s, nil
+		}})
+
+	mux.Handle(ControlJobEndpointSignal,
+		requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) {
+			type reqT struct {
+				Name string
+				Op   string
+			}
+			var req reqT
+			if decoder(&req) != nil {
+				return nil, errors.Errorf("decode failed")
+			}
+
+			var err error
+			switch req.Op {
+			case "wakeup":
+				err = j.jobs.wakeup(req.Name)
+			case "reset":
+				err = j.jobs.reset(req.Name)
+			default:
+				err = fmt.Errorf("operation %q is invalid", req.Op)
+			}
+
+			return struct{}{}, err
+		}}})
+	server := http.Server{
+		Handler: mux,
+		// control socket is local, 1s timeout should be more than sufficient, even on a loaded system
+		WriteTimeout: envconst.Duration("ZREPL_DAEMON_CONTROL_SERVER_WRITE_TIMEOUT", 1*time.Second),
+		ReadTimeout:  envconst.Duration("ZREPL_DAEMON_CONTROL_SERVER_READ_TIMEOUT", 1*time.Second),
+	}
+
+outer:
+	for {
+
+		served := make(chan error)
+		go func() {
+			served <- server.Serve(l)
+			close(served)
+		}()
+
+		select {
+		case <-ctx.Done():
+			log.WithError(ctx.Err()).Info("context done")
+			err := server.Shutdown(context.Background())
+			if err != nil {
+				log.WithError(err).Error("cannot shutdown server")
+			}
+			break outer
+		case err = <-served:
+			if err != nil {
+				log.WithError(err).Error("error serving")
+				break outer
+			}
+		}
+
+	}
+
+}
+
+type jsonResponder struct {
+	log      Logger
+	producer func() (interface{}, error)
+}
+
+func (j jsonResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	logIoErr := func(err error) {
+		if err != nil {
+			j.log.WithError(err).Error("control handler io error")
+		}
+	}
+	res, err := j.producer()
+	if err != nil {
+		j.log.WithError(err).Error("control handler error")
+		w.WriteHeader(http.StatusInternalServerError)
+		_, err = io.WriteString(w, err.Error())
+		logIoErr(err)
+		return
+	}
+	var buf bytes.Buffer
+	err = json.NewEncoder(&buf).Encode(res)
+	if err != nil {
+		j.log.WithError(err).Error("control handler json marshal error")
+		w.WriteHeader(http.StatusInternalServerError)
+		_, err = io.WriteString(w, err.Error())
+	} else {
+		_, err = io.Copy(w, &buf)
+	}
+	logIoErr(err)
+}
+
+type jsonDecoder = func(interface{}) error
+
+type jsonRequestResponder struct {
+	log      Logger
+	producer func(decoder jsonDecoder) (interface{}, error)
+}
+
+func (j jsonRequestResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	logIoErr := func(err error) {
+		if err != nil {
+			j.log.WithError(err).Error("control handler io error")
+		}
+	}
+
+	var decodeError error
+	decoder := func(i interface{}) error {
+		err := json.NewDecoder(r.Body).Decode(&i)
+		decodeError = err
+		return err
+	}
+	res, producerErr := j.producer(decoder)
+
+	//If we had a decode error ignore output of producer and return error
+	if decodeError != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		_, err := io.WriteString(w, decodeError.Error())
+		logIoErr(err)
+		return
+	}
+	if producerErr != nil {
+		j.log.WithError(producerErr).Error("control handler error")
+		w.WriteHeader(http.StatusInternalServerError)
+		_, err := io.WriteString(w, producerErr.Error())
+		logIoErr(err)
+		return
+	}
+
+	var buf bytes.Buffer
+	encodeErr := json.NewEncoder(&buf).Encode(res)
+	if encodeErr != nil {
+		j.log.WithError(producerErr).Error("control handler json marshal error")
+		w.WriteHeader(http.StatusInternalServerError)
+		_, err := io.WriteString(w, encodeErr.Error())
+		logIoErr(err)
+	} else {
+		_, err := io.Copy(w, &buf)
+		logIoErr(err)
+	}
+}
+
+type requestLogger struct {
+	log         logger.Logger
+	handler     http.Handler
+	handlerFunc http.HandlerFunc
+}
+
+func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	log := l.log.WithField("method", r.Method).WithField("url", r.URL)
+	log.Debug("start")
+	promControl.requestBegin.WithLabelValues(r.URL.Path).Inc()
+	defer prometheus.NewTimer(promControl.requestFinished.WithLabelValues(r.URL.Path)).ObserveDuration()
+	if l.handlerFunc != nil {
+		l.handlerFunc(w, r)
+	} else if l.handler != nil {
+		l.handler.ServeHTTP(w, r)
+	} else {
+		log.Error("no handler or handlerFunc configured")
+	}
+	log.Debug("finish")
+}

internal/daemon/daemon.go

@@ -0,0 +1,251 @@
+package daemon
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/zrepl/zrepl/internal/daemon/logging/trace"
+	"github.com/zrepl/zrepl/internal/endpoint"
+	"github.com/zrepl/zrepl/internal/util/envconst"
+
+	"github.com/zrepl/zrepl/internal/config"
+	"github.com/zrepl/zrepl/internal/daemon/job"
+	"github.com/zrepl/zrepl/internal/daemon/job/reset"
+	"github.com/zrepl/zrepl/internal/daemon/job/wakeup"
+	"github.com/zrepl/zrepl/internal/daemon/logging"
+	"github.com/zrepl/zrepl/internal/logger"
+	"github.com/zrepl/zrepl/internal/version"
+	"github.com/zrepl/zrepl/internal/zfs/zfscmd"
+)
+
+func Run(ctx context.Context, conf *config.Config) error {
+	ctx, cancel := context.WithCancel(ctx)
+
+	defer cancel()
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		<-sigChan
+		cancel()
+	}()
+
+	outlets, err := logging.OutletsFromConfig(*conf.Global.Logging)
+	if err != nil {
+		return errors.Wrap(err, "cannot build logging from config")
+	}
+	outlets.Add(newPrometheusLogOutlet(), logger.Debug)
+
+	confJobs, err := job.JobsFromConfig(conf, config.ParseFlagsNone)
+	if err != nil {
+		return errors.Wrap(err, "cannot build jobs from config")
+	}
+
+	log := logger.NewLogger(outlets, 1*time.Second)
+	log.Info(version.NewZreplVersionInformation().String())
+
+	ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
+	trace.RegisterCallback(trace.Callback{
+		OnBegin: func(ctx context.Context) { logging.GetLogger(ctx, logging.SubsysTraceData).Debug("begin span") },
+		OnEnd: func(ctx context.Context, spanInfo trace.SpanInfo) {
+			logging.
+				GetLogger(ctx, logging.SubsysTraceData).
+				WithField("duration_s", spanInfo.EndedAt().Sub(spanInfo.StartedAt()).Seconds()).
+				Debug("finished span " + spanInfo.TaskAndSpanStack(trace.SpanStackKindAnnotation))
+		},
+	})
+
+	for _, job := range confJobs {
+		if IsInternalJobName(job.Name()) {
+			panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME
+		}
+	}
+
+	jobs := newJobs()
+
+	// start control socket
+	controlJob, err := newControlJob(conf.Global.Control.SockPath, jobs)
+	if err != nil {
+		panic(err) // FIXME
+	}
+	jobs.start(ctx, controlJob, true)
+
+	for i, jc := range conf.Global.Monitoring {
+		var (
+			job job.Job
+			err error
+		)
+		switch v := jc.Ret.(type) {
+		case *config.PrometheusMonitoring:
+			job, err = newPrometheusJobFromConfig(v)
+		default:
+			return errors.Errorf("unknown monitoring job #%d (type %T)", i, v)
+		}
+		if err != nil {
+			return errors.Wrapf(err, "cannot build monitoring job #%d", i)
+		}
+		jobs.start(ctx, job, true)
+	}
+
+	// register global (=non job-local) metrics
+	version.PrometheusRegister(prometheus.DefaultRegisterer)
+	zfscmd.RegisterMetrics(prometheus.DefaultRegisterer)
+	trace.RegisterMetrics(prometheus.DefaultRegisterer)
+	endpoint.RegisterMetrics(prometheus.DefaultRegisterer)
+
+	log.Info("starting daemon")
+
+	// start regular jobs
+	for _, j := range confJobs {
+		jobs.start(ctx, j, false)
+	}
+
+	select {
+	case <-jobs.wait():
+		log.Info("all jobs finished")
+	case <-ctx.Done():
+		log.WithError(ctx.Err()).Info("context finished")
+	}
+	log.Info("waiting for jobs to finish")
+	<-jobs.wait()
+	log.Info("daemon exiting")
+	return nil
+}
+
+type jobs struct {
+	wg sync.WaitGroup
+
+	// m protects all fields below it
+	m       sync.RWMutex
+	wakeups map[string]wakeup.Func // by Job.Name
+	resets  map[string]reset.Func  // by Job.Name
+	jobs    map[string]job.Job
+}
+
+func newJobs() *jobs {
+	return &jobs{
+		wakeups: make(map[string]wakeup.Func),
+		resets:  make(map[string]reset.Func),
+		jobs:    make(map[string]job.Job),
+	}
+}
+
+func (s *jobs) wait() <-chan struct{} {
+	ch := make(chan struct{})
+	go func() {
+		s.wg.Wait()
+		close(ch)
+	}()
+	return ch
+}
+
+type Status struct {
+	Jobs   map[string]*job.Status
+	Global GlobalStatus
+}
+
+type GlobalStatus struct {
+	ZFSCmds   *zfscmd.Report
+	Envconst  *envconst.Report
+	OsEnviron []string
+}
+
+func (s *jobs) status() map[string]*job.Status {
+	s.m.RLock()
+	defer s.m.RUnlock()
+
+	type res struct {
+		name   string
+		status *job.Status
+	}
+	var wg sync.WaitGroup
+	c := make(chan res, len(s.jobs))
+	for name, j := range s.jobs {
+		wg.Add(1)
+		go func(name string, j job.Job) {
+			defer wg.Done()
+			c <- res{name: name, status: j.Status()}
+		}(name, j)
+	}
+	wg.Wait()
+	close(c)
+	ret := make(map[string]*job.Status, len(s.jobs))
+	for res := range c {
+		ret[res.name] = res.status
+	}
+	return ret
+}
+
+func (s *jobs) wakeup(job string) error {
+	s.m.RLock()
+	defer s.m.RUnlock()
+
+	wu, ok := s.wakeups[job]
+	if !ok {
+		return errors.Errorf("Job %s does not exist", job)
+	}
+	return wu()
+}
+
+func (s *jobs) reset(job string) error {
+	s.m.RLock()
+	defer s.m.RUnlock()
+
+	wu, ok := s.resets[job]
+	if !ok {
+		return errors.Errorf("Job %s does not exist", job)
+	}
+	return wu()
+}
+
+const (
+	jobNamePrometheus = "_prometheus"
+	jobNameControl    = "_control"
+)
+
+func IsInternalJobName(s string) bool {
+	return strings.HasPrefix(s, "_")
+}
+
+func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
+	s.m.Lock()
+	defer s.m.Unlock()
+
+	ctx = logging.WithInjectedField(ctx, logging.JobField, j.Name())
+
+	jobName := j.Name()
+	if !internal && IsInternalJobName(jobName) {
+		panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName))
+	}
+	if internal && !IsInternalJobName(jobName) {
+		panic(fmt.Sprintf("internal job does not use internal job name %s", jobName))
+	}
+	if _, ok := s.jobs[jobName]; ok {
+		panic(fmt.Sprintf("duplicate job name %s", jobName))
+	}
+
+	j.RegisterMetrics(prometheus.DefaultRegisterer)
+
+	s.jobs[jobName] = j
+	ctx = zfscmd.WithJobID(ctx, j.Name())
+	ctx, wakeup := wakeup.Context(ctx)
+	ctx, resetFunc := reset.Context(ctx)
+	s.wakeups[jobName] = wakeup
+	s.resets[jobName] = resetFunc
+
+	s.wg.Add(1)
+	go func() {
+		defer s.wg.Done()
+		job.GetLogger(ctx).Info("starting job")
+		defer job.GetLogger(ctx).Info("job exited")
+		j.Run(ctx)
+	}()
+}

internal/daemon/filters/fsmapfilter.go

@@ -0,0 +1,283 @@
+package filters
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/internal/endpoint"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+type DatasetMapFilter struct {
+	entries []datasetMapFilterEntry
+
+	// if set, only valid filter entries can be added using Add()
+	// and Map() will always return an error
+	filterMode bool
+}
+
+type datasetMapFilterEntry struct {
+	path *zfs.DatasetPath
+	// the mapping. since this datastructure acts as both mapping and filter
+	// we have to convert it to the desired rep dynamically
+	mapping      string
+	subtreeMatch bool
+}
+
+func NewDatasetMapFilter(capacity int, filterMode bool) *DatasetMapFilter {
+	return &DatasetMapFilter{
+		entries:    make([]datasetMapFilterEntry, 0, capacity),
+		filterMode: filterMode,
+	}
+}
+
+func (m *DatasetMapFilter) Add(pathPattern, mapping string) (err error) {
+
+	if m.filterMode {
+		if _, err = m.parseDatasetFilterResult(mapping); err != nil {
+			return
+		}
+	}
+
+	// assert path glob adheres to spec
+	const SUBTREE_PATTERN string = "<"
+	patternCount := strings.Count(pathPattern, SUBTREE_PATTERN)
+	switch {
+	case patternCount > 1:
+	case patternCount == 1 && !strings.HasSuffix(pathPattern, SUBTREE_PATTERN):
+		err = fmt.Errorf("pattern invalid: only one '<' at end of string allowed")
+		return
+	}
+
+	pathStr := strings.TrimSuffix(pathPattern, SUBTREE_PATTERN)
+	path, err := zfs.NewDatasetPath(pathStr)
+	if err != nil {
+		return fmt.Errorf("pattern is not a dataset path: %s", err)
+	}
+
+	entry := datasetMapFilterEntry{
+		path:         path,
+		mapping:      mapping,
+		subtreeMatch: patternCount > 0,
+	}
+	m.entries = append(m.entries, entry)
+	return
+
+}
+
+// find the most specific prefix mapping we have
+//
+// longer prefix wins over shorter prefix, direct wins over glob
+func (m DatasetMapFilter) mostSpecificPrefixMapping(path *zfs.DatasetPath) (idx int, found bool) {
+	lcp, lcp_entry_idx := -1, -1
+	direct_idx := -1
+	for e := range m.entries {
+		entry := m.entries[e]
+		ep := m.entries[e].path
+		lep := ep.Length()
+
+		switch {
+		case !entry.subtreeMatch && ep.Equal(path):
+			direct_idx = e
+			continue
+		case entry.subtreeMatch && path.HasPrefix(ep) && lep > lcp:
+			lcp = lep
+			lcp_entry_idx = e
+		default:
+			continue
+		}
+	}
+
+	if lcp_entry_idx >= 0 || direct_idx >= 0 {
+		found = true
+		switch {
+		case direct_idx >= 0:
+			idx = direct_idx
+		case lcp_entry_idx >= 0:
+			idx = lcp_entry_idx
+		}
+	}
+	return
+}
+
+// Returns target == nil if there is no mapping
+func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error) {
+
+	if m.filterMode {
+		err = fmt.Errorf("using a filter for mapping simply does not work")
+		return
+	}
+
+	mi, hasMapping := m.mostSpecificPrefixMapping(source)
+	if !hasMapping {
+		return nil, nil
+	}
+	me := m.entries[mi]
+
+	if me.mapping == "" {
+		// Special case treatment: 'foo/bar<' => ''
+		if !me.subtreeMatch {
+			return nil, fmt.Errorf("mapping to '' must be a subtree match")
+		}
+		// ok...
+	} else {
+		if strings.HasPrefix("!", me.mapping) {
+			// reject mapping
+			return nil, nil
+		}
+	}
+
+	target, err = zfs.NewDatasetPath(me.mapping)
+	if err != nil {
+		err = fmt.Errorf("mapping target is not a dataset path: %s", err)
+		return
+	}
+	if me.subtreeMatch {
+		// strip common prefix ('<' wildcards are no special case here)
+		extendComps := source.Copy()
+		extendComps.TrimPrefix(me.path)
+		target.Extend(extendComps)
+	}
+	return
+}
+
+func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
+
+	if !m.filterMode {
+		err = fmt.Errorf("using a mapping as a filter does not work")
+		return
+	}
+
+	mi, hasMapping := m.mostSpecificPrefixMapping(p)
+	if !hasMapping {
+		pass = false
+		return
+	}
+	me := m.entries[mi]
+	pass, err = m.parseDatasetFilterResult(me.mapping)
+	return
+}
+
+func (m DatasetMapFilter) UserSpecifiedDatasets() (datasets zfs.UserSpecifiedDatasetsSet) {
+	datasets = make(zfs.UserSpecifiedDatasetsSet)
+	for i := range m.entries {
+		datasets[m.entries[i].path.ToString()] = true
+	}
+	return
+}
+
+// Construct a new filter-only DatasetMapFilter from a mapping
+// The new filter allows exactly those paths that were not forbidden by the mapping.
+func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
+
+	if m.filterMode {
+		err = errors.Errorf("can only invert mappings")
+		return
+	}
+
+	inv = &DatasetMapFilter{
+		make([]datasetMapFilterEntry, len(m.entries)),
+		true,
+	}
+
+	for i, e := range m.entries {
+		inv.entries[i].path, err = zfs.NewDatasetPath(e.mapping)
+		if err != nil {
+			err = errors.Wrapf(err, "mapping cannot be inverted: '%s' is not a dataset path", e.mapping)
+			return
+		}
+		inv.entries[i].mapping = MapFilterResultOk
+		inv.entries[i].subtreeMatch = e.subtreeMatch
+	}
+
+	return inv, nil
+}
+
+// FIXME investigate whether we can support more...
+func (m DatasetMapFilter) Invert() (endpoint.FSMap, error) {
+
+	if m.filterMode {
+		return nil, errors.Errorf("can only invert mappings")
+	}
+
+	if len(m.entries) != 1 {
+		return nil, errors.Errorf("inversion of complicated mappings is not implemented") // FIXME
+	}
+
+	e := m.entries[0]
+
+	inv := &DatasetMapFilter{
+		make([]datasetMapFilterEntry, len(m.entries)),
+		false,
+	}
+	mp, err := zfs.NewDatasetPath(e.mapping)
+	if err != nil {
+		return nil, err
+	}
+
+	inv.entries[0] = datasetMapFilterEntry{
+		path:         mp,
+		mapping:      e.path.ToString(),
+		subtreeMatch: e.subtreeMatch,
+	}
+
+	return inv, nil
+}
+
+// Creates a new DatasetMapFilter in filter mode from a mapping
+// All accepting mapping results are mapped to accepting filter results
+// All rejecting mapping results are mapped to rejecting filter results
+func (m DatasetMapFilter) AsFilter() endpoint.FSFilter {
+
+	f := &DatasetMapFilter{
+		make([]datasetMapFilterEntry, len(m.entries)),
+		true,
+	}
+
+	for i, e := range m.entries {
+		var newe datasetMapFilterEntry = e
+		if strings.HasPrefix(newe.mapping, "!") {
+			newe.mapping = MapFilterResultOmit
+		} else {
+			newe.mapping = MapFilterResultOk
+		}
+		f.entries[i] = newe
+	}
+
+	return f
+}
+
+const (
+	MapFilterResultOk   string = "ok"
+	MapFilterResultOmit string = "!"
+)
+
+// Parse a dataset filter result
+func (m DatasetMapFilter) parseDatasetFilterResult(result string) (pass bool, err error) {
+	l := strings.ToLower(result)
+	if l == MapFilterResultOk {
+		return true, nil
+	}
+	if l == MapFilterResultOmit {
+		return false, nil
+	}
+	return false, fmt.Errorf("'%s' is not a valid filter result", result)
+}
+
+func DatasetMapFilterFromConfig(in map[string]bool) (f *DatasetMapFilter, err error) {
+
+	f = NewDatasetMapFilter(len(in), true)
+	for pathPattern, accept := range in {
+		mapping := MapFilterResultOmit
+		if accept {
+			mapping = MapFilterResultOk
+		}
+		if err = f.Add(pathPattern, mapping); err != nil {
+			err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err)
+			return
+		}
+	}
+	return
+}

internal/daemon/filters/fsmapfilter_test.go

@@ -0,0 +1,89 @@
+package filters
+
+import (
+	"testing"
+
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+func TestDatasetMapFilter(t *testing.T) {
+
+	type testCase struct {
+		name      string
+		filter    map[string]string
+		checkPass map[string]bool // each entry is checked to match the filter's `pass` return value
+	}
+
+	tcs := []testCase{
+		{
+			"default_no_match",
+			map[string]string{},
+			map[string]bool{
+				"":      false,
+				"foo":   false,
+				"zroot": false,
+			},
+		},
+		{
+			"more_specific_path_has_precedence",
+			map[string]string{
+				"tank<":         "ok",
+				"tank/tmp<":     "!",
+				"tank/home/x<":  "!",
+				"tank/home/x/1": "ok",
+			},
+			map[string]bool{
+				"zroot":         false,
+				"tank":          true,
+				"tank/tmp":      false,
+				"tank/tmp/foo":  false,
+				"tank/home/x":   false,
+				"tank/home/y":   true,
+				"tank/home/x/1": true,
+				"tank/home/x/2": false,
+			},
+		},
+		{
+			"precedence_of_specific_over_subtree_wildcard_on_same_path",
+			map[string]string{
+				"tank/home/bob":  "ok",
+				"tank/home/bob<": "!",
+			},
+			map[string]bool{
+				"tank/home/bob":           true,
+				"tank/home/bob/downloads": false,
+			},
+		},
+	}
+
+	for tc := range tcs {
+		t.Run(tcs[tc].name, func(t *testing.T) {
+			c := tcs[tc]
+			f := NewDatasetMapFilter(len(c.filter), true)
+			for p, a := range c.filter {
+				err := f.Add(p, a)
+				if err != nil {
+					t.Fatalf("incorrect filter spec: %s", err)
+				}
+			}
+			for p, checkPass := range c.checkPass {
+				zp, err := zfs.NewDatasetPath(p)
+				if err != nil {
+					t.Fatalf("incorrect path spec: %s", err)
+				}
+				pass, err := f.Filter(zp)
+				if err != nil {
+					t.Fatalf("unexpected filter error: %s", err)
+				}
+				ok := pass == checkPass
+				failstr := "OK"
+				if !ok {
+					failstr = "FAIL"
+					t.Fail()
+				}
+				t.Logf("%-40q  %5v  (exp=%v act=%v)", p, failstr, checkPass, pass)
+			}
+		})
+	}
+
+}

internal/daemon/hooks/edge_string.go

@@ -0,0 +1,35 @@
+// Code generated by "stringer -type=Edge"; DO NOT EDIT.
+
+package hooks
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[Pre-1]
+	_ = x[Callback-2]
+	_ = x[Post-4]
+}
+
+const (
+	_Edge_name_0 = "PreCallback"
+	_Edge_name_1 = "Post"
+)
+
+var (
+	_Edge_index_0 = [...]uint8{0, 3, 11}
+)
+
+func (i Edge) String() string {
+	switch {
+	case 1 <= i && i <= 2:
+		i -= 1
+		return _Edge_name_0[_Edge_index_0[i]:_Edge_index_0[i+1]]
+	case i == 4:
+		return _Edge_name_1
+	default:
+		return "Edge(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+}

internal/daemon/hooks/hook_config.go

@@ -0,0 +1,52 @@
+package hooks
+
+import (
+	"fmt"
+
+	"github.com/zrepl/zrepl/internal/config"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+type List []Hook
+
+func HookFromConfig(in config.HookEnum) (Hook, error) {
+	switch v := in.Ret.(type) {
+	case *config.HookCommand:
+		return NewCommandHook(v)
+	case *config.HookPostgresCheckpoint:
+		return PgChkptHookFromConfig(v)
+	case *config.HookMySQLLockTables:
+		return MyLockTablesFromConfig(v)
+	default:
+		return nil, fmt.Errorf("unknown hook type %T", v)
+	}
+}
+
+func ListFromConfig(in *config.HookList) (r *List, err error) {
+	hl := make(List, len(*in))
+
+	for i, h := range *in {
+		hl[i], err = HookFromConfig(h)
+		if err != nil {
+			return nil, fmt.Errorf("create hook #%d: %s", i+1, err)
+		}
+	}
+
+	return &hl, nil
+}
+
+func (l List) CopyFilteredForFilesystem(fs *zfs.DatasetPath) (ret List, err error) {
+	ret = make(List, 0, len(l))
+
+	for _, h := range l {
+		var passFilesystem bool
+		if passFilesystem, err = h.Filesystems().Filter(fs); err != nil {
+			return nil, err
+		}
+		if passFilesystem {
+			ret = append(ret, h)
+		}
+	}
+
+	return ret, nil
+}

internal/daemon/hooks/hook_docs.go

@@ -0,0 +1,33 @@
+// Package hooks implements pre- and post snapshot hooks.
+//
+// Plan is a generic executor for ExpectStepReports before and after an activity specified in a callback.
+// It provides a reporting facility that can be polled while the plan is executing to gather progress information.
+//
+// This package also provides all supported hook type implementations and abstractions around them.
+//
+// # Use For Other Kinds Of ExpectStepReports
+//
+// This package REQUIRES REFACTORING before it can be used for other activities than snapshots, e.g. pre- and post-replication:
+//
+// The Hook interface requires a hook to provide a Filesystems() filter, which doesn't make sense for
+// all kinds of activities.
+//
+// The hook implementations should move out of this package.
+// However, there is a lot of tight coupling which to untangle isn't worth it ATM.
+//
+// # How This Package Is Used By Package Snapper
+//
+// Deserialize a config.List using ListFromConfig().
+// Then it MUST filter the list to only contain hooks for a particular filesystem using
+// hooksList.CopyFilteredForFilesystem(fs).
+//
+// Then create a CallbackHook using NewCallbackHookForFilesystem().
+//
+// Pass all of the above to NewPlan() which provides a Report() and Run() method:
+//
+// Plan.Run(ctx context.Context,dryRun bool) executes the plan and take a context as argument that should contain a logger added using hooks.WithLogger()).
+// The value of dryRun is passed through to the hooks' Run() method.
+// Command hooks make it available in the environment variable ZREPL_DRYRUN.
+//
+// Plan.Report() can be called while Plan.Run() is executing to give an overview of plan execution progress (future use in "zrepl status").
+package hooks

internal/daemon/hooks/hook_exec.go

@@ -0,0 +1,292 @@
+package hooks
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+// Re-export type here so that
+// every file in package hooks doesn't
+// have to import github.com/zrepl/zrepl/zfs
+type Filter zfs.DatasetFilter
+
+type Hook interface {
+	Filesystems() Filter
+
+	// If true and the Pre edge invocation of Run fails, Post edge will not run and other Pre edges will not run.
+	ErrIsFatal() bool
+
+	// Run is invoked by HookPlan for a Pre edge.
+	// If HookReport.HadError() == false, the Post edge will be invoked, too.
+	Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport
+
+	String() string
+}
+
+type Phase string
+
+const (
+	PhaseSnapshot = Phase("snapshot")
+	PhaseTesting  = Phase("testing")
+)
+
+func (p Phase) String() string {
+	return string(p)
+}
+
+//go:generate stringer -type=Edge
+type Edge uint
+
+const (
+	Pre = Edge(1 << iota)
+	Callback
+	Post
+)
+
+func (e Edge) StringForPhase(phase Phase) string {
+	return fmt.Sprintf("%s_%s", e.String(), phase.String())
+}
+
+//go:generate enumer -type=StepStatus -trimprefix=Step
+type StepStatus int
+
+const (
+	StepPending StepStatus = 1 << iota
+	StepExec
+	StepOk
+	StepErr
+	StepSkippedDueToFatalErr
+	StepSkippedDueToPreErr
+)
+
+type HookReport interface {
+	String() string
+	HadError() bool
+	Error() string
+}
+
+type Step struct {
+	Hook       Hook
+	Edge       Edge
+	Status     StepStatus
+	Begin, End time.Time
+	// Report may be nil
+	// FIXME cannot serialize this for client status, but contains interesting info (like what error happened)
+	Report HookReport
+	state  map[interface{}]interface{}
+}
+
+func (s Step) String() (out string) {
+	fatal := "~"
+	if s.Hook.ErrIsFatal() && s.Edge == Pre {
+		fatal = "!"
+	}
+	runTime := "..."
+	if s.Status != StepPending {
+		t := s.End.Sub(s.Begin)
+		runTime = t.Round(time.Millisecond).String()
+	}
+	return fmt.Sprintf("[%s] [%5s] %s [%s]  %s", s.Status, runTime, fatal, s.Edge, s.Hook)
+}
+
+type Plan struct {
+	mtx sync.RWMutex
+
+	steps []*Step
+	pre   []*Step // protected by mtx
+	cb    *Step
+	post  []*Step // not reversed, i.e. entry at index i corresponds to pre-edge in pre[i]
+
+	phase Phase
+	env   Env
+}
+
+func NewPlan(hooks *List, phase Phase, cb *CallbackHook, extra Env) (*Plan, error) {
+
+	var pre, post []*Step
+	// TODO sanity check unique name of hook?
+	for _, hook := range *hooks {
+		state := make(map[interface{}]interface{})
+		preE := &Step{
+			Hook:   hook,
+			Edge:   Pre,
+			Status: StepPending,
+			state:  state,
+		}
+		pre = append(pre, preE)
+		postE := &Step{
+			Hook:   hook,
+			Edge:   Post,
+			Status: StepPending,
+			state:  state,
+		}
+		post = append(post, postE)
+	}
+
+	cbE := &Step{
+		Hook:   cb,
+		Edge:   Callback,
+		Status: StepPending,
+	}
+
+	steps := make([]*Step, 0, len(pre)+len(post)+1)
+	steps = append(steps, pre...)
+	steps = append(steps, cbE)
+	for i := len(post) - 1; i >= 0; i-- {
+		steps = append(steps, post[i])
+	}
+
+	plan := &Plan{
+		phase: phase,
+		env:   extra,
+		steps: steps,
+		pre:   pre,
+		post:  post,
+		cb:    cbE,
+	}
+
+	return plan, nil
+}
+
+type PlanReport []Step
+
+func (p *Plan) Report() PlanReport {
+	p.mtx.RLock()
+	defer p.mtx.RUnlock()
+	rep := make([]Step, len(p.steps))
+	for i := range rep {
+		rep[i] = *p.steps[i]
+	}
+	return rep
+}
+
+func (r PlanReport) HadError() bool {
+	for _, e := range r {
+		if e.Status == StepErr {
+			return true
+		}
+	}
+	return false
+}
+
+func (r PlanReport) HadFatalError() bool {
+	for _, e := range r {
+		if e.Status == StepSkippedDueToFatalErr {
+			return true
+		}
+	}
+	return false
+}
+
+func (r PlanReport) String() string {
+	stepStrings := make([]string, len(r))
+	for i, e := range r {
+		stepStrings[i] = fmt.Sprintf("%02d %s", i+1, e)
+	}
+	return strings.Join(stepStrings, "\n")
+}
+
+func (p *Plan) Run(ctx context.Context, dryRun bool) {
+	p.mtx.RLock()
+	defer p.mtx.RUnlock()
+	w := func(f func()) {
+		p.mtx.RUnlock()
+		defer p.mtx.RLock()
+		p.mtx.Lock()
+		defer p.mtx.Unlock()
+		f()
+	}
+	runHook := func(s *Step, ctx context.Context, edge Edge) HookReport {
+		w(func() { s.Status = StepExec })
+		begin := time.Now()
+		r := s.Hook.Run(ctx, edge, p.phase, dryRun, p.env, s.state)
+		end := time.Now()
+		w(func() {
+			s.Report = r
+			s.Status = StepOk
+			if r.HadError() {
+				s.Status = StepErr
+			}
+			s.Begin, s.End = begin, end
+		})
+		return r
+	}
+
+	l := getLogger(ctx)
+
+	// it's a stack, execute until we reach the end of the list (last item in)
+	// or fail inbetween
+	l.Info("run pre-edges in configuration order")
+	next := 0
+	for ; next < len(p.pre); next++ {
+		e := p.pre[next]
+		l := l.WithField("hook", e.Hook)
+		r := runHook(e, ctx, Pre)
+		if r.HadError() {
+			l.WithError(r).Error("hook invocation failed for pre-edge")
+			if e.Hook.ErrIsFatal() {
+				l.Error("the hook run was aborted due to a fatal error in this hook")
+				break
+			}
+		}
+	}
+
+	hadFatalErr := next != len(p.pre)
+	if hadFatalErr {
+		l.Error("fatal error in a pre-snapshot hook invocation")
+		l.Error("no snapshot will be taken")
+		l.Error("only running post-edges for successful pre-edges")
+		w(func() {
+			p.post[next].Status = StepSkippedDueToFatalErr
+			for i := next + 1; i < len(p.pre); i++ {
+				p.pre[i].Status = StepSkippedDueToFatalErr
+				p.post[i].Status = StepSkippedDueToFatalErr
+			}
+			p.cb.Status = StepSkippedDueToFatalErr
+		})
+		return
+	}
+
+	l.Info("running callback")
+	cbR := runHook(p.cb, ctx, Callback)
+	if cbR.HadError() {
+		l.WithError(cbR).Error("callback failed")
+	}
+
+	l.Info("run post-edges for successful pre-edges in reverse configuration order")
+
+	// the constructor produces pre and post entries
+	// post is NOT reversed
+	next-- // now at index of last executed pre-edge
+	for ; next >= 0; next-- {
+		e := p.post[next]
+		l := l.WithField("hook", e.Hook)
+
+		if p.pre[next].Status != StepOk {
+			if p.pre[next].Status != StepErr {
+				panic(fmt.Sprintf("expecting a pre-edge hook report to be either Ok or Err, got %s", p.pre[next].Status))
+			}
+			l.Info("skip post-edge because pre-edge failed")
+			w(func() {
+				e.Status = StepSkippedDueToPreErr
+			})
+			continue
+		}
+
+		report := runHook(e, ctx, Post)
+
+		if report.HadError() {
+			l.WithError(report).Error("hook invocation failed for post-edge")
+			l.Error("subsequent post-edges run regardless of this post-edge failure")
+		}
+
+		// ErrIsFatal is only relevant for Pre
+
+	}
+
+}

internal/daemon/hooks/hook_logging.go

@@ -0,0 +1,91 @@
+package hooks
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"sync"
+
+	"github.com/zrepl/zrepl/internal/daemon/logging"
+	"github.com/zrepl/zrepl/internal/logger"
+	"github.com/zrepl/zrepl/internal/util/envconst"
+)
+
+type Logger = logger.Logger
+
+func GetLogger(ctx context.Context) Logger { return getLogger(ctx) }
+
+func getLogger(ctx context.Context) Logger {
+	return logging.GetLogger(ctx, logging.SubsysHooks)
+}
+
+const MAX_HOOK_LOG_SIZE_DEFAULT int = 1 << 20
+
+type logWriter struct {
+	/*
+		Mutex prevents:
+			concurrent writes to buf, scanner in Write([]byte)
+			data race on scanner vs Write([]byte)
+				and concurrent write to buf (call to buf.Reset())
+				in Close()
+
+		(Also, Close() should generally block until any Write() call completes.)
+	*/
+	mtx     *sync.Mutex
+	buf     bytes.Buffer
+	scanner *bufio.Scanner
+	logger  Logger
+	level   logger.Level
+	field   string
+}
+
+func NewLogWriter(mtx *sync.Mutex, logger Logger, level logger.Level, field string) *logWriter {
+	w := new(logWriter)
+	w.mtx = mtx
+	w.scanner = bufio.NewScanner(&w.buf)
+	w.logger = logger
+	w.level = level
+	w.field = field
+	return w
+}
+
+func (w *logWriter) log(line string) {
+	w.logger.WithField(w.field, line).Log(w.level, "hook output")
+}
+
+func (w *logWriter) logUnreadBytes() error {
+	for w.scanner.Scan() {
+		w.log(w.scanner.Text())
+	}
+	if w.buf.Cap() > envconst.Int("ZREPL_MAX_HOOK_LOG_SIZE", MAX_HOOK_LOG_SIZE_DEFAULT) {
+		w.buf.Reset()
+	}
+
+	return nil
+}
+
+func (w *logWriter) Write(in []byte) (int, error) {
+	w.mtx.Lock()
+	defer w.mtx.Unlock()
+
+	n, bufErr := w.buf.Write(in)
+	if bufErr != nil {
+		return n, bufErr
+	}
+
+	err := w.logUnreadBytes()
+	if err != nil {
+		return n, err
+	}
+	// Always reset the scanner for the next Write
+	w.scanner = bufio.NewScanner(&w.buf)
+
+	return n, nil
+}
+
+func (w *logWriter) Close() (err error) {
+	w.mtx.Lock()
+	defer w.mtx.Unlock()
+
+	return w.logUnreadBytes()
+}

internal/daemon/hooks/hook_type_callback.go

@@ -0,0 +1,65 @@
+package hooks
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/zrepl/zrepl/internal/daemon/filters"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+type HookJobCallback func(ctx context.Context) error
+
+type CallbackHook struct {
+	cb            HookJobCallback
+	filter        Filter
+	displayString string
+}
+
+func NewCallbackHookForFilesystem(displayString string, fs *zfs.DatasetPath, cb HookJobCallback) *CallbackHook {
+	filter, _ := filters.DatasetMapFilterFromConfig(map[string]bool{fs.ToString(): true})
+	return NewCallbackHook(displayString, cb, filter)
+}
+
+func NewCallbackHook(displayString string, cb HookJobCallback, filter Filter) *CallbackHook {
+	return &CallbackHook{
+		cb:            cb,
+		filter:        filter,
+		displayString: displayString,
+	}
+}
+
+func (h *CallbackHook) Filesystems() Filter {
+	return h.filter
+}
+
+func (h *CallbackHook) ErrIsFatal() bool {
+	return false // callback is by definition
+}
+
+func (h *CallbackHook) String() string {
+	return h.displayString
+}
+
+type CallbackHookReport struct {
+	Name string
+	Err  error
+}
+
+func (r *CallbackHookReport) String() string {
+	if r.HadError() {
+		return r.Error()
+	}
+	return r.Name
+}
+
+func (r *CallbackHookReport) HadError() bool { return r.Err != nil }
+
+func (r *CallbackHookReport) Error() string {
+	return fmt.Sprintf("%s error: %s", r.Name, r.Err)
+}
+
+func (h *CallbackHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
+	err := h.cb(ctx)
+	return &CallbackHookReport{h.displayString, err}
+}

internal/daemon/hooks/hook_type_command.go

@@ -0,0 +1,199 @@
+package hooks
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"os"
+	"os/exec"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/zrepl/zrepl/internal/config"
+	"github.com/zrepl/zrepl/internal/daemon/filters"
+	"github.com/zrepl/zrepl/internal/logger"
+	"github.com/zrepl/zrepl/internal/util/circlog"
+	"github.com/zrepl/zrepl/internal/util/envconst"
+)
+
+type HookEnvVar string
+
+const (
+	EnvType     HookEnvVar = "ZREPL_HOOKTYPE"
+	EnvDryRun   HookEnvVar = "ZREPL_DRYRUN"
+	EnvFS       HookEnvVar = "ZREPL_FS"
+	EnvSnapshot HookEnvVar = "ZREPL_SNAPNAME"
+	EnvTimeout  HookEnvVar = "ZREPL_TIMEOUT"
+)
+
+type Env map[HookEnvVar]string
+
+func NewHookEnv(edge Edge, phase Phase, dryRun bool, timeout time.Duration, extra Env) Env {
+	r := Env{
+		EnvTimeout: fmt.Sprintf("%.f", math.Floor(timeout.Seconds())),
+	}
+
+	edgeString := edge.StringForPhase(phase)
+	r[EnvType] = strings.ToLower(edgeString)
+
+	var dryRunString string
+	if dryRun {
+		dryRunString = "true"
+	} else {
+		dryRunString = ""
+	}
+	r[EnvDryRun] = dryRunString
+
+	for k, v := range extra {
+		r[k] = v
+	}
+
+	return r
+}
+
+type CommandHook struct {
+	edge       Edge
+	filter     Filter
+	errIsFatal bool
+	command    string
+	timeout    time.Duration
+}
+
+type CommandHookReport struct {
+	Command                      string
+	Args                         []string // currently always empty
+	Env                          Env
+	Err                          error
+	CapturedStdoutStderrCombined []byte
+}
+
+func (r *CommandHookReport) String() string {
+	// Reproduces a POSIX shell-compatible command line
+	var cmdLine strings.Builder
+	sep := ""
+
+	// Make sure environment variables are always
+	// printed in the same order
+	var hookEnvKeys []HookEnvVar
+	for k := range r.Env {
+		hookEnvKeys = append(hookEnvKeys, k)
+	}
+	sort.Slice(hookEnvKeys, func(i, j int) bool { return string(hookEnvKeys[i]) < string(hookEnvKeys[j]) })
+
+	for _, k := range hookEnvKeys {
+		cmdLine.WriteString(fmt.Sprintf("%s%s='%s'", sep, k, r.Env[k]))
+		sep = " "
+	}
+
+	cmdLine.WriteString(fmt.Sprintf("%s%s", sep, r.Command))
+	for _, a := range r.Args {
+		cmdLine.WriteString(fmt.Sprintf("%s'%s'", sep, a))
+	}
+
+	var msg string
+	if r.Err == nil {
+		msg = "command hook"
+	} else {
+		msg = fmt.Sprintf("command hook failed with %q", r.Err)
+	}
+
+	return fmt.Sprintf("%s: \"%s\"", msg, cmdLine.String()) // no %q to make copy-pastable
+}
+func (r *CommandHookReport) Error() string {
+	if r.Err == nil {
+		return ""
+	}
+	return fmt.Sprintf("%s FAILED with error: %s", r.String(), r.Err)
+}
+
+func (r *CommandHookReport) HadError() bool {
+	return r.Err != nil
+}
+
+func NewCommandHook(in *config.HookCommand) (r *CommandHook, err error) {
+	r = &CommandHook{
+		errIsFatal: in.ErrIsFatal,
+		command:    in.Path,
+		timeout:    in.Timeout,
+	}
+
+	r.filter, err = filters.DatasetMapFilterFromConfig(in.Filesystems)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse filesystem filter: %s", err)
+	}
+
+	r.edge = Pre | Post
+
+	return r, nil
+}
+
+func (h *CommandHook) Filesystems() Filter {
+	return h.filter
+}
+
+func (h *CommandHook) ErrIsFatal() bool {
+	return h.errIsFatal
+}
+
+func (h *CommandHook) String() string {
+	return h.command
+}
+
+func (h *CommandHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
+	l := getLogger(ctx).WithField("command", h.command)
+
+	cmdCtx, cancel := context.WithTimeout(ctx, h.timeout)
+	defer cancel()
+
+	cmdExec := exec.CommandContext(cmdCtx, h.command)
+
+	hookEnv := NewHookEnv(edge, phase, dryRun, h.timeout, extra)
+	cmdEnv := os.Environ()
+	for k, v := range hookEnv {
+		cmdEnv = append(cmdEnv, fmt.Sprintf("%s=%s", k, v))
+	}
+	cmdExec.Env = cmdEnv
+
+	var scanMutex sync.Mutex
+	combinedOutput, err := circlog.NewCircularLog(envconst.Int("ZREPL_MAX_HOOK_LOG_SIZE", MAX_HOOK_LOG_SIZE_DEFAULT))
+	if err != nil {
+		return &CommandHookReport{Err: err}
+	}
+	logErrWriter := NewLogWriter(&scanMutex, l, logger.Warn, "stderr")
+	logOutWriter := NewLogWriter(&scanMutex, l, logger.Info, "stdout")
+	defer logErrWriter.Close()
+	defer logOutWriter.Close()
+
+	cmdExec.Stderr = io.MultiWriter(logErrWriter, combinedOutput)
+	cmdExec.Stdout = io.MultiWriter(logOutWriter, combinedOutput)
+
+	report := &CommandHookReport{
+		Command: h.command,
+		Env:     hookEnv,
+		// no report.Args
+	}
+
+	err = cmdExec.Start()
+	if err != nil {
+		report.Err = err
+		return report
+	}
+
+	err = cmdExec.Wait()
+	combinedOutputBytes := combinedOutput.Bytes()
+	report.CapturedStdoutStderrCombined = make([]byte, len(combinedOutputBytes))
+	copy(report.CapturedStdoutStderrCombined, combinedOutputBytes)
+	if err != nil {
+		if cmdCtx.Err() == context.DeadlineExceeded {
+			report.Err = fmt.Errorf("timed out after %s: %s", h.timeout, err)
+			return report
+		}
+		report.Err = err
+		return report
+	}
+
+	return report
+}

internal/daemon/hooks/hook_type_mysql_lock_tables.go

@@ -0,0 +1,148 @@
+package hooks
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strings"
+
+	sqldriver "database/sql/driver"
+
+	"github.com/go-sql-driver/mysql"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/internal/config"
+	"github.com/zrepl/zrepl/internal/daemon/filters"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+// Hook to implement the following recommmendation from MySQL docs
+// https://dev.mysql.com/doc/mysql-backup-excerpt/5.7/en/backup-methods.html
+//
+//	Making Backups Using a File System Snapshot:
+//
+//	If you are using a Veritas file system, you can make a backup like this:
+//
+//	   	From a client program, execute FLUSH TABLES WITH READ LOCK.
+//	   	From another shell, execute mount vxfs snapshot.
+//	   	From the first client, execute UNLOCK TABLES.
+//	   	Copy files from the snapshot.
+//	   	Unmount the snapshot.
+//
+//		Similar snapshot capabilities may be available in other file systems, such as LVM or ZFS.
+//
+
+type MySQLLockTables struct {
+	errIsFatal  bool
+	connector   sqldriver.Connector
+	filesystems Filter
+}
+
+type myLockTablesStateKey int
+
+const (
+	myLockTablesConnection myLockTablesStateKey = 1 + iota
+)
+
+func MyLockTablesFromConfig(in *config.HookMySQLLockTables) (*MySQLLockTables, error) {
+	conf, err := mysql.ParseDSN(in.DSN)
+	if err != nil {
+		return nil, errors.Wrap(err, "`dsn` invalid")
+	}
+	cn, err := mysql.NewConnector(conf)
+	if err != nil {
+		return nil, errors.Wrap(err, "`connect` invalid")
+	}
+
+	filesystems, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
+	if err != nil {
+		return nil, errors.Wrap(err, "`filesystems` invalid")
+	}
+
+	return &MySQLLockTables{
+		in.ErrIsFatal,
+		cn,
+		filesystems,
+	}, nil
+}
+
+func (h *MySQLLockTables) ErrIsFatal() bool    { return h.errIsFatal }
+func (h *MySQLLockTables) Filesystems() Filter { return h.filesystems }
+func (h *MySQLLockTables) String() string      { return "MySQL FLUSH TABLES WITH READ LOCK" }
+
+type MyLockTablesReport struct {
+	What string
+	Err  error
+}
+
+func (r *MyLockTablesReport) HadError() bool { return r.Err != nil }
+func (r *MyLockTablesReport) Error() string  { return r.String() }
+func (r *MyLockTablesReport) String() string {
+	var s strings.Builder
+	s.WriteString(r.What)
+	if r.Err != nil {
+		fmt.Fprintf(&s, ": %s", r.Err)
+	}
+	return s.String()
+}
+
+func (h *MySQLLockTables) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
+	fs, ok := extra[EnvFS]
+	if !ok {
+		panic(extra)
+	}
+	dp, err := zfs.NewDatasetPath(fs)
+	if err != nil {
+		panic(err)
+	}
+	if pass, err := h.filesystems.Filter(dp); err != nil {
+		return &MyLockTablesReport{What: "filesystem filter", Err: err}
+	} else if !pass {
+		getLogger(ctx).Debug("filesystem does not match filter, skipping")
+		return &MyLockTablesReport{What: "filesystem filter skipped this filesystem", Err: nil}
+	}
+
+	switch edge {
+	case Pre:
+		err := h.doRunPre(ctx, dp, dryRun, state)
+		return &MyLockTablesReport{"FLUSH TABLES WITH READ LOCK", err}
+	case Post:
+		err := h.doRunPost(ctx, dp, dryRun, state)
+		return &MyLockTablesReport{"UNLOCK TABLES", err}
+	}
+	return &MyLockTablesReport{What: "skipped this edge", Err: nil}
+}
+
+func (h *MySQLLockTables) doRunPre(ctx context.Context, fs *zfs.DatasetPath, dry bool, state map[interface{}]interface{}) (err error) {
+	db := sql.OpenDB(h.connector)
+	defer func(err *error) {
+		if *err != nil {
+			db.Close()
+		}
+	}(&err)
+
+	getLogger(ctx).Debug("do FLUSH TABLES WITH READ LOCK")
+	_, err = db.ExecContext(ctx, "FLUSH TABLES WITH READ LOCK")
+	if err != nil {
+		return
+	}
+
+	state[myLockTablesConnection] = db
+
+	return nil
+}
+
+func (h *MySQLLockTables) doRunPost(ctx context.Context, fs *zfs.DatasetPath, dry bool, state map[interface{}]interface{}) error {
+
+	db := state[myLockTablesConnection].(*sql.DB)
+	defer db.Close()
+
+	getLogger(ctx).Debug("do UNLOCK TABLES")
+	_, err := db.ExecContext(ctx, "UNLOCK TABLES")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

internal/daemon/hooks/hook_type_postgres_checkpoint.go

@@ -0,0 +1,98 @@
+package hooks
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"math"
+	"time"
+
+	"github.com/lib/pq"
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/internal/config"
+	"github.com/zrepl/zrepl/internal/daemon/filters"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+type PgChkptHook struct {
+	errIsFatal  bool
+	connector   *pq.Connector
+	filesystems Filter
+}
+
+func PgChkptHookFromConfig(in *config.HookPostgresCheckpoint) (*PgChkptHook, error) {
+	filesystems, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
+	if err != nil {
+		return nil, errors.Wrap(err, "`filesystems` invalid")
+	}
+	cn, err := pq.NewConnector(in.DSN)
+	if err != nil {
+		return nil, errors.Wrap(err, "`dsn` invalid")
+	}
+
+	return &PgChkptHook{
+		in.ErrIsFatal,
+		cn,
+		filesystems,
+	}, nil
+}
+
+func (h *PgChkptHook) ErrIsFatal() bool    { return h.errIsFatal }
+func (h *PgChkptHook) Filesystems() Filter { return h.filesystems }
+func (h *PgChkptHook) String() string      { return "postgres checkpoint" }
+
+type PgChkptHookReport struct{ Err error }
+
+func (r *PgChkptHookReport) HadError() bool { return r.Err != nil }
+func (r *PgChkptHookReport) Error() string  { return r.Err.Error() }
+func (r *PgChkptHookReport) String() string {
+	if r.Err != nil {
+		return fmt.Sprintf("postgres CHECKPOINT failed: %s", r.Err)
+	} else {
+		return "postgres CHECKPOINT completed"
+	}
+}
+
+func (h *PgChkptHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
+	if edge != Pre {
+		return &PgChkptHookReport{nil}
+	}
+	fs, ok := extra[EnvFS]
+	if !ok {
+		panic(extra)
+	}
+	dp, err := zfs.NewDatasetPath(fs)
+	if err != nil {
+		panic(err)
+	}
+	err = h.doRunPre(ctx, dp, dryRun)
+	return &PgChkptHookReport{err}
+}
+
+func (h *PgChkptHook) doRunPre(ctx context.Context, fs *zfs.DatasetPath, dry bool) error {
+
+	if pass, err := h.filesystems.Filter(fs); err != nil || !pass {
+		getLogger(ctx).Debug("filesystem does not match filter, skipping")
+		return err
+	}
+
+	db := sql.OpenDB(h.connector)
+	defer db.Close()
+	dl, ok := ctx.Deadline()
+	if ok {
+		timeout := uint64(math.Floor(time.Until(dl).Seconds() * 1000)) // TODO go1.13 milliseconds
+		getLogger(ctx).WithField("statement_timeout", timeout).Debug("setting statement timeout for CHECKPOINT")
+		_, err := db.ExecContext(ctx, "SET statement_timeout TO ?", timeout)
+		if err != nil {
+			return err
+		}
+	}
+	if dry {
+		getLogger(ctx).Debug("dry-run - use ping instead of CHECKPOINT")
+		return db.PingContext(ctx)
+	}
+	getLogger(ctx).Info("execute CHECKPOINT command")
+	_, err := db.ExecContext(ctx, "CHECKPOINT")
+	return err
+}

internal/daemon/hooks/hooks_test.go

@@ -0,0 +1,489 @@
+package hooks_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"regexp"
+	"testing"
+	"text/template"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/daemon/logging/trace"
+
+	"github.com/zrepl/zrepl/internal/config"
+	"github.com/zrepl/zrepl/internal/daemon/hooks"
+	"github.com/zrepl/zrepl/internal/daemon/logging"
+	"github.com/zrepl/zrepl/internal/logger"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+type comparisonAssertionFunc func(require.TestingT, interface{}, interface{}, ...interface{})
+type valueAssertionFunc func(require.TestingT, interface{}, ...interface{})
+
+type expectStep struct {
+	ExpectedEdge hooks.Edge
+	ExpectStatus hooks.StepStatus
+	OutputTest   valueAssertionFunc
+	ErrorTest    valueAssertionFunc
+}
+
+type testCase struct {
+	Name           string
+	Config         []string
+	IsSlow         bool
+	SuppressOutput bool
+
+	ExpectCallbackSkipped bool
+	ExpectHadFatalErr     bool
+	ExpectHadError        bool
+	ExpectStepReports     []expectStep
+}
+
+func curryLeft(f comparisonAssertionFunc, expected interface{}) valueAssertionFunc {
+	return curry(f, expected, false)
+}
+
+func curryRight(f comparisonAssertionFunc, expected interface{}) valueAssertionFunc {
+	return curry(f, expected, true)
+}
+
+func curry(f comparisonAssertionFunc, expected interface{}, right bool) (ret valueAssertionFunc) {
+	ret = func(t require.TestingT, s interface{}, v ...interface{}) {
+		var x interface{}
+		var y interface{}
+		if right {
+			x = s
+			y = expected
+		} else {
+			x = expected
+			y = s
+		}
+
+		if len(v) > 0 {
+			f(t, x, y, v)
+		} else {
+			f(t, x, y)
+		}
+	}
+	return
+}
+
+func TestHooks(t *testing.T) {
+	ctx, end := trace.WithTaskFromStack(context.Background())
+	defer end()
+
+	testFSName := "testpool/testdataset"
+	testSnapshotName := "testsnap"
+
+	tmpl, err := template.New("TestHooks").Parse(`
+jobs:
+- name: TestHooks
+  type: snap
+  filesystems: {"<": true}
+  snapshotting:
+    type: periodic
+    interval: 1m
+    prefix: zrepl_snapjob_
+    hooks:
+    {{- template "List" . }}
+  pruning:
+    keep:
+      - type: last_n
+        count: 10
+`)
+	if err != nil {
+		panic(err)
+	}
+
+	regexpTest := func(s string) valueAssertionFunc {
+		return curryLeft(require.Regexp, regexp.MustCompile(s))
+	}
+
+	containsTest := func(s string) valueAssertionFunc {
+		return curryRight(require.Contains, s)
+	}
+
+	testTable := []testCase{
+		testCase{
+			Name: "no_hooks",
+			ExpectStepReports: []expectStep{
+				{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+			},
+		},
+		testCase{
+			Name:           "timeout",
+			IsSlow:         true,
+			ExpectHadError: true,
+			Config:         []string{`{type: command, path: {{.WorkDir}}/test/test-timeout.sh, timeout: 2s}`},
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepErr,
+					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s ZREPL_TIMEOUT=2", testFSName, testSnapshotName)),
+					ErrorTest:    regexpTest(`timed out after 2(.\d+)?s`),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepSkippedDueToPreErr,
+				},
+			},
+		},
+		testCase{
+			Name:           "check_env",
+			Config:         []string{`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`},
+			ExpectHadError: false,
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
+				},
+			},
+		},
+
+		testCase{
+			Name:                  "nonfatal_pre_error_continues",
+			ExpectCallbackSkipped: false,
+			ExpectHadError:        true,
+			Config:                []string{`{type: command, path: {{.WorkDir}}/test/test-error.sh}`},
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepErr,
+					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
+					ErrorTest:    regexpTest("^command hook failed.*exit status 1$"),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepSkippedDueToPreErr, // post-edge is not executed for failing pre-edge
+				},
+			},
+		},
+
+		testCase{
+			Name:                  "pre_error_fatal_skips_subsequent_pre_edges_and_callback_and_its_post_edge_and_post_edges",
+			ExpectCallbackSkipped: true,
+			ExpectHadFatalErr:     true,
+			ExpectHadError:        true,
+			Config: []string{
+				`{type: command, path: {{.WorkDir}}/test/test-error.sh, err_is_fatal: true}`,
+				`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`,
+			},
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepErr,
+					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
+					ErrorTest:    regexpTest("^command hook failed.*exit status 1$"),
+				},
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepSkippedDueToFatalErr,
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepSkippedDueToFatalErr},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepSkippedDueToFatalErr,
+				},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepSkippedDueToFatalErr,
+				},
+			},
+		},
+
+		testCase{
+			Name:              "post_error_fails_are_ignored_even_if_fatal",
+			ExpectHadFatalErr: false, // only occurs during Post, so it's not a fatal error
+			ExpectHadError:    true,
+			Config: []string{
+				`{type: command, path: {{.WorkDir}}/test/test-post-error.sh, err_is_fatal: true}`,
+				`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`,
+			},
+			ExpectStepReports: []expectStep{
+				expectStep{
+					// No-action run of test-post-error.sh
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   require.Empty,
+				},
+				expectStep{
+					// Pre run of test-report-env.sh
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
+				},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepErr,
+					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR post_testing %s@%s", testFSName, testSnapshotName)),
+					ErrorTest:    regexpTest("^command hook failed.*exit status 1$"),
+				},
+			},
+		},
+
+		testCase{
+			Name:   "cleanup_check_env",
+			Config: []string{`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`},
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
+				},
+			},
+		},
+
+		testCase{
+			Name:              "pre_error_cancels_post",
+			Config:            []string{`{type: command, path: {{.WorkDir}}/test/test-pre-error-post-ok.sh}`},
+			ExpectHadError:    true,
+			ExpectHadFatalErr: false,
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepErr,
+					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
+					ErrorTest:    regexpTest("^command hook failed.*exit status 1$"),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepSkippedDueToPreErr,
+				},
+			},
+		},
+
+		testCase{
+			Name: "pre_error_does_not_cancel_other_posts_but_itself",
+			Config: []string{
+				`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`,
+				`{type: command, path: {{.WorkDir}}/test/test-pre-error-post-ok.sh}`,
+			},
+			ExpectHadError:    true,
+			ExpectHadFatalErr: false,
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
+				},
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepErr,
+					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
+					ErrorTest:    regexpTest("^command hook failed.*exit status 1$"),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepSkippedDueToPreErr,
+				},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
+				},
+			},
+		},
+
+		testCase{
+			Name:              "exceed_buffer_limit",
+			SuppressOutput:    true,
+			Config:            []string{`{type: command, path: {{.WorkDir}}/test/test-large-stdout.sh}`},
+			ExpectHadError:    false,
+			ExpectHadFatalErr: false,
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepOk,
+					OutputTest: func(t require.TestingT, s interface{}, v ...interface{}) {
+						require.Len(t, s, 1<<20)
+					},
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					// No-action run of above hook
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   require.Empty,
+				},
+			},
+		},
+
+		/*
+			Following not intended to test functionality of
+			filter package. Only to demonstrate that hook
+			filters are being applied. The following should
+			result in NO hooks running. If it does run, a
+			fatal hooks.RunReport will be returned.
+		*/
+		testCase{
+			Name: "exclude_all_filesystems",
+			Config: []string{
+				`{type: command, path: {{.WorkDir}}/test/test-error.sh, err_is_fatal: true, filesystems: {"<": false}}`,
+			},
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Callback,
+					ExpectStatus: hooks.StepOk,
+				},
+			},
+		},
+	}
+
+	parseHookConfig := func(t *testing.T, in string) *config.Config {
+		t.Helper()
+		conf, err := config.ParseConfigBytes([]byte(in))
+		require.NoError(t, err)
+		require.NotNil(t, conf)
+		return conf
+	}
+
+	fillHooks := func(tt *testCase) string {
+		// make hook path absolute
+		cwd, err := os.Getwd()
+		if err != nil {
+			panic("os.Getwd() failed")
+		}
+		var hooksTmpl string = "\n"
+		for _, l := range tt.Config {
+			hooksTmpl += fmt.Sprintf("    - %s\n", l)
+		}
+		tmpl.New("List").Parse(hooksTmpl)
+
+		var outBytes bytes.Buffer
+		data := struct {
+			WorkDir string
+		}{
+			WorkDir: cwd,
+		}
+		if err := tmpl.Execute(&outBytes, data); err != nil {
+			panic(err)
+		}
+
+		return outBytes.String()
+	}
+
+	var c *config.Config
+	fs, err := zfs.NewDatasetPath(testFSName)
+	require.NoError(t, err)
+
+	log := logger.NewTestLogger(t)
+
+	var cbReached bool
+	cb := hooks.NewCallbackHookForFilesystem("testcallback", fs, func(_ context.Context) error {
+		cbReached = true
+		return nil
+	})
+
+	hookEnvExtra := hooks.Env{
+		hooks.EnvFS:       fs.ToString(),
+		hooks.EnvSnapshot: testSnapshotName,
+	}
+
+	for _, tt := range testTable {
+		if testing.Short() && tt.IsSlow {
+			continue
+		}
+
+		t.Run(tt.Name, func(t *testing.T) {
+			c = parseHookConfig(t, fillHooks(&tt))
+			snp := c.Jobs[0].Ret.(*config.SnapJob).Snapshotting.Ret.(*config.SnapshottingPeriodic)
+			hookList, err := hooks.ListFromConfig(&snp.Hooks)
+			require.NoError(t, err)
+
+			filteredHooks, err := hookList.CopyFilteredForFilesystem(fs)
+			require.NoError(t, err)
+			plan, err := hooks.NewPlan(&filteredHooks, hooks.PhaseTesting, cb, hookEnvExtra)
+			require.NoError(t, err)
+			t.Logf("REPORT PRE EXECUTION:\n%s", plan.Report())
+
+			cbReached = false
+
+			if testing.Verbose() && !tt.SuppressOutput {
+				ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
+			}
+			plan.Run(ctx, false)
+			report := plan.Report()
+
+			t.Logf("REPORT POST EXECUTION:\n%s", report)
+
+			/*
+			 * TEST ASSERTIONS
+			 */
+
+			t.Logf("len(runReports)=%v", len(report))
+			t.Logf("len(tt.ExpectStepReports)=%v", len(tt.ExpectStepReports))
+			require.Equal(t, len(tt.ExpectStepReports), len(report), "ExpectStepReports must be same length as expected number of hook runs, excluding possible Callback")
+
+			// Check if callback ran, when required
+			if tt.ExpectCallbackSkipped {
+				require.False(t, cbReached, "callback ran but should not have run")
+			} else {
+				require.True(t, cbReached, "callback should have run but did not")
+			}
+
+			// Check if a fatal run error occurred and was expected
+			require.Equal(t, tt.ExpectHadFatalErr, report.HadFatalError(), "non-matching HadFatalError")
+			require.Equal(t, tt.ExpectHadError, report.HadError(), "non-matching HadError")
+
+			if tt.ExpectHadFatalErr {
+				require.True(t, tt.ExpectHadError, "ExpectHadFatalErr implies ExpectHadError")
+			}
+			if !tt.ExpectHadError {
+				require.False(t, tt.ExpectHadFatalErr, "!ExpectHadError implies !ExpectHadFatalErr")
+			}
+
+			// Iterate through each expected hook run
+			for i, hook := range tt.ExpectStepReports {
+				t.Logf("expecting report conforming to %v", hook)
+
+				exp, act := hook.ExpectStatus, report[i].Status
+				require.Equal(t, exp, act, "%s != %s", exp, act)
+
+				// Check for required ExpectedEdge
+				require.NotZero(t, hook.ExpectedEdge, "each hook must have an ExpectedEdge")
+				require.Equal(t, hook.ExpectedEdge, report[i].Edge,
+					"incorrect edge: expected %q, actual %q", hook.ExpectedEdge.String(), report[i].Edge.String(),
+				)
+
+				// Check for expected output
+				if hook.OutputTest != nil {
+					require.IsType(t, (*hooks.CommandHookReport)(nil), report[i].Report)
+					chr := report[i].Report.(*hooks.CommandHookReport)
+					hook.OutputTest(t, string(chr.CapturedStdoutStderrCombined))
+				}
+
+				// Check for expected errors
+				if hook.ErrorTest != nil {
+					hook.ErrorTest(t, string(report[i].Report.Error()))
+				}
+			}
+
+		})
+	}
+}

internal/daemon/hooks/stepstatus_enumer.go

@@ -0,0 +1,76 @@
+// Code generated by "enumer -type=StepStatus -trimprefix=Step"; DO NOT EDIT.
+
+package hooks
+
+import (
+	"fmt"
+)
+
+const (
+	_StepStatusName_0 = "PendingExec"
+	_StepStatusName_1 = "Ok"
+	_StepStatusName_2 = "Err"
+	_StepStatusName_3 = "SkippedDueToFatalErr"
+	_StepStatusName_4 = "SkippedDueToPreErr"
+)
+
+var (
+	_StepStatusIndex_0 = [...]uint8{0, 7, 11}
+	_StepStatusIndex_1 = [...]uint8{0, 2}
+	_StepStatusIndex_2 = [...]uint8{0, 3}
+	_StepStatusIndex_3 = [...]uint8{0, 20}
+	_StepStatusIndex_4 = [...]uint8{0, 18}
+)
+
+func (i StepStatus) String() string {
+	switch {
+	case 1 <= i && i <= 2:
+		i -= 1
+		return _StepStatusName_0[_StepStatusIndex_0[i]:_StepStatusIndex_0[i+1]]
+	case i == 4:
+		return _StepStatusName_1
+	case i == 8:
+		return _StepStatusName_2
+	case i == 16:
+		return _StepStatusName_3
+	case i == 32:
+		return _StepStatusName_4
+	default:
+		return fmt.Sprintf("StepStatus(%d)", i)
+	}
+}
+
+var _StepStatusValues = []StepStatus{1, 2, 4, 8, 16, 32}
+
+var _StepStatusNameToValueMap = map[string]StepStatus{
+	_StepStatusName_0[0:7]:  1,
+	_StepStatusName_0[7:11]: 2,
+	_StepStatusName_1[0:2]:  4,
+	_StepStatusName_2[0:3]:  8,
+	_StepStatusName_3[0:20]: 16,
+	_StepStatusName_4[0:18]: 32,
+}
+
+// StepStatusString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func StepStatusString(s string) (StepStatus, error) {
+	if val, ok := _StepStatusNameToValueMap[s]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to StepStatus values", s)
+}
+
+// StepStatusValues returns all values of the enum
+func StepStatusValues() []StepStatus {
+	return _StepStatusValues
+}
+
+// IsAStepStatus returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i StepStatus) IsAStepStatus() bool {
+	for _, v := range _StepStatusValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}

internal/daemon/hooks/test/test-error.sh

@@ -0,0 +1,5 @@
+#!/bin/sh -eu
+
+>&2 echo "TEST ERROR $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
+
+exit 1

internal/daemon/hooks/test/test-large-stdout.sh

@@ -0,0 +1,18 @@
+#!/bin/sh -eu
+
+# Exceed default hook log size of 1<<20 bytes by double
+# The Go test should fail if buffer size exceeds 1MB
+
+pre_testing() {
+	ln=0
+	while :; do printf '%06d: 012345678901234567890123456789012345678901234567890123456789012345678901\n' "$ln"; ln="$(($ln + 1))"; done \
+	| head -c$(( 2 << 20 ))
+}
+
+case "$ZREPL_HOOKTYPE" in
+	pre_testing)
+		"$ZREPL_HOOKTYPE";;
+	*)
+		# Not handled by this script
+		exit 0;;
+esac

internal/daemon/hooks/test/test-post-error.sh

@@ -0,0 +1,15 @@
+#!/bin/sh -eu
+
+post_testing() {
+	>&2 echo "TEST ERROR $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
+
+	exit 1
+}
+
+case "$ZREPL_HOOKTYPE" in
+	post_testing)
+		"$ZREPL_HOOKTYPE";;
+	*)
+		# Not handled by this script
+		exit 0;;
+esac

internal/daemon/hooks/test/test-pre-error-post-ok.sh

@@ -0,0 +1,11 @@
+#!/bin/sh -eu
+
+if [ "$ZREPL_HOOKTYPE" = "pre_testing" ]; then
+	>&2 echo "TEST ERROR $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
+	exit 1
+elif [ "$ZREPL_HOOKTYPE" = "post_testing" ]; then
+	echo "TEST $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
+else
+	printf "Unknown hook type: %s" "$ZREPL_HOOKTYPE" 
+	exit 255
+fi

internal/daemon/hooks/test/test-report-env.sh

@@ -0,0 +1,3 @@
+#!/bin/sh -eu
+
+echo "TEST $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME ${ZREPL_DRYRUN:+(dry run)}"

internal/daemon/hooks/test/test-timeout.sh

@@ -0,0 +1,5 @@
+#!/bin/sh -eu
+
+echo "TEST $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME ZREPL_TIMEOUT=$ZREPL_TIMEOUT"
+
+exec sleep $(($ZREPL_TIMEOUT + 1))

internal/daemon/job/active.go

@@ -0,0 +1,567 @@
+package job
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/zrepl/zrepl/internal/daemon/logging/trace"
+	"github.com/zrepl/zrepl/internal/util/envconst"
+
+	"github.com/zrepl/zrepl/internal/config"
+	"github.com/zrepl/zrepl/internal/daemon/job/reset"
+	"github.com/zrepl/zrepl/internal/daemon/job/wakeup"
+	"github.com/zrepl/zrepl/internal/daemon/pruner"
+	"github.com/zrepl/zrepl/internal/daemon/snapper"
+	"github.com/zrepl/zrepl/internal/endpoint"
+	"github.com/zrepl/zrepl/internal/replication"
+	"github.com/zrepl/zrepl/internal/replication/driver"
+	"github.com/zrepl/zrepl/internal/replication/logic"
+	"github.com/zrepl/zrepl/internal/replication/report"
+	"github.com/zrepl/zrepl/internal/rpc"
+	"github.com/zrepl/zrepl/internal/transport"
+	"github.com/zrepl/zrepl/internal/transport/fromconfig"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+type ActiveSide struct {
+	mode      activeMode
+	name      endpoint.JobID
+	connecter transport.Connecter
+
+	replicationDriverConfig driver.Config
+
+	prunerFactory *pruner.PrunerFactory
+
+	promRepStateSecs      *prometheus.HistogramVec // labels: state
+	promPruneSecs         *prometheus.HistogramVec // labels: prune_side
+	promBytesReplicated   *prometheus.CounterVec   // labels: filesystem
+	promReplicationErrors prometheus.Gauge
+	promLastSuccessful    prometheus.Gauge
+
+	tasksMtx sync.Mutex
+	tasks    activeSideTasks
+}
+
+//go:generate enumer -type=ActiveSideState
+type ActiveSideState int
+
+const (
+	ActiveSideReplicating ActiveSideState = 1 << iota
+	ActiveSidePruneSender
+	ActiveSidePruneReceiver
+	ActiveSideDone // also errors
+)
+
+type activeSideTasks struct {
+	state ActiveSideState
+
+	// valid for state ActiveSideReplicating, ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
+	replicationReport driver.ReportFunc
+	replicationCancel context.CancelFunc
+
+	// valid for state ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
+	prunerSender, prunerReceiver *pruner.Pruner
+
+	// valid for state ActiveSidePruneReceiver, ActiveSideDone
+	prunerSenderCancel, prunerReceiverCancel context.CancelFunc
+}
+
+func (a *ActiveSide) updateTasks(u func(*activeSideTasks)) activeSideTasks {
+	a.tasksMtx.Lock()
+	defer a.tasksMtx.Unlock()
+	copy := a.tasks
+	if u == nil {
+		return copy
+	}
+	u(&copy)
+	a.tasks = copy
+	return copy
+}
+
+type activeMode interface {
+	ConnectEndpoints(ctx context.Context, connecter transport.Connecter)
+	DisconnectEndpoints()
+	SenderReceiver() (logic.Sender, logic.Receiver)
+	Type() Type
+	PlannerPolicy() logic.PlannerPolicy
+	RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{})
+	SnapperReport() *snapper.Report
+	ResetConnectBackoff()
+}
+
+type modePush struct {
+	setupMtx      sync.Mutex
+	sender        *endpoint.Sender
+	receiver      *rpc.Client
+	senderConfig  *endpoint.SenderConfig
+	plannerPolicy *logic.PlannerPolicy
+	snapper       snapper.Snapper
+}
+
+func (m *modePush) ConnectEndpoints(ctx context.Context, connecter transport.Connecter) {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	if m.receiver != nil || m.sender != nil {
+		panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
+	}
+	m.sender = endpoint.NewSender(*m.senderConfig)
+	m.receiver = rpc.NewClient(connecter, rpc.GetLoggersOrPanic(ctx))
+}
+
+func (m *modePush) DisconnectEndpoints() {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	m.receiver.Close()
+	m.sender = nil
+	m.receiver = nil
+}
+
+func (m *modePush) SenderReceiver() (logic.Sender, logic.Receiver) {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	return m.sender, m.receiver
+}
+
+func (m *modePush) Type() Type { return TypePush }
+
+func (m *modePush) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
+
+func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
+	m.snapper.Run(ctx, wakeUpCommon)
+}
+
+func (m *modePush) SnapperReport() *snapper.Report {
+	r := m.snapper.Report()
+	return &r
+}
+
+func (m *modePush) ResetConnectBackoff() {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	if m.receiver != nil {
+		m.receiver.ResetConnectBackoff()
+	}
+}
+
+func modePushFromConfig(g *config.Global, in *config.PushJob, jobID endpoint.JobID) (*modePush, error) {
+	m := &modePush{}
+	var err error
+
+	m.senderConfig, err = buildSenderConfig(in, jobID)
+	if err != nil {
+		return nil, errors.Wrap(err, "sender config")
+	}
+
+	replicationConfig, err := logic.ReplicationConfigFromConfig(in.Replication)
+	if err != nil {
+		return nil, errors.Wrap(err, "field `replication`")
+	}
+
+	conflictResolution, err := logic.ConflictResolutionFromConfig(in.ConflictResolution)
+	if err != nil {
+		return nil, errors.Wrap(err, "field `conflict_resolution`")
+	}
+
+	m.plannerPolicy = &logic.PlannerPolicy{
+		ConflictResolution:        conflictResolution,
+		ReplicationConfig:         replicationConfig,
+		SizeEstimationConcurrency: in.Replication.Concurrency.SizeEstimates,
+	}
+	if err := m.plannerPolicy.Validate(); err != nil {
+		return nil, errors.Wrap(err, "cannot build planner policy")
+	}
+
+	if m.snapper, err = snapper.FromConfig(g, m.senderConfig.FSF, in.Snapshotting); err != nil {
+		return nil, errors.Wrap(err, "cannot build snapper")
+	}
+
+	return m, nil
+}
+
+type modePull struct {
+	setupMtx       sync.Mutex
+	receiver       *endpoint.Receiver
+	receiverConfig endpoint.ReceiverConfig
+	sender         *rpc.Client
+	plannerPolicy  *logic.PlannerPolicy
+	interval       config.PositiveDurationOrManual
+}
+
+func (m *modePull) ConnectEndpoints(ctx context.Context, connecter transport.Connecter) {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	if m.receiver != nil || m.sender != nil {
+		panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
+	}
+	m.receiver = endpoint.NewReceiver(m.receiverConfig)
+	m.sender = rpc.NewClient(connecter, rpc.GetLoggersOrPanic(ctx))
+}
+
+func (m *modePull) DisconnectEndpoints() {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	m.sender.Close()
+	m.sender = nil
+	m.receiver = nil
+}
+
+func (m *modePull) SenderReceiver() (logic.Sender, logic.Receiver) {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	return m.sender, m.receiver
+}
+
+func (*modePull) Type() Type { return TypePull }
+
+func (m *modePull) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
+
+func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
+	if m.interval.Manual {
+		GetLogger(ctx).Info("manual pull configured, periodic pull disabled")
+		// "waiting for wakeups" is printed in common ActiveSide.do
+		return
+	}
+	t := time.NewTicker(m.interval.Interval)
+	defer t.Stop()
+	for {
+		select {
+		case <-t.C:
+			select {
+			case wakeUpCommon <- struct{}{}:
+			default:
+				GetLogger(ctx).
+					WithField("pull_interval", m.interval).
+					Warn("pull job took longer than pull interval")
+				wakeUpCommon <- struct{}{} // block anyways, to queue up the wakeup
+			}
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (m *modePull) SnapperReport() *snapper.Report {
+	return nil
+}
+
+func (m *modePull) ResetConnectBackoff() {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	if m.sender != nil {
+		m.sender.ResetConnectBackoff()
+	}
+}
+
+func modePullFromConfig(g *config.Global, in *config.PullJob, jobID endpoint.JobID) (m *modePull, err error) {
+	m = &modePull{}
+	m.interval = in.Interval
+
+	replicationConfig, err := logic.ReplicationConfigFromConfig(in.Replication)
+	if err != nil {
+		return nil, errors.Wrap(err, "field `replication`")
+	}
+
+	conflictResolution, err := logic.ConflictResolutionFromConfig(in.ConflictResolution)
+	if err != nil {
+		return nil, errors.Wrap(err, "field `conflict_resolution`")
+	}
+
+	m.plannerPolicy = &logic.PlannerPolicy{
+		ConflictResolution:        conflictResolution,
+		ReplicationConfig:         replicationConfig,
+		SizeEstimationConcurrency: in.Replication.Concurrency.SizeEstimates,
+	}
+	if err := m.plannerPolicy.Validate(); err != nil {
+		return nil, errors.Wrap(err, "cannot build planner policy")
+	}
+
+	m.receiverConfig, err = buildReceiverConfig(in, jobID)
+	if err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}
+
+func replicationDriverConfigFromConfig(in *config.Replication) (c driver.Config, err error) {
+	c = driver.Config{
+		StepQueueConcurrency:     in.Concurrency.Steps,
+		MaxAttempts:              envconst.Int("ZREPL_REPLICATION_MAX_ATTEMPTS", 3),
+		ReconnectHardFailTimeout: envconst.Duration("ZREPL_REPLICATION_RECONNECT_HARD_FAIL_TIMEOUT", 10*time.Minute),
+	}
+	err = c.Validate()
+	return c, err
+}
+
+func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}, parseFlags config.ParseFlags) (j *ActiveSide, err error) {
+
+	j = &ActiveSide{}
+	j.name, err = endpoint.MakeJobID(in.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid job name")
+	}
+
+	switch v := configJob.(type) {
+	case *config.PushJob:
+		j.mode, err = modePushFromConfig(g, v, j.name) // shadow
+	case *config.PullJob:
+		j.mode, err = modePullFromConfig(g, v, j.name) // shadow
+	default:
+		panic(fmt.Sprintf("implementation error: unknown job type %T", v))
+	}
+	if err != nil {
+		return nil, err // no wrapping required
+	}
+
+	j.promRepStateSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace:   "zrepl",
+		Subsystem:   "replication",
+		Name:        "state_time",
+		Help:        "seconds spent during replication",
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
+	}, []string{"state"})
+	j.promBytesReplicated = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace:   "zrepl",
+		Subsystem:   "replication",
+		Name:        "bytes_replicated",
+		Help:        "number of bytes replicated from sender to receiver per filesystem",
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
+	}, []string{"filesystem"})
+	j.promReplicationErrors = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace:   "zrepl",
+		Subsystem:   "replication",
+		Name:        "filesystem_errors",
+		Help:        "number of filesystems that failed replication in the latest replication attempt, or -1 if the job failed before enumerating the filesystems",
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
+	})
+	j.promLastSuccessful = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace:   "zrepl",
+		Subsystem:   "replication",
+		Name:        "last_successful",
+		Help:        "timestamp of last successful replication",
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
+	})
+
+	j.connecter, err = fromconfig.ConnecterFromConfig(g, in.Connect, parseFlags)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot build client")
+	}
+
+	j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace:   "zrepl",
+		Subsystem:   "pruning",
+		Name:        "time",
+		Help:        "seconds spent in pruner",
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
+	}, []string{"prune_side"})
+	j.prunerFactory, err = pruner.NewPrunerFactory(in.Pruning, j.promPruneSecs)
+	if err != nil {
+		return nil, err
+	}
+
+	j.replicationDriverConfig, err = replicationDriverConfigFromConfig(in.Replication)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot build replication driver config")
+	}
+
+	return j, nil
+}
+
+func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) {
+	registerer.MustRegister(j.promRepStateSecs)
+	registerer.MustRegister(j.promPruneSecs)
+	registerer.MustRegister(j.promBytesReplicated)
+	registerer.MustRegister(j.promReplicationErrors)
+	registerer.MustRegister(j.promLastSuccessful)
+}
+
+func (j *ActiveSide) Name() string { return j.name.String() }
+
+type ActiveSideStatus struct {
+	Replication                    *report.Report
+	PruningSender, PruningReceiver *pruner.Report
+	Snapshotting                   *snapper.Report
+}
+
+func (j *ActiveSide) Status() *Status {
+	tasks := j.updateTasks(nil)
+
+	s := &ActiveSideStatus{}
+	t := j.mode.Type()
+	if tasks.replicationReport != nil {
+		s.Replication = tasks.replicationReport()
+	}
+	if tasks.prunerSender != nil {
+		s.PruningSender = tasks.prunerSender.Report()
+	}
+	if tasks.prunerReceiver != nil {
+		s.PruningReceiver = tasks.prunerReceiver.Report()
+	}
+	s.Snapshotting = j.mode.SnapperReport()
+	return &Status{Type: t, JobSpecific: s}
+}
+
+func (j *ActiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
+	pull, ok := j.mode.(*modePull)
+	if !ok {
+		_ = j.mode.(*modePush) // make sure we didn't introduce a new job type
+		return nil, false
+	}
+	return pull.receiverConfig.RootWithoutClientComponent.Copy(), true
+}
+
+func (j *ActiveSide) SenderConfig() *endpoint.SenderConfig {
+	push, ok := j.mode.(*modePush)
+	if !ok {
+		_ = j.mode.(*modePull) // make sure we didn't introduce a new job type
+		return nil
+	}
+	return push.senderConfig
+}
+
+// The active side of a replication uses one end (sender or receiver)
+// directly by method invocation, without going through a transport that
+// provides a client identity.
+// However, in order to avoid the need to distinguish between direct-method-invocating
+// clients and RPC client, we use an invalid client identity as a sentinel value.
+func FakeActiveSideDirectMethodInvocationClientIdentity(jobId endpoint.JobID) string {
+	return fmt.Sprintf("<local><active><job><client><identity><job=%q>", jobId.String())
+}
+
+func (j *ActiveSide) Run(ctx context.Context) {
+	ctx, endTask := trace.WithTaskAndSpan(ctx, "active-side-job", j.Name())
+	defer endTask()
+
+	ctx = context.WithValue(ctx, endpoint.ClientIdentityKey, FakeActiveSideDirectMethodInvocationClientIdentity(j.name))
+
+	log := GetLogger(ctx)
+
+	defer log.Info("job exiting")
+
+	periodicDone := make(chan struct{})
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	periodicCtx, endTask := trace.WithTask(ctx, "periodic")
+	defer endTask()
+	go j.mode.RunPeriodic(periodicCtx, periodicDone)
+
+	invocationCount := 0
+outer:
+	for {
+		log.Info("wait for wakeups")
+		select {
+		case <-ctx.Done():
+			log.WithError(ctx.Err()).Info("context")
+			break outer
+
+		case <-wakeup.Wait(ctx):
+			j.mode.ResetConnectBackoff()
+		case <-periodicDone:
+		}
+		invocationCount++
+		invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", invocationCount))
+		j.do(invocationCtx)
+		endSpan()
+	}
+}
+
+func (j *ActiveSide) do(ctx context.Context) {
+
+	j.mode.ConnectEndpoints(ctx, j.connecter)
+	defer j.mode.DisconnectEndpoints()
+
+	// allow cancellation of an invocation (this function)
+	ctx, cancelThisRun := context.WithCancel(ctx)
+	defer cancelThisRun()
+	go func() {
+		select {
+		case <-reset.Wait(ctx):
+			GetLogger(ctx).Info("reset received, cancelling current invocation")
+			cancelThisRun()
+		case <-ctx.Done():
+		}
+	}()
+
+	sender, receiver := j.mode.SenderReceiver()
+
+	{
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+		ctx, endSpan := trace.WithSpan(ctx, "replication")
+		ctx, repCancel := context.WithCancel(ctx)
+		var repWait driver.WaitFunc
+		j.updateTasks(func(tasks *activeSideTasks) {
+			// reset it
+			*tasks = activeSideTasks{}
+			tasks.replicationCancel = func() { repCancel(); endSpan() }
+			tasks.replicationReport, repWait = replication.Do(
+				ctx, j.replicationDriverConfig, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver, j.mode.PlannerPolicy()),
+			)
+			tasks.state = ActiveSideReplicating
+		})
+		GetLogger(ctx).Info("start replication")
+		repWait(true) // wait blocking
+		repCancel()   // always cancel to free up context resources
+
+		replicationReport := j.tasks.replicationReport()
+		var numErrors = replicationReport.GetFailedFilesystemsCountInLatestAttempt()
+		j.promReplicationErrors.Set(float64(numErrors))
+		if numErrors == 0 {
+			j.promLastSuccessful.SetToCurrentTime()
+		}
+
+		endSpan()
+	}
+
+	{
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+		ctx, endSpan := trace.WithSpan(ctx, "prune_sender")
+		ctx, senderCancel := context.WithCancel(ctx)
+		tasks := j.updateTasks(func(tasks *activeSideTasks) {
+			tasks.prunerSender = j.prunerFactory.BuildSenderPruner(ctx, sender, sender)
+			tasks.prunerSenderCancel = func() { senderCancel(); endSpan() }
+			tasks.state = ActiveSidePruneSender
+		})
+		GetLogger(ctx).Info("start pruning sender")
+		tasks.prunerSender.Prune()
+		GetLogger(ctx).Info("finished pruning sender")
+		senderCancel()
+		endSpan()
+	}
+	{
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+		ctx, endSpan := trace.WithSpan(ctx, "prune_recever")
+		ctx, receiverCancel := context.WithCancel(ctx)
+		tasks := j.updateTasks(func(tasks *activeSideTasks) {
+			tasks.prunerReceiver = j.prunerFactory.BuildReceiverPruner(ctx, receiver, sender)
+			tasks.prunerReceiverCancel = func() { receiverCancel(); endSpan() }
+			tasks.state = ActiveSidePruneReceiver
+		})
+		GetLogger(ctx).Info("start pruning receiver")
+		tasks.prunerReceiver.Prune()
+		GetLogger(ctx).Info("finished pruning receiver")
+		receiverCancel()
+		endSpan()
+	}
+
+	j.updateTasks(func(tasks *activeSideTasks) {
+		tasks.state = ActiveSideDone
+	})
+
+}

internal/daemon/job/active_test.go

@@ -0,0 +1,20 @@
+package job
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/endpoint"
+	"github.com/zrepl/zrepl/internal/transport"
+)
+
+func TestFakeActiveSideDirectMethodInvocationClientIdentityDoesNotPassValidityTest(t *testing.T) {
+	jobid, err := endpoint.MakeJobID("validjobname")
+	require.NoError(t, err)
+	clientIdentity := FakeActiveSideDirectMethodInvocationClientIdentity(jobid)
+	t.Logf("%v", clientIdentity)
+	err = transport.ValidateClientIdentity(clientIdentity)
+	assert.Error(t, err)
+}

internal/daemon/job/activesidestate_enumer.go

@@ -0,0 +1,66 @@
+// Code generated by "enumer -type=ActiveSideState"; DO NOT EDIT.
+
+package job
+
+import (
+	"fmt"
+)
+
+const (
+	_ActiveSideStateName_0 = "ActiveSideReplicatingActiveSidePruneSender"
+	_ActiveSideStateName_1 = "ActiveSidePruneReceiver"
+	_ActiveSideStateName_2 = "ActiveSideDone"
+)
+
+var (
+	_ActiveSideStateIndex_0 = [...]uint8{0, 21, 42}
+	_ActiveSideStateIndex_1 = [...]uint8{0, 23}
+	_ActiveSideStateIndex_2 = [...]uint8{0, 14}
+)
+
+func (i ActiveSideState) String() string {
+	switch {
+	case 1 <= i && i <= 2:
+		i -= 1
+		return _ActiveSideStateName_0[_ActiveSideStateIndex_0[i]:_ActiveSideStateIndex_0[i+1]]
+	case i == 4:
+		return _ActiveSideStateName_1
+	case i == 8:
+		return _ActiveSideStateName_2
+	default:
+		return fmt.Sprintf("ActiveSideState(%d)", i)
+	}
+}
+
+var _ActiveSideStateValues = []ActiveSideState{1, 2, 4, 8}
+
+var _ActiveSideStateNameToValueMap = map[string]ActiveSideState{
+	_ActiveSideStateName_0[0:21]:  1,
+	_ActiveSideStateName_0[21:42]: 2,
+	_ActiveSideStateName_1[0:23]:  4,
+	_ActiveSideStateName_2[0:14]:  8,
+}
+
+// ActiveSideStateString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func ActiveSideStateString(s string) (ActiveSideState, error) {
+	if val, ok := _ActiveSideStateNameToValueMap[s]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to ActiveSideState values", s)
+}
+
+// ActiveSideStateValues returns all values of the enum
+func ActiveSideStateValues() []ActiveSideState {
+	return _ActiveSideStateValues
+}
+
+// IsAActiveSideState returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i ActiveSideState) IsAActiveSideState() bool {
+	for _, v := range _ActiveSideStateValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}

internal/daemon/job/build_jobs.go

@@ -0,0 +1,105 @@
+package job
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/internal/config"
+	"github.com/zrepl/zrepl/internal/util/bandwidthlimit"
+)
+
+func JobsFromConfig(c *config.Config, parseFlags config.ParseFlags) ([]Job, error) {
+	js := make([]Job, len(c.Jobs))
+	for i := range c.Jobs {
+		j, err := buildJob(c.Global, c.Jobs[i], parseFlags)
+		if err != nil {
+			return nil, err
+		}
+		if j == nil || j.Name() == "" {
+			panic(fmt.Sprintf("implementation error: job builder returned nil job type %T", c.Jobs[i].Ret))
+		}
+		js[i] = j
+	}
+
+	return js, nil
+}
+
+func buildJob(c *config.Global, in config.JobEnum, parseFlags config.ParseFlags) (j Job, err error) {
+	cannotBuildJob := func(e error, name string) (Job, error) {
+		return nil, errors.Wrapf(e, "cannot build job %q", name)
+	}
+	// FIXME prettify this
+	switch v := in.Ret.(type) {
+	case *config.SinkJob:
+		j, err = passiveSideFromConfig(c, &v.PassiveJob, v, parseFlags)
+		if err != nil {
+			return cannotBuildJob(err, v.Name)
+		}
+	case *config.SourceJob:
+		j, err = passiveSideFromConfig(c, &v.PassiveJob, v, parseFlags)
+		if err != nil {
+			return cannotBuildJob(err, v.Name)
+		}
+	case *config.SnapJob:
+		j, err = snapJobFromConfig(c, v)
+		if err != nil {
+			return cannotBuildJob(err, v.Name)
+		}
+	case *config.PushJob:
+		j, err = activeSide(c, &v.ActiveJob, v, parseFlags)
+		if err != nil {
+			return cannotBuildJob(err, v.Name)
+		}
+	case *config.PullJob:
+		j, err = activeSide(c, &v.ActiveJob, v, parseFlags)
+		if err != nil {
+			return cannotBuildJob(err, v.Name)
+		}
+	default:
+		panic(fmt.Sprintf("implementation error: unknown job type %T", v))
+	}
+	return j, nil
+
+}
+
+func validateReceivingSidesDoNotOverlap(receivingRootFSs []string) error {
+	if len(receivingRootFSs) == 0 {
+		return nil
+	}
+	rfss := make([]string, len(receivingRootFSs))
+	copy(rfss, receivingRootFSs)
+	sort.Slice(rfss, func(i, j int) bool {
+		return strings.Compare(rfss[i], rfss[j]) == -1
+	})
+	// add tailing slash because of hierarchy-simulation
+	// rootfs/ is not root of rootfs2/
+	for i := 0; i < len(rfss); i++ {
+		rfss[i] += "/"
+	}
+	// idea:
+	//   no path in rfss must be prefix of another
+	//
+	// rfss is now lexicographically sorted, which means that
+	// if i is prefix of j, i < j (in lexicographical order)
+	// thus,
+	// if any i is prefix of i+n (n >= 1), there is overlap
+	for i := 0; i < len(rfss)-1; i++ {
+		if strings.HasPrefix(rfss[i+1], rfss[i]) {
+			return fmt.Errorf("receiving jobs with overlapping root filesystems are forbidden")
+		}
+	}
+	return nil
+}
+
+func buildBandwidthLimitConfig(in *config.BandwidthLimit) (c bandwidthlimit.Config, _ error) {
+	if in.Max.ToBytes() > 0 && int64(in.Max.ToBytes()) == 0 {
+		return c, fmt.Errorf("bandwidth limit `max` is too small, must at least specify one byte")
+	}
+	return bandwidthlimit.Config{
+		Max:            int64(in.Max.ToBytes()),
+		BucketCapacity: int64(in.BucketCapacity.ToBytes()),
+	}, nil
+}

internal/daemon/job/build_jobs_sendrecvoptions.go

@@ -0,0 +1,102 @@
+package job
+
+import (
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/internal/config"
+	"github.com/zrepl/zrepl/internal/daemon/filters"
+	"github.com/zrepl/zrepl/internal/endpoint"
+	"github.com/zrepl/zrepl/internal/util/nodefault"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+type SendingJobConfig interface {
+	GetFilesystems() config.FilesystemsFilter
+	GetSendOptions() *config.SendOptions // must not be nil
+}
+
+func buildSenderConfig(in SendingJobConfig, jobID endpoint.JobID) (*endpoint.SenderConfig, error) {
+
+	fsf, err := filters.DatasetMapFilterFromConfig(in.GetFilesystems())
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot build filesystem filter")
+	}
+	sendOpts := in.GetSendOptions()
+	bwlim, err := buildBandwidthLimitConfig(sendOpts.BandwidthLimit)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot build bandwith limit config")
+	}
+
+	sc := &endpoint.SenderConfig{
+		FSF:   fsf,
+		JobID: jobID,
+
+		Encrypt:              &nodefault.Bool{B: sendOpts.Encrypted},
+		SendRaw:              sendOpts.Raw,
+		SendProperties:       sendOpts.SendProperties,
+		SendBackupProperties: sendOpts.BackupProperties,
+		SendLargeBlocks:      sendOpts.LargeBlocks,
+		SendCompressed:       sendOpts.Compressed,
+		SendEmbeddedData:     sendOpts.EmbeddedData,
+		SendSaved:            sendOpts.Saved,
+
+		BandwidthLimit: bwlim,
+	}
+
+	if err := sc.Validate(); err != nil {
+		return nil, errors.Wrap(err, "cannot build sender config")
+	}
+
+	return sc, nil
+}
+
+type ReceivingJobConfig interface {
+	GetRootFS() string
+	GetAppendClientIdentity() bool
+	GetRecvOptions() *config.RecvOptions
+}
+
+func buildReceiverConfig(in ReceivingJobConfig, jobID endpoint.JobID) (rc endpoint.ReceiverConfig, err error) {
+	rootFs, err := zfs.NewDatasetPath(in.GetRootFS())
+	if err != nil {
+		return rc, errors.New("root_fs is not a valid zfs filesystem path")
+	}
+	if rootFs.Length() <= 0 {
+		return rc, errors.New("root_fs must not be empty") // duplicates error check of receiver
+	}
+
+	recvOpts := in.GetRecvOptions()
+
+	bwlim, err := buildBandwidthLimitConfig(recvOpts.BandwidthLimit)
+	if err != nil {
+		return rc, errors.Wrap(err, "cannot build bandwith limit config")
+	}
+
+	placeholderEncryption, err := endpoint.PlaceholderCreationEncryptionPropertyString(recvOpts.Placeholder.Encryption)
+	if err != nil {
+		options := []string{}
+		for _, v := range endpoint.PlaceholderCreationEncryptionPropertyValues() {
+			options = append(options, endpoint.PlaceholderCreationEncryptionProperty(v).String())
+		}
+		return rc, errors.Errorf("placeholder encryption value %q is invalid, must be one of %s",
+			recvOpts.Placeholder.Encryption, options)
+	}
+
+	rc = endpoint.ReceiverConfig{
+		JobID:                      jobID,
+		RootWithoutClientComponent: rootFs,
+		AppendClientIdentity:       in.GetAppendClientIdentity(),
+
+		InheritProperties:  recvOpts.Properties.Inherit,
+		OverrideProperties: recvOpts.Properties.Override,
+
+		BandwidthLimit: bwlim,
+
+		PlaceholderEncryption: placeholderEncryption,
+	}
+	if err := rc.Validate(); err != nil {
+		return rc, errors.Wrap(err, "cannot build receiver config")
+	}
+
+	return rc, nil
+}

internal/daemon/job/build_jobs_test.go

@@ -0,0 +1,318 @@
+package job
+
+import (
+	"fmt"
+	"path"
+	"path/filepath"
+	"testing"
+
+	"github.com/kr/pretty"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/config"
+)
+
+func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
+	type testCase struct {
+		err   bool
+		input []string
+	}
+	tcs := []testCase{
+		{false, nil},
+		{false, []string{}},
+		{false, []string{""}}, // not our job to determine valid paths
+		{false, []string{"a"}},
+		{false, []string{"some/path"}},
+		{false, []string{"zroot/sink1", "zroot/sink2", "zroot/sink3"}},
+		{false, []string{"zroot/foo", "zroot/foobar"}},
+		{true, []string{"zroot/b", "zroot/b"}},
+		{true, []string{"zroot/foo", "zroot/foo/bar", "zroot/baz"}},
+		{false, []string{"a/x", "b/x"}},
+		{false, []string{"a", "b"}},
+		{true, []string{"a", "a"}},
+		{true, []string{"a/x/y", "a/x"}},
+		{true, []string{"a/x", "a/x/y"}},
+		{true, []string{"a/x", "b/x", "a/x/y"}},
+		{true, []string{"a", "a/b", "a/c", "a/b"}},
+		{true, []string{"a/b", "a/c", "a/b", "a/d", "a/c"}},
+	}
+
+	for _, tc := range tcs {
+		t.Logf("input: %v", tc.input)
+		err := validateReceivingSidesDoNotOverlap(tc.input)
+		if tc.err {
+			assert.Error(t, err)
+		} else {
+			assert.NoError(t, err)
+		}
+	}
+}
+
+func TestJobIDErrorHandling(t *testing.T) {
+	tmpl := `
+jobs:
+- name: %s
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
+
+	type Case struct {
+		jobName string
+		valid   bool
+	}
+	cases := []Case{
+		{"validjobname", true},
+		{"valid with spaces", true},
+		{"invalid\twith\ttabs", false},
+		{"invalid#withdelimiter", false},
+		{"invalid@withdelimiter", false},
+		{"withnewline\\nmiddle", false},
+		{"withnewline\\n", false},
+		{"withslash/", false},
+		{"withslash/inthemiddle", false},
+		{"/", false},
+	}
+
+	for i := range cases {
+		t.Run(cases[i].jobName, func(t *testing.T) {
+			c := cases[i]
+
+			conf, err := config.ParseConfigBytes([]byte(fill(c.jobName)))
+			require.NoError(t, err, "not expecting yaml-config to know about job ids")
+			require.NotNil(t, conf)
+			jobs, err := JobsFromConfig(conf, config.ParseFlagsNone)
+
+			if c.valid {
+				assert.NoError(t, err)
+				require.Len(t, jobs, 1)
+				assert.Equal(t, c.jobName, jobs[0].Name())
+			} else {
+				t.Logf("error: %s", err)
+				assert.Error(t, err)
+				assert.Nil(t, jobs)
+			}
+
+		})
+	}
+
+}
+
+func TestSampleConfigsAreBuiltWithoutErrors(t *testing.T) {
+	paths, err := filepath.Glob("../../../internal/config/samples/*")
+	if err != nil {
+		t.Errorf("glob failed: %+v", err)
+	}
+
+	type additionalCheck struct {
+		state int
+		test  func(t *testing.T, jobs []Job)
+	}
+	additionalChecks := map[string]*additionalCheck{
+		"bandwidth_limit.yml": {test: testSampleConfig_BandwidthLimit},
+	}
+
+	for _, p := range paths {
+
+		if path.Ext(p) != ".yml" {
+			t.Logf("skipping file %s", p)
+			continue
+		}
+
+		filename := path.Base(p)
+		t.Logf("checking for presence additonal checks for file %q", filename)
+		additionalCheck := additionalChecks[filename]
+		if additionalCheck == nil {
+			t.Logf("no additional checks")
+		} else {
+			t.Logf("additional check present")
+			additionalCheck.state = 1
+		}
+
+		t.Run(p, func(t *testing.T) {
+			c, err := config.ParseConfig(p)
+			if err != nil {
+				t.Fatalf("error parsing %s:\n%+v", p, err)
+			}
+
+			t.Logf("file: %s", p)
+			t.Log(pretty.Sprint(c))
+
+			jobs, err := JobsFromConfig(c, config.ParseFlagsNoCertCheck)
+			t.Logf("jobs: %#v", jobs)
+			require.NoError(t, err)
+
+			if additionalCheck != nil {
+				additionalCheck.test(t, jobs)
+				additionalCheck.state = 2
+			}
+		})
+
+	}
+
+	for basename, c := range additionalChecks {
+		if c.state == 0 {
+			panic("univisited additional check " + basename)
+		}
+	}
+
+}
+
+func testSampleConfig_BandwidthLimit(t *testing.T, jobs []Job) {
+	require.Len(t, jobs, 3)
+
+	{
+		limitedSink, ok := jobs[0].(*PassiveSide)
+		require.True(t, ok, "%T", jobs[0])
+		limitedSinkMode, ok := limitedSink.mode.(*modeSink)
+		require.True(t, ok, "%T", limitedSink)
+
+		assert.Equal(t, int64(12345), limitedSinkMode.receiverConfig.BandwidthLimit.Max)
+		assert.Equal(t, int64(1<<17), limitedSinkMode.receiverConfig.BandwidthLimit.BucketCapacity)
+	}
+
+	{
+		limitedPush, ok := jobs[1].(*ActiveSide)
+		require.True(t, ok, "%T", jobs[1])
+		limitedPushMode, ok := limitedPush.mode.(*modePush)
+		require.True(t, ok, "%T", limitedPush)
+
+		assert.Equal(t, int64(54321), limitedPushMode.senderConfig.BandwidthLimit.Max)
+		assert.Equal(t, int64(1024), limitedPushMode.senderConfig.BandwidthLimit.BucketCapacity)
+	}
+
+	{
+		unlimitedSink, ok := jobs[2].(*PassiveSide)
+		require.True(t, ok, "%T", jobs[2])
+		unlimitedSinkMode, ok := unlimitedSink.mode.(*modeSink)
+		require.True(t, ok, "%T", unlimitedSink)
+
+		max := unlimitedSinkMode.receiverConfig.BandwidthLimit.Max
+		assert.Less(t, max, int64(0), max, "unlimited mode <=> negative value for .Max, see bandwidthlimit.Config")
+	}
+
+}
+
+func TestReplicationOptions(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  %s
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+
+	type Test struct {
+		name        string
+		input       string
+		expectOk    func(t *testing.T, a *ActiveSide, m *modePush)
+		expectError bool
+	}
+
+	tests := []Test{
+		{
+			name: "defaults",
+			input: `
+  replication: {}
+`,
+			expectOk: func(t *testing.T, a *ActiveSide, m *modePush) {},
+		},
+		{
+			name: "steps_zero",
+			input: `
+  replication:
+    concurrency:
+      steps: 0
+`,
+			expectError: true,
+		},
+		{
+			name: "size_estimates_zero",
+			input: `
+  replication:
+    concurrency:
+      size_estimates: 0
+`,
+			expectError: true,
+		},
+		{
+			name: "custom_values",
+			input: `
+  replication:
+    concurrency:
+      steps: 23
+      size_estimates: 42
+`,
+			expectOk: func(t *testing.T, a *ActiveSide, m *modePush) {
+				assert.Equal(t, 23, a.replicationDriverConfig.StepQueueConcurrency)
+				assert.Equal(t, 42, m.plannerPolicy.SizeEstimationConcurrency)
+			},
+		},
+		{
+			name: "negative_values_forbidden",
+			input: `
+  replication:
+    concurrency:
+      steps: -23
+      size_estimates: -42
+`,
+			expectError: true,
+		},
+	}
+
+	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
+
+	for _, ts := range tests {
+		t.Run(ts.name, func(t *testing.T) {
+			assert.True(t, (ts.expectError) != (ts.expectOk != nil))
+
+			cstr := fill(ts.input)
+			t.Logf("testing config:\n%s", cstr)
+			c, err := config.ParseConfigBytes([]byte(cstr))
+			require.NoError(t, err)
+			jobs, err := JobsFromConfig(c, config.ParseFlagsNone)
+			if ts.expectOk != nil {
+				require.NoError(t, err)
+				require.NotNil(t, c)
+				require.NoError(t, err)
+				require.Len(t, jobs, 1)
+				a := jobs[0].(*ActiveSide)
+				m := a.mode.(*modePush)
+				ts.expectOk(t, a, m)
+			} else if ts.expectError {
+				require.Error(t, err)
+			} else {
+				t.Fatalf("test must define expectOk or expectError")
+			}
+
+		})
+	}
+
+}

internal/daemon/job/job.go

@@ -0,0 +1,108 @@
+package job
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/zrepl/zrepl/internal/daemon/logging"
+	"github.com/zrepl/zrepl/internal/endpoint"
+	"github.com/zrepl/zrepl/internal/logger"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+type Logger = logger.Logger
+
+func GetLogger(ctx context.Context) Logger {
+	return logging.GetLogger(ctx, logging.SubsysJob)
+}
+
+type Job interface {
+	Name() string
+	Run(ctx context.Context)
+	Status() *Status
+	RegisterMetrics(registerer prometheus.Registerer)
+	// Jobs that return a subtree of the dataset hierarchy
+	// must return the root of that subtree as rfs and ok = true
+	OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool)
+	SenderConfig() *endpoint.SenderConfig
+}
+
+type Type string
+
+const (
+	TypeInternal Type = "internal"
+	TypeSnap     Type = "snap"
+	TypePush     Type = "push"
+	TypeSink     Type = "sink"
+	TypePull     Type = "pull"
+	TypeSource   Type = "source"
+)
+
+type Status struct {
+	Type        Type
+	JobSpecific interface{}
+}
+
+func (s *Status) MarshalJSON() ([]byte, error) {
+	typeJson, err := json.Marshal(s.Type)
+	if err != nil {
+		return nil, err
+	}
+	jobJSON, err := json.Marshal(s.JobSpecific)
+	if err != nil {
+		return nil, err
+	}
+	m := map[string]json.RawMessage{
+		"type":         typeJson,
+		string(s.Type): jobJSON,
+	}
+	return json.Marshal(m)
+}
+
+func (s *Status) UnmarshalJSON(in []byte) (err error) {
+	var m map[string]json.RawMessage
+	if err := json.Unmarshal(in, &m); err != nil {
+		return err
+	}
+	tJSON, ok := m["type"]
+	if !ok {
+		return fmt.Errorf("field 'type' not found")
+	}
+	if err := json.Unmarshal(tJSON, &s.Type); err != nil {
+		return err
+	}
+	key := string(s.Type)
+	jobJSON, ok := m[key]
+	if !ok {
+		return fmt.Errorf("field '%s', not found", key)
+	}
+	switch s.Type {
+	case TypeSnap:
+		var st SnapJobStatus
+		err = json.Unmarshal(jobJSON, &st)
+		s.JobSpecific = &st
+
+	case TypePull:
+		fallthrough
+	case TypePush:
+		var st ActiveSideStatus
+		err = json.Unmarshal(jobJSON, &st)
+		s.JobSpecific = &st
+
+	case TypeSource:
+		fallthrough
+	case TypeSink:
+		var st PassiveStatus
+		err = json.Unmarshal(jobJSON, &st)
+		s.JobSpecific = &st
+
+	case TypeInternal:
+		// internal jobs do not report specifics
+	default:
+		err = fmt.Errorf("unknown job type '%s'", key)
+	}
+	return err
+}

internal/daemon/job/passive.go

@@ -0,0 +1,192 @@
+package job
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/zrepl/zrepl/internal/daemon/logging/trace"
+
+	"github.com/zrepl/zrepl/internal/config"
+	"github.com/zrepl/zrepl/internal/daemon/logging"
+	"github.com/zrepl/zrepl/internal/daemon/snapper"
+	"github.com/zrepl/zrepl/internal/endpoint"
+	"github.com/zrepl/zrepl/internal/rpc"
+	"github.com/zrepl/zrepl/internal/transport"
+	"github.com/zrepl/zrepl/internal/transport/fromconfig"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+type PassiveSide struct {
+	mode   passiveMode
+	name   endpoint.JobID
+	listen transport.AuthenticatedListenerFactory
+}
+
+type passiveMode interface {
+	Handler() rpc.Handler
+	RunPeriodic(ctx context.Context)
+	SnapperReport() *snapper.Report // may be nil
+	Type() Type
+}
+
+type modeSink struct {
+	receiverConfig endpoint.ReceiverConfig
+}
+
+func (m *modeSink) Type() Type { return TypeSink }
+
+func (m *modeSink) Handler() rpc.Handler {
+	return endpoint.NewReceiver(m.receiverConfig)
+}
+
+func (m *modeSink) RunPeriodic(_ context.Context)  {}
+func (m *modeSink) SnapperReport() *snapper.Report { return nil }
+
+func modeSinkFromConfig(g *config.Global, in *config.SinkJob, jobID endpoint.JobID) (m *modeSink, err error) {
+	m = &modeSink{}
+
+	m.receiverConfig, err = buildReceiverConfig(in, jobID)
+	if err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}
+
+type modeSource struct {
+	senderConfig *endpoint.SenderConfig
+	snapper      snapper.Snapper
+}
+
+func modeSourceFromConfig(g *config.Global, in *config.SourceJob, jobID endpoint.JobID) (m *modeSource, err error) {
+	// FIXME exact dedup of modePush
+	m = &modeSource{}
+
+	m.senderConfig, err = buildSenderConfig(in, jobID)
+	if err != nil {
+		return nil, errors.Wrap(err, "send options")
+	}
+
+	if m.snapper, err = snapper.FromConfig(g, m.senderConfig.FSF, in.Snapshotting); err != nil {
+		return nil, errors.Wrap(err, "cannot build snapper")
+	}
+
+	return m, nil
+}
+
+func (m *modeSource) Type() Type { return TypeSource }
+
+func (m *modeSource) Handler() rpc.Handler {
+	return endpoint.NewSender(*m.senderConfig)
+}
+
+func (m *modeSource) RunPeriodic(ctx context.Context) {
+	m.snapper.Run(ctx, nil)
+}
+
+func (m *modeSource) SnapperReport() *snapper.Report {
+	r := m.snapper.Report()
+	return &r
+}
+
+func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, configJob interface{}, parseFlags config.ParseFlags) (s *PassiveSide, err error) {
+
+	s = &PassiveSide{}
+
+	s.name, err = endpoint.MakeJobID(in.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid job name")
+	}
+
+	switch v := configJob.(type) {
+	case *config.SinkJob:
+		s.mode, err = modeSinkFromConfig(g, v, s.name) // shadow
+	case *config.SourceJob:
+		s.mode, err = modeSourceFromConfig(g, v, s.name) // shadow
+	}
+	if err != nil {
+		return nil, err // no wrapping necessary
+	}
+
+	if s.listen, err = fromconfig.ListenerFactoryFromConfig(g, in.Serve, parseFlags); err != nil {
+		return nil, errors.Wrap(err, "cannot build listener factory")
+	}
+
+	return s, nil
+}
+
+func (j *PassiveSide) Name() string { return j.name.String() }
+
+type PassiveStatus struct {
+	Snapper *snapper.Report
+}
+
+func (s *PassiveSide) Status() *Status {
+	st := &PassiveStatus{
+		Snapper: s.mode.SnapperReport(),
+	}
+	return &Status{Type: s.mode.Type(), JobSpecific: st}
+}
+
+func (j *PassiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
+	sink, ok := j.mode.(*modeSink)
+	if !ok {
+		_ = j.mode.(*modeSource) // make sure we didn't introduce a new job type
+		return nil, false
+	}
+	return sink.receiverConfig.RootWithoutClientComponent.Copy(), true
+}
+
+func (j *PassiveSide) SenderConfig() *endpoint.SenderConfig {
+	source, ok := j.mode.(*modeSource)
+	if !ok {
+		_ = j.mode.(*modeSink) // make sure we didn't introduce a new job type
+		return nil
+	}
+	return source.senderConfig
+}
+
+func (*PassiveSide) RegisterMetrics(registerer prometheus.Registerer) {}
+
+func (j *PassiveSide) Run(ctx context.Context) {
+	ctx, endTask := trace.WithTaskAndSpan(ctx, "passive-side-job", j.Name())
+	defer endTask()
+	log := GetLogger(ctx)
+	defer log.Info("job exiting")
+	{
+		ctx, endTask := trace.WithTask(ctx, "periodic") // shadowing
+		defer endTask()
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+		go j.mode.RunPeriodic(ctx)
+	}
+
+	handler := j.mode.Handler()
+	if handler == nil {
+		panic(fmt.Sprintf("implementation error: j.mode.Handler() returned nil: %#v", j))
+	}
+
+	ctxInterceptor := func(handlerCtx context.Context, info rpc.HandlerContextInterceptorData, handler func(ctx context.Context)) {
+		// the handlerCtx is clean => need to inherit logging and tracing config from job context
+		handlerCtx = logging.WithInherit(handlerCtx, ctx)
+		handlerCtx = trace.WithInherit(handlerCtx, ctx)
+
+		handlerCtx, endTask := trace.WithTaskAndSpan(handlerCtx, "handler", fmt.Sprintf("job=%q client=%q method=%q", j.Name(), info.ClientIdentity(), info.FullMethod()))
+		defer endTask()
+		handler(handlerCtx)
+	}
+
+	rpcLoggers := rpc.GetLoggersOrPanic(ctx) // WithSubsystemLoggers above
+	server := rpc.NewServer(handler, rpcLoggers, ctxInterceptor)
+
+	listener, err := j.listen()
+	if err != nil {
+		log.WithError(err).Error("cannot listen")
+		return
+	}
+
+	server.Serve(ctx, listener)
+}

internal/daemon/job/reset/reset.go

@@ -0,0 +1,35 @@
+package reset
+
+import (
+	"context"
+	"errors"
+)
+
+type contextKey int
+
+const contextKeyReset contextKey = iota
+
+func Wait(ctx context.Context) <-chan struct{} {
+	wc, ok := ctx.Value(contextKeyReset).(chan struct{})
+	if !ok {
+		wc = make(chan struct{})
+	}
+	return wc
+}
+
+type Func func() error
+
+var AlreadyReset = errors.New("already reset")
+
+func Context(ctx context.Context) (context.Context, Func) {
+	wc := make(chan struct{})
+	wuf := func() error {
+		select {
+		case wc <- struct{}{}:
+			return nil
+		default:
+			return AlreadyReset
+		}
+	}
+	return context.WithValue(ctx, contextKeyReset, wc), wuf
+}

internal/daemon/job/snapjob.go

@@ -0,0 +1,196 @@
+package job
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"sync"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/zrepl/zrepl/internal/daemon/logging/trace"
+	"github.com/zrepl/zrepl/internal/util/bandwidthlimit"
+	"github.com/zrepl/zrepl/internal/util/nodefault"
+
+	"github.com/zrepl/zrepl/internal/config"
+	"github.com/zrepl/zrepl/internal/daemon/filters"
+	"github.com/zrepl/zrepl/internal/daemon/job/wakeup"
+	"github.com/zrepl/zrepl/internal/daemon/pruner"
+	"github.com/zrepl/zrepl/internal/daemon/snapper"
+	"github.com/zrepl/zrepl/internal/endpoint"
+	"github.com/zrepl/zrepl/internal/replication/logic/pdu"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+type SnapJob struct {
+	name     endpoint.JobID
+	fsfilter zfs.DatasetFilter
+	snapper  snapper.Snapper
+
+	prunerFactory *pruner.LocalPrunerFactory
+
+	promPruneSecs *prometheus.HistogramVec // labels: prune_side
+
+	prunerMtx sync.Mutex
+	pruner    *pruner.Pruner
+}
+
+func (j *SnapJob) Name() string { return j.name.String() }
+
+func (j *SnapJob) Type() Type { return TypeSnap }
+
+func snapJobFromConfig(g *config.Global, in *config.SnapJob) (j *SnapJob, err error) {
+	j = &SnapJob{}
+	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot build filesystem filter")
+	}
+	j.fsfilter = fsf
+
+	if j.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
+		return nil, errors.Wrap(err, "cannot build snapper")
+	}
+	j.name, err = endpoint.MakeJobID(in.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid job name")
+	}
+	j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace:   "zrepl",
+		Subsystem:   "pruning",
+		Name:        "time",
+		Help:        "seconds spent in pruner",
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
+	}, []string{"prune_side"})
+	j.prunerFactory, err = pruner.NewLocalPrunerFactory(in.Pruning, j.promPruneSecs)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot build snapjob pruning rules")
+	}
+	return j, nil
+}
+
+func (j *SnapJob) RegisterMetrics(registerer prometheus.Registerer) {
+	registerer.MustRegister(j.promPruneSecs)
+}
+
+type SnapJobStatus struct {
+	Pruning      *pruner.Report
+	Snapshotting *snapper.Report // may be nil
+}
+
+func (j *SnapJob) Status() *Status {
+	s := &SnapJobStatus{}
+	t := j.Type()
+	j.prunerMtx.Lock()
+	if j.pruner != nil {
+		s.Pruning = j.pruner.Report()
+	}
+	j.prunerMtx.Unlock()
+	r := j.snapper.Report()
+	s.Snapshotting = &r
+	return &Status{Type: t, JobSpecific: s}
+}
+
+func (j *SnapJob) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
+	return nil, false
+}
+
+func (j *SnapJob) SenderConfig() *endpoint.SenderConfig { return nil }
+
+func (j *SnapJob) Run(ctx context.Context) {
+	ctx, endTask := trace.WithTaskAndSpan(ctx, "snap-job", j.Name())
+	defer endTask()
+	log := GetLogger(ctx)
+
+	defer log.Info("job exiting")
+
+	periodicDone := make(chan struct{})
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	periodicCtx, endTask := trace.WithTask(ctx, "snapshotting")
+	defer endTask()
+	go j.snapper.Run(periodicCtx, periodicDone)
+
+	invocationCount := 0
+outer:
+	for {
+		log.Info("wait for wakeups")
+		select {
+		case <-ctx.Done():
+			log.WithError(ctx.Err()).Info("context")
+			break outer
+
+		case <-wakeup.Wait(ctx):
+		case <-periodicDone:
+		}
+		invocationCount++
+
+		invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", invocationCount))
+		j.doPrune(invocationCtx)
+		endSpan()
+	}
+}
+
+// Adaptor that implements pruner.History around a pruner.Target.
+// The ReplicationCursor method is Get-op only and always returns
+// the filesystem's most recent version's GUID.
+//
+// TODO:
+// This is a work-around for the current package daemon/pruner
+// and package pruning.Snapshot limitation: they require the
+// `Replicated` getter method be present, but obviously,
+// a local job like SnapJob can't deliver on that.
+// But the pruner.Pruner gives up on an FS if no replication
+// cursor is present, which is why this pruner returns the
+// most recent filesystem version.
+type alwaysUpToDateReplicationCursorHistory struct {
+	// the Target passed as Target to BuildLocalPruner
+	target pruner.Target
+}
+
+var _ pruner.Sender = (*alwaysUpToDateReplicationCursorHistory)(nil)
+
+func (h alwaysUpToDateReplicationCursorHistory) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
+	fsvReq := &pdu.ListFilesystemVersionsReq{
+		Filesystem: req.GetFilesystem(),
+	}
+	res, err := h.target.ListFilesystemVersions(ctx, fsvReq)
+	if err != nil {
+		return nil, err
+	}
+	fsvs := res.GetVersions()
+	if len(fsvs) <= 0 {
+		return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Notexist{Notexist: true}}, nil
+	}
+	// always return must recent version
+	sort.Slice(fsvs, func(i, j int) bool {
+		return fsvs[i].CreateTXG < fsvs[j].CreateTXG
+	})
+	mostRecent := fsvs[len(fsvs)-1]
+	return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Guid{Guid: mostRecent.GetGuid()}}, nil
+}
+
+func (h alwaysUpToDateReplicationCursorHistory) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error) {
+	return h.target.ListFilesystems(ctx, req)
+}
+
+func (j *SnapJob) doPrune(ctx context.Context) {
+	ctx, endSpan := trace.WithSpan(ctx, "snap-job-do-prune")
+	defer endSpan()
+	log := GetLogger(ctx)
+	sender := endpoint.NewSender(endpoint.SenderConfig{
+		JobID: j.name,
+		FSF:   j.fsfilter,
+		// FIXME the following config fields are irrelevant for SnapJob
+		// because the endpoint is only used as pruner.Target.
+		// However, the implementation requires them to be set.
+		Encrypt:        &nodefault.Bool{B: true},
+		BandwidthLimit: bandwidthlimit.NoLimitConfig(),
+	})
+	j.prunerMtx.Lock()
+	j.pruner = j.prunerFactory.BuildLocalPruner(ctx, sender, alwaysUpToDateReplicationCursorHistory{sender})
+	j.prunerMtx.Unlock()
+	log.Info("start pruning")
+	j.pruner.Prune()
+	log.Info("finished pruning")
+}

internal/daemon/job/wakeup/wakeup.go

@@ -0,0 +1,35 @@
+package wakeup
+
+import (
+	"context"
+	"errors"
+)
+
+type contextKey int
+
+const contextKeyWakeup contextKey = iota
+
+func Wait(ctx context.Context) <-chan struct{} {
+	wc, ok := ctx.Value(contextKeyWakeup).(chan struct{})
+	if !ok {
+		wc = make(chan struct{})
+	}
+	return wc
+}
+
+type Func func() error
+
+var AlreadyWokenUp = errors.New("already woken up")
+
+func Context(ctx context.Context) (context.Context, Func) {
+	wc := make(chan struct{})
+	wuf := func() error {
+		select {
+		case wc <- struct{}{}:
+			return nil
+		default:
+			return AlreadyWokenUp
+		}
+	}
+	return context.WithValue(ctx, contextKeyWakeup, wc), wuf
+}

internal/daemon/logging/build_logging.go

@@ -0,0 +1,303 @@
+package logging
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"log/syslog"
+	"os"
+
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/internal/config"
+	"github.com/zrepl/zrepl/internal/daemon/logging/trace"
+	"github.com/zrepl/zrepl/internal/logger"
+	"github.com/zrepl/zrepl/internal/tlsconf"
+)
+
+func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error) {
+
+	outlets := logger.NewOutlets()
+
+	if len(in) == 0 {
+		// Default config
+		out := WriterOutlet{&HumanFormatter{}, os.Stdout}
+		outlets.Add(out, logger.Warn)
+		return outlets, nil
+	}
+
+	var syslogOutlets, stdoutOutlets int
+	for lei, le := range in {
+
+		outlet, minLevel, err := ParseOutlet(le)
+		if err != nil {
+			return nil, errors.Wrapf(err, "cannot parse outlet #%d", lei)
+		}
+		var _ logger.Outlet = WriterOutlet{}
+		var _ logger.Outlet = &SyslogOutlet{}
+		switch outlet.(type) {
+		case *SyslogOutlet:
+			syslogOutlets++
+		case WriterOutlet:
+			stdoutOutlets++
+		}
+
+		outlets.Add(outlet, minLevel)
+
+	}
+
+	if syslogOutlets > 1 {
+		return nil, errors.Errorf("can only define one 'syslog' outlet")
+	}
+	if stdoutOutlets > 1 {
+		return nil, errors.Errorf("can only define one 'stdout' outlet")
+	}
+
+	return outlets, nil
+
+}
+
+type Subsystem string
+
+const (
+	SubsysMeta         Subsystem = "meta"
+	SubsysJob          Subsystem = "job"
+	SubsysReplication  Subsystem = "repl"
+	SubsysEndpoint     Subsystem = "endpoint"
+	SubsysPruning      Subsystem = "pruning"
+	SubsysSnapshot     Subsystem = "snapshot"
+	SubsysHooks        Subsystem = "hook"
+	SubsysTransport    Subsystem = "transport"
+	SubsysTransportMux Subsystem = "transportmux"
+	SubsysRPC          Subsystem = "rpc"
+	SubsysRPCControl   Subsystem = "rpc.ctrl"
+	SubsysRPCData      Subsystem = "rpc.data"
+	SubsysZFSCmd       Subsystem = "zfs.cmd"
+	SubsysTraceData    Subsystem = "trace.data"
+	SubsysPlatformtest Subsystem = "platformtest"
+)
+
+var AllSubsystems = []Subsystem{
+	SubsysMeta,
+	SubsysJob,
+	SubsysReplication,
+	SubsysEndpoint,
+	SubsysPruning,
+	SubsysSnapshot,
+	SubsysHooks,
+	SubsysTransport,
+	SubsysTransportMux,
+	SubsysRPC,
+	SubsysRPCControl,
+	SubsysRPCData,
+	SubsysZFSCmd,
+	SubsysTraceData,
+	SubsysPlatformtest,
+}
+
+type injectedField struct {
+	field  string
+	value  interface{}
+	parent *injectedField
+}
+
+func WithInjectedField(ctx context.Context, field string, value interface{}) context.Context {
+	var parent *injectedField
+	parentI := ctx.Value(contextKeyInjectedField)
+	if parentI != nil {
+		parent = parentI.(*injectedField)
+	}
+	// TODO sanity-check `field` now
+	this := &injectedField{field, value, parent}
+	return context.WithValue(ctx, contextKeyInjectedField, this)
+}
+
+func iterInjectedFields(ctx context.Context, cb func(field string, value interface{})) {
+	injI := ctx.Value(contextKeyInjectedField)
+	if injI == nil {
+		return
+	}
+	inj := injI.(*injectedField)
+	for ; inj != nil; inj = inj.parent {
+		cb(inj.field, inj.value)
+	}
+}
+
+type SubsystemLoggers map[Subsystem]logger.Logger
+
+func SubsystemLoggersWithUniversalLogger(l logger.Logger) SubsystemLoggers {
+	loggers := make(SubsystemLoggers)
+	for _, s := range AllSubsystems {
+		loggers[s] = l
+	}
+	return loggers
+}
+
+func WithLoggers(ctx context.Context, loggers SubsystemLoggers) context.Context {
+	return context.WithValue(ctx, contextKeyLoggers, loggers)
+}
+
+func GetLoggers(ctx context.Context) SubsystemLoggers {
+	loggers, ok := ctx.Value(contextKeyLoggers).(SubsystemLoggers)
+	if !ok {
+		return nil
+	}
+	return loggers
+}
+
+func GetLogger(ctx context.Context, subsys Subsystem) logger.Logger {
+	return getLoggerImpl(ctx, subsys, true)
+}
+
+func getLoggerImpl(ctx context.Context, subsys Subsystem, panicIfEnded bool) logger.Logger {
+	loggers, ok := ctx.Value(contextKeyLoggers).(SubsystemLoggers)
+	if !ok || loggers == nil {
+		return logger.NewNullLogger()
+	}
+	l, ok := loggers[subsys]
+	if !ok {
+		return logger.NewNullLogger()
+	}
+
+	l = l.WithField(SubsysField, subsys)
+
+	l = l.WithField(SpanField, trace.GetSpanStackOrDefault(ctx, *trace.StackKindId, "NOSPAN"))
+
+	fields := make(logger.Fields)
+	iterInjectedFields(ctx, func(field string, value interface{}) {
+		fields[field] = value
+	})
+	l = l.WithFields(fields)
+
+	return l
+}
+
+func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
+	var is string
+	switch j := i.(type) {
+	case string:
+		is = j
+	default:
+		return nil, errors.Errorf("invalid log format: wrong type: %T", i)
+	}
+
+	switch is {
+	case "human":
+		return &HumanFormatter{}, nil
+	case "logfmt":
+		return &LogfmtFormatter{}, nil
+	case "json":
+		return &JSONFormatter{}, nil
+	default:
+		return nil, errors.Errorf("invalid log format: '%s'", is)
+	}
+
+}
+
+func ParseOutlet(in config.LoggingOutletEnum) (o logger.Outlet, level logger.Level, err error) {
+
+	parseCommon := func(common config.LoggingOutletCommon) (logger.Level, EntryFormatter, error) {
+		if common.Level == "" || common.Format == "" {
+			return 0, nil, errors.Errorf("must specify 'level' and 'format' field")
+		}
+
+		minLevel, err := logger.ParseLevel(common.Level)
+		if err != nil {
+			return 0, nil, errors.Wrap(err, "cannot parse 'level' field")
+		}
+		formatter, err := parseLogFormat(common.Format)
+		if err != nil {
+			return 0, nil, errors.Wrap(err, "cannot parse 'formatter' field")
+		}
+		return minLevel, formatter, nil
+	}
+
+	var f EntryFormatter
+
+	switch v := in.Ret.(type) {
+	case *config.StdoutLoggingOutlet:
+		level, f, err = parseCommon(v.LoggingOutletCommon)
+		if err != nil {
+			break
+		}
+		o, err = parseStdoutOutlet(v, f)
+	case *config.TCPLoggingOutlet:
+		level, f, err = parseCommon(v.LoggingOutletCommon)
+		if err != nil {
+			break
+		}
+		o, err = parseTCPOutlet(v, f)
+	case *config.SyslogLoggingOutlet:
+		level, f, err = parseCommon(v.LoggingOutletCommon)
+		if err != nil {
+			break
+		}
+		o, err = parseSyslogOutlet(v, f)
+	default:
+		panic(v)
+	}
+	return o, level, err
+}
+
+func parseStdoutOutlet(in *config.StdoutLoggingOutlet, formatter EntryFormatter) (WriterOutlet, error) {
+	flags := MetadataAll
+	writer := os.Stdout
+	if !isatty.IsTerminal(writer.Fd()) && !in.Time {
+		flags &= ^MetadataTime
+	}
+	if isatty.IsTerminal(writer.Fd()) && !in.Color {
+		flags &= ^MetadataColor
+	}
+
+	formatter.SetMetadataFlags(flags)
+	return WriterOutlet{
+		formatter,
+		os.Stdout,
+	}, nil
+}
+
+func parseTCPOutlet(in *config.TCPLoggingOutlet, formatter EntryFormatter) (out *TCPOutlet, err error) {
+	var tlsConfig *tls.Config
+	if in.TLS != nil {
+		tlsConfig, err = func(m *config.TCPLoggingOutletTLS, host string) (*tls.Config, error) {
+			clientCert, err := tls.LoadX509KeyPair(m.Cert, m.Key)
+			if err != nil {
+				return nil, errors.Wrap(err, "cannot load client cert")
+			}
+
+			var rootCAs *x509.CertPool
+			if m.CA == "" {
+				if rootCAs, err = x509.SystemCertPool(); err != nil {
+					return nil, errors.Wrap(err, "cannot open system cert pool")
+				}
+			} else {
+				rootCAs, err = tlsconf.ParseCAFile(m.CA)
+				if err != nil {
+					return nil, errors.Wrap(err, "cannot parse CA cert")
+				}
+			}
+			if rootCAs == nil {
+				panic("invariant violated")
+			}
+
+			return tlsconf.ClientAuthClient(host, rootCAs, clientCert)
+		}(in.TLS, in.Address)
+		if err != nil {
+			return nil, errors.New("cannot not parse TLS config in field 'tls'")
+		}
+	}
+
+	formatter.SetMetadataFlags(MetadataAll)
+	return NewTCPOutlet(formatter, in.Net, in.Address, tlsConfig, in.RetryInterval), nil
+
+}
+
+func parseSyslogOutlet(in *config.SyslogLoggingOutlet, formatter EntryFormatter) (out *SyslogOutlet, err error) {
+	out = &SyslogOutlet{}
+	out.Formatter = formatter
+	out.Formatter.SetMetadataFlags(MetadataNone)
+	out.Facility = syslog.Priority(*in.Facility)
+	out.RetryInterval = in.RetryInterval
+	return out, nil
+}

internal/daemon/logging/logging_context.go

@@ -0,0 +1,24 @@
+package logging
+
+import "context"
+
+type contextKey int
+
+const (
+	contextKeyLoggers contextKey = 1 + iota
+	contextKeyInjectedField
+)
+
+var contextKeys = []contextKey{
+	contextKeyLoggers,
+	contextKeyInjectedField,
+}
+
+func WithInherit(ctx, inheritFrom context.Context) context.Context {
+	for _, k := range contextKeys {
+		if v := inheritFrom.Value(k); v != nil {
+			ctx = context.WithValue(ctx, k, v) // no shadow
+		}
+	}
+	return ctx
+}

internal/daemon/logging/logging_formatters.go

@@ -0,0 +1,221 @@
+package logging
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/fatih/color"
+	"github.com/go-logfmt/logfmt"
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/internal/logger"
+)
+
+const (
+	FieldLevel   = "level"
+	FieldMessage = "msg"
+	FieldTime    = "time"
+)
+
+const (
+	JobField    string = "job"
+	SubsysField string = "subsystem"
+	SpanField   string = "span"
+)
+
+type MetadataFlags int64
+
+const (
+	MetadataTime MetadataFlags = 1 << iota
+	MetadataLevel
+	MetadataColor
+
+	MetadataNone MetadataFlags = 0
+	MetadataAll  MetadataFlags = ^0
+)
+
+type NoFormatter struct{}
+
+func (f NoFormatter) SetMetadataFlags(flags MetadataFlags) {}
+
+func (f NoFormatter) Format(e *logger.Entry) ([]byte, error) {
+	return []byte(e.Message), nil
+}
+
+type HumanFormatter struct {
+	metadataFlags MetadataFlags
+	ignoreFields  map[string]bool
+}
+
+const HumanFormatterDateFormat = time.RFC3339
+
+func (f *HumanFormatter) SetMetadataFlags(flags MetadataFlags) {
+	f.metadataFlags = flags
+}
+
+func (f *HumanFormatter) SetIgnoreFields(ignore []string) {
+	if ignore == nil {
+		f.ignoreFields = nil
+		return
+	}
+	f.ignoreFields = make(map[string]bool, len(ignore))
+
+	for _, field := range ignore {
+		f.ignoreFields[field] = true
+	}
+}
+
+func (f *HumanFormatter) ignored(field string) bool {
+	return f.ignoreFields != nil && f.ignoreFields[field]
+}
+
+func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
+
+	var line bytes.Buffer
+	col := color.New()
+	if f.metadataFlags&MetadataColor != 0 {
+		col = e.Color()
+	}
+
+	if f.metadataFlags&MetadataTime != 0 {
+		fmt.Fprintf(&line, "%s ", e.Time.Format(HumanFormatterDateFormat))
+	}
+	if f.metadataFlags&MetadataLevel != 0 {
+		fmt.Fprintf(&line, "[%s]", col.Sprint(e.Level.Short()))
+	}
+
+	prefixFields := []string{JobField, SubsysField, SpanField}
+	prefixed := make(map[string]bool, len(prefixFields)+2)
+	for _, field := range prefixFields {
+		val, ok := e.Fields[field]
+		if !ok {
+			continue
+		}
+		if !f.ignored(field) {
+			fmt.Fprintf(&line, "[%s]", col.Sprint(val))
+			prefixed[field] = true
+		}
+	}
+
+	if line.Len() > 0 {
+		fmt.Fprint(&line, ": ")
+	}
+	fmt.Fprint(&line, e.Message)
+
+	if len(e.Fields)-len(prefixed) > 0 {
+		for field, value := range e.Fields {
+			if prefixed[field] || f.ignored(field) {
+				continue
+			}
+			fmt.Fprintf(&line, " %s=%q", col.Sprint(field), fmt.Sprint(value))
+		}
+	}
+
+	return line.Bytes(), nil
+}
+
+type JSONFormatter struct {
+	metadataFlags MetadataFlags
+}
+
+func (f *JSONFormatter) SetMetadataFlags(flags MetadataFlags) {
+	f.metadataFlags = flags
+}
+
+func (f *JSONFormatter) Format(e *logger.Entry) ([]byte, error) {
+	data := make(logger.Fields, len(e.Fields)+3)
+	for k, v := range e.Fields {
+		switch v := v.(type) {
+		case error:
+			// Otherwise errors are ignored by `encoding/json`
+			// https://github.com/sirupsen/logrus/issues/137
+			data[k] = v.Error()
+		default:
+			_, err := json.Marshal(v)
+			if err != nil {
+				return nil, errors.Errorf("field is not JSON encodable: %s", k)
+			}
+			data[k] = v
+		}
+	}
+
+	data[FieldMessage] = e.Message
+	data[FieldTime] = e.Time.Format(time.RFC3339)
+	data[FieldLevel] = e.Level
+
+	return json.Marshal(data)
+
+}
+
+type LogfmtFormatter struct {
+	metadataFlags MetadataFlags
+}
+
+func (f *LogfmtFormatter) SetMetadataFlags(flags MetadataFlags) {
+	f.metadataFlags = flags
+}
+
+func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
+	var buf bytes.Buffer
+	enc := logfmt.NewEncoder(&buf)
+
+	if f.metadataFlags&MetadataTime != 0 {
+		err := enc.EncodeKeyval(FieldTime, e.Time)
+		if err != nil {
+			return nil, errors.Wrap(err, "logfmt: encode time")
+		}
+	}
+	if f.metadataFlags&MetadataLevel != 0 {
+		err := enc.EncodeKeyval(FieldLevel, e.Level)
+		if err != nil {
+			return nil, errors.Wrap(err, "logfmt: encode level")
+		}
+	}
+
+	// at least try and put job and task in front
+	prefixed := make(map[string]bool, 3)
+	prefix := []string{JobField, SubsysField, SpanField}
+	for _, pf := range prefix {
+		v, ok := e.Fields[pf]
+		if !ok {
+			break
+		}
+		if err := logfmtTryEncodeKeyval(enc, pf, v); err != nil {
+			return nil, err // unlikely
+		}
+		prefixed[pf] = true
+	}
+
+	err := enc.EncodeKeyval(FieldMessage, e.Message)
+	if err != nil {
+		return nil, errors.Wrap(err, "logfmt: encode message")
+	}
+	for k, v := range e.Fields {
+		if !prefixed[k] {
+			if err := logfmtTryEncodeKeyval(enc, k, v); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return buf.Bytes(), nil
+}
+
+func logfmtTryEncodeKeyval(enc *logfmt.Encoder, field, value interface{}) error {
+
+	err := enc.EncodeKeyval(field, value)
+	switch err {
+	case nil: // ok
+		return nil
+	case logfmt.ErrUnsupportedValueType:
+		err := enc.EncodeKeyval(field, fmt.Sprintf("<%T>", value))
+		if err != nil {
+			return errors.Wrap(err, "cannot encode unsupported value type Go type")
+		}
+		return nil
+	}
+	return errors.Wrapf(err, "cannot encode field '%s'", field)
+
+}

internal/daemon/logging/logging_outlets.go

@@ -0,0 +1,174 @@
+package logging
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"io"
+	"log/syslog"
+	"net"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/internal/logger"
+)
+
+type EntryFormatter interface {
+	SetMetadataFlags(flags MetadataFlags)
+	Format(e *logger.Entry) ([]byte, error)
+}
+
+type WriterOutlet struct {
+	formatter EntryFormatter
+	writer    io.Writer
+}
+
+func (h WriterOutlet) WriteEntry(entry logger.Entry) error {
+	bytes, err := h.formatter.Format(&entry)
+	if err != nil {
+		return err
+	}
+	_, err = h.writer.Write(bytes)
+	if err != nil {
+		return err
+	}
+	_, err = h.writer.Write([]byte("\n"))
+	return err
+}
+
+type TCPOutlet struct {
+	formatter EntryFormatter
+	// Specifies how much time must pass between a connection error and a reconnection attempt
+	// Log entries written to the outlet during this time interval are silently dropped.
+	connect   func(ctx context.Context) (net.Conn, error)
+	entryChan chan *bytes.Buffer
+}
+
+func NewTCPOutlet(formatter EntryFormatter, network, address string, tlsConfig *tls.Config, retryInterval time.Duration) *TCPOutlet {
+
+	connect := func(ctx context.Context) (conn net.Conn, err error) {
+		deadl, ok := ctx.Deadline()
+		if !ok {
+			deadl = time.Time{}
+		}
+		dialer := net.Dialer{
+			Deadline: deadl,
+		}
+		if tlsConfig != nil {
+			conn, err = tls.DialWithDialer(&dialer, network, address, tlsConfig)
+		} else {
+			conn, err = dialer.DialContext(ctx, network, address)
+		}
+		return
+	}
+
+	entryChan := make(chan *bytes.Buffer, 1) // allow one message in flight while previous is in io.Copy()
+
+	o := &TCPOutlet{
+		formatter: formatter,
+		connect:   connect,
+		entryChan: entryChan,
+	}
+
+	go o.outLoop(retryInterval)
+
+	return o
+}
+
+// FIXME: use this method
+func (h *TCPOutlet) Close() {
+	close(h.entryChan)
+}
+
+func (h *TCPOutlet) outLoop(retryInterval time.Duration) {
+
+	var retry time.Time
+	var conn net.Conn
+	for msg := range h.entryChan {
+		var err error
+		for conn == nil {
+			time.Sleep(time.Until(retry))
+			ctx, cancel := context.WithDeadline(context.TODO(), time.Now().Add(retryInterval))
+			conn, err = h.connect(ctx)
+			cancel()
+			if err != nil {
+				retry = time.Now().Add(retryInterval)
+				conn = nil
+			}
+		}
+		err = conn.SetWriteDeadline(time.Now().Add(retryInterval))
+		if err == nil {
+			_, err = io.Copy(conn, msg)
+		}
+		if err != nil {
+			retry = time.Now().Add(retryInterval)
+			conn.Close()
+			conn = nil
+		}
+	}
+}
+
+func (h *TCPOutlet) WriteEntry(e logger.Entry) error {
+
+	ebytes, err := h.formatter.Format(&e)
+	if err != nil {
+		return err
+	}
+
+	buf := new(bytes.Buffer)
+	buf.Write(ebytes)
+	buf.WriteString("\n")
+
+	select {
+	case h.entryChan <- buf:
+		return nil
+	default:
+		return errors.New("connection broken or not fast enough")
+	}
+}
+
+type SyslogOutlet struct {
+	Formatter          EntryFormatter
+	RetryInterval      time.Duration
+	Facility           syslog.Priority
+	writer             *syslog.Writer
+	lastConnectAttempt time.Time
+}
+
+func (o *SyslogOutlet) WriteEntry(entry logger.Entry) error {
+
+	bytes, err := o.Formatter.Format(&entry)
+	if err != nil {
+		return err
+	}
+
+	s := string(bytes)
+
+	if o.writer == nil {
+		now := time.Now()
+		if now.Sub(o.lastConnectAttempt) < o.RetryInterval {
+			return nil // not an error toward logger
+		}
+		o.writer, err = syslog.New(o.Facility, "zrepl")
+		o.lastConnectAttempt = time.Now()
+		if err != nil {
+			o.writer = nil
+			return err
+		}
+	}
+
+	switch entry.Level {
+	case logger.Debug:
+		return o.writer.Debug(s)
+	case logger.Info:
+		return o.writer.Info(s)
+	case logger.Warn:
+		return o.writer.Warning(s)
+	case logger.Error:
+		return o.writer.Err(s)
+	default:
+		return o.writer.Err(s) // write as error as reaching this case is in fact an error
+	}
+
+}

internal/daemon/logging/trace/trace.go

@@ -0,0 +1,422 @@
+// package trace provides activity tracing via ctx through Tasks and Spans
+//
+// # Basic Concepts
+//
+// Tracing can be used to identify where a piece of code spends its time.
+//
+// The Go standard library provides package runtime/trace which is useful to identify CPU bottlenecks or
+// to understand what happens inside the Go runtime.
+// However, it is not ideal for application level tracing, in particular if those traces should be understandable
+// to tech-savvy users (albeit not developers).
+//
+// This package provides the concept of Tasks and Spans to express what activity is happening within an application:
+// - Neither task nor span is really tangible but instead contained within the context.Context tree
+// - Tasks represent concurrent activity (i.e. goroutines).
+// - Spans represent a semantic stack trace within a task.
+// As a consequence, whenever a context is propagated across goroutine boundary, you need to create a child task:
+//
+//	go func(ctx context.Context) {
+//	  ctx, endTask = WithTask(ctx, "what-happens-inside-the-child-task")
+//	  defer endTask()
+//	  // ...
+//	}(ctx)
+//
+// Within the task, you can open up a hierarchy of spans.
+// In contrast to tasks, which have can multiple concurrently running child tasks,
+// spans must nest and not cross the goroutine boundary.
+//
+//	ctx, endSpan = WithSpan(ctx, "copy-dir")
+//	defer endSpan()
+//	for _, f := range dir.Files() {
+//	  func() {
+//	    ctx, endSpan := WithSpan(ctx, fmt.Sprintf("copy-file %q", f))
+//	    defer endspan()
+//	    b, _ := ioutil.ReadFile(f)
+//	    _ = ioutil.WriteFile(f + ".copy", b, 0600)
+//	  }()
+//	}
+//
+// In combination:
+//
+//	ctx, endTask = WithTask(ctx, "copy-dirs")
+//	defer endTask()
+//	for i := range dirs {
+//	  go func(dir string) {
+//	    ctx, endTask := WithTask(ctx, "copy-dir")
+//	    defer endTask()
+//	    for _, f := range filesIn(dir) {
+//	      func() {
+//	        ctx, endSpan := WithSpan(ctx, fmt.Sprintf("copy-file %q", f))
+//	        defer endspan()
+//	        b, _ := ioutil.ReadFile(f)
+//	        _ = ioutil.WriteFile(f + ".copy", b, 0600)
+//	      }()
+//	    }
+//	  }()
+//	}
+//
+// Note that a span ends at the time you call endSpan - not before and not after that.
+// If you violate the stack-like nesting of spans by forgetting an endSpan() invocation,
+// the out-of-order endSpan() will panic.
+//
+// A similar rule applies to the endTask closure returned by WithTask:
+// If a task has live child tasks at the time you call endTask(), the call will panic.
+//
+// Recovering from endSpan() or endTask() panics will corrupt the trace stack and lead to corrupt tracefile output.
+//
+// # Best Practices For Naming Tasks And Spans
+//
+// Tasks should always have string constants as names, and must not contain the `#` character. WHy?
+// First, the visualization by chrome://tracing draws a horizontal bar for each task in the trace.
+// Also, the package appends `#NUM` for each concurrently running instance of a task name.
+// Note that the `#NUM` suffix will be reused if a task has ended, in order to avoid an
+// infinite number of horizontal bars in the visualization.
+//
+// # Chrome-compatible Tracefile Support
+//
+// The activity trace generated by usage of WithTask and WithSpan can be rendered to a JSON output file
+// that can be loaded into chrome://tracing .
+// Apart from function GetSpanStackOrDefault, this is the main benefit of this package.
+//
+// First, there is a convenience environment variable 'ZREPL_ACTIVITY_TRACE' that can be set to an output path.
+// From process start onward, a trace is written to that path.
+//
+// More consumers can attach to the activity trace through the ChrometraceClientWebsocketHandler websocket handler.
+//
+// If a write error is encountered with any consumer (including the env-var based one), the consumer is closed and
+// will not receive further trace output.
+package trace
+
+import (
+	"context"
+	"fmt"
+	"os"
+	runtimedebug "runtime/debug"
+	"strings"
+	"time"
+
+	"github.com/kr/pretty"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/zrepl/zrepl/internal/util/chainlock"
+)
+
+var metrics struct {
+	activeTasks prometheus.Gauge
+}
+var taskNamer *uniqueConcurrentTaskNamer = newUniqueTaskNamer()
+
+func init() {
+	metrics.activeTasks = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "zrepl",
+		Subsystem: "trace",
+		Name:      "active_tasks",
+		Help:      "number of active (tracing-level) tasks in the daemon",
+	})
+}
+
+func RegisterMetrics(r prometheus.Registerer) {
+	r.MustRegister(metrics.activeTasks)
+}
+
+type traceNode struct {
+	// NOTE: members with prefix debug are only valid if the package variable debugEnabled is true.
+
+	id                 string
+	annotation         string
+	parentTask         *traceNode
+	debugCreationStack string // debug: stack trace of when the traceNode was created
+
+	mtx chainlock.L
+
+	activeChildTasks      int32               // only for task nodes, insignificant for span nodes
+	debugActiveChildTasks map[*traceNode]bool // debug: set of active child tasks
+	parentSpan            *traceNode
+	activeChildSpan       *traceNode // nil if task or span doesn't have an active child span
+
+	startedAt time.Time
+	endedAt   time.Time
+}
+
+func (s *traceNode) StartedAt() time.Time { return s.startedAt }
+func (s *traceNode) EndedAt() time.Time   { return s.endedAt }
+
+// caller must hold mtx
+func (s *traceNode) debugString() string {
+	if !debugEnabled {
+		return "<debugEnabled=false>"
+	}
+	return pretty.Sprint(s)
+}
+
+// Returned from WithTask or WithSpan.
+// Must be called once the task or span ends.
+// See package-level docs for nesting rules.
+// Wrong call order / forgetting to call it will result in panics.
+type DoneFunc func()
+
+var ErrTaskStillHasActiveChildTasks = fmt.Errorf("end task: task still has active child tasks")
+
+// Start a new root task or create a child task of an existing task.
+//
+// This is required when starting a new goroutine and
+// passing an existing task context to it.
+//
+// taskName should be a constantand must not contain '#'
+//
+// The implementation ensures that,
+// if multiple tasks with the same name exist simultaneously,
+// a unique suffix is appended to uniquely identify the task opened with this function.
+func WithTask(ctx context.Context, taskName string) (context.Context, DoneFunc) {
+
+	var parentTask *traceNode
+	nodeI := ctx.Value(contextKeyTraceNode)
+	if nodeI != nil {
+		node := nodeI.(*traceNode)
+		if node.parentSpan != nil {
+			parentTask = node.parentTask
+		} else {
+			parentTask = node
+		}
+	}
+	// find the first ancestor that hasn't ended yet (nil if need be)
+	if parentTask != nil {
+		parentTask.mtx.Lock()
+	}
+	for parentTask != nil && !parentTask.endedAt.IsZero() {
+		thisParent := parentTask
+		parentTask = parentTask.parentTask
+		// lock parent first such that it isn't modified by other callers to thisParent
+		if parentTask != nil {
+			parentTask.mtx.Lock()
+		}
+		thisParent.mtx.Unlock()
+	}
+	// invariant: either parentTask != nil and we hold the lock on parentTask, or parentTask is nil
+
+	taskName, taskNameDone := taskNamer.UniqueConcurrentTaskName(taskName)
+
+	this := &traceNode{
+		id:               genID(),
+		annotation:       taskName,
+		parentTask:       parentTask,
+		activeChildTasks: 0,
+		parentSpan:       nil,
+		activeChildSpan:  nil,
+
+		startedAt: time.Now(),
+		endedAt:   time.Time{},
+	}
+
+	if debugEnabled {
+		this.debugCreationStack = string(runtimedebug.Stack())
+		this.debugActiveChildTasks = map[*traceNode]bool{}
+	}
+
+	if parentTask != nil {
+		this.parentTask.activeChildTasks++
+		if debugEnabled {
+			this.parentTask.debugActiveChildTasks[this] = true
+		}
+		parentTask.mtx.Unlock()
+	}
+
+	ctx = context.WithValue(ctx, contextKeyTraceNode, this)
+
+	chrometraceBeginTask(this)
+
+	metrics.activeTasks.Inc()
+
+	endTaskFunc := func() {
+
+		// only hold locks while manipulating the tree
+		// (trace writer might block too long and unlike spans, tasks are updated concurrently)
+		alreadyEnded := func() (alreadyEnded bool) {
+			if this.parentTask != nil {
+				defer this.parentTask.mtx.Lock().Unlock()
+			}
+			defer this.mtx.Lock().Unlock()
+
+			if this.activeChildTasks != 0 {
+				if debugEnabled {
+					// the debugString can be quite long and panic won't print it completely
+					fmt.Fprintf(os.Stderr, "going to panic due to activeChildTasks:\n%s\n", this.debugString())
+				}
+				panic(errors.WithMessagef(ErrTaskStillHasActiveChildTasks, "end task: %v active child tasks (run daemon with env var %s=1 for more details)\n", this.activeChildTasks, debugEnabledEnvVar))
+			}
+
+			// support idempotent task ends
+			if !this.endedAt.IsZero() {
+				return true
+			}
+			this.endedAt = time.Now()
+
+			if this.parentTask != nil {
+				this.parentTask.activeChildTasks--
+				if debugEnabled {
+					delete(this.parentTask.debugActiveChildTasks, this)
+				}
+				if this.parentTask.activeChildTasks < 0 {
+					if debugEnabled {
+						// the debugString can be quite long and panic won't print it completely
+						fmt.Fprintf(os.Stderr, "going to panic due to activeChildTasks < 0:\n%s\n", this.parentTask.debugString())
+					}
+					panic(fmt.Sprintf("impl error: parent task with negative activeChildTasks count: %v", this.parentTask.activeChildTasks))
+				}
+			}
+			return false
+		}()
+		if alreadyEnded {
+			return
+		}
+
+		chrometraceEndTask(this)
+
+		metrics.activeTasks.Dec()
+
+		taskNameDone()
+	}
+
+	return ctx, endTaskFunc
+}
+
+var ErrAlreadyActiveChildSpan = fmt.Errorf("create child span: span already has an active child span")
+var ErrSpanStillHasActiveChildSpan = fmt.Errorf("end span: span still has active child spans")
+
+// Start a new span.
+// Important: ctx must have an active task (see WithTask)
+func WithSpan(ctx context.Context, annotation string) (context.Context, DoneFunc) {
+	var parentSpan, parentTask *traceNode
+	nodeI := ctx.Value(contextKeyTraceNode)
+	if nodeI != nil {
+		parentSpan = nodeI.(*traceNode)
+		if parentSpan.parentSpan == nil {
+			parentTask = parentSpan
+		} else {
+			parentTask = parentSpan.parentTask
+		}
+	} else {
+		panic("must be called from within a task")
+	}
+
+	this := &traceNode{
+		id:              genID(),
+		annotation:      annotation,
+		parentTask:      parentTask,
+		parentSpan:      parentSpan,
+		activeChildSpan: nil,
+
+		startedAt: time.Now(),
+		endedAt:   time.Time{},
+	}
+
+	if debugEnabled {
+		this.debugCreationStack = string(runtimedebug.Stack())
+	}
+
+	parentSpan.mtx.HoldWhile(func() {
+		if parentSpan.activeChildSpan != nil {
+			panic(ErrAlreadyActiveChildSpan)
+		}
+		parentSpan.activeChildSpan = this
+	})
+
+	ctx = context.WithValue(ctx, contextKeyTraceNode, this)
+	chrometraceBeginSpan(this)
+	callbackEndSpan := callbackBeginSpan(ctx)
+
+	endTaskFunc := func() {
+
+		defer parentSpan.mtx.Lock().Unlock()
+		if parentSpan.activeChildSpan != this && this.endedAt.IsZero() {
+			panic("impl error: activeChildSpan should not change while != nil because there can only be one")
+		}
+
+		defer this.mtx.Lock().Unlock()
+		if this.activeChildSpan != nil {
+			panic(ErrSpanStillHasActiveChildSpan)
+		}
+
+		if !this.endedAt.IsZero() {
+			return // support idempotent span ends
+		}
+
+		parentSpan.activeChildSpan = nil
+		this.endedAt = time.Now()
+
+		chrometraceEndSpan(this)
+		callbackEndSpan(this)
+	}
+
+	return ctx, endTaskFunc
+}
+
+type StackKind struct {
+	symbolizeTask func(t *traceNode) string
+	symbolizeSpan func(s *traceNode) string
+}
+
+var (
+	StackKindId = &StackKind{
+		symbolizeTask: func(t *traceNode) string { return t.id },
+		symbolizeSpan: func(s *traceNode) string { return s.id },
+	}
+	SpanStackKindCombined = &StackKind{
+		symbolizeTask: func(t *traceNode) string { return fmt.Sprintf("(%s %q)", t.id, t.annotation) },
+		symbolizeSpan: func(s *traceNode) string { return fmt.Sprintf("(%s %q)", s.id, s.annotation) },
+	}
+	SpanStackKindAnnotation = &StackKind{
+		symbolizeTask: func(t *traceNode) string { return t.annotation },
+		symbolizeSpan: func(s *traceNode) string { return s.annotation },
+	}
+)
+
+func (n *traceNode) task() *traceNode {
+	task := n.parentTask
+	if n.parentSpan == nil {
+		task = n
+	}
+	return task
+}
+
+func (n *traceNode) TaskName() string {
+	task := n.task()
+	return task.annotation
+}
+
+func (this *traceNode) TaskAndSpanStack(kind *StackKind) (spanIdStack string) {
+	task := this.task()
+
+	var spansInTask []*traceNode
+	for s := this; s != nil; s = s.parentSpan {
+		spansInTask = append(spansInTask, s)
+	}
+
+	var tasks []*traceNode
+	for t := task; t != nil; t = t.parentTask {
+		tasks = append(tasks, t)
+	}
+
+	var taskIdsRev []string
+	for i := len(tasks) - 1; i >= 0; i-- {
+		taskIdsRev = append(taskIdsRev, kind.symbolizeTask(tasks[i]))
+	}
+
+	var spanIdsRev []string
+	for i := len(spansInTask) - 1; i >= 0; i-- {
+		spanIdsRev = append(spanIdsRev, kind.symbolizeSpan(spansInTask[i]))
+	}
+
+	taskStack := strings.Join(taskIdsRev, "$")
+	return fmt.Sprintf("%s$%s", taskStack, strings.Join(spanIdsRev, "."))
+}
+
+func GetSpanStackOrDefault(ctx context.Context, kind StackKind, def string) string {
+	if nI := ctx.Value(contextKeyTraceNode); nI != nil {
+		n := nI.(*traceNode)
+		return n.TaskAndSpanStack(StackKindId)
+	} else {
+		return def
+	}
+}

internal/daemon/logging/trace/trace_callback.go

@@ -0,0 +1,54 @@
+package trace
+
+import (
+	"context"
+	"time"
+
+	"github.com/zrepl/zrepl/internal/util/chainlock"
+)
+
+type SpanInfo interface {
+	StartedAt() time.Time
+	EndedAt() time.Time
+	TaskAndSpanStack(kind *StackKind) string
+}
+
+type Callback struct {
+	OnBegin func(ctx context.Context)
+	OnEnd   func(ctx context.Context, spanInfo SpanInfo)
+}
+
+var callbacks struct {
+	mtx chainlock.L
+	cs  []Callback
+}
+
+func RegisterCallback(c Callback) {
+	callbacks.mtx.HoldWhile(func() {
+		callbacks.cs = append(callbacks.cs, c)
+	})
+}
+
+func callbackBeginSpan(ctx context.Context) func(SpanInfo) {
+	// capture the current state of callbacks into a local variable
+	// this is safe because the slice is append-only and immutable
+
+	// (it is important that a callback registered _after_ callbackBeginSpin is called does not get called on OnEnd)
+
+	var cbs []Callback
+	callbacks.mtx.HoldWhile(func() {
+		cbs = callbacks.cs
+	})
+	for _, cb := range cbs {
+		if cb.OnBegin != nil {
+			cb.OnBegin(ctx)
+		}
+	}
+	return func(spanInfo SpanInfo) {
+		for _, cb := range cbs {
+			if cb.OnEnd != nil {
+				cb.OnEnd(ctx, spanInfo)
+			}
+		}
+	}
+}

internal/daemon/logging/trace/trace_chrometrace.go

@@ -0,0 +1,230 @@
+package trace
+
+// The functions in this file are concerned with the generation
+// of trace files based on the information from WithTask and WithSpan.
+//
+// The emitted trace files are open-ended array of JSON objects
+// that follow the Chrome trace file format:
+//   https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview
+//
+// The emitted JSON can be loaded into Chrome's chrome://tracing view.
+//
+// The trace file can be written to a file whose path is specified in an env file,
+// and be written to web sockets established on ChrometraceHttpHandler
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"sync"
+	"sync/atomic"
+
+	"golang.org/x/net/websocket"
+
+	"github.com/zrepl/zrepl/internal/util/envconst"
+)
+
+var chrometracePID string
+
+func init() {
+	var err error
+	chrometracePID, err = os.Hostname()
+	if err != nil {
+		panic(err)
+	}
+	chrometracePID = fmt.Sprintf("%q", chrometracePID)
+}
+
+type chrometraceEvent struct {
+	Cat                       string   `json:"cat,omitempty"`
+	Name                      string   `json:"name"`
+	Stack                     []string `json:"stack,omitempty"`
+	Phase                     string   `json:"ph"`
+	TimestampUnixMicroseconds int64    `json:"ts"`
+	DurationMicroseconds      int64    `json:"dur,omitempty"`
+	Pid                       string   `json:"pid"`
+	Tid                       string   `json:"tid"`
+	Id                        string   `json:"id,omitempty"`
+}
+
+func chrometraceBeginSpan(s *traceNode) {
+	taskName := s.TaskName()
+	chrometraceWrite(chrometraceEvent{
+		Name:                      s.annotation,
+		Phase:                     "B",
+		TimestampUnixMicroseconds: s.startedAt.UnixNano() / 1000,
+		Pid:                       chrometracePID,
+		Tid:                       taskName,
+	})
+}
+
+func chrometraceEndSpan(s *traceNode) {
+	taskName := s.TaskName()
+	chrometraceWrite(chrometraceEvent{
+		Name:                      s.annotation,
+		Phase:                     "E",
+		TimestampUnixMicroseconds: s.endedAt.UnixNano() / 1000,
+		Pid:                       chrometracePID,
+		Tid:                       taskName,
+	})
+}
+
+var chrometraceFlowId uint64
+
+func chrometraceBeginTask(s *traceNode) {
+	chrometraceBeginSpan(s)
+
+	if s.parentTask == nil {
+		return
+	}
+	// beginning of a task that has a parent
+	// => use flow events to link parent and child
+
+	flowId := atomic.AddUint64(&chrometraceFlowId, 1)
+	flowIdStr := fmt.Sprintf("%x", flowId)
+
+	parentTask := s.parentTask.TaskName()
+	chrometraceWrite(chrometraceEvent{
+		Cat:                       "task", // seems to be necessary, otherwise the GUI shows some `indexOf` JS error
+		Name:                      "child-task",
+		Phase:                     "s",
+		TimestampUnixMicroseconds: s.startedAt.UnixNano() / 1000, // yes, the child's timestamp (=> from-point of the flow line is at right x-position of parent's bar)
+		Pid:                       chrometracePID,
+		Tid:                       parentTask,
+		Id:                        flowIdStr,
+	})
+
+	childTask := s.TaskName()
+	if parentTask == childTask {
+		panic(parentTask)
+	}
+	chrometraceWrite(chrometraceEvent{
+		Cat:                       "task", // seems to be necessary, otherwise the GUI shows some `indexOf` JS error
+		Name:                      "child-task",
+		Phase:                     "f",
+		TimestampUnixMicroseconds: s.startedAt.UnixNano() / 1000,
+		Pid:                       chrometracePID,
+		Tid:                       childTask,
+		Id:                        flowIdStr,
+	})
+}
+
+func chrometraceEndTask(s *traceNode) {
+	chrometraceEndSpan(s)
+}
+
+type chrometraceConsumerRegistration struct {
+	w io.Writer
+	// errored must have capacity 1, the writer thread will send to it non-blocking, then close it
+	errored chan error
+}
+
+var chrometraceConsumers struct {
+	register  chan chrometraceConsumerRegistration
+	consumers map[chrometraceConsumerRegistration]bool
+	write     chan []byte
+}
+
+func init() {
+	chrometraceConsumers.register = make(chan chrometraceConsumerRegistration)
+	chrometraceConsumers.consumers = make(map[chrometraceConsumerRegistration]bool)
+	chrometraceConsumers.write = make(chan []byte)
+	go func() {
+		kickConsumer := func(c chrometraceConsumerRegistration, err error) {
+			debug("chrometrace kicking consumer %#v after error %v", c, err)
+			select {
+			case c.errored <- err:
+			default:
+			}
+			close(c.errored)
+			delete(chrometraceConsumers.consumers, c)
+		}
+		for {
+			select {
+			case reg := <-chrometraceConsumers.register:
+				debug("registered chrometrace consumer %#v", reg)
+				chrometraceConsumers.consumers[reg] = true
+				n, err := reg.w.Write([]byte("[\n"))
+				if err != nil {
+					kickConsumer(reg, err)
+				} else if n != 2 {
+					kickConsumer(reg, fmt.Errorf("short write: %v", n))
+				}
+				// successfully registered
+
+			case buf := <-chrometraceConsumers.write:
+				debug("chrometrace write request: %s", string(buf))
+				var r bytes.Reader
+				for c := range chrometraceConsumers.consumers {
+					r.Reset(buf)
+					n, err := io.Copy(c.w, &r)
+					debug("chrometrace wrote n=%v bytes to consumer %#v", n, c)
+					if err != nil {
+						kickConsumer(c, err)
+					}
+				}
+			}
+		}
+	}()
+}
+
+func chrometraceWrite(i interface{}) {
+	var buf bytes.Buffer
+	err := json.NewEncoder(&buf).Encode(i)
+	if err != nil {
+		panic(err)
+	}
+	buf.WriteString(",")
+	chrometraceConsumers.write <- buf.Bytes()
+}
+
+func ChrometraceClientWebsocketHandler(conn *websocket.Conn) {
+	defer conn.Close()
+
+	var wg sync.WaitGroup
+	defer wg.Wait()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		r := bufio.NewReader(conn)
+		_, _, _ = r.ReadLine() // ignore errors
+		conn.Close()
+	}()
+
+	errored := make(chan error, 1)
+	chrometraceConsumers.register <- chrometraceConsumerRegistration{
+		w:       conn,
+		errored: errored,
+	}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		<-errored
+		conn.Close()
+	}()
+}
+
+var chrometraceFileConsumerPath = envconst.String("ZREPL_ACTIVITY_TRACE", "")
+
+func init() {
+	if chrometraceFileConsumerPath != "" {
+		var err error
+		f, err := os.Create(chrometraceFileConsumerPath)
+		if err != nil {
+			panic(err)
+		}
+		errored := make(chan error, 1)
+		chrometraceConsumers.register <- chrometraceConsumerRegistration{
+			w:       f,
+			errored: errored,
+		}
+		go func() {
+			<-errored
+			f.Close()
+		}()
+	}
+}

internal/daemon/logging/trace/trace_context.go

@@ -0,0 +1,27 @@
+package trace
+
+import "context"
+
+type contextKey int
+
+const (
+	contextKeyTraceNode contextKey = 1 + iota
+)
+
+var contextKeys = []contextKey{
+	contextKeyTraceNode,
+}
+
+// WithInherit inherits the task hierarchy from inheritFrom into ctx.
+// The returned context is a child of ctx, but its task and span are those of inheritFrom.
+//
+// Note that in most use cases, callers most likely want to call WithTask since it will most likely
+// be in some sort of connection handler context.
+func WithInherit(ctx, inheritFrom context.Context) context.Context {
+	for _, k := range contextKeys {
+		if v := inheritFrom.Value(k); v != nil {
+			ctx = context.WithValue(ctx, k, v) // no shadow
+		}
+	}
+	return ctx
+}

internal/daemon/logging/trace/trace_convenience.go

@@ -0,0 +1,76 @@
+package trace
+
+import (
+	"context"
+	"fmt"
+	"runtime"
+	"strings"
+	"sync"
+)
+
+// use like this:
+//
+//	defer WithSpanFromStackUpdateCtx(&existingCtx)()
+func WithSpanFromStackUpdateCtx(ctx *context.Context) DoneFunc {
+	childSpanCtx, end := WithSpan(*ctx, getMyCallerOrPanic())
+	*ctx = childSpanCtx
+	return end
+}
+
+// derive task name from call stack (caller's name)
+func WithTaskFromStack(ctx context.Context) (context.Context, DoneFunc) {
+	return WithTask(ctx, getMyCallerOrPanic())
+}
+
+// derive task name from call stack (caller's name) and update *ctx
+// to point to be the child task ctx
+func WithTaskFromStackUpdateCtx(ctx *context.Context) DoneFunc {
+	child, end := WithTask(*ctx, getMyCallerOrPanic())
+	*ctx = child
+	return end
+}
+
+// create a task and a span within it in one call
+func WithTaskAndSpan(ctx context.Context, task string, span string) (context.Context, DoneFunc) {
+	ctx, endTask := WithTask(ctx, task)
+	ctx, endSpan := WithSpan(ctx, fmt.Sprintf("%s %s", task, span))
+	return ctx, func() {
+		endSpan()
+		endTask()
+	}
+}
+
+// create a span during which several child tasks are spawned using the `add` function
+//
+// IMPORTANT FOR USERS: Caller must ensure that the capturing behavior is correct, the Go linter doesn't catch this.
+func WithTaskGroup(ctx context.Context, taskGroup string) (_ context.Context, add func(f func(context.Context)), waitEnd DoneFunc) {
+	var wg sync.WaitGroup
+	ctx, endSpan := WithSpan(ctx, taskGroup)
+	add = func(f func(context.Context)) {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			ctx, endTask := WithTask(ctx, taskGroup)
+			defer endTask()
+			f(ctx)
+		}()
+	}
+	waitEnd = func() {
+		wg.Wait()
+		endSpan()
+	}
+	return ctx, add, waitEnd
+}
+
+func getMyCallerOrPanic() string {
+	pc, _, _, ok := runtime.Caller(2)
+	if !ok {
+		panic("cannot get caller")
+	}
+	details := runtime.FuncForPC(pc)
+	if ok && details != nil {
+		const prefix = "github.com/zrepl/zrepl"
+		return strings.TrimPrefix(strings.TrimPrefix(details.Name(), prefix), "/")
+	}
+	return ""
+}

internal/daemon/logging/trace/trace_convenience_test.go

@@ -0,0 +1,62 @@
+package trace
+
+import (
+	"context"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetCallerOrPanic(t *testing.T) {
+	withStackFromCtxMock := func() string {
+		return getMyCallerOrPanic()
+	}
+	ret := withStackFromCtxMock()
+	// zrepl prefix is stripped
+	assert.Equal(t, "internal/daemon/logging/trace.TestGetCallerOrPanic", ret)
+}
+
+func TestWithTaskGroupRunTasksConcurrently(t *testing.T) {
+
+	// spawn a task group where each task waits for the other to start
+	// => without concurrency, they would hang
+
+	rootCtx, endRoot := WithTaskFromStack(context.Background())
+	defer endRoot()
+
+	_, add, waitEnd := WithTaskGroup(rootCtx, "test-task-group")
+
+	schedulerTimeout := 2 * time.Second
+	timeout := time.After(schedulerTimeout)
+	var hadTimeout uint32
+	started0, started1 := make(chan struct{}), make(chan struct{})
+	for i := 0; i < 2; i++ {
+		i := i // capture by copy
+		add(func(ctx context.Context) {
+			switch i {
+			case 0:
+				close(started0)
+				select {
+				case <-started1:
+				case <-timeout:
+					atomic.AddUint32(&hadTimeout, 1)
+				}
+			case 1:
+				close(started1)
+				select {
+				case <-started0:
+				case <-timeout:
+					atomic.AddUint32(&hadTimeout, 1)
+				}
+			default:
+				panic("unreachable")
+			}
+		})
+	}
+
+	waitEnd()
+	assert.Zero(t, hadTimeout, "either bad impl or scheduler timeout (which is %v)", schedulerTimeout)
+
+}

internal/daemon/logging/trace/trace_debug.go

@@ -0,0 +1,19 @@
+package trace
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/zrepl/zrepl/internal/util/envconst"
+)
+
+const debugEnabledEnvVar = "ZREPL_TRACE_DEBUG_ENABLED"
+
+var debugEnabled = envconst.Bool(debugEnabledEnvVar, false)
+
+func debug(format string, args ...interface{}) {
+	if !debugEnabled {
+		return
+	}
+	fmt.Fprintf(os.Stderr, format+"\n", args...)
+}