Pruner: parallel pruning

refs #62

.gitmodules

@@ -1,4 +0,0 @@
-[submodule "docs/themes/docdock"]
-	path = docs/themes/docdock
-	url = git@github.com:zrepl/hugo-theme-docdock.git
-

Gopkg.lock

@@ -8,10 +8,10 @@
   version = "v1.1.0"
 
 [[projects]]
-  name = "github.com/fsnotify/fsnotify"
+  branch = "master"
+  name = "github.com/dustin/go-humanize"
   packages = ["."]
-  revision = "629574ca2a5df945712d3079857300b5e4da0236"
-  version = "v1.4.2"
+  revision = "bb3d318650d48840a39aa21a027c6630e198e626"
 
 [[projects]]
   branch = "master"
@@ -31,18 +31,6 @@
   packages = ["."]
   revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"
 
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/hcl"
-  packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"]
-  revision = "68e816d1c783414e79bc65b3994d9ab6b0a722ab"
-
-[[projects]]
-  name = "github.com/inconshreveable/mousetrap"
-  packages = ["."]
-  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
-  version = "v1.0"
-
 [[projects]]
   branch = "master"
   name = "github.com/jinzhu/copier"
@@ -68,16 +56,10 @@
   revision = "7cafcd837844e784b526369c9bce262804aebc60"
 
 [[projects]]
-  name = "github.com/magiconair/properties"
+  name = "github.com/mattn/go-isatty"
   packages = ["."]
-  revision = "be5ece7dd465ab0765a9682137865547526d1dfb"
-  version = "v1.7.3"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/mitchellh/go-homedir"
-  packages = ["."]
-  revision = "b8bc1bf767474819792c23f32d8286a45736f1c6"
+  revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
+  version = "v0.0.3"
 
 [[projects]]
   branch = "master"
@@ -85,18 +67,6 @@
   packages = ["."]
   revision = "d0303fe809921458f417bcf828397a65db30a7e4"
 
-[[projects]]
-  name = "github.com/pelletier/go-buffruneio"
-  packages = ["."]
-  revision = "c37440a7cf42ac63b919c752ca73a85067e05992"
-  version = "v0.2.0"
-
-[[projects]]
-  name = "github.com/pelletier/go-toml"
-  packages = ["."]
-  revision = "5ccdfb18c776b740aecaf085c4d9a2779199c279"
-  version = "v1.0.0"
-
 [[projects]]
   name = "github.com/pkg/errors"
   packages = ["."]
@@ -111,15 +81,15 @@
 
 [[projects]]
   branch = "master"
-  name = "github.com/spf13/afero"
-  packages = [".","mem"]
-  revision = "ee1bd8ee15a1306d1f9201acc41ef39cd9f99a1b"
+  name = "github.com/problame/go-netssh"
+  packages = ["."]
+  revision = "ffa145d2506e222977205e7666a9722d6b9959ac"
 
 [[projects]]
-  name = "github.com/spf13/cast"
+  branch = "master"
+  name = "github.com/problame/go-rwccmd"
   packages = ["."]
-  revision = "acbeb36b902d72a7a4c18e8f3241075e7ab763e4"
-  version = "v1.1.0"
+  revision = "391d2c78c8404a9683d79f75dd24ab53040f89f7"
 
 [[projects]]
   branch = "master"
@@ -127,24 +97,12 @@
   packages = ["."]
   revision = "b78744579491c1ceeaaa3b40205e56b0591b93a3"
 
-[[projects]]
-  branch = "master"
-  name = "github.com/spf13/jwalterweatherman"
-  packages = ["."]
-  revision = "12bd96e66386c1960ab0f74ced1362f66f552f7b"
-
 [[projects]]
   name = "github.com/spf13/pflag"
   packages = ["."]
   revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66"
   version = "v1.0.0"
 
-[[projects]]
-  name = "github.com/spf13/viper"
-  packages = ["."]
-  revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7"
-  version = "v1.0.0"
-
 [[projects]]
   name = "github.com/stretchr/testify"
   packages = ["assert"]
@@ -155,23 +113,11 @@
   branch = "master"
   name = "golang.org/x/sys"
   packages = ["unix"]
-  revision = "429f518978ab01db8bb6f44b66785088e7fba58b"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/text"
-  packages = ["internal/gen","internal/triegen","internal/ucd","transform","unicode/cldr","unicode/norm"]
-  revision = "1cbadb444a806fd9430d14ad08967ed91da4fa0a"
-
-[[projects]]
-  branch = "v2"
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"
+  revision = "bf42f188b9bc6f2cf5b8ee5a912ef1aedd0eba4c"
 
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "7ce2ead5225e4bb72a34132538171a649b26de574596e909730658ddbc904cd5"
+  inputs-digest = "fcfc66322b5f9dbe329411b7688a4e9dcebe3c529c1dadf4f78bcdb2bcfb8fbf"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -1,25 +1,4 @@
-
-# Gopkg.toml example
-#
-# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
-# for detailed Gopkg.toml documentation.
-#
-# required = ["github.com/user/thing/cmd/thing"]
-# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
-#
-# [[constraint]]
-#   name = "github.com/user/project"
-#   version = "1.0.0"
-#
-# [[constraint]]
-#   name = "github.com/user/project2"
-#   branch = "dev"
-#   source = "github.com/myfork/project2"
-#
-# [[override]]
-#  name = "github.com/x/y"
-#  version = "2.4.0"
-
+ignored = [ "github.com/inconshreveable/mousetrap" ]
 
 [[constraint]]
   branch = "master"
@@ -64,3 +43,11 @@
 [[constraint]]
   name = "github.com/go-logfmt/logfmt"
   version = "*"
+
+[[constraint]]
+  name = "github.com/problame/go-rwccmd"
+  branch = "master"
+
+[[constraint]]
+  name = "github.com/problame/go-netssh"
+  branch = "master"

Makefile

@@ -1,20 +1,39 @@
-.PHONY: generate build test vet cover release clean
+.PHONY: generate build test vet cover release docs docs-clean clean vendordeps
+.DEFAULT_GOAL := build
 
 ROOT := github.com/zrepl/zrepl
-SUBPKGS := cmd logger rpc sshbytestream util
+SUBPKGS := cmd logger rpc util zfs
 
 _TESTPKGS := $(ROOT) $(foreach p,$(SUBPKGS),$(ROOT)/$(p))
 
 ARTIFACTDIR := artifacts
 
-build: generate
-		go build -o $(ARTIFACTDIR)/zrepl
+ifndef ZREPL_VERSION
+    ZREPL_VERSION := $(shell git describe --dirty 2>/dev/null || echo "ZREPL_BUILD_INVALID_VERSION" )
+    ifeq ($(ZREPL_VERSION),ZREPL_BUILD_INVALID_VERSION) # can't use .SHELLSTATUS because Debian Stretch is still on gmake 4.1
+        $(error cannot infer variable ZREPL_VERSION using git and variable is not overriden by make invocation)
+    endif
+endif
+GO_LDFLAGS := "-X github.com/zrepl/zrepl/cmd.zreplVersion=$(ZREPL_VERSION)"
 
-generate:
+GO_BUILD := go build -ldflags $(GO_LDFLAGS)
+
+RELEASE_BINS := $(ARTIFACTDIR)/zrepl-freebsd-amd64 $(ARTIFACTDIR)/zrepl-linux-amd64
+RELEASE_NOARCH := $(ARTIFACTDIR)/zrepl-noarch.tar
+THIS_PLATFORM_RELEASE_BIN := $(shell bash -c 'source <(go env) && echo "zrepl-$${GOOS}-$${GOARCH}"' )
+
+vendordeps:
+	dep ensure -v -vendor-only
+
+generate: #not part of the build, must do that manually
 	@for pkg in $(_TESTPKGS); do\
 		go generate "$$pkg" || exit 1; \
 	done;
 
+build:
+	@echo "INFO: In case of missing dependencies, run 'make vendordeps'"
+	$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl"
+
 test:
 	@for pkg in $(_TESTPKGS); do \
 		echo "Testing $$pkg"; \
@@ -36,12 +55,52 @@ cover: artifacts
 		fi; \
 	done;
 
-artifacts:
-	mkdir artifacts
+$(ARTIFACTDIR):
+	mkdir -p "$@"
 
-release: generate artifacts vet test
-	GOOS=linux GOARCH=amd64   go build -o "$(ARTIFACTDIR)/zrepl-linux-amd64"
-	GOOS=freebsd GOARCH=amd64 go build -o "$(ARTIFACTDIR)/zrepl-freebsd-amd64"
+$(ARTIFACTDIR)/docs: $(ARTIFACTDIR)
+	mkdir -p "$@"
 
-clean:
+$(ARTIFACTDIR)/bash_completion: $(RELEASE_BINS)
+	artifacts/$(THIS_PLATFORM_RELEASE_BIN) bashcomp "$@"
+
+$(ARTIFACTDIR)/go_version.txt:
+	go version > $@
+
+docs: $(ARTIFACTDIR)/docs
+	make -C docs \
+		html \
+		BUILDDIR=../artifacts/docs \
+
+docs-clean:
+	make -C docs \
+		clean \
+		BUILDDIR=../artifacts/docs
+
+
+.PHONY: $(RELEASE_BINS)
+# TODO: two wildcards possible
+$(RELEASE_BINS): $(ARTIFACTDIR)/zrepl-%-amd64: generate $(ARTIFACTDIR) vet test
+	@echo "INFO: In case of missing dependencies, run 'make vendordeps'"
+	GOOS=$* GOARCH=amd64   $(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$*-amd64"
+
+$(RELEASE_NOARCH): docs $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_version.txt
+	tar --mtime='1970-01-01' --sort=name \
+		--transform 's/$(ARTIFACTDIR)/zrepl-$(ZREPL_VERSION)-noarch/' \
+		-acf $@ \
+		$(ARTIFACTDIR)/docs/html \
+		$(ARTIFACTDIR)/bash_completion \
+		$(ARTIFACTDIR)/go_version.txt
+
+release: $(RELEASE_BINS) $(RELEASE_NOARCH)
+	rm -rf "$(ARTIFACTDIR)/release"
+	mkdir -p "$(ARTIFACTDIR)/release"
+	cp $^ "$(ARTIFACTDIR)/release"
+	cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
+	@if echo "$(ZREPL_VERSION)" | grep dirty > /dev/null; then\
+		echo '[WARN] Do not publish the artifacts, make variable ZREPL_VERSION=$(ZREPL_VERSION) indicates they are dirty!'; \
+		exit 1; \
+	fi
+
+clean: docs-clean
 	rm -rf "$(ARTIFACTDIR)"

README.md

@@ -3,7 +3,7 @@ zrepl is a ZFS filesystem backup & replication solution written in Go.
 
 ## User Documentation
 
-**User Documentation** cab be found at [zrepl.github.io](https://zrepl.github.io).
+**User Documentation** can be found at [zrepl.github.io](https://zrepl.github.io).
 
 ## Bug Reports
 
@@ -20,8 +20,20 @@ zrepl is a ZFS filesystem backup & replication solution written in Go.
 The above does not apply if you already implemented everything.
 Check out the *Coding Workflow* section below for details.
 
+## Package Maintainer Information
+
+* Follow the steps in `docs/installation.rst -> Compiling from Source` and read the Makefile / shell scripts used in this process.
+* Make sure your distro is compatible with the paths in `docs/installation.rst`.
+* Ship a default config that adheres to your distro's `hier` and logging system.
+* Ship a service manager file and _please_ try to upstream it to this repository.
+* Use `make release ZREPL_VERSION='mydistro-1.2.3_1'`
+    * Your distro's name and any versioning supplemental to zrepl's (e.g. package revision) should be in this string
+* Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
+
 ## Developer Documentation
 
+First, use `./lazy.sh devsetup` to install build dependencies and read `docs/installation.rst -> Compiling from Source`.
+
 ### Overall Architecture
 
 The application architecture is documented as part of the user docs in the *Implementation* section (`docs/content/impl`).
@@ -32,17 +44,15 @@ Make sure to develop an understanding how zrepl is typically used by studying th
 ```
 ├── cmd
 │   ├── sampleconf          # example configuration
-├── docs
-│   ├── content             # hugo-based documentation -> sources for ./public_git
-│   ├── deploy.sh           # shell script for automated rendering & deploy to zrepl.github.io repo
-│   ├── public_git          # used by above shell script
-│   └── themes
-│       └── docdock         # submodule of our docdock theme fork
-├── jobrun                  # OBSOLETE
+├── docs                    # sphinx-based documentation
+│   ├── **/*.rst            # documentation in reStructuredText
+│   ├── sphinxconf
+│   │   └── conf.py         # sphinx config (see commit 445a280 why its not in docs/)
+│   ├── requirements.txt    # pip3 requirements to build documentation
+│   ├── publish.sh          # shell script for automated rendering & deploy to zrepl.github.io repo
+│   ├── public_git          # checkout of zrepl.github.io managed by above shell script
 ├── logger                  # logger package used by zrepl
 ├── rpc                     # rpc protocol implementation
-├── scratchpad              # small example programs demoing some internal packages. probably OBSOLETE
-├── sshbytestream           # io.ReadWriteCloser over SSH
 ├── util
 └── zfs                     # ZFS wrappers, filesystemm diffing
 ```
@@ -52,3 +62,28 @@ Make sure to develop an understanding how zrepl is typically used by studying th
 * Open an issue when starting to hack on a new feature
 * Commits should reference the issue they are related to
 * Docs improvements not documenting new features do not require an issue.
+
+### Breaking Changes
+
+Backward-incompatible changes must be documented in the git commit message and are listed in `docs/changelog.rst`.
+
+* Config-breaking changes must contain a line `BREAK CONFIG` in the commit message
+* Other breaking changes must contain a line `BREAK` in the commit message
+
+### Glossary & Naming Inconsistencies
+
+In ZFS, *dataset* refers to the objects *filesystem*, *ZVOL* and *snapshot*. <br />
+However, we need a word for *filesystem* & *ZVOL* but not a snapshot, bookmark, etc.
+
+Toward the user, the following terminology is used:
+
+* **filesystem**: a ZFS filesystem or a ZVOL
+* **filesystem version**: a ZFS snapshot or a bookmark
+
+Sadly, the zrepl implementation is inconsistent in its use of these words:
+variables and types are often named *dataset* when they in fact refer to a *filesystem*.
+
+There will not be a big refactoring (an attempt was made, but it's destroying too much history without much gain).
+
+However, new contributions & patches should fix naming without further notice in the commit message.
+

build.Dockerfile

@@ -0,0 +1,16 @@
+FROM golang:latest
+
+RUN apt-get update && apt-get install -y \
+    python3-pip
+
+ADD lazy.sh /tmp/lazy.sh
+ADD docs/requirements.txt /tmp/requirements.txt
+ENV ZREPL_LAZY_DOCS_REQPATH=/tmp/requirements.txt
+RUN /tmp/lazy.sh devsetup
+
+# prepare volume mount of git checkout to /zrepl
+RUN mkdir -p /go/src/github.com/zrepl/zrepl
+RUN chmod -R 0777 /go
+WORKDIR /go/src/github.com/zrepl/zrepl
+
+

cmd/autosnap.go

@@ -5,56 +5,59 @@ import (
 	"fmt"
 	"github.com/zrepl/zrepl/zfs"
 	"sort"
+	"sync"
 	"time"
 )
 
 type IntervalAutosnap struct {
+	task             *Task
 	DatasetFilter    zfs.DatasetFilter
 	Prefix           string
 	SnapshotInterval time.Duration
-
-	log       Logger
-	snaptimes []snapTime
 }
 
-type snapTime struct {
-	ds   *zfs.DatasetPath
-	time time.Time
-}
-
-func (a *IntervalAutosnap) Run(ctx context.Context, didSnaps chan struct{}) {
-
-	a.log = ctx.Value(contextKeyLog).(Logger)
-
-	const LOG_TIME_FMT string = time.ANSIC
-
-	ds, err := zfs.ZFSListMapping(a.DatasetFilter)
+func (a *IntervalAutosnap) filterFilesystems() (fss []*zfs.DatasetPath, stop bool) {
+	a.task.Enter("filter_filesystems")
+	defer a.task.Finish()
+	fss, err := zfs.ZFSListMapping(a.DatasetFilter)
+	stop = err != nil
 	if err != nil {
-		a.log.WithError(err).Error("cannot list datasets")
-		return
+		a.task.Log().WithError(err).Error("cannot list datasets")
 	}
-	if len(ds) == 0 {
-		a.log.WithError(err).Error("no datasets matching dataset filter")
-		return
+	if len(fss) == 0 {
+		a.task.Log().Warn("no filesystem matching filesystem filter")
+	}
+	return fss, stop
+}
+
+func (a *IntervalAutosnap) findSyncPoint(fss []*zfs.DatasetPath) (syncPoint time.Time, err error) {
+	a.task.Enter("find_sync_point")
+	defer a.task.Finish()
+	type snapTime struct {
+		ds   *zfs.DatasetPath
+		time time.Time
 	}
 
-	a.snaptimes = make([]snapTime, len(ds))
+	if len(fss) == 0 {
+		return time.Now(), nil
+	}
+
+	snaptimes := make([]snapTime, 0, len(fss))
 
 	now := time.Now()
 
-	a.log.Debug("examine filesystem state")
-	for i, d := range ds {
+	a.task.Log().Debug("examine filesystem state")
+	for _, d := range fss {
 
-		l := a.log.WithField(logFSField, d.ToString())
+		l := a.task.Log().WithField(logFSField, d.ToString())
 
-		fsvs, err := zfs.ZFSListFilesystemVersions(d, &PrefixSnapshotFilter{a.Prefix})
+		fsvs, err := zfs.ZFSListFilesystemVersions(d, NewPrefixFilter(a.Prefix))
 		if err != nil {
 			l.WithError(err).Error("cannot list filesystem versions")
 			continue
 		}
 		if len(fsvs) <= 0 {
 			l.WithField("prefix", a.Prefix).Info("no filesystem versions with prefix")
-			a.snaptimes[i] = snapTime{d, now}
 			continue
 		}
 
@@ -78,34 +81,75 @@ func (a *IntervalAutosnap) Run(ctx context.Context, didSnaps chan struct{}) {
 		if since < a.SnapshotInterval {
 			next = latest.Creation.Add(a.SnapshotInterval)
 		}
-		a.snaptimes[i] = snapTime{d, next}
+		snaptimes = append(snaptimes, snapTime{d, next})
 	}
 
-	sort.Slice(a.snaptimes, func(i, j int) bool {
-		return a.snaptimes[i].time.Before(a.snaptimes[j].time)
+	if len(snaptimes) == 0 {
+		snaptimes = append(snaptimes, snapTime{nil, now})
+	}
+
+	sort.Slice(snaptimes, func(i, j int) bool {
+		return snaptimes[i].time.Before(snaptimes[j].time)
 	})
 
-	syncPoint := a.snaptimes[0]
-	a.log.WithField("sync_point", syncPoint.time.Format(LOG_TIME_FMT)).
+	return snaptimes[0].time, nil
+
+}
+
+func (a *IntervalAutosnap) waitForSyncPoint(ctx context.Context, syncPoint time.Time) {
+	a.task.Enter("wait_sync_point")
+	defer a.task.Finish()
+
+	const LOG_TIME_FMT string = time.ANSIC
+
+	a.task.Log().WithField("sync_point", syncPoint.Format(LOG_TIME_FMT)).
 		Info("wait for sync point")
 
 	select {
 	case <-ctx.Done():
-		a.log.WithError(ctx.Err()).Info("context done")
+		a.task.Log().WithError(ctx.Err()).Info("context done")
 		return
+	case <-time.After(syncPoint.Sub(time.Now())):
+	}
+}
 
-	case <-time.After(syncPoint.time.Sub(now)):
-		a.log.Debug("snapshot all filesystems to enable further snaps in lockstep")
-		a.doSnapshots(didSnaps)
+func (a *IntervalAutosnap) syncUpRun(ctx context.Context, didSnaps chan struct{}) (stop bool) {
+	a.task.Enter("sync_up")
+	defer a.task.Finish()
+
+	fss, stop := a.filterFilesystems()
+	if stop {
+		return true
 	}
 
-	ticker := time.NewTicker(a.SnapshotInterval)
+	syncPoint, err := a.findSyncPoint(fss)
+	if err != nil {
+		return true
+	}
 
+	a.waitForSyncPoint(ctx, syncPoint)
+
+	a.task.Log().Debug("snapshot all filesystems to enable further snaps in lockstep")
+	a.doSnapshots(didSnaps)
+	return false
+}
+
+func (a *IntervalAutosnap) Run(ctx context.Context, didSnaps chan struct{}) {
+
+	if a.syncUpRun(ctx, didSnaps) {
+		a.task.Log().Error("stoppping autosnap after error in sync up")
+		return
+	}
+
+	// task drops back to idle here
+
+	a.task.Log().Debug("setting up ticker in SnapshotInterval")
+	ticker := time.NewTicker(a.SnapshotInterval)
 	for {
 		select {
 		case <-ctx.Done():
 			ticker.Stop()
-			a.log.WithError(ctx.Err()).Info("context done")
+			a.task.Log().WithError(ctx.Err()).Info("context done")
 			return
 
 		case <-ticker.C:
@@ -117,32 +161,55 @@ func (a *IntervalAutosnap) Run(ctx context.Context, didSnaps chan struct{}) {
 
 func (a *IntervalAutosnap) doSnapshots(didSnaps chan struct{}) {
 
-	// fetch new dataset list in case user added new dataset
-	ds, err := zfs.ZFSListMapping(a.DatasetFilter)
-	if err != nil {
-		a.log.WithError(err).Error("cannot list datasets")
+	a.task.Enter("do_snapshots")
+	defer a.task.Finish()
+
+	// don't cache the result from previous run in case the user added
+	// a new dataset in the meantime
+	fss, stop := a.filterFilesystems()
+	if stop {
 		return
 	}
 
+	a.task.Log().Info("beginning parallel snapshots")
+
 	// TODO channel programs -> allow a little jitter?
-	for _, d := range ds {
-		suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
-		snapname := fmt.Sprintf("%s%s", a.Prefix, suffix)
+	var wg sync.WaitGroup
+	for fsi := range fss {
+		wg.Add(1)
+		go func(fs *zfs.DatasetPath) {
+			defer wg.Done()
 
-		a.log.WithField(logFSField, d.ToString()).
-			WithField("snapname", snapname).
-			Info("create snapshot")
+			suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
+			snapname := fmt.Sprintf("%s%s", a.Prefix, suffix)
 
-		err := zfs.ZFSSnapshot(d, snapname, false)
-		if err != nil {
-			a.log.WithError(err).Error("cannot create snapshot")
-		}
+			l := a.task.Log().WithField(logFSField, fs.ToString()).
+				WithField("snapname", snapname)
+
+			l.Info("create snapshot")
+			err := zfs.ZFSSnapshot(fs, snapname, false)
+			if err != nil {
+				l.WithError(err).Error("cannot create snapshot")
+				return
+			}
+
+			l.Info("create corresponding bookmark")
+			err = zfs.ZFSBookmark(fs, snapname, snapname)
+			if err != nil {
+				l.WithError(err).Error("cannot create bookmark")
+			}
+
+		}(fss[fsi])
 	}
 
+	a.task.Log().Info("waiting for parallel snapshots to finish")
+	wg.Wait()
+	a.task.Log().Info("snapshots finished")
+
 	select {
 	case didSnaps <- struct{}{}:
 	default:
-		a.log.Error("warning: callback channel is full, discarding")
+		a.task.Log().Error("warning: callback channel is full, discarding")
 	}
 
 }

cmd/bashcomp.go

@@ -0,0 +1,29 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"os"
+)
+
+var bashcompCmd = &cobra.Command{
+	Use:   "bashcomp path/to/out/file",
+	Short: "generate bash completions",
+	Run: func(cmd *cobra.Command, args []string) {
+		if len(args) != 1 {
+			fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
+			cmd.Usage()
+			os.Exit(1)
+		}
+		if err := RootCmd.GenBashCompletionFile(args[0]); err != nil {
+			fmt.Fprintf(os.Stderr, "error generating bash completion: %s", err)
+			os.Exit(1)
+		}
+	},
+	Hidden: true,
+}
+
+func init() {
+	RootCmd.AddCommand(bashcompCmd)
+}

cmd/config.go

@@ -59,11 +59,13 @@ type SSHStdinServerConnectDescr struct {
 }
 
 type PrunePolicy interface {
+	// Prune filters versions and decide which to keep and which to remove.
+	// Prune **does not** implement the actual removal of the versions.
 	Prune(fs *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error)
 }
 
 type PruningJob interface {
-	Pruner(side PrunePolicySide, dryRun bool) (Pruner, error)
+	Pruner(task *Task, side PrunePolicySide, dryRun bool) (Pruner, error)
 }
 
 // A type for constants describing different prune policies of a PruningJob

cmd/config_connect.go

@@ -4,10 +4,11 @@ import (
 	"fmt"
 	"io"
 
+	"context"
 	"github.com/jinzhu/copier"
 	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/sshbytestream"
+	"github.com/problame/go-netssh"
 )
 
 type SSHStdinserverConnecter struct {
@@ -34,11 +35,12 @@ func parseSSHStdinserverConnecter(i map[string]interface{}) (c *SSHStdinserverCo
 }
 
 func (c *SSHStdinserverConnecter) Connect() (rwc io.ReadWriteCloser, err error) {
-	var rpcTransport sshbytestream.SSHTransport
-	if err = copier.Copy(&rpcTransport, c); err != nil {
+
+	var endpoint netssh.Endpoint
+	if err = copier.Copy(&endpoint, c); err != nil {
 		return
 	}
-	if rwc, err = sshbytestream.Outgoing(rpcTransport); err != nil {
+	if rwc, err = netssh.Dial(context.TODO(), endpoint); err != nil {
 		err = errors.WithStack(err)
 		return
 	}

cmd/config_fsvfilter.go

@@ -6,8 +6,18 @@ import (
 	"strings"
 )
 
-type PrefixSnapshotFilter struct {
-	Prefix string
+type PrefixFilter struct {
+	prefix    string
+	fstype    zfs.VersionType
+	fstypeSet bool // optionals anyone?
+}
+
+func NewPrefixFilter(prefix string) *PrefixFilter {
+	return &PrefixFilter{prefix: prefix}
+}
+
+func NewTypedPrefixFilter(prefix string, versionType zfs.VersionType) *PrefixFilter {
+	return &PrefixFilter{prefix, versionType, true}
 }
 
 func parseSnapshotPrefix(i string) (p string, err error) {
@@ -19,15 +29,8 @@ func parseSnapshotPrefix(i string) (p string, err error) {
 	return
 }
 
-func parsePrefixSnapshotFilter(i string) (f *PrefixSnapshotFilter, err error) {
-	if !(len(i) > 0) {
-		err = errors.Errorf("snapshot prefix must be longer than 0 characters")
-		return
-	}
-	f = &PrefixSnapshotFilter{i}
-	return
-}
-
-func (f *PrefixSnapshotFilter) Filter(fsv zfs.FilesystemVersion) (accept bool, err error) {
-	return fsv.Type == zfs.Snapshot && strings.HasPrefix(fsv.Name, f.Prefix), nil
+func (f *PrefixFilter) Filter(fsv zfs.FilesystemVersion) (accept bool, err error) {
+	fstypeMatches := (!f.fstypeSet || fsv.Type == f.fstype)
+	prefixMatches := strings.HasPrefix(fsv.Name, f.prefix)
+	return fstypeMatches && prefixMatches, nil
 }

cmd/config_job_control.go

@@ -1,11 +1,13 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"github.com/pkg/errors"
+	"io"
 	"net"
 	"net/http"
-	"net/http/pprof"
 )
 
 type ControlJob struct {
@@ -29,8 +31,14 @@ func (j *ControlJob) JobName() string {
 	return j.Name
 }
 
+func (j *ControlJob) JobStatus(ctx context.Context) (*JobStatus, error) {
+	return &JobStatus{Tasks: nil}, nil
+}
+
 const (
-	ControlJobEndpointProfile string = "/debug/pprof/profile"
+	ControlJobEndpointPProf   string = "/debug/pprof"
+	ControlJobEndpointVersion string = "/version"
+	ControlJobEndpointStatus  string = "/status"
 )
 
 func (j *ControlJob) JobStart(ctx context.Context) {
@@ -38,14 +46,35 @@ func (j *ControlJob) JobStart(ctx context.Context) {
 	log := ctx.Value(contextKeyLog).(Logger)
 	defer log.Info("control job finished")
 
+	daemon := ctx.Value(contextKeyDaemon).(*Daemon)
+
 	l, err := ListenUnixPrivate(j.sockaddr)
 	if err != nil {
 		log.WithError(err).Error("error listening")
 		return
 	}
 
+	pprofServer := NewPProfServer(ctx)
+
 	mux := http.NewServeMux()
-	mux.Handle(ControlJobEndpointProfile, requestLogger{log, pprof.Profile})
+	mux.Handle(ControlJobEndpointPProf, requestLogger{log: log, handlerFunc: func(w http.ResponseWriter, r *http.Request) {
+		var msg PprofServerControlMsg
+		err := json.NewDecoder(r.Body).Decode(&msg)
+		if err != nil {
+			log.WithError(err).Error("bad pprof request from client")
+			w.WriteHeader(http.StatusBadRequest)
+		}
+		pprofServer.Control(msg)
+		w.WriteHeader(200)
+	}})
+	mux.Handle(ControlJobEndpointVersion,
+		requestLogger{log: log, handler: jsonResponder{func() (interface{}, error) {
+			return NewZreplVersionInformation(), nil
+		}}})
+	mux.Handle(ControlJobEndpointStatus,
+		requestLogger{log: log, handler: jsonResponder{func() (interface{}, error) {
+			return daemon.Status(), nil
+		}}})
 	server := http.Server{Handler: mux}
 
 outer:
@@ -59,7 +88,7 @@ outer:
 
 		select {
 		case <-ctx.Done():
-			log.WithError(err).Info("contex done")
+			log.WithError(ctx.Err()).Info("context done")
 			server.Shutdown(context.Background())
 			break outer
 		case err = <-served:
@@ -73,14 +102,42 @@ outer:
 
 }
 
+type jsonResponder struct {
+	producer func() (interface{}, error)
+}
+
+func (j jsonResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	res, err := j.producer()
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		io.WriteString(w, err.Error())
+		return
+	}
+	var buf bytes.Buffer
+	err = json.NewEncoder(&buf).Encode(res)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		io.WriteString(w, err.Error())
+	} else {
+		io.Copy(w, &buf)
+	}
+}
+
 type requestLogger struct {
 	log         Logger
-	handlerFunc func(w http.ResponseWriter, r *http.Request)
+	handler     http.Handler
+	handlerFunc http.HandlerFunc
 }
 
 func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	log := l.log.WithField("method", r.Method).WithField("url", r.URL)
 	log.Info("start")
-	l.handlerFunc(w, r)
+	if l.handlerFunc != nil {
+		l.handlerFunc(w, r)
+	} else if l.handler != nil {
+		l.handler.ServeHTTP(w, r)
+	} else {
+		log.Error("no handler or handlerFunc configured")
+	}
 	log.Info("finish")
 }

cmd/config_job_local.go

@@ -20,6 +20,11 @@ type LocalJob struct {
 	PruneLHS          PrunePolicy
 	PruneRHS          PrunePolicy
 	Debug             JobDebugSettings
+	snapperTask       *Task
+	mainTask          *Task
+	handlerTask       *Task
+	pruneRHSTask      *Task
+	pruneLHSTask      *Task
 }
 
 func parseLocalJob(c JobParsingContext, name string, i map[string]interface{}) (j *LocalJob, err error) {
@@ -49,7 +54,7 @@ func parseLocalJob(c JobParsingContext, name string, i map[string]interface{}) (
 		return
 	}
 
-	if j.Interval, err = time.ParseDuration(asMap.Interval); err != nil {
+	if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
 		err = errors.Wrap(err, "cannot parse interval")
 		return
 	}
@@ -58,11 +63,11 @@ func parseLocalJob(c JobParsingContext, name string, i map[string]interface{}) (
 		return
 	}
 
-	if j.PruneLHS, err = parsePrunePolicy(asMap.PruneLHS); err != nil {
+	if j.PruneLHS, err = parsePrunePolicy(asMap.PruneLHS, true); err != nil {
 		err = errors.Wrap(err, "cannot parse 'prune_lhs'")
 		return
 	}
-	if j.PruneRHS, err = parsePrunePolicy(asMap.PruneRHS); err != nil {
+	if j.PruneRHS, err = parsePrunePolicy(asMap.PruneRHS, false); err != nil {
 		err = errors.Wrap(err, "cannot parse 'prune_rhs'")
 		return
 	}
@@ -81,8 +86,13 @@ func (j *LocalJob) JobName() string {
 
 func (j *LocalJob) JobStart(ctx context.Context) {
 
-	log := ctx.Value(contextKeyLog).(Logger)
-	defer log.Info("exiting")
+	rootLog := ctx.Value(contextKeyLog).(Logger)
+
+	j.snapperTask = NewTask("snapshot", rootLog)
+	j.mainTask = NewTask("main", rootLog)
+	j.handlerTask = NewTask("handler", rootLog)
+	j.pruneRHSTask = NewTask("prune_rhs", rootLog)
+	j.pruneLHSTask = NewTask("prune_lhs", rootLog)
 
 	local := rpc.NewLocalRPC()
 	// Allow access to any dataset since we control what mapping
@@ -90,78 +100,69 @@ func (j *LocalJob) JobStart(ctx context.Context) {
 	// All local datasets will be passed to its Map() function,
 	// but only those for which a mapping exists will actually be pulled.
 	// We can pay this small performance penalty for now.
-	handler := NewHandler(log.WithField(logTaskField, "handler"), localPullACL{}, &PrefixSnapshotFilter{j.SnapshotPrefix})
+	handler := NewHandler(j.handlerTask.Log(), localPullACL{}, NewPrefixFilter(j.SnapshotPrefix))
 
 	registerEndpoints(local, handler)
 
 	snapper := IntervalAutosnap{
+		task:             j.snapperTask,
 		DatasetFilter:    j.Mapping.AsFilter(),
 		Prefix:           j.SnapshotPrefix,
 		SnapshotInterval: j.Interval,
 	}
 
-	plhs, err := j.Pruner(PrunePolicySideLeft, false)
+	plhs, err := j.Pruner(j.pruneLHSTask, PrunePolicySideLeft, false)
 	if err != nil {
-		log.WithError(err).Error("error creating lhs pruner")
+		rootLog.WithError(err).Error("error creating lhs pruner")
 		return
 	}
-	prhs, err := j.Pruner(PrunePolicySideRight, false)
+	prhs, err := j.Pruner(j.pruneRHSTask, PrunePolicySideRight, false)
 	if err != nil {
-		log.WithError(err).Error("error creating rhs pruner")
+		rootLog.WithError(err).Error("error creating rhs pruner")
 		return
 	}
 
-	makeCtx := func(parent context.Context, taskName string) (ctx context.Context) {
-		return context.WithValue(parent, contextKeyLog, log.WithField(logTaskField, taskName))
-	}
-	var snapCtx, plCtx, prCtx, pullCtx context.Context
-	snapCtx = makeCtx(ctx, "autosnap")
-	plCtx = makeCtx(ctx, "prune_lhs")
-	prCtx = makeCtx(ctx, "prune_rhs")
-	pullCtx = makeCtx(ctx, "repl")
-
 	didSnaps := make(chan struct{})
-	go snapper.Run(snapCtx, didSnaps)
+	go snapper.Run(ctx, didSnaps)
 
 outer:
 	for {
 
 		select {
 		case <-ctx.Done():
+			j.mainTask.Log().WithError(ctx.Err()).Info("context")
 			break outer
 		case <-didSnaps:
-			log.Debug("finished taking snapshots")
-			log.Info("starting replication procedure")
+			j.mainTask.Log().Debug("finished taking snapshots")
+			j.mainTask.Log().Info("starting replication procedure")
 		}
 
-		{
-			log := pullCtx.Value(contextKeyLog).(Logger)
-			log.Debug("replicating from lhs to rhs")
-			err := doPull(PullContext{local, log, j.Mapping, j.InitialReplPolicy})
-			if err != nil {
-				log.WithError(err).Error("error replicating lhs to rhs")
-			}
-			// use a ctx as soon as doPull gains ctx support
-			select {
-			case <-ctx.Done():
-				break outer
-			default:
-			}
+		j.mainTask.Log().Debug("replicating from lhs to rhs")
+		j.mainTask.Enter("replicate")
+		puller := Puller{j.mainTask, local, j.Mapping, j.InitialReplPolicy}
+		puller.Pull()
+		j.mainTask.Finish()
+
+		// use a ctx as soon as Pull gains ctx support
+		select {
+		case <-ctx.Done():
+			break outer
+		default:
 		}
 
 		var wg sync.WaitGroup
 
-		log.Info("pruning lhs")
+		j.mainTask.Log().Info("pruning lhs")
 		wg.Add(1)
 		go func() {
-			plhs.Run(plCtx)
+			plhs.Run(ctx)
 			wg.Done()
 		}()
 
-		log.Info("pruning rhs")
+		j.mainTask.Log().Info("pruning rhs")
 		wg.Add(1)
 		go func() {
-			prhs.Run(prCtx)
+			prhs.Run(ctx)
 			wg.Done()
 		}()
 
@@ -169,11 +170,18 @@ outer:
 
 	}
 
-	log.WithError(ctx.Err()).Info("context")
-
 }
 
-func (j *LocalJob) Pruner(side PrunePolicySide, dryRun bool) (p Pruner, err error) {
+func (j *LocalJob) JobStatus(ctxt context.Context) (*JobStatus, error) {
+	return &JobStatus{Tasks: []*TaskStatus{
+		j.snapperTask.Status(),
+		j.pruneLHSTask.Status(),
+		j.pruneRHSTask.Status(),
+		j.mainTask.Status(),
+	}}, nil
+}
+
+func (j *LocalJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) {
 
 	var dsfilter zfs.DatasetFilter
 	var pp PrunePolicy
@@ -194,6 +202,7 @@ func (j *LocalJob) Pruner(side PrunePolicySide, dryRun bool) (p Pruner, err erro
 	}
 
 	p = Pruner{
+		task,
 		time.Now(),
 		dryRun,
 		dsfilter,

cmd/config_job_pull.go

@@ -4,6 +4,7 @@ import (
 	"time"
 
 	"context"
+	"fmt"
 	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
 	"github.com/zrepl/zrepl/rpc"
@@ -21,6 +22,8 @@ type PullJob struct {
 	InitialReplPolicy InitialReplPolicy
 	Prune             PrunePolicy
 	Debug             JobDebugSettings
+
+	task *Task
 }
 
 func parsePullJob(c JobParsingContext, name string, i map[string]interface{}) (j *PullJob, err error) {
@@ -48,7 +51,7 @@ func parsePullJob(c JobParsingContext, name string, i map[string]interface{}) (j
 		return nil, err
 	}
 
-	if j.Interval, err = time.ParseDuration(asMap.Interval); err != nil {
+	if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
 		err = errors.Wrap(err, "cannot parse 'interval'")
 		return nil, err
 	}
@@ -74,7 +77,7 @@ func parsePullJob(c JobParsingContext, name string, i map[string]interface{}) (j
 		return
 	}
 
-	if j.Prune, err = parsePrunePolicy(asMap.Prune); err != nil {
+	if j.Prune, err = parsePrunePolicy(asMap.Prune, false); err != nil {
 		err = errors.Wrap(err, "cannot parse prune policy")
 		return
 	}
@@ -95,15 +98,31 @@ func (j *PullJob) JobStart(ctx context.Context) {
 
 	log := ctx.Value(contextKeyLog).(Logger)
 	defer log.Info("exiting")
+	j.task = NewTask("main", log)
+
+	// j.task is idle here idle here
 
 	ticker := time.NewTicker(j.Interval)
+	for {
+		j.doRun(ctx)
+		select {
+		case <-ctx.Done():
+			j.task.Log().WithError(ctx.Err()).Info("context")
+			return
+		case <-ticker.C:
+		}
+	}
+}
 
-start:
+func (j *PullJob) doRun(ctx context.Context) {
 
-	log.Info("connecting")
+	j.task.Enter("run")
+	defer j.task.Finish()
+
+	j.task.Log().Info("connecting")
 	rwc, err := j.Connect.Connect()
 	if err != nil {
-		log.WithError(err).Error("error connecting")
+		j.task.Log().WithError(err).Error("error connecting")
 		return
 	}
 
@@ -114,43 +133,34 @@ start:
 
 	client := rpc.NewClient(rwc)
 	if j.Debug.RPC.Log {
-		client.SetLogger(log, true)
+		client.SetLogger(j.task.Log(), true)
 	}
 
-	log.Info("starting pull")
+	j.task.Enter("pull")
+	puller := Puller{j.task, client, j.Mapping, j.InitialReplPolicy}
+	puller.Pull()
+	closeRPCWithTimeout(j.task, client, time.Second*1, "")
+	rwc.Close()
+	j.task.Finish()
 
-	pullLog := log.WithField(logTaskField, "pull")
-	err = doPull(PullContext{client, pullLog, j.Mapping, j.InitialReplPolicy})
+	j.task.Enter("prune")
+	pruner, err := j.Pruner(j.task, PrunePolicySideDefault, false)
 	if err != nil {
-		log.WithError(err).Error("error doing pull")
-	}
-
-	closeRPCWithTimeout(log, client, time.Second*10, "")
-
-	log.Info("starting prune")
-	prunectx := context.WithValue(ctx, contextKeyLog, log.WithField(logTaskField, "prune"))
-	pruner, err := j.Pruner(PrunePolicySideDefault, false)
-	if err != nil {
-		log.WithError(err).Error("error creating pruner")
-		return
-	}
-
-	pruner.Run(prunectx)
-	log.Info("finish prune")
-
-	log.Info("wait for next interval")
-	select {
-	case <-ctx.Done():
-		log.WithError(ctx.Err()).Info("context")
-		return
-	case <-ticker.C:
-		goto start
+		j.task.Log().WithError(err).Error("error creating pruner")
+	} else {
+		pruner.Run(ctx)
 	}
+	j.task.Finish()
 
 }
 
-func (j *PullJob) Pruner(side PrunePolicySide, dryRun bool) (p Pruner, err error) {
+func (j *PullJob) JobStatus(ctxt context.Context) (*JobStatus, error) {
+	return &JobStatus{Tasks: []*TaskStatus{j.task.Status()}}, nil
+}
+
+func (j *PullJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) {
 	p = Pruner{
+		task,
 		time.Now(),
 		dryRun,
 		j.pruneFilter,
@@ -160,6 +170,26 @@ func (j *PullJob) Pruner(side PrunePolicySide, dryRun bool) (p Pruner, err error
 	return
 }
 
-func (j *PullJob) doRun(ctx context.Context) {
+func closeRPCWithTimeout(task *Task, remote rpc.RPCClient, timeout time.Duration, goodbye string) {
 
+	task.Log().Info("closing rpc connection")
+
+	ch := make(chan error)
+	go func() {
+		ch <- remote.Close()
+		close(ch)
+	}()
+
+	var err error
+	select {
+	case <-time.After(timeout):
+		err = fmt.Errorf("timeout exceeded (%s)", timeout)
+	case closeRequestErr := <-ch:
+		err = closeRequestErr
+	}
+
+	if err != nil {
+		task.Log().WithError(err).Error("error closing connection")
+	}
+	return
 }

cmd/config_job_source.go

@@ -2,29 +2,33 @@ package cmd
 
 import (
 	"context"
+	"io"
+	"time"
+
 	mapstructure "github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
 	"github.com/zrepl/zrepl/rpc"
 	"github.com/zrepl/zrepl/util"
-	"io"
-	"time"
 )
 
 type SourceJob struct {
 	Name           string
 	Serve          AuthenticatedChannelListenerFactory
-	Datasets       *DatasetMapFilter
+	Filesystems    *DatasetMapFilter
 	SnapshotPrefix string
 	Interval       time.Duration
 	Prune          PrunePolicy
 	Debug          JobDebugSettings
+	serveTask      *Task
+	autosnapTask   *Task
+	pruneTask      *Task
 }
 
 func parseSourceJob(c JobParsingContext, name string, i map[string]interface{}) (j *SourceJob, err error) {
 
 	var asMap struct {
 		Serve          map[string]interface{}
-		Datasets       map[string]string
+		Filesystems    map[string]string
 		SnapshotPrefix string `mapstructure:"snapshot_prefix"`
 		Interval       string
 		Prune          map[string]interface{}
@@ -42,7 +46,7 @@ func parseSourceJob(c JobParsingContext, name string, i map[string]interface{})
 		return
 	}
 
-	if j.Datasets, err = parseDatasetMapFilter(asMap.Datasets, true); err != nil {
+	if j.Filesystems, err = parseDatasetMapFilter(asMap.Filesystems, true); err != nil {
 		return
 	}
 
@@ -50,12 +54,12 @@ func parseSourceJob(c JobParsingContext, name string, i map[string]interface{})
 		return
 	}
 
-	if j.Interval, err = time.ParseDuration(asMap.Interval); err != nil {
+	if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
 		err = errors.Wrap(err, "cannot parse 'interval'")
 		return
 	}
 
-	if j.Prune, err = parsePrunePolicy(asMap.Prune); err != nil {
+	if j.Prune, err = parsePrunePolicy(asMap.Prune, true); err != nil {
 		err = errors.Wrap(err, "cannot parse 'prune'")
 		return
 	}
@@ -77,20 +81,22 @@ func (j *SourceJob) JobStart(ctx context.Context) {
 	log := ctx.Value(contextKeyLog).(Logger)
 	defer log.Info("exiting")
 
-	a := IntervalAutosnap{DatasetFilter: j.Datasets, Prefix: j.SnapshotPrefix, SnapshotInterval: j.Interval}
-	p, err := j.Pruner(PrunePolicySideDefault, false)
+	j.autosnapTask = NewTask("autosnap", log)
+	j.pruneTask = NewTask("prune", log)
+	j.serveTask = NewTask("serve", log)
+
+	a := IntervalAutosnap{j.autosnapTask, j.Filesystems, j.SnapshotPrefix, j.Interval}
+	p, err := j.Pruner(j.pruneTask, PrunePolicySideDefault, false)
+
 	if err != nil {
 		log.WithError(err).Error("error creating pruner")
 		return
 	}
 
-	snapContext := context.WithValue(ctx, contextKeyLog, log.WithField(logTaskField, "autosnap"))
-	prunerContext := context.WithValue(ctx, contextKeyLog, log.WithField(logTaskField, "prune"))
-	serveContext := context.WithValue(ctx, contextKeyLog, log.WithField(logTaskField, "serve"))
 	didSnaps := make(chan struct{})
 
-	go j.serve(serveContext)
-	go a.Run(snapContext, didSnaps)
+	go j.serve(ctx, j.serveTask)
+	go a.Run(ctx, didSnaps)
 
 outer:
 	for {
@@ -99,36 +105,48 @@ outer:
 			break outer
 		case <-didSnaps:
 			log.Info("starting pruner")
-			p.Run(prunerContext)
+			p.Run(ctx)
 			log.Info("pruner done")
 		}
 	}
-	log.WithError(prunerContext.Err()).Info("context")
+	log.WithError(ctx.Err()).Info("context")
 
 }
 
-func (j *SourceJob) Pruner(side PrunePolicySide, dryRun bool) (p Pruner, err error) {
+func (j *SourceJob) JobStatus(ctxt context.Context) (*JobStatus, error) {
+	return &JobStatus{
+		Tasks: []*TaskStatus{
+			j.autosnapTask.Status(),
+			j.pruneTask.Status(),
+			j.serveTask.Status(),
+		}}, nil
+}
+
+func (j *SourceJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) {
 	p = Pruner{
+		task,
 		time.Now(),
 		dryRun,
-		j.Datasets,
+		j.Filesystems,
 		j.SnapshotPrefix,
 		j.Prune,
 	}
 	return
 }
 
-func (j *SourceJob) serve(ctx context.Context) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
+func (j *SourceJob) serve(ctx context.Context, task *Task) {
 
 	listener, err := j.Serve.Listen()
 	if err != nil {
-		log.WithError(err).Error("error listening")
+		task.Log().WithError(err).Error("error listening")
 		return
 	}
 
-	rwcChan := make(chan io.ReadWriteCloser)
+	type rwcChanMsg struct {
+		rwc io.ReadWriteCloser
+		err error
+	}
+	rwcChan := make(chan rwcChanMsg)
 
 	// Serve connections until interrupted or error
 outer:
@@ -137,55 +155,96 @@ outer:
 		go func() {
 			rwc, err := listener.Accept()
 			if err != nil {
-				log.WithError(err).Error("error accepting connection")
+				rwcChan <- rwcChanMsg{rwc, err}
 				close(rwcChan)
 				return
 			}
-			rwcChan <- rwc
+			rwcChan <- rwcChanMsg{rwc, err}
 		}()
 
 		select {
 
-		case rwc, notClosed := <-rwcChan:
+		case rwcMsg := <-rwcChan:
 
-			if !notClosed {
-				break outer // closed because of accept error
+			if rwcMsg.err != nil {
+				task.Log().WithError(err).Error("error accepting connection")
+				break outer
 			}
 
-			rwc, err := util.NewReadWriteCloserLogger(rwc, j.Debug.Conn.ReadDump, j.Debug.Conn.WriteDump)
-			if err != nil {
-				panic(err)
-			}
-
-			// construct connection handler
-			handler := NewHandler(log, j.Datasets, &PrefixSnapshotFilter{j.SnapshotPrefix})
-
-			// handle connection
-			rpcServer := rpc.NewServer(rwc)
-			if j.Debug.RPC.Log {
-				rpclog := log.WithField("subsystem", "rpc")
-				rpcServer.SetLogger(rpclog, true)
-			}
-			registerEndpoints(rpcServer, handler)
-			if err = rpcServer.Serve(); err != nil {
-				log.WithError(err).Error("error serving connection")
-			}
-			rwc.Close()
+			j.handleConnection(rwcMsg.rwc, task)
 
 		case <-ctx.Done():
-			log.WithError(ctx.Err()).Info("context")
+			task.Log().WithError(ctx.Err()).Info("context")
 			break outer
 
 		}
 
 	}
 
-	log.Info("closing listener")
+	task.Enter("close_listener")
+	defer task.Finish()
 	err = listener.Close()
 	if err != nil {
-		log.WithError(err).Error("error closing listener")
+		task.Log().WithError(err).Error("error closing listener")
 	}
 
 	return
 
 }
+
+func (j *SourceJob) handleConnection(rwc io.ReadWriteCloser, task *Task) {
+
+	task.Enter("handle_connection")
+	defer task.Finish()
+
+	task.Log().Info("handling client connection")
+
+	rwc, err := util.NewReadWriteCloserLogger(rwc, j.Debug.Conn.ReadDump, j.Debug.Conn.WriteDump)
+	if err != nil {
+		panic(err)
+	}
+
+	// construct connection handler
+	handler := NewHandler(task.Log(), j.Filesystems, NewPrefixFilter(j.SnapshotPrefix))
+
+	// handle connection
+	rpcServer := rpc.NewServer(rwc)
+	if j.Debug.RPC.Log {
+		rpclog := task.Log().WithField("subsystem", "rpc")
+		rpcServer.SetLogger(rpclog, true)
+	}
+	registerEndpoints(rpcServer, handler)
+	if err = rpcServer.Serve(); err != nil {
+		task.Log().WithError(err).Error("error serving connection")
+	}
+
+	// wait for client to close connection
+	// FIXME: we cannot just close it like we would to with a TCP socket because
+	// FIXME: go-nettsh's Close() may overtake the remaining data in the pipe
+	const CLIENT_HANGUP_TIMEOUT = 1 * time.Second
+	task.Log().
+		WithField("timeout", CLIENT_HANGUP_TIMEOUT).
+		Debug("waiting for client to hang up")
+
+	wchan := make(chan error)
+	go func() {
+		var pseudo [1]byte
+		_, err := io.ReadFull(rwc, pseudo[:])
+		wchan <- err
+	}()
+	var werr error
+	select {
+	case werr = <-wchan:
+		// all right
+	case <-time.After(CLIENT_HANGUP_TIMEOUT):
+		werr = errors.New("client did not close connection within timeout")
+	}
+	if werr != nil && werr != io.EOF {
+		task.Log().WithError(werr).
+			Error("error waiting for client to hang up")
+	}
+	task.Log().Info("closing client connection")
+	if err = rwc.Close(); err != nil {
+		task.Log().WithError(err).Error("error force-closing connection")
+	}
+}

cmd/config_logging.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"crypto/tls"
 	"crypto/x509"
+	"github.com/mattn/go-isatty"
 	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
 	"github.com/zrepl/zrepl/logger"
@@ -12,156 +13,60 @@ import (
 )
 
 type LoggingConfig struct {
-	Outlets logger.Outlets
+	Outlets *logger.Outlets
 }
 
-type SetNoMetadataFormatter interface {
-	SetNoMetadata(noMetadata bool)
-}
+type MetadataFlags int64
+
+const (
+	MetadataTime MetadataFlags = 1 << iota
+	MetadataLevel
+
+	MetadataNone MetadataFlags = 0
+	MetadataAll  MetadataFlags = ^0
+)
 
 func parseLogging(i interface{}) (c *LoggingConfig, err error) {
 
 	c = &LoggingConfig{}
-	if i == nil {
-		return c, nil
-	}
-
-	var asMap struct {
-		Stdout struct {
-			Level  string
-			Format string
-		}
-		TCP struct {
-			Level         string
-			Format        string
-			Net           string
-			Address       string
-			RetryInterval string `mapstructure:"retry_interval"`
-			TLS           *struct {
-				CA   string
-				Cert string
-				Key  string
-			}
-		}
-		Syslog struct {
-			Enable        bool
-			Format        string
-			RetryInterval string `mapstructure:"retry_interval"`
-		}
-	}
-	if err = mapstructure.Decode(i, &asMap); err != nil {
-		return nil, errors.Wrap(err, "mapstructure error")
-	}
-
 	c.Outlets = logger.NewOutlets()
 
-	if asMap.Stdout.Level != "" {
+	var asList []interface{}
+	if err = mapstructure.Decode(i, &asList); err != nil {
+		return nil, errors.Wrap(err, "mapstructure error")
+	}
+	if len(asList) == 0 {
+		// Default config
+		out := WriterOutlet{&HumanFormatter{}, os.Stdout}
+		c.Outlets.Add(out, logger.Warn)
+		return
+	}
 
-		out := WriterOutlet{
-			&HumanFormatter{},
-			os.Stdout,
-		}
+	var syslogOutlets, stdoutOutlets int
+	for lei, le := range asList {
 
-		level, err := logger.ParseLevel(asMap.Stdout.Level)
+		outlet, minLevel, err := parseOutlet(le)
 		if err != nil {
-			return nil, errors.Wrap(err, "cannot parse 'level'")
+			return nil, errors.Wrapf(err, "cannot parse outlet #%d", lei)
+		}
+		var _ logger.Outlet = WriterOutlet{}
+		var _ logger.Outlet = &SyslogOutlet{}
+		switch outlet.(type) {
+		case *SyslogOutlet:
+			syslogOutlets++
+		case WriterOutlet:
+			stdoutOutlets++
 		}
 
-		if asMap.Stdout.Format != "" {
-			out.Formatter, err = parseLogFormat(asMap.Stdout.Format)
-			if err != nil {
-				return nil, errors.Wrap(err, "cannot parse 'format'")
-			}
-		}
-
-		c.Outlets.Add(out, level)
+		c.Outlets.Add(outlet, minLevel)
 
 	}
 
-	if asMap.TCP.Address != "" {
-
-		out := &TCPOutlet{}
-
-		out.Formatter, err = parseLogFormat(asMap.TCP.Format)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot parse 'format'")
-		}
-
-		lvl, err := logger.ParseLevel(asMap.TCP.Level)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot parse 'level'")
-		}
-
-		out.RetryInterval, err = time.ParseDuration(asMap.TCP.RetryInterval)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
-		}
-
-		out.Net, out.Address = asMap.TCP.Net, asMap.TCP.Address
-
-		if asMap.TCP.TLS != nil {
-
-			cert, err := tls.LoadX509KeyPair(asMap.TCP.TLS.Cert, asMap.TCP.TLS.Key)
-			if err != nil {
-				return nil, errors.Wrap(err, "cannot load client cert")
-			}
-
-			var rootCAs *x509.CertPool
-			if asMap.TCP.TLS.CA == "" {
-				if rootCAs, err = x509.SystemCertPool(); err != nil {
-					return nil, errors.Wrap(err, "cannot open system cert pool")
-				}
-			} else {
-				rootCAs = x509.NewCertPool()
-				rootCAPEM, err := ioutil.ReadFile(asMap.TCP.TLS.CA)
-				if err != nil {
-					return nil, errors.Wrap(err, "cannot load CA cert")
-				}
-				if !rootCAs.AppendCertsFromPEM(rootCAPEM) {
-					return nil, errors.New("cannot parse CA cert")
-				}
-			}
-			if err != nil && asMap.TCP.TLS.CA == "" {
-				return nil, errors.Wrap(err, "cannot load root ca pool")
-			}
-
-			out.TLS = &tls.Config{
-				Certificates: []tls.Certificate{cert},
-				RootCAs:      rootCAs,
-			}
-
-			out.TLS.BuildNameToCertificate()
-		}
-
-		c.Outlets.Add(out, lvl)
+	if syslogOutlets > 1 {
+		return nil, errors.Errorf("can only define one 'syslog' outlet")
 	}
-
-	if asMap.Syslog.Enable {
-
-		out := &SyslogOutlet{}
-
-		out.Formatter = &HumanFormatter{}
-		if asMap.Syslog.Format != "" {
-			out.Formatter, err = parseLogFormat(asMap.Syslog.Format)
-			if err != nil {
-				return nil, errors.Wrap(err, "cannot parse 'format'")
-			}
-		}
-
-		if f, ok := out.Formatter.(SetNoMetadataFormatter); ok {
-			f.SetNoMetadata(true)
-		}
-
-		out.RetryInterval = 0 // default to 0 as we assume local syslog will just work
-		if asMap.Syslog.RetryInterval != "" {
-			out.RetryInterval, err = time.ParseDuration(asMap.Syslog.RetryInterval)
-			if err != nil {
-				return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
-			}
-		}
-
-		c.Outlets.Add(out, logger.Debug)
-
+	if stdoutOutlets > 1 {
+		return nil, errors.Errorf("can only define one 'stdout' outlet")
 	}
 
 	return c, nil
@@ -189,3 +94,158 @@ func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
 	}
 
 }
+
+func parseOutlet(i interface{}) (o logger.Outlet, level logger.Level, err error) {
+
+	var in struct {
+		Outlet string
+		Level  string
+		Format string
+	}
+	if err = mapstructure.Decode(i, &in); err != nil {
+		err = errors.Wrap(err, "mapstructure error")
+		return
+	}
+	if in.Outlet == "" || in.Level == "" || in.Format == "" {
+		err = errors.Errorf("must specify 'outlet', 'level' and 'format' field")
+		return
+	}
+
+	minLevel, err := logger.ParseLevel(in.Level)
+	if err != nil {
+		err = errors.Wrap(err, "cannot parse 'level' field")
+		return
+	}
+	formatter, err := parseLogFormat(in.Format)
+	if err != nil {
+		err = errors.Wrap(err, "cannot parse")
+		return
+	}
+
+	switch in.Outlet {
+	case "stdout":
+		o, err = parseStdoutOutlet(i, formatter)
+	case "tcp":
+		o, err = parseTCPOutlet(i, formatter)
+	case "syslog":
+		o, err = parseSyslogOutlet(i, formatter)
+	default:
+		err = errors.Errorf("unknown outlet type '%s'", in.Outlet)
+	}
+	return o, minLevel, err
+
+}
+
+func parseStdoutOutlet(i interface{}, formatter EntryFormatter) (WriterOutlet, error) {
+
+	var in struct {
+		Time bool
+	}
+	if err := mapstructure.Decode(i, &in); err != nil {
+		return WriterOutlet{}, errors.Wrap(err, "invalid structure for stdout outlet")
+	}
+
+	flags := MetadataAll
+	writer := os.Stdout
+	if !isatty.IsTerminal(writer.Fd()) && !in.Time {
+		flags &= ^MetadataTime
+	}
+
+	formatter.SetMetadataFlags(flags)
+	return WriterOutlet{
+		formatter,
+		os.Stdout,
+	}, nil
+}
+
+func parseTCPOutlet(i interface{}, formatter EntryFormatter) (out *TCPOutlet, err error) {
+
+	var in struct {
+		Net           string
+		Address       string
+		RetryInterval string `mapstructure:"retry_interval"`
+		TLS           *struct {
+			CA   string
+			Cert string
+			Key  string
+		}
+	}
+	if err = mapstructure.Decode(i, &in); err != nil {
+		return nil, errors.Wrap(err, "mapstructure error")
+	}
+
+	retryInterval, err := time.ParseDuration(in.RetryInterval)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
+	}
+
+	if len(in.Net) == 0 {
+		return nil, errors.New("field 'net' must not be empty")
+	}
+	if len(in.Address) == 0 {
+		return nil, errors.New("field 'address' must not be empty")
+	}
+
+	var tlsConfig *tls.Config
+	if in.TLS != nil {
+
+		cert, err := tls.LoadX509KeyPair(in.TLS.Cert, in.TLS.Key)
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot load client cert")
+		}
+
+		var rootCAs *x509.CertPool
+		if in.TLS.CA == "" {
+			if rootCAs, err = x509.SystemCertPool(); err != nil {
+				return nil, errors.Wrap(err, "cannot open system cert pool")
+			}
+		} else {
+			rootCAs = x509.NewCertPool()
+			rootCAPEM, err := ioutil.ReadFile(in.TLS.CA)
+			if err != nil {
+				return nil, errors.Wrap(err, "cannot load CA cert")
+			}
+			if !rootCAs.AppendCertsFromPEM(rootCAPEM) {
+				return nil, errors.New("cannot parse CA cert")
+			}
+		}
+		if err != nil && in.TLS.CA == "" {
+			return nil, errors.Wrap(err, "cannot load root ca pool")
+		}
+
+		tlsConfig = &tls.Config{
+			Certificates: []tls.Certificate{cert},
+			RootCAs:      rootCAs,
+		}
+
+		tlsConfig.BuildNameToCertificate()
+	}
+
+	formatter.SetMetadataFlags(MetadataAll)
+	return NewTCPOutlet(formatter, in.Net, in.Address, tlsConfig, retryInterval), nil
+
+}
+
+func parseSyslogOutlet(i interface{}, formatter EntryFormatter) (out *SyslogOutlet, err error) {
+
+	var in struct {
+		RetryInterval string `mapstructure:"retry_interval"`
+	}
+	if err = mapstructure.Decode(i, &in); err != nil {
+		return nil, errors.Wrap(err, "mapstructure error")
+	}
+
+	out = &SyslogOutlet{}
+	out.Formatter = formatter
+	out.Formatter.SetMetadataFlags(MetadataNone)
+
+	out.RetryInterval = 0 // default to 0 as we assume local syslog will just work
+	if in.RetryInterval != "" {
+		out.RetryInterval, err = time.ParseDuration(in.RetryInterval)
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
+		}
+	}
+
+	return
+}

cmd/config_mapfilter.go

@@ -124,15 +124,10 @@ func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath,
 		err = fmt.Errorf("mapping target is not a dataset path: %s", err)
 		return
 	}
-	if m.entries[mi].subtreeMatch {
-		// strip common prefix
+	if me.subtreeMatch {
+		// strip common prefix ('<' wildcards are no special case here)
 		extendComps := source.Copy()
-		if me.path.Empty() {
-			// special case: trying to map the root => strip first component
-			extendComps.TrimNPrefixComps(1)
-		} else {
-			extendComps.TrimPrefix(me.path)
-		}
+		extendComps.TrimPrefix(me.path)
 		target.Extend(extendComps)
 	}
 	return

cmd/config_parse.go

@@ -8,6 +8,9 @@ import (
 	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
 	"os"
+	"regexp"
+	"strconv"
+	"time"
 )
 
 var ConfigFileDefaultLocations []string = []string{
@@ -217,7 +220,7 @@ err:
 	return
 }
 
-func parsePrunePolicy(v map[string]interface{}) (p PrunePolicy, err error) {
+func parsePrunePolicy(v map[string]interface{}, willSeeBookmarks bool) (p PrunePolicy, err error) {
 
 	policyName, err := extractStringField(v, "policy", true)
 	if err != nil {
@@ -226,14 +229,13 @@ func parsePrunePolicy(v map[string]interface{}) (p PrunePolicy, err error) {
 
 	switch policyName {
 	case "grid":
-		return parseGridPrunePolicy(v)
+		return parseGridPrunePolicy(v, willSeeBookmarks)
 	case "noprune":
 		return NoPrunePolicy{}, nil
 	default:
 		err = errors.Errorf("unknown policy '%s'", policyName)
 		return
 	}
-
 }
 
 func parseAuthenticatedChannelListenerFactory(c JobParsingContext, v map[string]interface{}) (p AuthenticatedChannelListenerFactory, err error) {
@@ -252,3 +254,41 @@ func parseAuthenticatedChannelListenerFactory(c JobParsingContext, v map[string]
 	}
 
 }
+
+var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
+
+func parsePostitiveDuration(e string) (d time.Duration, err error) {
+	comps := durationStringRegex.FindStringSubmatch(e)
+	if len(comps) != 3 {
+		err = fmt.Errorf("does not match regex: %s %#v", e, comps)
+		return
+	}
+
+	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+	if durationFactor <= 0 {
+		return 0, errors.New("duration must be positive integer")
+	}
+
+	var durationUnit time.Duration
+	switch comps[2] {
+	case "s":
+		durationUnit = time.Second
+	case "m":
+		durationUnit = time.Minute
+	case "h":
+		durationUnit = time.Hour
+	case "d":
+		durationUnit = 24 * time.Hour
+	case "w":
+		durationUnit = 24 * 7 * time.Hour
+	default:
+		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
+		return
+	}
+
+	d = time.Duration(durationFactor) * durationUnit
+	return
+}

cmd/config_prune_grid.go

@@ -6,6 +6,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/zrepl/zrepl/util"
 	"github.com/zrepl/zrepl/zfs"
+	"math"
 	"regexp"
 	"sort"
 	"strconv"
@@ -15,8 +16,11 @@ import (
 
 type GridPrunePolicy struct {
 	RetentionGrid *util.RetentionGrid
+	MaxBookmarks  int
 }
 
+const GridPrunePolicyMaxBookmarksKeepAll = -1
+
 type retentionGridAdaptor struct {
 	zfs.FilesystemVersion
 }
@@ -29,12 +33,27 @@ func (a retentionGridAdaptor) LessThan(b util.RetentionGridEntry) bool {
 	return a.CreateTXG < b.(retentionGridAdaptor).CreateTXG
 }
 
+// Prune filters snapshots with the retention grid.
+// Bookmarks are deleted such that KeepBookmarks are kept in the end.
+// The oldest bookmarks are removed first.
 func (p *GridPrunePolicy) Prune(_ *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error) {
+	skeep, sremove := p.pruneSnapshots(versions)
+	keep, remove = p.pruneBookmarks(skeep)
+	remove = append(remove, sremove...)
+	return keep, remove, nil
+}
+
+func (p *GridPrunePolicy) pruneSnapshots(versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion) {
 
 	// Build adaptors for retention grid
-	adaptors := make([]util.RetentionGridEntry, len(versions))
+	keep = []zfs.FilesystemVersion{}
+	adaptors := make([]util.RetentionGridEntry, 0)
 	for fsv := range versions {
-		adaptors[fsv] = retentionGridAdaptor{versions[fsv]}
+		if versions[fsv].Type != zfs.Snapshot {
+			keep = append(keep, versions[fsv])
+			continue
+		}
+		adaptors = append(adaptors, retentionGridAdaptor{versions[fsv]})
 	}
 
 	sort.SliceStable(adaptors, func(i, j int) bool {
@@ -46,9 +65,8 @@ func (p *GridPrunePolicy) Prune(_ *zfs.DatasetPath, versions []zfs.FilesystemVer
 	keepa, removea := p.RetentionGrid.FitEntries(now, adaptors)
 
 	// Revert adaptors
-	keep = make([]zfs.FilesystemVersion, len(keepa))
 	for i := range keepa {
-		keep[i] = keepa[i].(retentionGridAdaptor).FilesystemVersion
+		keep = append(keep, keepa[i].(retentionGridAdaptor).FilesystemVersion)
 	}
 	remove = make([]zfs.FilesystemVersion, len(removea))
 	for i := range removea {
@@ -58,20 +76,60 @@ func (p *GridPrunePolicy) Prune(_ *zfs.DatasetPath, versions []zfs.FilesystemVer
 
 }
 
-func parseGridPrunePolicy(e map[string]interface{}) (p *GridPrunePolicy, err error) {
+func (p *GridPrunePolicy) pruneBookmarks(versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion) {
 
-	var i struct {
-		Grid string
+	if p.MaxBookmarks == GridPrunePolicyMaxBookmarksKeepAll {
+		return versions, []zfs.FilesystemVersion{}
 	}
 
-	if err = mapstructure.Decode(e, &i); err != nil {
+	keep = []zfs.FilesystemVersion{}
+	bookmarks := make([]zfs.FilesystemVersion, 0)
+	for fsv := range versions {
+		if versions[fsv].Type != zfs.Bookmark {
+			keep = append(keep, versions[fsv])
+			continue
+		}
+		bookmarks = append(bookmarks, versions[fsv])
+	}
+
+	if len(bookmarks) == 0 {
+		return keep, []zfs.FilesystemVersion{}
+	}
+	if len(bookmarks) < p.MaxBookmarks {
+		keep = append(keep, bookmarks...)
+		return keep, []zfs.FilesystemVersion{}
+	}
+
+	// NOTE: sorting descending by descending by createtxg <=> sorting ascending wrt creation time
+	sort.SliceStable(bookmarks, func(i, j int) bool {
+		return (bookmarks[i].CreateTXG > bookmarks[j].CreateTXG)
+	})
+
+	keep = append(keep, bookmarks[:p.MaxBookmarks]...)
+	remove = bookmarks[p.MaxBookmarks:]
+
+	return keep, remove
+}
+
+func parseGridPrunePolicy(e map[string]interface{}, willSeeBookmarks bool) (p *GridPrunePolicy, err error) {
+
+	const KeepBookmarksAllString = "all"
+	var i struct {
+		Grid          string
+		KeepBookmarks string `mapstructure:"keep_bookmarks"`
+	}
+
+	dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{Result: &i, WeaklyTypedInput: true})
+	if err != nil {
+		err = errors.Wrap(err, "mapstructure error")
+		return
+	}
+	if err = dec.Decode(e); err != nil {
 		err = errors.Wrapf(err, "mapstructure error")
 		return
 	}
 
-	p = &GridPrunePolicy{}
-
-	// Parse grid policy
+	// Parse grid
 	intervals, err := parseRetentionGridIntervalsString(i.Grid)
 	if err != nil {
 		err = fmt.Errorf("cannot parse retention grid: %s", err)
@@ -80,53 +138,39 @@ func parseGridPrunePolicy(e map[string]interface{}) (p *GridPrunePolicy, err err
 	// Assert intervals are of increasing length (not necessarily required, but indicates config mistake)
 	lastDuration := time.Duration(0)
 	for i := range intervals {
+
 		if intervals[i].Length < lastDuration {
-			err = fmt.Errorf("retention grid interval length must be monotonically increasing:"+
-				"interval %d is shorter than %d", i+1, i)
+			// If all intervals before were keep=all, this is ok
+			allPrevKeepCountAll := true
+			for j := i - 1; allPrevKeepCountAll && j >= 0; j-- {
+				allPrevKeepCountAll = intervals[j].KeepCount == util.RetentionGridKeepCountAll
+			}
+			if allPrevKeepCountAll {
+				goto isMonotonicIncrease
+			}
+			err = errors.New("retention grid interval length must be monotonically increasing")
 			return
-		} else {
-			lastDuration = intervals[i].Length
 		}
-	}
-	p.RetentionGrid = util.NewRetentionGrid(intervals)
+	isMonotonicIncrease:
+		lastDuration = intervals[i].Length
 
-	return
-}
-
-var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
-
-func parseDuration(e string) (d time.Duration, err error) {
-	comps := durationStringRegex.FindStringSubmatch(e)
-	if len(comps) != 3 {
-		err = fmt.Errorf("does not match regex: %s %#v", e, comps)
-		return
 	}
 
-	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
-	if err != nil {
-		return
+	// Parse KeepBookmarks
+	keepBookmarks := 0
+	if i.KeepBookmarks == KeepBookmarksAllString || (i.KeepBookmarks == "" && !willSeeBookmarks) {
+		keepBookmarks = GridPrunePolicyMaxBookmarksKeepAll
+	} else {
+		i, err := strconv.ParseInt(i.KeepBookmarks, 10, 32)
+		if err != nil || i <= 0 || i > math.MaxInt32 {
+			return nil, errors.Errorf("keep_bookmarks must be positive integer or 'all'")
+		}
+		keepBookmarks = int(i)
 	}
-
-	var durationUnit time.Duration
-	switch comps[2] {
-	case "s":
-		durationUnit = time.Second
-	case "m":
-		durationUnit = time.Minute
-	case "h":
-		durationUnit = time.Hour
-	case "d":
-		durationUnit = 24 * time.Hour
-	case "w":
-		durationUnit = 24 * 7 * time.Hour
-	default:
-		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
-		return
-	}
-
-	d = time.Duration(durationFactor) * durationUnit
-	return
-
+	return &GridPrunePolicy{
+		util.NewRetentionGrid(intervals),
+		keepBookmarks,
+	}, nil
 }
 
 var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
@@ -146,7 +190,7 @@ func parseRetentionGridIntervalString(e string) (intervals []util.RetentionInter
 		return nil, fmt.Errorf("contains factor <= 0")
 	}
 
-	duration, err := parseDuration(comps[2])
+	duration, err := parsePostitiveDuration(comps[2])
 	if err != nil {
 		return nil, err
 	}

cmd/config_serve_stdinserver.go

@@ -1,18 +1,16 @@
 package cmd
 
 import (
-	"github.com/ftrvxmtrx/fd"
 	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
+	"github.com/problame/go-netssh"
 	"io"
-	"net"
-	"os"
 	"path"
 )
 
 type StdinserverListenerFactory struct {
 	ClientIdentity string `mapstructure:"client_identity"`
-	sockaddr       *net.UnixAddr
+	sockpath       string
 }
 
 func parseStdinserverListenerFactory(c JobParsingContext, i map[string]interface{}) (f *StdinserverListenerFactory, err error) {
@@ -27,78 +25,32 @@ func parseStdinserverListenerFactory(c JobParsingContext, i map[string]interface
 		return
 	}
 
-	f.sockaddr, err = stdinserverListenerSocket(c.Global.Serve.Stdinserver.SockDir, f.ClientIdentity)
-	if err != nil {
-		return
-	}
+	f.sockpath = path.Join(c.Global.Serve.Stdinserver.SockDir, f.ClientIdentity)
 
 	return
 }
 
-func stdinserverListenerSocket(sockdir, clientIdentity string) (addr *net.UnixAddr, err error) {
-	sockpath := path.Join(sockdir, clientIdentity)
-	addr, err = net.ResolveUnixAddr("unix", sockpath)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot resolve unix address")
-	}
-	return addr, nil
-}
-
 func (f *StdinserverListenerFactory) Listen() (al AuthenticatedChannelListener, err error) {
 
-	ul, err := ListenUnixPrivate(f.sockaddr)
-	if err != nil {
-		return nil, errors.Wrapf(err, "cannot listen on unix socket %s", f.sockaddr)
+	if err = PreparePrivateSockpath(f.sockpath); err != nil {
+		return nil, err
 	}
 
-	l := &StdinserverListener{ul}
-
-	return l, nil
+	l, err := netssh.Listen(f.sockpath)
+	if err != nil {
+		return nil, err
+	}
+	return StdinserverListener{l}, nil
 }
 
 type StdinserverListener struct {
-	l *net.UnixListener
+	l *netssh.Listener
 }
 
-type fdRWC struct {
-	stdin, stdout *os.File
-	control       *net.UnixConn
+func (l StdinserverListener) Accept() (ch io.ReadWriteCloser, err error) {
+	return l.l.Accept()
 }
 
-func (f fdRWC) Read(p []byte) (n int, err error) {
-	return f.stdin.Read(p)
-}
-
-func (f fdRWC) Write(p []byte) (n int, err error) {
-	return f.stdout.Write(p)
-}
-
-func (f fdRWC) Close() (err error) {
-	f.stdin.Close()
-	f.stdout.Close()
-	return f.control.Close()
-}
-
-func (l *StdinserverListener) Accept() (ch io.ReadWriteCloser, err error) {
-	c, err := l.l.Accept()
-	if err != nil {
-		err = errors.Wrap(err, "error accepting on unix listener")
-		return
-	}
-
-	// Read the stdin and stdout of the stdinserver command
-	files, err := fd.Get(c.(*net.UnixConn), 2, []string{"stdin", "stdout"})
-	if err != nil {
-		err = errors.Wrap(err, "error receiving fds from stdinserver command")
-		c.Close()
-	}
-
-	rwc := fdRWC{files[0], files[1], c.(*net.UnixConn)}
-
-	return rwc, nil
-
-}
-
-func (l *StdinserverListener) Close() (err error) {
-	return l.l.Close() // removes socket file automatically
+func (l StdinserverListener) Close() (err error) {
+	return l.l.Close()
 }

cmd/config_test.go

@@ -129,8 +129,8 @@ func TestDatasetMapFilter(t *testing.T) {
 	expectMapping(map1, "a/b/c/d/e", "")
 	expectMapping(map1, "a/b/e", "root2/e")
 	expectMapping(map1, "a/b", "root2")
-	expectMapping(map1, "x", "root3/b/c")
-	expectMapping(map1, "x/y", "root3/b/c/y")
+	expectMapping(map1, "x", "root3/b/c/x")
+	expectMapping(map1, "x/y", "root3/b/c/x/y")
 	expectMapping(map1, "q", "root4/1/2")
 	expectMapping(map1, "b", "")
 	expectMapping(map1, "q/r", "root4/1/2/r")

cmd/control.go

@@ -1,15 +1,22 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
+	"github.com/dustin/go-humanize"
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/logger"
 	"io"
 	golog "log"
 	"net"
 	"net/http"
-	"net/url"
 	"os"
+	"sort"
+	"strings"
+	"time"
 )
 
 var controlCmd = &cobra.Command{
@@ -18,18 +25,78 @@ var controlCmd = &cobra.Command{
 }
 
 var pprofCmd = &cobra.Command{
-	Use:   "pprof cpu OUTFILE",
-	Short: "pprof CPU of daemon to OUTFILE (- for stdout)",
+	Use:   "pprof off | [on TCP_LISTEN_ADDRESS]",
+	Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
 	Run:   doControlPProf,
+	PreRunE: func(cmd *cobra.Command, args []string) error {
+		if cmd.Flags().NArg() < 1 {
+			goto enargs
+		}
+		switch cmd.Flags().Arg(0) {
+		case "on":
+			pprofCmdArgs.msg.Run = true
+			if cmd.Flags().NArg() != 2 {
+				return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
+			}
+			pprofCmdArgs.msg.HttpListenAddress = cmd.Flags().Arg(1)
+		case "off":
+			if cmd.Flags().NArg() != 1 {
+				goto enargs
+			}
+			pprofCmdArgs.msg.Run = false
+		}
+		return nil
+	enargs:
+		return errors.New("invalid number of positional arguments")
+
+	},
 }
 var pprofCmdArgs struct {
-	seconds int64
+	msg PprofServerControlMsg
+}
+
+var controlVersionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "print version of running zrepl daemon",
+	Run:   doControLVersionCmd,
+}
+
+var controlStatusCmdArgs struct {
+	format      string
+	level       logger.Level
+	onlyShowJob string
+}
+
+var controlStatusCmd = &cobra.Command{
+	Use:   "status [JOB_NAME]",
+	Short: "get current status",
+	Run:   doControlStatusCmd,
 }
 
 func init() {
 	RootCmd.AddCommand(controlCmd)
 	controlCmd.AddCommand(pprofCmd)
-	pprofCmd.Flags().Int64Var(&pprofCmdArgs.seconds, "seconds", 30, "seconds to profile")
+	controlCmd.AddCommand(controlVersionCmd)
+	controlCmd.AddCommand(controlStatusCmd)
+	controlStatusCmd.Flags().StringVar(&controlStatusCmdArgs.format, "format", "human", "output format (human|raw)")
+	controlStatusCmdArgs.level = logger.Warn
+	controlStatusCmd.Flags().Var(&controlStatusCmdArgs.level, "level", "minimum log level to show")
+}
+
+func controlHttpClient() (client http.Client, err error) {
+
+	conf, err := ParseConfig(rootArgs.configFile)
+	if err != nil {
+		return http.Client{}, err
+	}
+
+	return http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", conf.Global.Control.Sockpath)
+			},
+		},
+	}, nil
 }
 
 func doControlPProf(cmd *cobra.Command, args []string) {
@@ -41,60 +108,198 @@ func doControlPProf(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	conf, err := ParseConfig(rootArgs.configFile)
+	log.Printf("connecting to zrepl daemon")
+	httpc, err := controlHttpClient()
 	if err != nil {
 		log.Printf("error parsing config: %s", err)
 		die()
 	}
 
-	if cmd.Flags().Arg(0) != "cpu" {
-		log.Printf("only CPU profiles are supported")
-		log.Printf("%s", cmd.UsageString())
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(&pprofCmdArgs.msg); err != nil {
+		log.Printf("error marshaling request: %s", err)
 		die()
 	}
-
-	outfn := cmd.Flags().Arg(1)
-	if outfn == "" {
-		log.Printf("must specify output filename")
-		log.Printf("%s", cmd.UsageString())
-		die()
-	}
-	var out io.Writer
-	if outfn == "-" {
-		out = os.Stdout
-	} else {
-		out, err = os.Create(outfn)
-		if err != nil {
-			log.Printf("error creating output file: %s", err)
-			die()
-		}
-	}
-
-	log.Printf("connecting to daemon")
-	httpc := http.Client{
-		Transport: &http.Transport{
-			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
-				return net.Dial("unix", conf.Global.Control.Sockpath)
-			},
-		},
-	}
-
-	log.Printf("profiling...")
-	v := url.Values{}
-	v.Set("seconds", fmt.Sprintf("%d", pprofCmdArgs.seconds))
-	v.Encode()
-	resp, err := httpc.Get("http://unix" + ControlJobEndpointProfile + "?" + v.Encode())
+	_, err = httpc.Post("http://unix"+ControlJobEndpointPProf, "application/json", &buf)
 	if err != nil {
 		log.Printf("error: %s", err)
 		die()
 	}
 
-	_, err = io.Copy(out, resp.Body)
+	log.Printf("finished")
+}
+
+func doControLVersionCmd(cmd *cobra.Command, args []string) {
+
+	log := golog.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Printf("exiting after error")
+		os.Exit(1)
+	}
+
+	httpc, err := controlHttpClient()
 	if err != nil {
-		log.Printf("error writing profile: %s", err)
+		log.Printf("could not connect to daemon: %s", err)
 		die()
 	}
 
-	log.Printf("finished")
+	resp, err := httpc.Get("http://unix" + ControlJobEndpointVersion)
+	if err != nil {
+		log.Printf("error: %s", err)
+		die()
+	} else if resp.StatusCode != http.StatusOK {
+		var msg bytes.Buffer
+		io.CopyN(&msg, resp.Body, 4096)
+		log.Printf("error: %s", msg.String())
+		die()
+	}
+
+	var info ZreplVersionInformation
+	err = json.NewDecoder(resp.Body).Decode(&info)
+	if err != nil {
+		log.Printf("error unmarshaling response: %s", err)
+		die()
+	}
+
+	fmt.Println(info.String())
+
+}
+
+func doControlStatusCmd(cmd *cobra.Command, args []string) {
+
+	log := golog.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Print("exiting after error")
+		os.Exit(1)
+	}
+
+	if len(args) == 1 {
+		controlStatusCmdArgs.onlyShowJob = args[0]
+	} else if len(args) > 1 {
+		log.Print("can only specify one job as positional argument")
+		cmd.Usage()
+		die()
+	}
+
+	httpc, err := controlHttpClient()
+	if err != nil {
+		log.Printf("could not connect to daemon: %s", err)
+		die()
+	}
+
+	resp, err := httpc.Get("http://unix" + ControlJobEndpointStatus)
+	if err != nil {
+		log.Printf("error: %s", err)
+		die()
+	} else if resp.StatusCode != http.StatusOK {
+		var msg bytes.Buffer
+		io.CopyN(&msg, resp.Body, 4096)
+		log.Printf("error: %s", msg.String())
+		die()
+	}
+
+	var status DaemonStatus
+	err = json.NewDecoder(resp.Body).Decode(&status)
+	if err != nil {
+		log.Printf("error unmarshaling response: %s", err)
+		die()
+	}
+
+	switch controlStatusCmdArgs.format {
+	case "raw":
+		enc := json.NewEncoder(os.Stdout)
+		enc.SetIndent("", "  ")
+		if err := enc.Encode(status); err != nil {
+			log.Panic(err)
+		}
+	case "human":
+
+		formatter := HumanFormatter{}
+		formatter.SetMetadataFlags(MetadataAll)
+		formatter.SetIgnoreFields([]string{
+			logJobField,
+		})
+		jobNames := make([]string, 0, len(status.Jobs))
+		for name, _ := range status.Jobs {
+			jobNames = append(jobNames, name)
+		}
+		sort.Slice(jobNames, func(i, j int) bool {
+			return strings.Compare(jobNames[i], jobNames[j]) == -1
+		})
+		now := time.Now()
+		for _, name := range jobNames {
+
+			if controlStatusCmdArgs.onlyShowJob != "" && name != controlStatusCmdArgs.onlyShowJob {
+				continue
+			}
+
+			job := status.Jobs[name]
+			jobLogEntries := make([]logger.Entry, 0)
+			informAboutError := false
+
+			fmt.Printf("Job '%s':\n", name)
+			for _, task := range job.Tasks {
+
+				var header bytes.Buffer
+				fmt.Fprintf(&header, "  Task '%s': ", task.Name)
+				if !task.Idle {
+					fmt.Fprint(&header, strings.Join(task.ActivityStack, "."))
+				} else {
+					fmt.Fprint(&header, "<idle>")
+				}
+				fmt.Fprint(&header, " ")
+				const TASK_STALLED_HOLDOFF_DURATION = 10 * time.Second
+				sinceLastUpdate := now.Sub(task.LastUpdate)
+				if !task.Idle || task.ProgressRx != 0 || task.ProgressTx != 0 {
+					fmt.Fprintf(&header, "(%s / %s , Rx/Tx",
+						humanize.Bytes(uint64(task.ProgressRx)),
+						humanize.Bytes(uint64(task.ProgressTx)))
+					if task.Idle {
+						fmt.Fprint(&header, ", values from last run")
+					}
+					fmt.Fprint(&header, ")")
+				}
+				fmt.Fprint(&header, "\n")
+				if !task.Idle && !task.LastUpdate.IsZero() && sinceLastUpdate >= TASK_STALLED_HOLDOFF_DURATION {
+					informAboutError = true
+					fmt.Fprintf(&header, "    WARNING: last update %s ago at %s)",
+						sinceLastUpdate.String(),
+						task.LastUpdate.Format(HumanFormatterDateFormat))
+					fmt.Fprint(&header, "\n")
+				}
+				io.Copy(os.Stdout, &header)
+
+				jobLogEntries = append(jobLogEntries, task.LogEntries...)
+				informAboutError = informAboutError || task.MaxLogLevel >= logger.Warn
+			}
+
+			sort.Slice(jobLogEntries, func(i, j int) bool {
+				return jobLogEntries[i].Time.Before(jobLogEntries[j].Time)
+			})
+			if informAboutError {
+				fmt.Println("    WARNING: Some tasks encountered problems since the last time they left idle state:")
+				fmt.Println("             check the logs below or your log file for more information.")
+				fmt.Println("             Use the --level flag if you need debug information.")
+				fmt.Println()
+			}
+			for _, e := range jobLogEntries {
+				if e.Level < controlStatusCmdArgs.level {
+					continue
+				}
+				formatted, err := formatter.Format(&e)
+				if err != nil {
+					panic(err)
+				}
+				fmt.Printf("      %s\n", string(formatted))
+			}
+			fmt.Println()
+
+		}
+	default:
+		log.Printf("invalid output format '%s'", controlStatusCmdArgs.format)
+		die()
+	}
 
 }

cmd/control_pprof.go

@@ -0,0 +1,80 @@
+package cmd
+
+import (
+	"net/http"
+	// FIXME: importing this package has the side-effect of poisoning the http.DefaultServeMux
+	// FIXME: with the /debug/pprof endpoints
+	"context"
+	"net"
+	"net/http/pprof"
+)
+
+type PProfServer struct {
+	cc       chan PprofServerControlMsg
+	state    PprofServerControlMsg
+	listener net.Listener
+}
+
+type PprofServerControlMsg struct {
+	// Whether the server should listen for requests on the given address
+	Run bool
+	// Must be set if Run is true, undefined otherwise
+	HttpListenAddress string
+}
+
+func NewPProfServer(ctx context.Context) *PProfServer {
+
+	s := &PProfServer{
+		cc: make(chan PprofServerControlMsg),
+	}
+
+	go s.controlLoop(ctx)
+	return s
+}
+
+func (s *PProfServer) controlLoop(ctx context.Context) {
+outer:
+	for {
+
+		var msg PprofServerControlMsg
+		select {
+		case <-ctx.Done():
+			if s.listener != nil {
+				s.listener.Close()
+			}
+			break outer
+		case msg = <-s.cc:
+			// proceed
+		}
+
+		var err error
+		if msg.Run && s.listener == nil {
+
+			s.listener, err = net.Listen("tcp", msg.HttpListenAddress)
+			if err != nil {
+				s.listener = nil
+				continue
+			}
+
+			// FIXME: because net/http/pprof does not provide a mux,
+			mux := http.NewServeMux()
+			mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
+			mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
+			mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
+			mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
+			mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
+			go http.Serve(s.listener, mux)
+			continue
+		}
+
+		if !msg.Run && s.listener != nil {
+			s.listener.Close()
+			s.listener = nil
+			continue
+		}
+	}
+}
+
+func (s *PProfServer) Control(msg PprofServerControlMsg) {
+	s.cc <- msg
+}

cmd/daemon.go

@@ -1,12 +1,16 @@
 package cmd
 
 import (
+	"container/list"
 	"context"
 	"fmt"
 	"github.com/spf13/cobra"
 	"github.com/zrepl/zrepl/logger"
+	"io"
 	"os"
 	"os/signal"
+	"strings"
+	"sync"
 	"syscall"
 	"time"
 )
@@ -25,18 +29,20 @@ func init() {
 type Job interface {
 	JobName() string
 	JobStart(ctxt context.Context)
+	JobStatus(ctxt context.Context) (*JobStatus, error)
 }
 
 func doDaemon(cmd *cobra.Command, args []string) {
 
 	conf, err := ParseConfig(rootArgs.configFile)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "error parsing config: %s", err)
+		fmt.Fprintf(os.Stderr, "error parsing config: %s\n", err)
 		os.Exit(1)
 	}
 
 	log := logger.NewLogger(conf.Global.logging.Outlets, 1*time.Second)
 
+	log.Info(NewZreplVersionInformation().String())
 	log.Debug("starting daemon")
 	ctx := context.WithValue(context.Background(), contextKeyLog, log)
 	ctx = context.WithValue(ctx, contextKeyLog, log)
@@ -49,22 +55,27 @@ func doDaemon(cmd *cobra.Command, args []string) {
 type contextKey string
 
 const (
-	contextKeyLog contextKey = contextKey("log")
+	contextKeyLog    contextKey = contextKey("log")
+	contextKeyDaemon contextKey = contextKey("daemon")
 )
 
 type Daemon struct {
-	conf *Config
+	conf      *Config
+	startedAt time.Time
 }
 
 func NewDaemon(initialConf *Config) *Daemon {
-	return &Daemon{initialConf}
+	return &Daemon{conf: initialConf}
 }
 
 func (d *Daemon) Loop(ctx context.Context) {
 
+	d.startedAt = time.Now()
+
 	log := ctx.Value(contextKeyLog).(Logger)
 
 	ctx, cancel := context.WithCancel(ctx)
+	ctx = context.WithValue(ctx, contextKeyDaemon, d)
 
 	sigChan := make(chan os.Signal, 1)
 	finishs := make(chan Job)
@@ -108,3 +119,276 @@ outer:
 	log.Info("exiting")
 
 }
+
+// Representation of a Job's status that is composed of Tasks
+type JobStatus struct {
+	// Statuses of all tasks of this job
+	Tasks []*TaskStatus
+	// Error != "" if JobStatus() returned an error
+	JobStatusError string
+}
+
+// Representation of a Daemon's status that is composed of Jobs
+type DaemonStatus struct {
+	StartedAt time.Time
+	Jobs      map[string]*JobStatus
+}
+
+func (d *Daemon) Status() (s *DaemonStatus) {
+
+	s = &DaemonStatus{}
+	s.StartedAt = d.startedAt
+
+	s.Jobs = make(map[string]*JobStatus, len(d.conf.Jobs))
+
+	for name, j := range d.conf.Jobs {
+		status, err := j.JobStatus(context.TODO())
+		if err != nil {
+			s.Jobs[name] = &JobStatus{nil, err.Error()}
+			continue
+		}
+		s.Jobs[name] = status
+	}
+
+	return
+}
+
+// Representation of a Task's status
+type TaskStatus struct {
+	Name string
+	// Whether the task is idle.
+	Idle bool
+	// The stack of activities the task is currently executing.
+	// The first element is the root activity and equal to Name.
+	ActivityStack []string
+	// Number of bytes received by the task since it last left idle state.
+	ProgressRx int64
+	// Number of bytes sent by the task since it last left idle state.
+	ProgressTx int64
+	// Log entries emitted by the task since it last left idle state.
+	// Only contains the log entries emitted through the task's logger
+	// (provided by Task.Log()).
+	LogEntries []logger.Entry
+	// The maximum log level of LogEntries.
+	// Only valid if len(LogEntries) > 0.
+	MaxLogLevel logger.Level
+	// Last time something about the Task changed
+	LastUpdate time.Time
+}
+
+// An instance of Task tracks  a single thread of activity that is part of a Job.
+type Task struct {
+	// Stack of activities the task is currently in
+	// Members are instances of taskActivity
+	activities *list.List
+	// Last time activities was changed (not the activities inside, the list)
+	activitiesLastUpdate time.Time
+	// Protects Task members from modification
+	rwl sync.RWMutex
+}
+
+// Structure that describes the progress a Task has made
+type taskProgress struct {
+	rx         int64
+	tx         int64
+	lastUpdate time.Time
+	logEntries []logger.Entry
+	mtx        sync.RWMutex
+}
+
+func newTaskProgress() (p *taskProgress) {
+	return &taskProgress{
+		logEntries: make([]logger.Entry, 0),
+	}
+}
+
+func (p *taskProgress) UpdateIO(drx, dtx int64) {
+	p.mtx.Lock()
+	defer p.mtx.Unlock()
+	p.rx += drx
+	p.tx += dtx
+	p.lastUpdate = time.Now()
+}
+
+func (p *taskProgress) UpdateLogEntry(entry logger.Entry) {
+	p.mtx.Lock()
+	defer p.mtx.Unlock()
+	// FIXME: ensure maximum size (issue #48)
+	p.logEntries = append(p.logEntries, entry)
+	p.lastUpdate = time.Now()
+}
+
+func (p *taskProgress) DeepCopy() (out taskProgress) {
+	p.mtx.RLock()
+	defer p.mtx.RUnlock()
+	out.rx, out.tx = p.rx, p.tx
+	out.lastUpdate = p.lastUpdate
+	out.logEntries = make([]logger.Entry, len(p.logEntries))
+	for i := range p.logEntries {
+		out.logEntries[i] = p.logEntries[i]
+	}
+	return
+}
+
+// returns a copy of this taskProgress, the mutex carries no semantic value
+func (p *taskProgress) Read() (out taskProgress) {
+	p.mtx.RLock()
+	defer p.mtx.RUnlock()
+	return p.DeepCopy()
+}
+
+// Element of a Task's activity stack
+type taskActivity struct {
+	name   string
+	idle   bool
+	logger *logger.Logger
+	// The progress of the task that is updated by UpdateIO() and UpdateLogEntry()
+	//
+	// Progress happens on a task-level and is thus global to the task.
+	// That's why progress is just a pointer to the current taskProgress:
+	// we reset progress when leaving the idle root activity
+	progress *taskProgress
+}
+
+func NewTask(name string, lg *logger.Logger) *Task {
+	t := &Task{
+		activities: list.New(),
+	}
+	rootLogger := lg.ReplaceField(logTaskField, name).
+		WithOutlet(t, logger.Debug)
+	rootAct := &taskActivity{name, true, rootLogger, newTaskProgress()}
+	t.activities.PushFront(rootAct)
+	return t
+}
+
+// callers must hold t.rwl
+func (t *Task) cur() *taskActivity {
+	return t.activities.Front().Value.(*taskActivity)
+}
+
+// buildActivityStack returns the stack of activity names
+// t.rwl must be held, but the slice can be returned since strings are immutable
+func (t *Task) buildActivityStack() []string {
+	comps := make([]string, 0, t.activities.Len())
+	for e := t.activities.Back(); e != nil; e = e.Prev() {
+		act := e.Value.(*taskActivity)
+		comps = append(comps, act.name)
+	}
+	return comps
+}
+
+// Start a sub-activity.
+// Must always be matched with a call to t.Finish()
+// --- consider using defer for this purpose.
+func (t *Task) Enter(activity string) {
+	t.rwl.Lock()
+	defer t.rwl.Unlock()
+
+	prev := t.cur()
+	if prev.idle {
+		// reset progress when leaving idle task
+		// we leave the old progress dangling to have the user not worry about
+		prev.progress = newTaskProgress()
+	}
+	act := &taskActivity{activity, false, nil, prev.progress}
+	t.activities.PushFront(act)
+	stack := t.buildActivityStack()
+	activityField := strings.Join(stack, ".")
+	act.logger = prev.logger.ReplaceField(logTaskField, activityField)
+
+	t.activitiesLastUpdate = time.Now()
+}
+
+func (t *Task) UpdateProgress(dtx, drx int64) {
+	t.rwl.RLock()
+	p := t.cur().progress // protected by own rwlock
+	t.rwl.RUnlock()
+	p.UpdateIO(dtx, drx)
+}
+
+// Returns a wrapper io.Reader that updates this task's _current_ progress value.
+// Progress updates after this task resets its progress value are discarded.
+func (t *Task) ProgressUpdater(r io.Reader) *IOProgressUpdater {
+	t.rwl.RLock()
+	defer t.rwl.RUnlock()
+	return &IOProgressUpdater{r, t.cur().progress}
+}
+
+func (t *Task) Status() *TaskStatus {
+	t.rwl.RLock()
+	defer t.rwl.RUnlock()
+	// NOTE
+	// do not return any state in TaskStatus that is protected by t.rwl
+
+	cur := t.cur()
+	stack := t.buildActivityStack()
+	prog := cur.progress.Read()
+
+	var maxLevel logger.Level
+	for _, entry := range prog.logEntries {
+		if maxLevel < entry.Level {
+			maxLevel = entry.Level
+		}
+	}
+
+	lastUpdate := prog.lastUpdate
+	if lastUpdate.Before(t.activitiesLastUpdate) {
+		lastUpdate = t.activitiesLastUpdate
+	}
+
+	s := &TaskStatus{
+		Name:          stack[0],
+		ActivityStack: stack,
+		Idle:          cur.idle,
+		ProgressRx:    prog.rx,
+		ProgressTx:    prog.tx,
+		LogEntries:    prog.logEntries,
+		MaxLogLevel:   maxLevel,
+		LastUpdate:    lastUpdate,
+	}
+
+	return s
+}
+
+// Finish a sub-activity.
+// Corresponds to a preceding call to t.Enter()
+func (t *Task) Finish() {
+	t.rwl.Lock()
+	defer t.rwl.Unlock()
+	top := t.activities.Front()
+	if top.Next() == nil {
+		return // cannot remove root activity
+	}
+	t.activities.Remove(top)
+	t.activitiesLastUpdate = time.Now()
+
+}
+
+// Returns a logger derived from the logger passed to the constructor function.
+// The logger's task field contains the current activity stack joined by '.'.
+func (t *Task) Log() *logger.Logger {
+	t.rwl.RLock()
+	defer t.rwl.RUnlock()
+	// FIXME should influence TaskStatus's LastUpdate field
+	return t.cur().logger
+}
+
+// implement logger.Outlet interface
+func (t *Task) WriteEntry(entry logger.Entry) error {
+	t.rwl.RLock()
+	defer t.rwl.RUnlock()
+	t.cur().progress.UpdateLogEntry(entry)
+	return nil
+}
+
+type IOProgressUpdater struct {
+	r io.Reader
+	p *taskProgress
+}
+
+func (u *IOProgressUpdater) Read(p []byte) (n int, err error) {
+	n, err = u.r.Read(p)
+	u.p.UpdateIO(int64(n), 0)
+	return
+
+}

cmd/handler.go

@@ -99,7 +99,7 @@ func (h Handler) HandleFilesystemVersionsRequest(r *FilesystemVersionsRequest, v
 		return
 	}
 
-	log.WithField("resposne", vs).Debug("response")
+	log.WithField("response", vs).Debug("response")
 
 	*versions = vs
 	return

cmd/helpers.go

@@ -7,28 +7,42 @@ import (
 	"path/filepath"
 )
 
-func ListenUnixPrivate(sockaddr *net.UnixAddr) (*net.UnixListener, error) {
-
-	sockdir := filepath.Dir(sockaddr.Name)
+func PreparePrivateSockpath(sockpath string) error {
+	sockdir := filepath.Dir(sockpath)
 	sdstat, err := os.Stat(sockdir)
 	if err != nil {
-		return nil, errors.Wrapf(err, "cannot stat(2) '%s'", sockdir)
+		return errors.Wrapf(err, "cannot stat(2) '%s'", sockdir)
 	}
 	if !sdstat.IsDir() {
-		return nil, errors.Errorf("not a directory: %s", sockdir)
+		return errors.Errorf("not a directory: %s", sockdir)
 	}
 	p := sdstat.Mode().Perm()
 	if p&0007 != 0 {
-		return nil, errors.Errorf("socket directory not be world-accessible: %s (permissions are %#o)", sockdir, p)
+		return errors.Errorf("socket directory must not be world-accessible: %s (permissions are %#o)", sockdir, p)
 	}
 
 	// Maybe things have not been cleaned up before
-	s, err := os.Stat(sockaddr.Name)
-	if err == nil {
-		if s.Mode()&os.ModeSocket != 0 {
-			// opportunistically try to remove it, but if this fails, it is not an error
-			os.Remove(sockaddr.Name)
-		}
+	s, err := os.Stat(sockpath)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return errors.Wrapf(err, "cannot stat(2) '%s'", sockpath)
+	}
+	if s.Mode()&os.ModeSocket == 0 {
+		return errors.Errorf("unexpected file type at path '%s'", sockpath)
+	}
+	err = os.Remove(sockpath)
+	if err != nil {
+		return errors.Wrapf(err, "cannot remove presumably stale socket '%s'", sockpath)
+	}
+	return nil
+}
+
+func ListenUnixPrivate(sockaddr *net.UnixAddr) (*net.UnixListener, error) {
+
+	if err := PreparePrivateSockpath(sockaddr.Name); err != nil {
+		return nil, err
 	}
 
 	return net.ListenUnix("unix", sockaddr)

cmd/logging_formatters.go

@@ -7,11 +7,11 @@ import (
 	"github.com/go-logfmt/logfmt"
 	"github.com/pkg/errors"
 	"github.com/zrepl/zrepl/logger"
-	"strings"
 	"time"
 )
 
 type EntryFormatter interface {
+	SetMetadataFlags(flags MetadataFlags)
 	Format(e *logger.Entry) ([]byte, error)
 }
 
@@ -33,25 +33,47 @@ const (
 
 type NoFormatter struct{}
 
+func (f NoFormatter) SetMetadataFlags(flags MetadataFlags) {}
+
 func (f NoFormatter) Format(e *logger.Entry) ([]byte, error) {
 	return []byte(e.Message), nil
 }
 
 type HumanFormatter struct {
-	NoMetadata bool
+	metadataFlags MetadataFlags
+	ignoreFields  map[string]bool
 }
 
-var _ SetNoMetadataFormatter = &HumanFormatter{}
+const HumanFormatterDateFormat = time.RFC3339
 
-func (f *HumanFormatter) SetNoMetadata(noMetadata bool) {
-	f.NoMetadata = noMetadata
+func (f *HumanFormatter) SetMetadataFlags(flags MetadataFlags) {
+	f.metadataFlags = flags
+}
+
+func (f *HumanFormatter) SetIgnoreFields(ignore []string) {
+	if ignore == nil {
+		f.ignoreFields = nil
+		return
+	}
+	f.ignoreFields = make(map[string]bool, len(ignore))
+
+	for _, field := range ignore {
+		f.ignoreFields[field] = true
+	}
+}
+
+func (f *HumanFormatter) ignored(field string) bool {
+	return f.ignoreFields != nil && f.ignoreFields[field]
 }
 
 func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
 
 	var line bytes.Buffer
 
-	if !f.NoMetadata {
+	if f.metadataFlags&MetadataTime != 0 {
+		fmt.Fprintf(&line, "%s ", e.Time.Format(HumanFormatterDateFormat))
+	}
+	if f.metadataFlags&MetadataLevel != 0 {
 		fmt.Fprintf(&line, "[%s]", e.Level.Short())
 	}
 
@@ -60,8 +82,10 @@ func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
 	for _, field := range prefixFields {
 		val, ok := e.Fields[field].(string)
 		if ok {
-			fmt.Fprintf(&line, "[%s]", val)
-			prefixed[field] = true
+			if !f.ignored(field) {
+				fmt.Fprintf(&line, "[%s]", val)
+				prefixed[field] = true
+			}
 		} else {
 			break
 		}
@@ -69,13 +93,13 @@ func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
 	// even more prefix fields
 	mapFrom, mapFromOk := e.Fields[logMapFromField].(string)
 	mapTo, mapToOk := e.Fields[logMapToField].(string)
-	if mapFromOk && mapToOk {
+	if mapFromOk && mapToOk && !f.ignored(logMapFromField) && !f.ignored(logMapToField) {
 		fmt.Fprintf(&line, "[%s => %s]", mapFrom, mapTo)
 		prefixed[logMapFromField], prefixed[logMapToField] = true, true
 	}
 	incFrom, incFromOk := e.Fields[logIncFromField].(string)
 	incTo, incToOk := e.Fields[logIncToField].(string)
-	if incFromOk && incToOk {
+	if incFromOk && incToOk && !f.ignored(logIncFromField) && !f.ignored(logMapToField) {
 		fmt.Fprintf(&line, "[%s => %s]", incFrom, incTo)
 		prefixed[logIncFromField], prefixed[logIncToField] = true, true
 	}
@@ -85,22 +109,29 @@ func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
 	}
 	fmt.Fprint(&line, e.Message)
 
-	for field, value := range e.Fields {
-
-		if prefixed[field] {
-			continue
+	if len(e.Fields)-len(prefixed) > 0 {
+		fmt.Fprint(&line, " ")
+		enc := logfmt.NewEncoder(&line)
+		for field, value := range e.Fields {
+			if prefixed[field] || f.ignored(field) {
+				continue
+			}
+			if err := logfmtTryEncodeKeyval(enc, field, value); err != nil {
+				return nil, err
+			}
 		}
-
-		if strings.ContainsAny(field, " \t") {
-			return nil, errors.Errorf("field must not contain whitespace: '%s'", field)
-		}
-		fmt.Fprintf(&line, " %s=\"%s\"", field, value)
 	}
 
 	return line.Bytes(), nil
 }
 
-type JSONFormatter struct{}
+type JSONFormatter struct {
+	metadataFlags MetadataFlags
+}
+
+func (f *JSONFormatter) SetMetadataFlags(flags MetadataFlags) {
+	f.metadataFlags = flags
+}
 
 func (f *JSONFormatter) Format(e *logger.Entry) ([]byte, error) {
 	data := make(logger.Fields, len(e.Fields)+3)
@@ -128,21 +159,21 @@ func (f *JSONFormatter) Format(e *logger.Entry) ([]byte, error) {
 }
 
 type LogfmtFormatter struct {
-	NoMetadata bool
+	metadataFlags MetadataFlags
 }
 
-var _ SetNoMetadataFormatter = &LogfmtFormatter{}
-
-func (f *LogfmtFormatter) SetNoMetadata(noMetadata bool) {
-	f.NoMetadata = noMetadata
+func (f *LogfmtFormatter) SetMetadataFlags(flags MetadataFlags) {
+	f.metadataFlags = flags
 }
 
 func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
 	var buf bytes.Buffer
 	enc := logfmt.NewEncoder(&buf)
 
-	if !f.NoMetadata {
+	if f.metadataFlags&MetadataTime != 0 {
 		enc.EncodeKeyval(FieldTime, e.Time)
+	}
+	if f.metadataFlags&MetadataLevel != 0 {
 		enc.EncodeKeyval(FieldLevel, e.Level)
 	}
 
@@ -154,7 +185,9 @@ func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
 		if !ok {
 			break
 		}
-		enc.EncodeKeyval(pf, v)
+		if err := logfmtTryEncodeKeyval(enc, pf, v); err != nil {
+			return nil, err // unlikely
+		}
 		prefixed[pf] = true
 	}
 
@@ -162,13 +195,25 @@ func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
 
 	for k, v := range e.Fields {
 		if !prefixed[k] {
-			enc.EncodeKeyval(k, v)
+			if err := logfmtTryEncodeKeyval(enc, k, v); err != nil {
+				return nil, err
+			}
 		}
 	}
 
-	if err := enc.EndRecord(); err != nil {
-		return nil, err
-	}
-
 	return buf.Bytes(), nil
 }
+
+func logfmtTryEncodeKeyval(enc *logfmt.Encoder, field, value interface{}) error {
+
+	err := enc.EncodeKeyval(field, value)
+	switch err {
+	case nil: // ok
+		return nil
+	case logfmt.ErrUnsupportedValueType:
+		enc.EncodeKeyval(field, fmt.Sprintf("<%T>", value))
+		return nil
+	}
+	return errors.Wrapf(err, "cannot encode field '%s'", field)
+
+}

cmd/logging_outlets.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"crypto/tls"
 	"github.com/pkg/errors"
@@ -16,7 +17,7 @@ type WriterOutlet struct {
 	Writer    io.Writer
 }
 
-func (h WriterOutlet) WriteEntry(ctx context.Context, entry logger.Entry) error {
+func (h WriterOutlet) WriteEntry(entry logger.Entry) error {
 	bytes, err := h.Formatter.Format(&entry)
 	if err != nil {
 		return err
@@ -27,56 +28,92 @@ func (h WriterOutlet) WriteEntry(ctx context.Context, entry logger.Entry) error
 }
 
 type TCPOutlet struct {
-	Formatter    EntryFormatter
-	Net, Address string
-	Dialer       net.Dialer
-	TLS          *tls.Config
+	formatter EntryFormatter
 	// Specifies how much time must pass between a connection error and a reconnection attempt
 	// Log entries written to the outlet during this time interval are silently dropped.
-	RetryInterval time.Duration
-	// nil if there was an error sending / connecting to remote server
-	conn net.Conn
-	// Last time an error occurred when sending / connecting to remote server
-	retry time.Time
+	connect   func(ctx context.Context) (net.Conn, error)
+	entryChan chan *bytes.Buffer
 }
 
-func (h *TCPOutlet) WriteEntry(ctx context.Context, e logger.Entry) error {
+func NewTCPOutlet(formatter EntryFormatter, network, address string, tlsConfig *tls.Config, retryInterval time.Duration) *TCPOutlet {
 
-	b, err := h.Formatter.Format(&e)
+	connect := func(ctx context.Context) (conn net.Conn, err error) {
+		deadl, ok := ctx.Deadline()
+		if !ok {
+			deadl = time.Time{}
+		}
+		dialer := net.Dialer{
+			Deadline: deadl,
+		}
+		if tlsConfig != nil {
+			conn, err = tls.DialWithDialer(&dialer, network, address, tlsConfig)
+		} else {
+			conn, err = dialer.DialContext(ctx, network, address)
+		}
+		return
+	}
+
+	entryChan := make(chan *bytes.Buffer, 1) // allow one message in flight while previos is in io.Copy()
+
+	o := &TCPOutlet{
+		formatter: formatter,
+		connect:   connect,
+		entryChan: entryChan,
+	}
+
+	go o.outLoop(retryInterval)
+
+	return o
+}
+
+// FIXME: use this method
+func (h *TCPOutlet) Close() {
+	close(h.entryChan)
+}
+
+func (h *TCPOutlet) outLoop(retryInterval time.Duration) {
+
+	var retry time.Time
+	var conn net.Conn
+	for msg := range h.entryChan {
+		var err error
+		for conn == nil {
+			time.Sleep(time.Until(retry))
+			ctx, cancel := context.WithDeadline(context.TODO(), time.Now().Add(retryInterval))
+			conn, err = h.connect(ctx)
+			cancel()
+			if err != nil {
+				retry = time.Now().Add(retryInterval)
+				conn = nil
+			}
+		}
+		conn.SetWriteDeadline(time.Now().Add(retryInterval))
+		_, err = io.Copy(conn, msg)
+		if err != nil {
+			retry = time.Now().Add(retryInterval)
+			conn.Close()
+			conn = nil
+		}
+	}
+}
+
+func (h *TCPOutlet) WriteEntry(e logger.Entry) error {
+
+	ebytes, err := h.formatter.Format(&e)
 	if err != nil {
 		return err
 	}
 
-	if h.conn == nil {
-		if time.Now().Sub(h.retry) < h.RetryInterval {
-			// cool-down phase, drop the log entry
-			return nil
-		}
+	buf := new(bytes.Buffer)
+	buf.Write(ebytes)
+	buf.WriteString("\n")
 
-		if h.TLS != nil {
-			h.conn, err = tls.DialWithDialer(&h.Dialer, h.Net, h.Address, h.TLS)
-		} else {
-			h.conn, err = h.Dialer.DialContext(ctx, h.Net, h.Address)
-		}
-		if err != nil {
-			h.conn = nil
-			h.retry = time.Now()
-			return errors.Wrap(err, "cannot dial")
-		}
+	select {
+	case h.entryChan <- buf:
+		return nil
+	default:
+		return errors.New("connection broken or not fast enough")
 	}
-
-	_, err = h.conn.Write(b)
-	if err == nil {
-		_, err = h.conn.Write([]byte("\n"))
-	}
-	if err != nil {
-		h.conn.Close()
-		h.conn = nil
-		h.retry = time.Now()
-		return errors.Wrap(err, "cannot write")
-	}
-
-	return nil
 }
 
 type SyslogOutlet struct {
@@ -86,7 +123,7 @@ type SyslogOutlet struct {
 	lastConnectAttempt time.Time
 }
 
-func (o *SyslogOutlet) WriteEntry(ctx context.Context, entry logger.Entry) error {
+func (o *SyslogOutlet) WriteEntry(entry logger.Entry) error {
 
 	bytes, err := o.Formatter.Format(&entry)
 	if err != nil {

cmd/main.go

@@ -20,6 +20,10 @@ import (
 //	Printf(format string, v ...interface{})
 //}
 
+var (
+	zreplVersion string // set by build infrastructure
+)
+
 type Logger = *logger.Logger
 
 var RootCmd = &cobra.Command{

cmd/prune.go

@@ -3,11 +3,14 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"github.com/zrepl/zrepl/util"
 	"github.com/zrepl/zrepl/zfs"
+	"sync"
 	"time"
 )
 
 type Pruner struct {
+	task           *Task
 	Now            time.Time
 	DryRun         bool
 	DatasetFilter  zfs.DatasetFilter
@@ -22,55 +25,66 @@ type PruneResult struct {
 	Remove     []zfs.FilesystemVersion
 }
 
-func (p *Pruner) Run(ctx context.Context) (r []PruneResult, err error) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-
-	if p.DryRun {
-		log.Info("doing dry run")
-	}
-
+// FIXME must not call p.task.Enter because it runs in parallel
+func (p *Pruner) filterFilesystems() (filesystems []*zfs.DatasetPath, stop bool) {
 	filesystems, err := zfs.ZFSListMapping(p.DatasetFilter)
 	if err != nil {
-		log.WithError(err).Error("error applying filesystem filter")
-		return nil, err
+		p.task.Log().WithError(err).Error("error applying filesystem filter")
+		return nil, true
 	}
 	if len(filesystems) <= 0 {
-		log.Info("no filesystems matching filter")
-		return nil, err
+		p.task.Log().Info("no filesystems matching filter")
+		return nil, true
+	}
+	return filesystems, false
+}
+
+// FIXME must not call p.task.Enter because it runs in parallel
+func (p *Pruner) filterVersions(fs *zfs.DatasetPath) (fsversions []zfs.FilesystemVersion, stop bool) {
+	log := p.task.Log().WithField(logFSField, fs.ToString())
+
+	filter := NewPrefixFilter(p.SnapshotPrefix)
+	fsversions, err := zfs.ZFSListFilesystemVersions(fs, filter)
+	if err != nil {
+		log.WithError(err).Error("error listing filesytem versions")
+		return nil, true
+	}
+	if len(fsversions) == 0 {
+		log.WithField("prefix", p.SnapshotPrefix).Info("no filesystem versions matching prefix")
+		return nil, true
+	}
+	return fsversions, false
+}
+
+// FIXME must not call p.task.Enter because it runs in parallel
+func (p *Pruner) pruneFilesystem(fs *zfs.DatasetPath, destroySemaphore util.Semaphore) (r PruneResult, valid bool) {
+	log := p.task.Log().WithField(logFSField, fs.ToString())
+
+	fsversions, stop := p.filterVersions(fs)
+	if stop {
+		return
 	}
 
-	r = make([]PruneResult, 0, len(filesystems))
+	keep, remove, err := p.PrunePolicy.Prune(fs, fsversions)
+	if err != nil {
+		log.WithError(err).Error("error evaluating prune policy")
+		return
+	}
 
-	for _, fs := range filesystems {
+	log.WithField("fsversions", fsversions).
+		WithField("keep", keep).
+		WithField("remove", remove).
+		Debug("prune policy debug dump")
 
-		log := log.WithField(logFSField, fs.ToString())
+	r = PruneResult{fs, fsversions, keep, remove}
 
-		fsversions, err := zfs.ZFSListFilesystemVersions(fs, &PrefixSnapshotFilter{p.SnapshotPrefix})
-		if err != nil {
-			log.WithError(err).Error("error listing filesytem versions")
-			continue
-		}
-		if len(fsversions) == 0 {
-			log.WithField("prefix", p.SnapshotPrefix).Info("no filesystem versions matching prefix")
-			continue
-		}
-
-		keep, remove, err := p.PrunePolicy.Prune(fs, fsversions)
-		if err != nil {
-			log.WithError(err).Error("error evaluating prune policy")
-			continue
-		}
-
-		log.WithField("fsversions", fsversions).
-			WithField("keep", keep).
-			WithField("remove", remove).
-			Debug("prune policy debug dump")
-
-		r = append(r, PruneResult{fs, fsversions, keep, remove})
-
-		makeFields := func(v zfs.FilesystemVersion) (fields map[string]interface{}) {
-			fields = make(map[string]interface{})
+	var wg sync.WaitGroup
+	for v := range remove {
+		wg.Add(1)
+		go func(v zfs.FilesystemVersion) {
+			defer wg.Done()
+			// log fields
+			fields := make(map[string]interface{})
 			fields["version"] = v.ToAbsPath(fs)
 			timeSince := v.Creation.Sub(p.Now)
 			fields["age_ns"] = timeSince
@@ -78,24 +92,64 @@ func (p *Pruner) Run(ctx context.Context) (r []PruneResult, err error) {
 			days := timeSince / day
 			remainder := timeSince % day
 			fields["age_str"] = fmt.Sprintf("%dd%s", days, remainder)
-			return
-		}
 
-		for _, v := range remove {
-			fields := makeFields(v)
 			log.WithFields(fields).Info("destroying version")
 			// echo what we'll do and exec zfs destroy if not dry run
 			// TODO special handling for EBUSY (zfs hold)
 			// TODO error handling for clones? just echo to cli, skip over, and exit with non-zero status code (we're idempotent)
 			if !p.DryRun {
+				destroySemaphore.Down()
 				err := zfs.ZFSDestroyFilesystemVersion(fs, v)
+				destroySemaphore.Up()
 				if err != nil {
-					// handle
 					log.WithFields(fields).WithError(err).Error("error destroying version")
 				}
 			}
-		}
+		}(remove[v])
+	}
+	wg.Wait()
 
+	return r, true
+}
+
+func (p *Pruner) Run(ctx context.Context) (r []PruneResult, err error) {
+	p.task.Enter("run")
+	defer p.task.Finish()
+
+	if p.DryRun {
+		p.task.Log().Info("doing dry run")
+	}
+
+	filesystems, stop := p.filterFilesystems()
+	if stop {
+		return
+	}
+
+	maxConcurrentDestroy := len(filesystems)
+	p.task.Log().WithField("max_concurrent_destroy", maxConcurrentDestroy).Info("begin concurrent destroy")
+	destroySem := util.NewSemaphore(maxConcurrentDestroy)
+
+	resChan := make(chan PruneResult, len(filesystems))
+	var wg sync.WaitGroup
+	for _, fs := range filesystems {
+		wg.Add(1)
+		go func(fs *zfs.DatasetPath) {
+			defer wg.Done()
+			res, ok := p.pruneFilesystem(fs, destroySem)
+			if ok {
+				resChan <- res
+			}
+
+		}(fs)
+	}
+	wg.Wait()
+	close(resChan)
+
+	p.task.Log().Info("destroys done")
+
+	r = make([]PruneResult, 0, len(filesystems))
+	for res := range resChan {
+		r = append(r, res)
 	}
 
 	return

cmd/replication.go

@@ -3,12 +3,10 @@ package cmd
 import (
 	"fmt"
 	"io"
-	"time"
 
 	"bytes"
 	"encoding/json"
 	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/util"
 	"github.com/zrepl/zrepl/zfs"
 )
 
@@ -18,8 +16,6 @@ func (a localPullACL) Filter(p *zfs.DatasetPath) (pass bool, err error) {
 	return true, nil
 }
 
-const LOCAL_TRANSPORT_IDENTITY string = "local"
-
 const DEFAULT_INITIAL_REPL_POLICY = InitialReplPolicyMostRecent
 
 type InitialReplPolicy string
@@ -29,85 +25,273 @@ const (
 	InitialReplPolicyAll        InitialReplPolicy = "all"
 )
 
-func closeRPCWithTimeout(log Logger, remote rpc.RPCClient, timeout time.Duration, goodbye string) {
-	log.Info("closing rpc connection")
-
-	ch := make(chan error)
-	go func() {
-		ch <- remote.Close()
-		close(ch)
-	}()
-
-	var err error
-	select {
-	case <-time.After(timeout):
-		err = fmt.Errorf("timeout exceeded (%s)", timeout)
-	case closeRequestErr := <-ch:
-		err = closeRequestErr
-	}
-
-	if err != nil {
-		log.WithError(err).Error("error closing connection")
-	}
-	return
-}
-
-type PullContext struct {
+type Puller struct {
+	task              *Task
 	Remote            rpc.RPCClient
-	Log               Logger
 	Mapping           DatasetMapping
 	InitialReplPolicy InitialReplPolicy
 }
 
-func doPull(pull PullContext) (err error) {
+type remoteLocalMapping struct {
+	Remote *zfs.DatasetPath
+	Local  *zfs.DatasetPath
+}
 
-	remote := pull.Remote
-	log := pull.Log
+func (p *Puller) getRemoteFilesystems() (rfs []*zfs.DatasetPath, ok bool) {
+	p.task.Enter("fetch_remote_fs_list")
+	defer p.task.Finish()
 
-	log.Info("request remote filesystem list")
 	fsr := FilesystemRequest{}
-	var remoteFilesystems []*zfs.DatasetPath
-	if err = remote.Call("FilesystemRequest", &fsr, &remoteFilesystems); err != nil {
-		return
+	if err := p.Remote.Call("FilesystemRequest", &fsr, &rfs); err != nil {
+		p.task.Log().WithError(err).Error("cannot fetch remote filesystem list")
+		return nil, false
 	}
+	return rfs, true
+}
 
-	log.Debug("map remote filesystems to local paths and determine order for per-filesystem sync")
-	type RemoteLocalMapping struct {
-		Remote *zfs.DatasetPath
-		Local  *zfs.DatasetPath
-	}
-	replMapping := make(map[string]RemoteLocalMapping, len(remoteFilesystems))
-	localTraversal := zfs.NewDatasetPathForest()
+func (p *Puller) buildReplMapping(remoteFilesystems []*zfs.DatasetPath) (replMapping map[string]remoteLocalMapping, ok bool) {
+	p.task.Enter("build_repl_mapping")
+	defer p.task.Finish()
+
+	replMapping = make(map[string]remoteLocalMapping, len(remoteFilesystems))
 	for fs := range remoteFilesystems {
 		var err error
 		var localFs *zfs.DatasetPath
-		localFs, err = pull.Mapping.Map(remoteFilesystems[fs])
+		localFs, err = p.Mapping.Map(remoteFilesystems[fs])
 		if err != nil {
 			err := fmt.Errorf("error mapping %s: %s", remoteFilesystems[fs], err)
-			log.WithError(err).WithField(logMapFromField, remoteFilesystems[fs]).Error("cannot map")
-			return err
+			p.task.Log().WithError(err).WithField(logMapFromField, remoteFilesystems[fs]).Error("cannot map")
+			return nil, false
 		}
 		if localFs == nil {
 			continue
 		}
-		log.WithField(logMapFromField, remoteFilesystems[fs].ToString()).
+		p.task.Log().WithField(logMapFromField, remoteFilesystems[fs].ToString()).
 			WithField(logMapToField, localFs.ToString()).Debug("mapping")
-		m := RemoteLocalMapping{remoteFilesystems[fs], localFs}
+		m := remoteLocalMapping{remoteFilesystems[fs], localFs}
 		replMapping[m.Local.ToString()] = m
+	}
+	return replMapping, true
+}
+
+// returns true if the receiving filesystem (local side) exists and can have child filesystems
+func (p *Puller) replFilesystem(m remoteLocalMapping, localFilesystemState map[string]zfs.FilesystemState) (localExists bool) {
+
+	p.task.Enter("repl_fs")
+	defer p.task.Finish()
+	var err error
+	remote := p.Remote
+
+	log := p.task.Log().
+		WithField(logMapFromField, m.Remote.ToString()).
+		WithField(logMapToField, m.Local.ToString())
+
+	log.Debug("examining local filesystem state")
+	localState, localExists := localFilesystemState[m.Local.ToString()]
+	var versions []zfs.FilesystemVersion
+	switch {
+	case !localExists:
+		log.Info("local filesystem does not exist")
+	case localState.Placeholder:
+		log.Info("local filesystem is marked as placeholder")
+	default:
+		log.Debug("local filesystem exists")
+		log.Debug("requesting local filesystem versions")
+		if versions, err = zfs.ZFSListFilesystemVersions(m.Local, nil); err != nil {
+			log.WithError(err).Error("cannot get local filesystem versions")
+			return false
+		}
+	}
+
+	log.Info("requesting remote filesystem versions")
+	r := FilesystemVersionsRequest{
+		Filesystem: m.Remote,
+	}
+	var theirVersions []zfs.FilesystemVersion
+	if err = remote.Call("FilesystemVersionsRequest", &r, &theirVersions); err != nil {
+		log.WithError(err).Error("cannot get remote filesystem versions")
+		log.Warn("stopping replication for all filesystems mapped as children of receiving filesystem")
+		return false
+	}
+
+	log.Debug("computing diff between remote and local filesystem versions")
+	diff := zfs.MakeFilesystemDiff(versions, theirVersions)
+	log.WithField("diff", diff).Debug("diff between local and remote filesystem")
+
+	if localState.Placeholder && diff.Conflict != zfs.ConflictAllRight {
+		panic("internal inconsistency: local placeholder implies ConflictAllRight")
+	}
+
+	switch diff.Conflict {
+	case zfs.ConflictAllRight:
+
+		log.WithField("replication_policy", p.InitialReplPolicy).Info("performing initial sync, following policy")
+
+		if p.InitialReplPolicy != InitialReplPolicyMostRecent {
+			panic(fmt.Sprintf("policy '%s' not implemented", p.InitialReplPolicy))
+		}
+
+		snapsOnly := make([]zfs.FilesystemVersion, 0, len(diff.MRCAPathRight))
+		for s := range diff.MRCAPathRight {
+			if diff.MRCAPathRight[s].Type == zfs.Snapshot {
+				snapsOnly = append(snapsOnly, diff.MRCAPathRight[s])
+			}
+		}
+
+		if len(snapsOnly) < 1 {
+			log.Warn("cannot perform initial sync: no remote snapshots")
+			return false
+		}
+
+		r := InitialTransferRequest{
+			Filesystem:        m.Remote,
+			FilesystemVersion: snapsOnly[len(snapsOnly)-1],
+		}
+
+		log.WithField("version", r.FilesystemVersion).Debug("requesting snapshot stream")
+
+		var stream io.Reader
+
+		if err = remote.Call("InitialTransferRequest", &r, &stream); err != nil {
+			log.WithError(err).Error("cannot request initial transfer")
+			return false
+		}
+		log.Debug("received initial transfer request response")
+
+		log.Debug("invoke zfs receive")
+		recvArgs := []string{"-u"}
+		if localState.Placeholder {
+			log.Info("receive with forced rollback to replace placeholder filesystem")
+			recvArgs = append(recvArgs, "-F")
+		}
+		progressStream := p.task.ProgressUpdater(stream)
+		if err = zfs.ZFSRecv(m.Local, progressStream, recvArgs...); err != nil {
+			log.WithError(err).Error("cannot receive stream")
+			return false
+		}
+		log.Info("finished receiving stream") // TODO rx delta
+
+		// TODO unify with recv path of ConflictIncremental
+		log.Debug("configuring properties of received filesystem")
+		props := zfs.NewZFSProperties()
+		props.Set("readonly", "on")
+		if err = zfs.ZFSSet(m.Local, props); err != nil {
+			log.WithError(err).Error("cannot set readonly property")
+		}
+
+		log.Info("finished initial transfer")
+		return true
+
+	case zfs.ConflictIncremental:
+
+		if len(diff.IncrementalPath) < 2 {
+			log.Info("remote and local are in sync")
+			return true
+		}
+
+		log.Info("following incremental path from diff")
+		for i := 0; i < len(diff.IncrementalPath)-1; i++ {
+
+			from, to := diff.IncrementalPath[i], diff.IncrementalPath[i+1]
+
+			log, _ := log.WithField(logIncFromField, from.Name).WithField(logIncToField, to.Name), 0
+
+			log.Debug("requesting incremental snapshot stream")
+			r := IncrementalTransferRequest{
+				Filesystem: m.Remote,
+				From:       from,
+				To:         to,
+			}
+			var stream io.Reader
+			if err = remote.Call("IncrementalTransferRequest", &r, &stream); err != nil {
+				log.WithError(err).Error("cannot request incremental snapshot stream")
+				return false
+			}
+
+			log.Debug("invoking zfs receive")
+			progressStream := p.task.ProgressUpdater(stream)
+			// TODO protect against malicious incremental stream
+			if err = zfs.ZFSRecv(m.Local, progressStream); err != nil {
+				log.WithError(err).Error("cannot receive stream")
+				return false
+			}
+			log.Info("finished incremental transfer") // TODO increment rx
+
+		}
+		log.Info("finished following incremental path") // TODO path rx
+		return true
+
+	case zfs.ConflictNoCommonAncestor:
+		fallthrough
+	case zfs.ConflictDiverged:
+
+		var jsonDiff bytes.Buffer
+		if err := json.NewEncoder(&jsonDiff).Encode(diff); err != nil {
+			log.WithError(err).Error("cannot JSON-encode diff")
+			return false
+		}
+
+		var problem, resolution string
+
+		switch diff.Conflict {
+		case zfs.ConflictNoCommonAncestor:
+			problem = "remote and local filesystem have snapshots, but no common one"
+			resolution = "perform manual establish a common snapshot history"
+		case zfs.ConflictDiverged:
+			problem = "remote and local filesystem share a history but have diverged"
+			resolution = "perform manual replication or delete snapshots on the receiving" +
+				"side  to establish an incremental replication parse"
+		}
+
+		log.WithField("diff", jsonDiff.String()).
+			WithField("problem", problem).
+			WithField("resolution", resolution).
+			Error("manual conflict resolution required")
+
+		return false
+
+	}
+
+	panic("should not be reached")
+}
+
+func (p *Puller) Pull() {
+	p.task.Enter("run")
+	defer p.task.Finish()
+
+	p.task.Log().Info("request remote filesystem list")
+	remoteFilesystems, ok := p.getRemoteFilesystems()
+	if !ok {
+		return
+	}
+
+	p.task.Log().Debug("map remote filesystems to local paths and determine order for per-filesystem sync")
+	replMapping, ok := p.buildReplMapping(remoteFilesystems)
+	if !ok {
+
+	}
+
+	p.task.Log().Debug("build cache for already present local filesystem state")
+	p.task.Enter("cache_local_fs_state")
+	localFilesystemState, err := zfs.ZFSListFilesystemState()
+	p.task.Finish()
+	if err != nil {
+		p.task.Log().WithError(err).Error("cannot request local filesystem state")
+		return
+	}
+
+	localTraversal := zfs.NewDatasetPathForest()
+	for _, m := range replMapping {
 		localTraversal.Add(m.Local)
 	}
 
-	log.Debug("build cache for already present local filesystem state")
-	localFilesystemState, err := zfs.ZFSListFilesystemState()
-	if err != nil {
-		log.WithError(err).Error("cannot request local filesystem state")
-		return err
-	}
-
-	log.Info("start per-filesystem sync")
+	p.task.Log().Info("start per-filesystem sync")
 	localTraversal.WalkTopDown(func(v zfs.DatasetPathVisit) bool {
 
-		log := log.WithField(logFSField, v.Path.ToString())
+		p.task.Enter("tree_walk")
+		defer p.task.Finish()
+
+		log := p.task.Log().WithField(logFSField, v.Path.ToString())
 
 		if v.FilledIn {
 			if _, exists := localFilesystemState[v.Path.ToString()]; exists {
@@ -116,7 +300,9 @@ func doPull(pull PullContext) (err error) {
 				return true
 			}
 			log.Debug("create placeholder filesystem")
+			p.task.Enter("create_placeholder")
 			err = zfs.ZFSCreatePlaceholderFilesystem(v.Path)
+			p.task.Finish()
 			if err != nil {
 				log.Error("cannot create placeholder filesystem")
 				return false
@@ -129,189 +315,7 @@ func doPull(pull PullContext) (err error) {
 			panic("internal inconsistency: replMapping should contain mapping for any path that was not filled in by WalkTopDown()")
 		}
 
-		log = log.WithField(logMapToField, m.Remote.ToString()).
-			WithField(logMapFromField, m.Local.ToString())
-
-		log.Debug("examing local filesystem state")
-		localState, localExists := localFilesystemState[m.Local.ToString()]
-		var versions []zfs.FilesystemVersion
-		switch {
-		case !localExists:
-			log.Info("local filesystem does not exist")
-		case localState.Placeholder:
-			log.Info("local filesystem is marked as placeholder")
-		default:
-			log.Debug("local filesystem exists")
-			log.Debug("requesting local filesystem versions")
-			if versions, err = zfs.ZFSListFilesystemVersions(m.Local, nil); err != nil {
-				log.WithError(err).Error("cannot get local filesystem versions")
-				return false
-			}
-		}
-
-		log.Info("requesting remote filesystem versions")
-		r := FilesystemVersionsRequest{
-			Filesystem: m.Remote,
-		}
-		var theirVersions []zfs.FilesystemVersion
-		if err = remote.Call("FilesystemVersionsRequest", &r, &theirVersions); err != nil {
-			log.WithError(err).Error("cannot get remote filesystem versions")
-			log.Warn("stopping replication for all filesystems mapped as children of receiving filesystem")
-			return false
-		}
-
-		log.Debug("computing diff between remote and local filesystem versions")
-		diff := zfs.MakeFilesystemDiff(versions, theirVersions)
-		log.WithField("diff", diff).Debug("diff between local and remote filesystem")
-
-		if localState.Placeholder && diff.Conflict != zfs.ConflictAllRight {
-			panic("internal inconsistency: local placeholder implies ConflictAllRight")
-		}
-
-		switch diff.Conflict {
-		case zfs.ConflictAllRight:
-
-			log.WithField("replication_policy", pull.InitialReplPolicy).Info("performing initial sync, following policy")
-
-			if pull.InitialReplPolicy != InitialReplPolicyMostRecent {
-				panic(fmt.Sprintf("policy '%s' not implemented", pull.InitialReplPolicy))
-			}
-
-			snapsOnly := make([]zfs.FilesystemVersion, 0, len(diff.MRCAPathRight))
-			for s := range diff.MRCAPathRight {
-				if diff.MRCAPathRight[s].Type == zfs.Snapshot {
-					snapsOnly = append(snapsOnly, diff.MRCAPathRight[s])
-				}
-			}
-
-			if len(snapsOnly) < 1 {
-				log.Warn("cannot perform initial sync: no remote snapshots")
-				return false
-			}
-
-			r := InitialTransferRequest{
-				Filesystem:        m.Remote,
-				FilesystemVersion: snapsOnly[len(snapsOnly)-1],
-			}
-
-			log.WithField("version", r.FilesystemVersion).Debug("requesting snapshot stream")
-
-			var stream io.Reader
-
-			if err = remote.Call("InitialTransferRequest", &r, &stream); err != nil {
-				log.WithError(err).Error("cannot request initial transfer")
-				return false
-			}
-			log.Debug("received initial transfer request response")
-
-			log.Debug("invoke zfs receive")
-			watcher := util.IOProgressWatcher{Reader: stream}
-			watcher.KickOff(1*time.Second, func(p util.IOProgress) {
-				log.WithField("total_rx", p.TotalRX).Info("progress on receive operation")
-			})
-
-			recvArgs := []string{"-u"}
-			if localState.Placeholder {
-				log.Info("receive with forced rollback to replace placeholder filesystem")
-				recvArgs = append(recvArgs, "-F")
-			}
-
-			if err = zfs.ZFSRecv(m.Local, &watcher, recvArgs...); err != nil {
-				log.WithError(err).Error("canot receive stream")
-				return false
-			}
-			log.WithField("total_rx", watcher.Progress().TotalRX).
-				Info("finished receiving stream")
-
-			log.Debug("configuring properties of received filesystem")
-			if err = zfs.ZFSSet(m.Local, "readonly", "on"); err != nil {
-				log.WithError(err).Error("cannot set readonly property")
-			}
-
-			log.Info("finished initial transfer")
-			return true
-
-		case zfs.ConflictIncremental:
-
-			if len(diff.IncrementalPath) < 2 {
-				log.Info("remote and local are in sync")
-				return true
-			}
-
-			log.Info("following incremental path from diff")
-			var pathRx uint64
-
-			for i := 0; i < len(diff.IncrementalPath)-1; i++ {
-
-				from, to := diff.IncrementalPath[i], diff.IncrementalPath[i+1]
-
-				log, _ := log.WithField(logIncFromField, from.Name).WithField(logIncToField, to.Name), 0
-
-				log.Debug("requesting incremental snapshot stream")
-				r := IncrementalTransferRequest{
-					Filesystem: m.Remote,
-					From:       from,
-					To:         to,
-				}
-				var stream io.Reader
-				if err = remote.Call("IncrementalTransferRequest", &r, &stream); err != nil {
-					log.WithError(err).Error("cannot request incremental snapshot stream")
-					return false
-				}
-
-				log.Debug("invoking zfs receive")
-				watcher := util.IOProgressWatcher{Reader: stream}
-				watcher.KickOff(1*time.Second, func(p util.IOProgress) {
-					log.WithField("total_rx", p.TotalRX).Info("progress on receive operation")
-				})
-
-				if err = zfs.ZFSRecv(m.Local, &watcher); err != nil {
-					log.WithError(err).Error("cannot receive stream")
-					return false
-				}
-
-				totalRx := watcher.Progress().TotalRX
-				pathRx += totalRx
-				log.WithField("total_rx", totalRx).Info("finished incremental transfer")
-
-			}
-
-			log.WithField("total_rx", pathRx).Info("finished following incremental path")
-			return true
-
-		case zfs.ConflictNoCommonAncestor:
-			fallthrough
-		case zfs.ConflictDiverged:
-
-			var jsonDiff bytes.Buffer
-			if err := json.NewEncoder(&jsonDiff).Encode(diff); err != nil {
-				log.WithError(err).Error("cannot JSON-encode diff")
-				return false
-			}
-
-			var problem, resolution string
-
-			switch diff.Conflict {
-			case zfs.ConflictNoCommonAncestor:
-				problem = "remote and local filesystem have snapshots, but no common one"
-				resolution = "perform manual establish a common snapshot history"
-			case zfs.ConflictDiverged:
-				problem = "remote and local filesystem share a history but have diverged"
-				resolution = "perform manual replication or delete snapshots on the receiving" +
-					"side  to establish an incremental replication parse"
-			}
-
-			log.WithField("diff", jsonDiff.String()).
-				WithField("problem", problem).
-				WithField("resolution", resolution).
-				Error("manual conflict resolution required")
-
-			return false
-
-		}
-
-		panic("should not be reached")
-
+		return p.replFilesystem(m, localFilesystemState)
 	})
 
 	return

cmd/sampleconf/localbackup/host1.yml

@@ -7,7 +7,8 @@ jobs:
   mapping: {
     "zroot/var/db<":    "storage/backups/local/zroot/var/db",
     "zroot/usr/home<":  "storage/backups/local/zroot/usr/home",
-    "zroot/var/tmp": "!", #don't backup /tmp
+    "zroot/usr/home/paranoid":  "!", #don't backup paranoid user
+    "zroot/poudriere/ports<": "!", #don't backup the ports trees
   }
   snapshot_prefix: zrepl_
   interval: 10m
@@ -18,6 +19,7 @@ jobs:
   prune_lhs:
     policy: grid
     grid: 1x1h(keep=all)
+    keep_bookmarks: all
 
   # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
   prune_rhs:

cmd/sampleconf/pullbackup/productionhost.yml

@@ -28,7 +28,7 @@ jobs:
     client_identity: fullbackup_prod1
 
   # snapshot these filesystems every 10m with zrepl_ as prefix
-  datasets: {
+  filesystems: {
     "zroot/var/db<": "ok",
     "zroot/usr/home<": "ok",
     "zroot/var/tmp": "!", #don't backup /tmp
@@ -37,9 +37,11 @@ jobs:
   interval: 10m
 
 
-  # keep a one day window 10m interval snapshots in case pull doesn't work (link down, etc)
-  # (we cannot keep more than one day because this host will run out of disk space)
+  # keep 1 hour of snapshots (6 at 10m interval)
+  # and  one day of bookmarks in case pull doesn't work (link down, etc)
+  # => keep_bookmarks = 24h / interval = 24h / 10m = 144
   prune:
     policy: grid
-    grid: 1x1d(keep=all)
+    grid: 1x1h(keep=all)
+    keep_bookmarks: 144
 

cmd/sampleconf/pushbackup/productionhost.yml

@@ -11,7 +11,7 @@ jobs:
       identity_file: /root/.ssh/id_ed25519
 
   # snapshot these datsets every 10m with zrepl_ as prefix
-  datasets: {
+  filesystems: {
     "zroot/var/db<": "ok",
     "zroot/usr/home<": "!",
   }

cmd/sampleconf/random/debugging.yml

@@ -22,7 +22,7 @@ jobs:
   serve:
     type: stdinserver
     client_identity: debian2
-  datasets: {
+  filesystems: {
     "pool1/db<": ok
   }
   snapshot_prefix: zrepl_
@@ -30,4 +30,4 @@ jobs:
   prune:
     policy: grid
     grid: 1x10s(keep=all)
-
+    keep_bookmarks: all

cmd/sampleconf/random/logging.yml

@@ -1,9 +1,11 @@
 global:
   logging:
-    stdout:
+
+    - outlet: stdout
       level: warn
       format: human
-    tcp:
+
+    - outlet: tcp
       level: debug
       format: json
       net: tcp
@@ -13,8 +15,10 @@ global:
         ca: sampleconf/random/logging/logserver.crt
         cert: sampleconf/random/logging/client.crt
         key: sampleconf/random/logging/client.key
-    syslog:
-      enable: true
+
+    - outlet: syslog
+      level: debug
       format: logfmt
 
+
 jobs: []

cmd/sampleconf/zrepl.yml

@@ -1,111 +0,0 @@
-remotes:
-  offsite_backups:
-    transport:
-      ssh:
-        host: 192.168.122.6
-        user: root
-        port: 22
-        identity_file: /etc/zrepl/identities/offsite_backups
-
-push:
-
-  offsite:
-    to: offsite_backups
-    filter: {
-      # like in pull_acls
-      "tank/var/db<":   ok,
-      "tank/usr/home<": ok,
-    }
-
-pull:
-
-  offsite:
-    from: offsite_backups
-    mapping: {
-      # like in sinks
-    }
-
-  # local replication, only allowed in pull mode
-  # the from name 'local' is reserved for this purpose
-  homemirror:
-    from: local
-    repeat:
-      interval: 15m
-    mapping: {
-      "tank/usr/home":"mirrorpool/foo/bar"
-    }
-
-sink:
-
-  db1:
-    mapping: {
-      # direct mapping
-      "ssdpool/var/db/postgresql9.6":"zroot/backups/db1/pg_data"
-    }
-
-  mirror1:
-    mapping: {
-      # "<" subtree wildcard matches the dataset left of < and all its children
-      "tank/foo/bar<":"zroot/backups/mirror1" 
-    }
-
-  mirror2:
-    # more specific path patterns win over less specific ones
-    # direct mappings win over subtree wildcards
-    # detailed rule precedence: check unit tests & docs for exact behavior
-    # TODO subcommand to test a mapping & filter
-    mapping: {
-      "tank<":                       "zroot/backups/mirror1/tank1",
-      "tank/cdn/root<":              "storage/cdn/root",
-      "tank/legacydb":               "legacypool/backups/legacydb",
-    }
-
-
-pull_acl:
-
-  office_backup:
-    filter: {
-      # valid filter results (right hand side): ok, omit
-      # default is to omit
-      # rule precedence is same as for mappings
-      "tank<":         omit,
-      "tank/usr/home": ok,
-    }
-
-
-prune:
-
-  clean_backups:
-    policy: grid
-    grid: 6x10m | 24x1h | 7x1d | 5 x 1w | 4 x 5w
-    dataset_filter: {
-      "tank/backups/legacyscript<": omit,
-      "tank/backups<":              ok,
-    }
-    snapshot_filter: {
-      prefix: zrepl_
-    }
-
-  hfbak_prune: # cleans up after hfbak autosnap job
-    policy: grid
-    grid: 1x1m(keep=all)
-    dataset_filter: {
-      "pool1<": ok
-    }
-    snapshot_filter: {
-      prefix: zrepl_hfbak_
-    }
-    repeat:
-      interval: 10s
-
-autosnap:
-
-  hfbak:
-    prefix: zrepl_hfbak_
-    interval: 1s
-    dataset_filter: {
-      "pool1<": ok
-    }
-    # prune: hfbak_prune
-    # future versions may inline the retention policy here, but for now,
-    # pruning has to be triggered manually (it's safe to run autosnap + prune in parallel)

cmd/stdinserver.go

@@ -1,14 +1,13 @@
 package cmd
 
 import (
-	"fmt"
 	"os"
 
-	"github.com/ftrvxmtrx/fd"
+	"context"
+	"github.com/problame/go-netssh"
 	"github.com/spf13/cobra"
-	"io"
 	"log"
-	"net"
+	"path"
 )
 
 var StdinserverCmd = &cobra.Command{
@@ -23,71 +22,34 @@ func init() {
 
 func cmdStdinServer(cmd *cobra.Command, args []string) {
 
-	log := log.New(os.Stderr, "", log.LUTC|log.Ldate|log.Ltime)
+	// NOTE: the netssh proxying protocol requires exiting with non-zero status if anything goes wrong
+	defer os.Exit(1)
 
-	die := func() {
-		log.Printf("stdinserver exiting after fatal error")
-		os.Exit(1)
-	}
+	log := log.New(os.Stderr, "", log.LUTC|log.Ldate|log.Ltime)
 
 	conf, err := ParseConfig(rootArgs.configFile)
 	if err != nil {
 		log.Printf("error parsing config: %s", err)
-		die()
+		return
 	}
 
 	if len(args) != 1 || args[0] == "" {
-		err = fmt.Errorf("must specify client_identity as positional argument")
-		die()
+		log.Print("must specify client_identity as positional argument")
+		return
 	}
+
 	identity := args[0]
+	unixaddr := path.Join(conf.Global.Serve.Stdinserver.SockDir, identity)
 
-	unixaddr, err := stdinserverListenerSocket(conf.Global.Serve.Stdinserver.SockDir, identity)
-	if err != nil {
-		log.Printf("%s", err)
-		os.Exit(1)
+	log.Printf("proxying client identity '%s' to zrepl daemon '%s'", identity, unixaddr)
+
+	ctx := netssh.ContextWithLog(context.TODO(), log)
+
+	err = netssh.Proxy(ctx, unixaddr)
+	if err == nil {
+		log.Print("proxying finished successfully, exiting with status 0")
+		os.Exit(0)
 	}
-
-	log.Printf("opening control connection to zrepld via %s", unixaddr)
-	conn, err := net.DialUnix("unix", nil, unixaddr)
-	if err != nil {
-		log.Printf("error connecting to zrepld: %s", err)
-		die()
-	}
-
-	log.Printf("sending stdin and stdout fds to zrepld")
-	err = fd.Put(conn, os.Stdin, os.Stdout)
-	if err != nil {
-		log.Printf("error: %s", err)
-		die()
-	}
-
-	log.Printf("waiting for zrepld to close control connection")
-	for {
-
-		var buf [64]byte
-		n, err := conn.Read(buf[:])
-		if err == nil && n != 0 {
-			log.Printf("protocol error: read expected to timeout or EOF returned bytes")
-		}
-
-		if err == io.EOF {
-			log.Printf("zrepld closed control connection, terminating")
-			break
-		}
-
-		neterr, ok := err.(net.Error)
-		if !ok {
-			log.Printf("received unexpected error type: %T %s", err, err)
-			die()
-		}
-		if !neterr.Timeout() {
-			log.Printf("receivd unexpected net.Error (not a timeout): %s", neterr)
-			die()
-		}
-		// Read timed out, as expected
-	}
-
-	return
+	log.Printf("error proxying: %s", err)
 
 }

cmd/test.go

@@ -36,7 +36,7 @@ var testConfigSyntaxCmd = &cobra.Command{
 var testDatasetMapFilter = &cobra.Command{
 	Use:     "pattern jobname test/zfs/dataset/path",
 	Short:   "test dataset mapping / filter specified in config",
-	Example: ` zrepl test pattern prune.clean_backups tank/backups/legacyscript/foo`,
+	Example: ` zrepl test pattern my_pull_job tank/tmp`,
 	Run:     doTestDatasetMapFilter,
 }
 
@@ -109,7 +109,7 @@ func doTestDatasetMapFilter(cmd *cobra.Command, args []string) {
 	case *PullJob:
 		mf = j.Mapping
 	case *SourceJob:
-		mf = j.Datasets
+		mf = j.Filesystems
 	case *LocalJob:
 		mf = j.Mapping
 	default:
@@ -170,7 +170,8 @@ func doTestPrunePolicy(cmd *cobra.Command, args []string) {
 
 	log.Printf("job dump:\n%s", pretty.Sprint(jobp))
 
-	pruner, err := jobp.Pruner(testPrunePolicyArgs.side, true)
+	task := NewTask("", log)
+	pruner, err := jobp.Pruner(task, testPrunePolicyArgs.side, true)
 	if err != nil {
 		log.Printf("cannot create test pruner: %s", err)
 		os.Exit(1)

cmd/version.go

@@ -0,0 +1,42 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"runtime"
+)
+
+var versionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "print version of zrepl binary (for running daemon 'zrepl control version' command)",
+	Run:   doVersion,
+}
+
+func init() {
+	RootCmd.AddCommand(versionCmd)
+}
+
+func doVersion(cmd *cobra.Command, args []string) {
+	fmt.Println(NewZreplVersionInformation().String())
+}
+
+type ZreplVersionInformation struct {
+	Version         string
+	RuntimeGOOS     string
+	RuntimeGOARCH   string
+	RUNTIMECompiler string
+}
+
+func NewZreplVersionInformation() *ZreplVersionInformation {
+	return &ZreplVersionInformation{
+		Version:         zreplVersion,
+		RuntimeGOOS:     runtime.GOOS,
+		RuntimeGOARCH:   runtime.GOARCH,
+		RUNTIMECompiler: runtime.Compiler,
+	}
+}
+
+func (i *ZreplVersionInformation) String() string {
+	return fmt.Sprintf("zrepl version=%s GOOS=%s GOARCH=%s Compiler=%s",
+		i.Version, i.RuntimeGOOS, i.RuntimeGOARCH, i.RUNTIMECompiler)
+}

docs/.gitignore

@@ -1 +1,2 @@
-public_git
+_build/
+public_git/

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = zrepl
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" -c sphinxconf $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" -c sphinxconf $(SPHINXOPTS) $(O)

docs/_static/zrepl.svg

@@ -0,0 +1,147 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   id="svg8"
+   version="1.1"
+   viewBox="0 0 70.555558 24.694445"
+   height="70pt"
+   width="200pt"
+   sodipodi:docname="zrepl.svg"
+   inkscape:version="0.92.2 5c3e80d, 2017-08-06">
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="2560"
+     inkscape:window-height="1405"
+     id="namedview10"
+     showgrid="false"
+     inkscape:zoom="3.1350001"
+     inkscape:cx="6.2268563"
+     inkscape:cy="-2.6953765"
+     inkscape:window-x="0"
+     inkscape:window-y="35"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="layer1" />
+  <defs
+     id="defs2" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     transform="translate(0,-272.30556)"
+     id="layer1">
+    <g
+       aria-label="zrepl"
+       transform="matrix(0.26458334,0,0,0.26458334,0.52916655,273.36389)"
+       style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:40px;line-height:25px;font-family:monospace;-inkscape-font-specification:'monospace, Bold';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:center;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       id="flowRoot829">
+      <path
+         d="m 74.504883,26.943359 11.523437,-13.125 H 74.973633 V 9.5410156 h 17.480469 v 4.4726564 l -11.503907,13.125 h 11.503907 v 4.277344 H 74.504883 Z"
+         style="stroke:#f8f8f8;stroke-opacity:1;fill:#f8f8f8;fill-opacity:1"
+         id="path873" />
+      <path
+         d="m 100.11035,9.5410156 h 5.15625 l 0.56641,3.3984374 q 0.83984,-1.875 2.57812,-2.910156 1.73828,-1.0156251 4.00391,-1.0156251 1.26953,0 2.36328,0.2734375 1.25,0.3515625 2.12891,0.8398436 v 5.449219 q -0.9375,-0.859375 -2.1875,-1.289063 -1.23047,-0.429687 -2.75391,-0.429687 -1.81641,0 -3.18359,0.625 -1.36719,0.605469 -2.10938,1.835937 -0.19531,0.3125 -0.37109,0.761719 -0.17578,0.449219 -0.27344,1.054688 -0.0977,0.566406 -0.15625,1.308593 -0.0391,0.742188 -0.0391,1.894532 v 10.078125 h -5.72266 z"
+         style="stroke:#f8f8f8;stroke-opacity:1;fill:#f8f8f8;fill-opacity:1"
+         id="path875" />
+      <path
+         d="m 132.4541,31.982422 q -2.85156,0 -4.9414,-0.722656 -2.07032,-0.742188 -3.53516,-2.226563 -2.91016,-2.96875 -2.91016,-8.4375 0,-5.371094 2.79297,-8.476562 2.79297,-3.1054691 7.65625,-3.1054691 3.10547,-0.097656 5.42969,1.2500001 2.34375,1.347656 3.59375,3.867187 1.26953,2.519532 1.17187,5.878907 v 2.324218 H 126.8877 q 0.0195,2.578125 1.52343,3.847657 1.50391,1.269531 4.47266,1.269531 1.93359,0 3.90625,-0.566406 0.89844,-0.273438 1.89453,-0.703125 0.99609,-0.429688 2.10938,-1.09375 v 5.253906 q -1.99219,0.820312 -4.0625,1.230469 -1.05469,0.214843 -2.14844,0.3125 -1.07422,0.09766 -2.12891,0.09766 z m 3.4961,-13.945313 q -0.0195,-2.285156 -1.15235,-3.4375 -1.11328,-1.152343 -3.28125,-1.152343 -2.01172,0 -3.18359,1.191406 -1.17188,1.210937 -1.38672,3.417969 z"
+         style="stroke:#f8f8f8;stroke-opacity:1;fill:#f8f8f8;fill-opacity:1"
+         id="path877" />
+      <path
+         d="m 146.45801,9.5410156 h 5.13672 l 0.5664,3.2812504 q 0.89844,-1.894532 2.36328,-2.8515629 1.42579,-0.9570312 3.45704,-0.9570312 3.71093,0 5.80078,2.9882811 1.05468,1.523438 1.5625,3.691406 0.50781,2.148438 0.50781,4.824219 0,2.851563 -0.54688,4.980469 -0.54687,2.109375 -1.54296,3.515625 -1.07422,1.523437 -2.59766,2.246094 -1.50391,0.722656 -3.28125,0.722656 -0.91797,0 -1.73828,-0.214844 -0.80078,-0.195312 -1.54297,-0.703125 -0.625,-0.410156 -1.28906,-1.09375 -0.64453,-0.703125 -1.15235,-1.640625 v 11.40625 h -5.70312 z m 9.70703,17.6757814 q 1.85547,0 2.91016,-1.796875 1.05468,-1.777344 1.05468,-4.960938 0,-3.164062 -1.05468,-4.960937 -1.05469,-1.796875 -2.91016,-1.796875 -1.83594,0 -2.92969,1.796875 -1.07422,1.777344 -1.07422,4.960937 0,3.183594 1.07422,4.960938 1.09375,1.796875 2.92969,1.796875 z"
+         style="stroke:#f8f8f8;stroke-opacity:1;fill:#f8f8f8;fill-opacity:1"
+         id="path879" />
+      <path
+         d="m 182.98145,31.572266 q -4.0625,0 -5.85938,-2.109375 -1.77734,-2.070313 -1.77734,-7.011719 V 5.4199219 h -5.80078 V 1.0253906 h 11.52343 V 22.451172 q 0,2.558594 0.80078,3.652344 0.78125,1.074218 2.71485,1.074218 h 4.57031 v 4.394532 z"
+         style="stroke:#f8f8f8;stroke-opacity:1;fill:#f8f8f8;fill-opacity:1"
+         id="path881" />
+    </g>
+    <path
+       inkscape:connector-curvature="0"
+       id="path837"
+       d="M 26.170418,286.30982 H 43.35751"
+       style="font-style:normal;font-weight:normal;font-size:medium;line-height:25px;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#f8f8f8;fill-opacity:1;fill-rule:evenodd;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+    <g
+       aria-label="ZFS replication"
+       style="font-style:normal;font-weight:normal;font-size:10.58333397px;line-height:6.61458349px;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       id="text820">
+      <path
+         d="m 14.758548,294.66467 h -3.493327 v -0.47887 l 2.652724,-3.99286 h -2.570041 v -0.565 h 3.338297 v 0.47887 l -2.652723,3.99287 h 2.72507 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path844" />
+      <path
+         d="M 16.353627,294.66467 H 15.71973 v -5.03673 h 2.814643 v 0.55811 h -2.180746 v 1.78456 h 2.046387 v 0.55466 h -2.046387 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path846" />
+      <path
+         d="m 22.241296,293.31764 q 0,0.65801 -0.475423,1.03697 -0.475424,0.37896 -1.322917,0.37896 -0.895725,0 -1.384929,-0.23771 v -0.60633 q 0.320394,0.1378 0.699355,0.21704 0.382406,0.0792 0.713135,0.0792 0.564995,0 0.850938,-0.21704 0.285943,-0.21704 0.285943,-0.596 0,-0.25149 -0.103353,-0.41341 -0.103352,-0.16192 -0.347954,-0.30317 -0.241157,-0.14125 -0.733806,-0.32039 -0.699354,-0.25494 -0.999077,-0.59945 -0.299724,-0.34795 -0.299724,-0.89572 0,-0.58911 0.440973,-0.93707 0.444417,-0.34795 1.167887,-0.34795 0.754476,0 1.388373,0.28249 l -0.19637,0.54433 q -0.647678,-0.26872 -1.205783,-0.26872 -0.447863,0 -0.7028,0.19293 -0.251492,0.19292 -0.251492,0.54088 0,0.24804 0.09646,0.41341 0.09991,0.16192 0.316949,0.29628 0.220486,0.13436 0.689019,0.30661 0.56155,0.20671 0.837158,0.40308 0.275608,0.19292 0.406521,0.44786 0.130914,0.25149 0.130914,0.60289 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path848" />
+      <path
+         d="m 26.768151,290.81306 q 0.251492,0 0.444418,0.0413 l -0.07579,0.57188 q -0.21015,-0.0482 -0.409966,-0.0482 -0.299723,0 -0.558105,0.16537 -0.254937,0.16536 -0.399631,0.45819 -0.144694,0.28939 -0.144694,0.64424 v 2.01882 h -0.620118 v -3.78271 h 0.509874 l 0.0689,0.69246 h 0.02756 q 0.217041,-0.37551 0.513319,-0.56844 0.296279,-0.19292 0.644233,-0.19292 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path850" />
+      <path
+         d="m 29.465661,294.73357 q -0.850939,0 -1.336697,-0.51331 -0.482313,-0.51677 -0.482313,-1.41938 0,-0.90951 0.451307,-1.44694 0.451308,-0.54088 1.219564,-0.54088 0.713135,0 1.136881,0.46164 0.423747,0.4582 0.423747,1.24368 v 0.37551 h -2.590712 q 0.01723,0.64424 0.327284,0.97841 0.310059,0.33418 0.8785,0.33418 0.303168,0 0.575331,-0.0517 0.272162,-0.0551 0.637342,-0.21015 v 0.54432 q -0.313503,0.13436 -0.589111,0.18948 -0.275608,0.0551 -0.651123,0.0551 z m -0.155029,-3.41408 q -0.444418,0 -0.7028,0.28594 -0.258382,0.28594 -0.306613,0.79582 h 1.925808 q -0.0069,-0.53055 -0.244602,-0.80616 -0.237711,-0.2756 -0.671793,-0.2756 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path852" />
+      <path
+         d="m 33.630782,294.73357 q -0.761366,0 -1.171333,-0.54777 h -0.04134 l 0.01378,0.14125 q 0.02756,0.26872 0.02756,0.47887 v 1.55374 h -0.620117 v -5.4777 h 0.509874 l 0.08268,0.51676 h 0.02756 q 0.220486,-0.31006 0.509874,-0.44786 0.289388,-0.1378 0.668349,-0.1378 0.73725,0 1.143772,0.51332 0.409966,0.50987 0.409966,1.44005 0,0.92673 -0.409966,1.44694 -0.409967,0.5202 -1.150662,0.5202 z m -0.103353,-3.4003 q -0.561551,0 -0.813043,0.31694 -0.248047,0.31695 -0.254937,0.99564 v 0.12058 q 0,0.76825 0.254937,1.11276 0.254937,0.34107 0.826823,0.34107 0.475423,0 0.740696,-0.38586 0.268717,-0.38585 0.268717,-1.07487 0,-0.69246 -0.268717,-1.05764 -0.265273,-0.36862 -0.754476,-0.36862 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path854" />
+      <path
+         d="m 36.800269,294.66467 h -0.620117 v -5.36057 h 0.620117 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path856" />
+      <path
+         d="m 38.619281,294.66467 h -0.620117 v -3.78271 h 0.620117 z m -0.668348,-4.80591 q 0,-0.2067 0.103352,-0.29972 0.106798,-0.093 0.261828,-0.093 0.144694,0 0.251492,0.093 0.110243,0.093 0.110243,0.29972 0,0.20327 -0.110243,0.29973 -0.106798,0.0965 -0.251492,0.0965 -0.15503,0 -0.261828,-0.0965 -0.103352,-0.0965 -0.103352,-0.29973 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path858" />
+      <path
+         d="m 41.334016,294.73357 q -0.816488,0 -1.27124,-0.49953 -0.454753,-0.50299 -0.454753,-1.44005 0,-0.95085 0.458198,-1.46417 0.461642,-0.51676 1.312581,-0.51676 0.275608,0 0.544325,0.0586 0.272163,0.0551 0.447863,0.1447 l -0.189481,0.51676 q -0.478868,-0.17914 -0.816487,-0.17914 -0.571886,0 -0.844049,0.36173 -0.272162,0.35829 -0.272162,1.07143 0,0.68557 0.272162,1.05075 0.272163,0.36173 0.806153,0.36173 0.499538,0 0.981852,-0.22048 v 0.55121 q -0.392741,0.20326 -0.974962,0.20326 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path860" />
+      <path
+         d="m 45.543923,294.66467 -0.120578,-0.53743 h -0.02756 q -0.282498,0.35484 -0.564996,0.48231 -0.279052,0.12402 -0.706244,0.12402 -0.558106,0 -0.875055,-0.29283 -0.316948,-0.29283 -0.316948,-0.82682 0,-1.15066 1.815565,-1.20578 l 0.640788,-0.0241 v -0.22393 q 0,-0.43064 -0.186035,-0.6339 -0.186035,-0.2067 -0.596002,-0.2067 -0.299723,0 -0.568441,0.0896 -0.265272,0.0896 -0.499538,0.19981 l -0.189481,-0.46508 q 0.285943,-0.15159 0.623563,-0.23772 0.337619,-0.0861 0.668348,-0.0861 0.685574,0 1.019748,0.30316 0.334175,0.30317 0.334175,0.96463 v 2.57693 z m -1.27813,-0.43063 q 0.520209,0 0.816487,-0.27906 0.299724,-0.28249 0.299724,-0.79926 v -0.34106 l -0.558106,0.0241 q -0.651123,0.0241 -0.950846,0.20671 -0.296278,0.18259 -0.296278,0.57533 0,0.29628 0.179145,0.45475 0.18259,0.15848 0.509874,0.15848 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path862" />
+      <path
+         d="m 48.427466,294.22715 q 0.120578,0 0.282498,-0.0241 0.161919,-0.0241 0.244602,-0.0551 v 0.47542 q -0.08613,0.0379 -0.268718,0.0723 -0.179145,0.0379 -0.36518,0.0379 -1.109321,0 -1.109321,-1.16788 v -2.20142 h -0.53399 v -0.29628 l 0.54088,-0.24804 0.248047,-0.80616 h 0.368625 v 0.86817 h 1.092096 v 0.48231 h -1.092096 v 2.18419 q 0,0.32729 0.15503,0.50299 0.158474,0.1757 0.437527,0.1757 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path864" />
+      <path
+         d="m 50.332607,294.66467 h -0.620118 v -3.78271 h 0.620118 z m -0.668349,-4.80591 q 0,-0.2067 0.103353,-0.29972 0.106798,-0.093 0.261827,-0.093 0.144694,0 0.251492,0.093 0.110243,0.093 0.110243,0.29972 0,0.20327 -0.110243,0.29973 -0.106798,0.0965 -0.251492,0.0965 -0.155029,0 -0.261827,-0.0965 -0.103353,-0.0965 -0.103353,-0.29973 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path866" />
+      <path
+         d="m 54.818119,292.76643 q 0,0.92673 -0.468533,1.44694 -0.468533,0.5202 -1.295356,0.5202 -0.51332,0 -0.909506,-0.23771 -0.396186,-0.24115 -0.609782,-0.68902 -0.213595,-0.44786 -0.213595,-1.04041 0,-0.92329 0.461642,-1.43661 0.461643,-0.51676 1.295356,-0.51676 0.799262,0 1.267795,0.5271 0.471979,0.52365 0.471979,1.42627 z m -2.855985,0 q 0,0.71313 0.279053,1.0852 0.282498,0.36863 0.830268,0.36863 0.54088,0 0.823378,-0.36863 0.282498,-0.37207 0.282498,-1.0852 0,-0.71314 -0.285943,-1.07143 -0.282498,-0.36173 -0.830268,-0.36173 -1.098986,0 -1.098986,1.43316 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path868" />
+      <path
+         d="m 58.380348,294.66467 v -2.41845 q 0,-0.46165 -0.206706,-0.68558 -0.20326,-0.22737 -0.640787,-0.22737 -0.582222,0 -0.847494,0.32383 -0.261827,0.3204 -0.261827,1.04731 v 1.96026 h -0.620117 v -3.78271 h 0.499538 l 0.09302,0.51676 h 0.03445 q 0.1757,-0.27905 0.492649,-0.43063 0.316948,-0.15503 0.699354,-0.15503 0.695909,0 1.033529,0.33762 0.337619,0.33761 0.337619,1.05075 v 2.46324 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path870" />
+    </g>
+  </g>
+</svg>

docs/archetypes/default.md

@@ -1,6 +0,0 @@
----
-title: "{{ replace .TranslationBaseName "-" " " | title }}"
-date: {{ .Date }}
-draft: true
----
-

docs/changelog.rst

@@ -0,0 +1,68 @@
+.. |break_config| replace:: **[BREAK]**
+.. |break| replace:: **[BREAK]**
+.. |bugfix| replace:: [BUG]
+.. |feature| replace:: [FEATURE]
+
+Changelog
+=========
+
+The changelog summarized bugfixes that are deemed relevant for users.
+Developers should consult the git commit log or GitHub issue tracker.
+
+0.0.3
+-----
+
+* |break_config| |feature| :issue:`34`: automatic bookmarking of snapshots
+
+  * Snapshots are automatically bookmarked and pruning of bookmarks **must** be configured.
+  * This breaks existing configuration: ``grid`` :ref:`prune policy <prune-retention-grid>`  specifications require the new ``keep_bookmarks`` parameter.
+  * Make sure to understand the meaning bookmarks have for :ref:`maximum replication downtime <replication-downtime>`.
+  * Example: :sampleconf:`pullbackup/productionhost.yml`
+
+* |break| :commit:`ccd062e`: both sides of a replication setup must be updated and restarted. Otherwise the connecting side will hang and not time out.
+* |break_config| :commit:`2bfcfa5`: first outlet in ``global.logging`` is now used for logging meta-errors, for example problems encountered when writing to other outlets.
+* |feature| :issue:`10`: ``zrepl control status`` subcommand
+
+  * Allows inspection of job activity per task and their log output at runtime.
+  * Supports ``--format raw`` option for JSON output, usable for monitoring from scripts.
+
+* |feature| :commit:`d7f3fb9`: subcommand bash completions
+
+  * Package maintainers should install this as appropriate.
+
+* |bugfix| :issue:`61`: fix excessive memory usage
+* |bugfix| :issue:`8` and :issue:`56`: ``ssh+stdinserver`` transport properly reaps SSH child processes
+* |bugfix| :commit:`cef63ac`: ``human`` format now prints non-string values correctly
+* |bugfix| :issue:`26`: slow TCP outlets no longer block the daemon
+
+0.0.2
+-----
+
+* |break_config| :commit:`b95260f`: ``global.logging`` is no longer a dictionary but a list
+
+* |break_config| :commit:`3e647c1`: ``source`` job field ``datasets`` renamed to ``filesystems``
+
+  * **NOTE**: zrepl will parse missing ``filesystems`` field as an empty filter,
+    i.e. no filesystems are presented to the other side.
+
+* |bugfix| :commit:`72d2885` fix aliasing bug with root `<` subtree wildcard
+
+  * Filesystems paths with final match at blank `s` subtree wildcard are now appended to the target path
+  * Non-root subtree wildcards, e.g. `zroot/foo/bar<` still map directrly onto the target path
+
+* Support days (``d``) and weeks (``w``) in durations
+
+* Docs
+
+  * Ditch Hugo, move to Python Sphinx
+  * Improve & simplify tutorial (single SSH key per installation)
+  * Document pruning policies
+  * Document job types
+  * Document logging
+  * Start updating implementation overview
+
+
+0.0.1
+-----
+
+* Initial release

docs/conf.py

@@ -0,0 +1 @@
+sphinxconf/conf.py

docs/config.toml

@@ -1,17 +0,0 @@
-baseURL = "https://zrepl.github.io"
-languageCode = "en-us"
-title = "zrepl - ZFS replication"
-theme = "docdock"
-
-[[menu.shortcuts]]
-pre = "<h3>More</h3>"
-name = "<i class='fa fa-github'></i> GitHub repo"
-identifier = "ds"
-url = "https://github.com/zrepl/zrepl"
-weight = 10
-
-[[menu.shortcuts]]
-name = "<i class='fa fa-twitter'></i> Primary Author on Twitter"
-url = "https://twitter.com/problame"
-weight = 11
-

docs/configuration.rst

@@ -0,0 +1,16 @@
+
+.. _configuration_toc:
+
+*************
+Configuration
+*************
+
+.. toctree::
+
+    configuration/preface
+    configuration/jobs
+    configuration/transports
+    configuration/map_filter_syntax
+    configuration/prune
+    configuration/logging
+    configuration/misc

docs/configuration/jobs.rst

@@ -0,0 +1,187 @@
+.. include:: ../global.rst.inc
+
+.. |patient| replace:: :ref:`patient <job-term-patient>`
+.. |serve-transport| replace:: :ref:`serve transport<transport-ssh+stdinserver-serve>`
+.. |connect-transport| replace:: :ref:`connect transport<transport-ssh+stdinserver-connect>`
+.. |mapping| replace:: :ref:`mapping <pattern-mapping>`
+.. |filter| replace:: :ref:`filter <pattern-filter>`
+.. |prune| replace:: :ref:`prune <prune>`
+
+.. _job:
+
+Job Types
+=========
+
+A *job* is the unit of activity tracked by the zrepl daemon and configured in the |mainconfig|.
+Every job has a unique ``name``, a ``type`` and type-dependent fields which are documented on this page.
+Check out the :ref:`tutorial` and :sampleconf:`/` for examples on how job types are actually used.
+
+.. ATTENTION::
+
+    Currently, zrepl does not replicate filesystem properties.
+    Whe receiving a filesystem, it is never mounted (`-u` flag)  and `mountpoint=none` is set.
+    This is temporary and being worked on :issue:`24`.
+
+.. _job-source:
+
+Source Job
+----------
+
+Example: :sampleconf:`pullbackup/productionhost.yml`.
+
+.. list-table::
+    :widths: 20 80
+    :header-rows: 1
+
+    * - Parameter
+      - Comment
+    * - ``type``
+      - = ``source``
+    * - ``name``
+      - unique name of the job
+    * - ``serve``
+      - |serve-transport|
+    * - ``filesystems``
+      - |filter| for filesystems to expose to client
+    * - ``snapshot_prefix``
+      - prefix for ZFS snapshots taken by this job
+    * - ``interval``
+      - snapshotting interval
+    * - ``prune``
+      - |prune| for versions of filesytems in ``filesystems``, versions prefixed with ``snapshot_prefix``
+
+
+- Snapshotting Task (every ``interval``, |patient|)
+
+  - A snapshot of filesystems matched by ``filesystems`` is taken every ``interval`` with prefix ``snapshot_prefix``.
+  - A bookmark of that snapshot is created with the same name.
+  - The ``prune`` policy is evaluated for versions of filesystems matched by ``filesystems``, versions prefixed with ``snapshot_prefix``.
+
+- Serve Task
+
+  - Wait for connections from pull job using ``serve``.
+
+A source job is the counterpart to a :ref:`job-pull`.
+
+Make sure you read the |prune| policy documentation.
+
+.. _job-pull:
+
+Pull Job
+--------
+
+Example: :sampleconf:`pullbackup/backuphost.yml`
+
+.. list-table::
+    :widths: 20 80
+    :header-rows: 1
+
+    * - Parameter
+      - Comment
+    * - ``type``
+      - = ``pull``
+    * - ``name``
+      - unqiue name of the job
+    * - ``connect``
+      - |connect-transport|
+    * - ``interval``
+      - Interval between pull attempts
+    * - ``mapping``
+      - |mapping| for remote to local filesystems
+    * - ``initial_repl_policy``
+      - default = ``most_recent``, initial replication policy
+    * - ``snapshot_prefix``
+      - prefix snapshots must match to be considered for replication & pruning
+    * - ``prune``
+      - |prune| policy for versions of filesystems of local filesystems reachable by ``mapping``, versions prefixed with ``snapshot_prefix``
+
+* Main Task (every ``interval``, |patient|)
+ 
+  #. A connection to the remote source job is established using the strategy in ``connect``
+  #. ``mapping`` maps filesystems presented by the remote side to local *target filesystems*
+  #. Those remote filesystems with a local *target filesystem* are replicated
+ 
+     #. Only snapshots with prefix ``snapshot_prefix`` are replicated.
+     #. If possible, incremental replication takes place.
+     #. If the local target filesystem does not exist, ``initial_repl_policy`` is used.
+     #. On conflicts, an error is logged but replication of other filesystems with mapping continues.
+  
+  #. The ``prune`` policy is evaluated for all *target filesystems*
+
+A pull job is the counterpart to a :ref:`job-source`.
+
+Make sure you read the |prune| policy documentation.
+
+.. _job-local:
+
+Local Job
+---------
+
+Example: :sampleconf:`localbackup/host1.yml`
+
+.. list-table::
+    :widths: 20 80
+    :header-rows: 1
+
+    * - Parameter
+      - Comment
+    * - ``type``
+      - = ``local``
+    * - ``name``
+      - unqiue name of the job
+    * - ``mapping``
+      - |mapping| from source to target filesystem (both local)
+    * - ``snapshot_prefix``
+      - prefix for ZFS snapshots taken by this job
+    * - ``interval``
+      - snapshotting & replication interval
+    * - ``initial_repl_policy``
+      - default = ``most_recent``, initial replication policy
+    * - ``prune_lhs``
+      -  pruning policy on left-hand-side (source)
+    * - ``prune_rhs``
+      - pruning policy on right-hand-side (target)
+
+* Main Task (every ``interval``, |patient|)
+
+  #. Evaluate ``mapping`` for local filesystems, those with a *target filesystem* are called *mapped filesystems*.
+  #. Snapshot *mapped filesystems* with ``snapshot_prefix``.
+  #. Bookmark the snapshot created above.
+  #. Replicate *mapped filesystems* to their respective *target filesystems*:
+
+     #. Only snapshots with prefix ``snapshot_prefix`` are replicated.
+     #. If possible, incremental replication takes place.
+     #. If the *target filesystem* does not exist, ``initial_repl_policy`` is used.
+     #. On conflicts, an error is logged but replication of other *mapped filesystems* continues.
+
+  #. The ``prune_lhs`` policy is triggered for all *mapped filesystems*
+  #. The ``prune_rhs`` policy is triggered for all *target filesystems*
+
+A local job is combination of source & pull job executed on the same machine.
+
+Terminology
+-----------
+
+task
+
+    A job consists of one or more tasks and a task consists of one or more steps.
+    Some tasks may be periodic while others wait for an event to occur.
+
+
+patient task
+
+    .. _job-term-patient:
+
+    A patient task is supposed to execute some task every `interval`.
+    We call the start of the task an *invocation*.
+
+    * If the task completes in less than `interval`, the task is restarted at `last_invocation + interval`.
+    * Otherwise, a patient job
+        * logs a warning as soon as a task exceeds its configured `interval`
+        * waits for the last invocation to finish
+        * logs a warning with the effective task duration
+        * immediately starts a new invocation of the task
+
+filesystem version
+
+    A snapshot or a bookmark.

docs/configuration/logging.rst

@@ -0,0 +1,213 @@
+.. include:: ../global.rst.inc
+
+.. _logging:
+
+Logging
+=======
+
+zrepl uses structured logging to provide users with easily processable log messages.
+
+Logging outlets are configured in the ``global`` section of the |mainconfig|.
+Check out :sampleconf:`random/logging.yml` for an example on how to configure multiple outlets:
+
+::
+
+    global:
+      logging:
+    
+        - outlet: OUTLET_TYPE
+          level: MINIMUM_LEVEL
+          format: FORMAT
+    
+        - outlet: OUTLET_TYPE
+          level: MINIMUM_LEVEL
+          format: FORMAT
+    
+        ...
+    
+    jobs: ...
+
+.. _logging-error-outlet:
+
+.. ATTENTION::
+    The **first outlet is special**: if an error writing to any outlet occurs, the first outlet receives the error and can print it.
+    Thus, the first outlet must be the one that always works and does not block, e.g. ``stdout``, which is the default.
+
+.. _logging-default-config:
+
+Default Configuration
+---------------------
+
+By default, the following logging configuration is used
+
+::
+
+    global:
+      logging:
+    
+        - outlet: "stdout"
+          level:  "warn"
+          format: "human"
+
+Building Blocks
+---------------
+
+The following sections document the semantics of the different log levels, formats and outlet types.
+
+.. _logging-levels:
+
+Levels
+~~~~~~
+
+.. list-table::
+    :widths: 10 10 80
+    :header-rows: 1
+
+    * - Level
+      - SHORT
+      - Description
+    * - ``error``
+      - ``ERRO``
+      - immediate action required
+    * - ``warn``
+      - ``WARN``
+      - symptoms for misconfiguration, soon expected failure, etc.
+    * - ``info``
+      - ``INFO``
+      - explains what happens without too much detail
+    * - ``debug``
+      - ``DEBG``
+      - tracing information, state dumps, etc. useful for debugging.
+
+Incorrectly classified messages are considered a bug and should be reported.
+
+.. _logging-formats:
+
+Formats
+~~~~~~~
+
+.. list-table::
+    :widths: 10 90
+    :header-rows: 1
+
+    * - Format
+      - Description
+    * - ``human``
+      - emphasizes context by putting job, task, step and other context variables into brackets
+        before the actual message, followed by remaining fields in logfmt style|
+    * - ``logfmt``
+      - `logfmt <https://brandur.org/logfmt>`_ output. zrepl uses `this Go package <https://github.com/go-logfmt/logfmt>`_.
+    * - ``json``
+      - JSON formatted output. Each line is a valid JSON document. Fields are marshaled by
+        ``encoding/json.Marshal()``, which is particularly useful for processing in
+        log aggregation or when processing state dumps.
+
+Outlets
+~~~~~~~
+
+Outlets are the destination for log entries.
+
+.. _logging-outlet-stdout:
+
+``stdout`` Outlet
+-----------------
+
+.. list-table::
+    :widths: 10 90
+    :header-rows: 1
+
+    * - Parameter
+      - Comment
+    * - ``outlet``
+      - ``stdout``
+    * - ``level``
+      -  minimum  :ref:`log level <logging-levels>`
+    * - ``format``
+      - output :ref:`format <logging-formats>`
+    * - ``time``
+      - always include time in output (``true`` or ``false``)
+
+Writes all log entries with minimum level ``level`` formatted by ``format`` to stdout.
+If stdout is a tty, interactive usage is assumed and the current time is included in the output.
+
+Can only be specified once.
+
+``syslog`` Outlet
+-----------------
+.. list-table::
+    :widths: 10 90
+    :header-rows: 1
+
+    * - Parameter
+      - Comment
+    * - ``outlet``
+      - ``syslog``
+    * - ``level``
+      -  minimum  :ref:`log level <logging-levels>`
+    * - ``format``
+      - output :ref:`format <logging-formats>`
+    * - ``retry_interval``
+      - Interval between reconnection attempts to syslog (default = 0)
+
+Writes all log entries formatted by ``format`` to syslog.
+On normal setups, you should not need to change the ``retry_interval``.
+
+Can only be specified once.
+
+``tcp`` Outlet
+--------------
+
+.. list-table::
+    :widths: 10 90
+    :header-rows: 1
+
+    * - Parameter
+      - Comment
+    * - ``outlet``
+      - ``tcp``
+    * - ``level``
+      -  minimum  :ref:`log level <logging-levels>`
+    * - ``format``
+      - output :ref:`format <logging-formats>`
+    * - ``net``
+      - ``tcp`` in most cases
+    * - ``address``
+      - remote network, e.g. ``logs.example.com:10202``
+    * - ``retry_interval``
+      - Interval between reconnection attempts to ``address``
+    * - ``tls``
+      - TLS config (see below)
+
+Establishes a TCP connection to ``address`` and sends log messages with minimum level ``level`` formatted by ``format``.
+
+If ``tls`` is not specified, an unencrypted connection is established.
+
+If ``tls`` is specified, the TCP connection is secured with TLS + Client Authentication.
+This is particularly useful in combination with log aggregation services that run on an other machine.
+
+.. list-table::
+    :widths: 10 90
+    :header-rows: 1
+
+    * - Parameter
+      - Description
+    * - ``ca``
+      - PEM-encoded certificate authority that signed the remote server's TLS certificate
+    * - ``cert``
+      - PEM-encoded client certificate identifying this zrepl daemon toward the remote server
+    * - ``key``
+      - PEM-encoded, unencrypted client private key identifying this zrepl daemon toward the remote server
+
+.. WARNING::
+
+    zrepl drops log messages to the TCP outlet if the underlying connection is not fast enough.
+    Note that TCP buffering in the kernel must first run full becfore messages are dropped.
+
+    Make sure to always configure a ``stdout`` outlet as the special error outlet to be informed about problems
+    with the TCP outlet (see :ref:`above <logging-error-outlet>` ).
+
+
+.. NOTE::
+
+    zrepl uses Go's ``crypto/tls`` and ``crypto/x509`` packages and leaves all but the required fields in ``tls.Config`` at their default values.
+    In case of a security defect in these packages, zrepl has to be rebuilt because Go binaries are statically linked.

docs/configuration/map_filter_syntax.rst

@@ -0,0 +1,108 @@
+.. include:: ../global.rst.inc
+
+Mapping & Filter Syntax
+=======================
+
+For various job types, a filesystem ``mapping`` or ``filter`` needs to be
+specified.
+
+Both have in common that they take a filesystem path (in the ZFS filesystem hierarchy)as parameters and return something.
+Mappings return a *target filesystem* and filters return a *filter result*.
+
+The pattern syntax is the same for mappings and filters and is documented in the following section.
+
+Common Pattern Syntax
+---------------------
+
+A mapping / filter is specified as a **YAML dictionary** with patterns as keys and
+results as values.
+The following rules determine which result is chosen for a given filesystem path:
+
+* More specific path patterns win over less specific ones
+* Non-wildcard patterns (full path patterns) win over *subtree wildcards* (`<` at end of pattern)
+
+The **subtree wildcard** ``<`` means "the dataset left of ``<`` and all its children".
+
+Example
+~~~~~~~
+
+::
+
+    # Rule number and its pattern
+    1: tank<            # tank and all its children
+    2: tank/foo/bar     # full path pattern (no wildcard)
+    3: tank/foo<        # tank/foo and all its children
+    
+    # Which rule applies to given path?
+    tank/foo/bar/loo => 3
+    tank/bar         => 1
+    tank/foo/bar     => 2
+    zroot            => NO MATCH
+    tank/var/log     => 1
+
+.. _pattern-mapping:
+
+Mappings
+--------
+
+Mappings map a *source filesystem path* to a *target filesystem path*.
+Per pattern, either a target filesystem path or ``"!"`` is specified as a result.
+
+* If no pattern matches, there exists no target filesystem (``NO MATCH``).
+* If the result is a ``"!"``, there exists no target filesystem (``NO MATCH``).
+* If the pattern is a non-wildcard pattern, the source path is mapped to the target path on the right.
+* If the pattern ends with a *subtree wildcard* (``<``), the source path is **prefix-trimmed** with the path specified left of ``<``.
+
+  * Note: this means that only for *wildcard-only* patterns (pattern= ``<`` ) is the source path simply appended to the target path.
+
+The example is from the :sampleconf:`localbackup/host1.yml` example config.
+
+::
+
+    jobs:
+    - name: mirror_local
+      type: local
+      mapping: {
+        "zroot/var/db<":    "storage/backups/local/zroot/var/db",
+        "zroot/usr/home<":  "storage/backups/local/zroot/usr/home",
+        "zroot/usr/home/paranoid":  "!", #don't backup paranoid user
+        "zroot/poudriere/ports<": "!", #don't backup the ports trees
+      }
+      ...
+
+
+::
+
+    zroot/var/db                        => storage/backups/local/zroot/var/db
+    zroot/var/db/a/child                => storage/backups/local/zroot/var/db/a/child
+    zroot/usr/home                      => storage/backups/local/zroot/usr/home
+    zroot/usr/home/paranoid             => NOT MAPPED
+    zroot/usr/home/bob                  => storage/backups/local/zroot/usr/home/bob
+    zroot/usr/src                       => NOT MAPPED
+    zroot/poudriere/ports/2017Q3        => NOT MAPPED
+    zroot/poudriere/ports/HEAD          => NOT MAPPED
+
+.. TIP::
+
+   You can try out patterns for a configured job using the ``zrepl test`` subcommand.
+
+.. _pattern-filter:
+
+Filters
+-------
+
+Valid filter results: ``ok`` or ``!``.
+
+The example below show the source job from the :ref:`tutorial <tutorial-configure-app-srv>`:
+The corresponding pull job is allowed access to ``zroot/var/db``, ``zroot/usr/home`` + children except ``zroot/usr/home/paranoid``::
+
+    jobs:
+    - name: pull_backup
+      type: source
+      ...
+      filesystems: {
+        "zroot/var/db": "ok",
+        "zroot/usr/home<": "ok",
+        "zroot/usr/home/paranoid": "!",
+      }
+      ...

docs/configuration/misc.rst

@@ -0,0 +1,62 @@
+Miscellaneous
+=============
+
+.. _conf-runtime-directories:
+
+Runtime Directories & UNIX Sockets
+----------------------------------
+
+zrepl daemon creates various UNIX sockets to allow communicating with it:
+
+* the :ref:`transport-ssh+stdinserver` transport connects to a socket named after ``client_identity`` parameter
+* the ``control`` CLI subcommand connects to a defined control socket
+
+There is no further authentication on these sockets.
+Therefore we have to make sure they can only be created and accessed by ``zrepl daemon``.
+In fact, ``zrepl daemon`` will not bind a socket to a path in a directory that is world-accessible.
+
+The directories can be configured in the main configuration file, the defaults are provided below:
+
+::
+
+    global:
+      control:
+        sockpath: /var/run/zrepl/control
+      serve:
+        stdinserver:
+          sockdir: /var/run/zrepl/stdinserver
+
+
+Durations & Intervals
+---------------------
+
+Interval & duration fields in job definitions, pruning configurations, etc. must match the following regex:
+
+::
+
+    var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
+    // s = second, m = minute, h = hour, d = day, w = week (7 days)
+
+Super-Verbose Job Debugging
+---------------------------
+
+You have probably landed here because you opened an issue on GitHub and some developer told you to do this...
+So just read the annotated comments ;)
+
+::
+
+    job:
+    - name: ...
+      ...
+     # JOB DEBUGGING OPTIONS
+      # should be equal for all job types, but each job implements the debugging itself
+      debug:
+        conn: # debug the io.ReadWriteCloser connection
+          read_dump: /tmp/connlog_read   # dump results of Read() invocations to this file
+          write_dump: /tmp/connlog_write # dump results of Write() invocations to this file
+        rpc: # debug the RPC protocol implementation
+          log: true # log output from rpc layer to the job log
+
+.. ATTENTION::
+
+    Connection dumps will almost certainly contain your or other's private data. Do not share it in a bug report.

docs/configuration/preface.rst

@@ -0,0 +1,39 @@
+
+.. _configuration_preface:
+
+=======
+Preface
+=======
+
+-----------------------
+Configuration File Path
+-----------------------
+
+zrepl searches for its main configuration file in the following locations (in that order):
+
+* If set, the location specified via the global ``--config`` flag
+* ``/etc/zrepl/zrepl.yml``
+* ``/usr/local/etc/zrepl/zrepl.yml``
+
+The examples in the :ref:`tutorial` or the ``cmd/sampleconf`` directory should provide a good starting point.
+
+-------------------
+Runtime Directories
+-------------------
+
+zrepl requires runtime directories for various UNIX sockets --- they are documented in the :ref:`config file<conf-runtime-directories>`.
+Your package maintainer / init script should take care of creating them.
+Alternatively, for default settings, the following should to the trick.
+
+::
+
+    mkdir -p /var/run/zrepl/stdinserver
+    chmod -R 0700 /var/run/zrepl
+
+
+----------
+Validating
+----------
+
+The config can be validated using the ``zrepl test`` subcommand.
+

docs/configuration/prune.rst

@@ -0,0 +1,84 @@
+.. _prune:
+
+Pruning Policies
+================
+
+In zrepl, *pruning* means *destroying filesystem versions by some policy* where filesystem versions are bookmarks and snapshots.
+
+A *pruning policy* takes a list of filesystem versions and decides for each whether it should be kept or destroyed.
+
+The job context defines which snapshots are even considered for pruning, for example through the ``snapshot_prefix`` variable.
+Check the respective :ref:`job definition <job>` for details.
+
+Currently, the :ref:`prune-retention-grid` is the only supported pruning policy.
+
+.. TIP::
+
+    You can perform a dry-run of a job's pruning policy using the ``zrepl test`` subcommand.
+
+.. _prune-retention-grid:
+
+Retention Grid
+--------------
+
+::
+
+    jobs:
+    - name: pull_app-srv
+      type: pull
+      ...
+      prune:
+        policy: grid
+        grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+                │               │
+                └─ one hour interval
+                                │
+                                └─ 24 adjacent one-hour intervals
+
+    - name: pull_backup
+      type: source
+      interval: 10m
+      prune:
+        policy: grid
+        grid: 1x1d(keep=all)
+        keep_bookmarks: 144
+
+
+The retention grid can be thought of as a time-based sieve:
+The ``grid`` field specifies a list of adjacent time intervals:
+the left edge of the leftmost (first) interval is the ``creation`` date of the youngest snapshot.
+All intervals to its right describe time intervals further in the past.
+
+Each interval carries a maximum number of snapshots to keep.
+It is secified via ``(keep=N)``, where ``N`` is either ``all`` (all snapshots are kept) or a positive integer.
+The default value is **1**.
+
+Bookmarks are not affected by the above.
+Instead, the ``keep_bookmarks`` field specifies the number of bookmarks to be kept per filesystem.
+You only need to specify ``keep_bookmarks`` at the source-side of a replication setup since the destination side does not receive bookmarks.
+You can specify ``all`` as a value to keep all bookmarks, but be warned that you should install some other way to prune unneeded ones then (see below).
+
+The following procedure happens during pruning:
+
+#. The list of snapshots eligible for pruning is sorted by ``creation``
+#. The left edge of the first interval is aligned to the ``creation`` date of the youngest snapshot
+#. A list of buckets is created, one for each interval
+#. The list of snapshots is split up into the buckets.
+#. For each bucket
+
+   #. the contained snapshot list is sorted by creation.
+   #. snapshots from the list, oldest first, are destroyed until the specified ``keep`` count is reached.
+   #. all remaining snapshots on the list are kept.
+#. The list of bookmarks eligible for pruning is sorted by ``createtxg`` and the most recent ``keep_bookmarks`` bookmarks are kept.
+
+.. _replication-downtime:
+
+.. ATTENTION::
+
+    Be aware that ``keep_bookmarks x interval`` (interval of the job level) controls the **maximum allowable replication downtime** between source and destination.
+    If replication does not work for whatever reason, source and destination will eventually run out of sync because the source will continue pruning snapshots.
+    The only recovery in that case is full replication, which may not always be viable due to disk space or traffic constraints.
+
+    Further note that while bookmarks consume a constant amount of disk space, listing them requires temporary dynamic **kernel memory** proportional to the number of bookmarks.
+    Thus, do not use ``all`` or an inappropriately high value without good reason.
+

docs/configuration/transports.rst

@@ -0,0 +1,112 @@
+.. highlight:: bash
+
+.. _transport:
+
+Transports
+==========
+
+A transport provides an authenticated `io.ReadWriteCloser <https://golang.org/pkg/io/#ReadWriteCloser>`_ to the RPC layer.
+(An ``io.ReadWriteCloser`` is essentially a bidirectional reliable communication channel.)
+
+Currently, only the ``ssh+stdinserver`` transport is supported.
+
+.. _transport-ssh+stdinserver:
+
+``ssh+stdinserver`` Transport
+-----------------------------
+
+The way the ``ssh+stdinserver`` transport works is inspired by `git shell <https://git-scm.com/docs/git-shell>`_ and `Borg Backup <https://borgbackup.readthedocs.io/en/stable/deployment.html>`_.
+It is provided by the Go package ``github.com/problame/go-netssh``.
+The config excerpts are taken from the :ref:`tutorial` which you should complete before reading further.
+
+.. _transport-ssh+stdinserver-serve:
+
+Serve Mode
+~~~~~~~~~~
+
+::
+
+    jobs:
+    - name: pull_backup
+      type: source
+      serve:
+        type: stdinserver
+        client_identity: backup-srv.example.com
+      ...
+
+The serving job opens a UNIX socket named after ``client_identity`` in the runtime directory, e.g. ``/var/run/zrepl/stdinserver/backup-srv.example.com``.
+
+On the same machine, the ``zrepl stdinserver $client_identity`` command connects to that socket.
+For example, ``zrepl stdinserver backup-srv.example.com`` connects to the UNIX socket ``/var/run/zrepl/stdinserver/backup-srv.example.com``.
+
+It then passes its stdin and stdout file descriptors to the zrepl daemon via *cmsg(3)*.
+zrepl daemon in turn combines them into an ``io.ReadWriteCloser``:
+a ``Write()`` turns into a write to stdout, a ``Read()`` turns into a read from stdin.
+
+Interactive use of the ``stdinserver`` subcommand does not make much sense.
+However, we can force its execution when a user with a particular SSH pubkey connects via SSH.
+This can be achieved with an entry in the ``authorized_keys`` file of the serving zrepl daemon.
+
+::
+
+    # for OpenSSH >= 7.2
+    command="zrepl stdinserver CLIENT_IDENTITY",restrict CLIENT_SSH_KEY
+    # for older OpenSSH versions
+    command="zrepl stdinserver CLIENT_IDENTITY",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc CLIENT_SSH_KEY
+
+* CLIENT_IDENTITY is substituted with ``backup-srv.example.com`` in our example
+* CLIENT_SSH_KEY is substituted with the public part of the SSH keypair specified in the ``connect`` directive on the connecting host.
+
+.. NOTE::
+
+    You may need to adjust the ``PermitRootLogin`` option in ``/etc/ssh/sshd_config`` to ``forced-commands-only`` or higher for this to work.
+    Refer to sshd_config(5) for details.
+
+To recap, this is of how client authentication works with the ``ssh+stdinserver`` transport:
+
+* Connections to the ``client_identity`` UNIX socket are blindly trusted by zrepl daemon.
+* Thus, the runtime directory must be private to the zrepl user (checked by zrepl daemon)
+* The admin of the host with the serving zrepl daemon controls the ``authorized_keys`` file.
+* Thus, the administrator controls the mapping ``PUBKEY -> CLIENT_IDENTITY``.
+
+.. _transport-ssh+stdinserver-connect:
+
+Connect Mode
+~~~~~~~~~~~~
+
+::
+
+    jobs:
+    - name: pull_app-srv
+      type: pull
+      connect:
+        type: ssh+stdinserver
+        host: app-srv.example.com
+        user: root
+        port: 22
+        identity_file: /etc/zrepl/ssh/identity
+        options: # optional
+        - "Compression=on"
+
+The connecting zrepl daemon
+
+#. Creates a pipe
+#. Forks
+#. In the forked process
+
+   #. Replaces forked stdin and stdout with the corresponding pipe ends
+   #. Executes the ``ssh`` binary found in ``$PATH``.
+
+      #. The identity file (``-i``) is set to ``$identity_file``.
+      #. The remote user, host and port correspond to those configured.
+      #. Further options can be specified using the ``options`` field, which appends each entry in the list to the command line using ``-o $entry``.
+
+1. Wraps the pipe ends in an ``io.ReadWriteCloser`` and uses it for RPC.
+
+As discussed in the section above, the connecting zrepl daemon expects that ``zrepl stdinserver $client_identity`` is  executed automatically via an ``authorized_keys`` file entry.
+
+.. NOTE::
+
+    The environment variables of the underlying SSH process are cleared. ``$SSH_AUTH_SOCK`` will not be available.
+    It is suggested to create a separate, unencrypted SSH key solely for that purpose.
+

docs/content/_header.md

@@ -1 +0,0 @@
-zrepl Documentation

docs/content/_index.md

@@ -1,50 +0,0 @@
-+++
-title = "zrepl - ZFS replication"
-+++
-
-# zrepl - ZFS replication
-
-{{% notice info %}}
-zrepl as well as this documentation is still under active development.
-Use & test at your own risk ;)
-{{% /notice %}}
-
-## Getting started
-
-The [5 minute tutorial setup](/tutorial/) gives you a first impression.
-
-## Main Features
-
-* Filesystem Replication
-    * [x] Local & Remote
-    * [x] Pull mode
-    * [ ] Push mode
-    * [x] Access control checks when pulling datasets
-    * [x] [Flexible mapping]({{< ref "configuration/map_filter_syntax.md" >}}) rules
-    * [x] Bookmarks support
-    * [ ] Feature-negotiation for
-        * Resumable `send & receive`
-        * Compressed `send & receive`
-        * Raw encrypted `send & receive` (as soon as it is available)
-* Automatic snapshot creation
-    * [x] Ensure fixed time interval between snapshots
-* Automatic snapshot pruning
-    * [x] <i class="fa fa-arrow-right" aria-hidden="true"></i> [Retention Grid]({{< ref "configuration/prune.md#retention-grid" >}})
-* Maintainable implementation in Go
-    * [x] Cross platform
-    * [x] Type safe & testable code
-
-## Contributing
-
-zrepl is usable but nowhere near a stable release right now -  we are happy about contributors!
-
-* Explore the codebase
-    * These docs live in the `docs/` subdirectory
-* Document any non-obvious / confusing / plain broken behavior you encounter when setting up zrepl for the first time
-* Check the *Issues* and *Projects* sections for things to do
-
-{{% panel header="<i class='fa fa-github'></i> Getting your code merged"%}}
-[The <i class='fa fa-github'></i> GitHub repository](https://github.com/zrepl/zrepl) is where all development happens.
-Open your issue / PR there.
-{{% /panel %}}
-

docs/content/configuration/_index.md

@@ -1,8 +0,0 @@
-+++
-title = "Configuration"
-alwaysopen = true
-+++
-
-{{% alert theme="warning" %}}Under Construction{{% /alert %}}
-
-{{% children description="true" %}}

docs/content/configuration/jobs.md

@@ -1,14 +0,0 @@
-+++
-title = "Job Types"
-weight = 10
-+++
-
-{{% alert theme="warning" %}}Under Construction{{% /alert %}}
-
-See `sampleconf/*/*.yml` for examples.
-
-## Source
-
-## Pull
-
-## Local

docs/content/configuration/logging.md

@@ -1,8 +0,0 @@
-+++
-title = "Logging"
-weight = 90
-+++
-
-{{% alert theme="warning" %}}Under Construction{{% /alert %}}
-
-See `cmd/sampleconf/random/logging.yml`.

docs/content/configuration/map_filter_syntax.md

@@ -1,92 +0,0 @@
-+++
-title = "Mapping & Filter Syntax"
-weight = 20
-description = "How to specify mappings & filters"
-+++
-
-{{% alert theme="warning" %}}Under Construction{{% /alert %}}
-
-## Mapping & Filter Syntax
-
-For various job types, a filesystem `mapping` or `filter` needs to be
-specified.
-
-Both have in common that they return a result for a given filesystem path (in
-the ZFS filesystem hierarchy): mappings return a *target filesystem*, filters
-return a *filter result* (`omit`, `ok`).
-
-The pattern syntax is the same for mappings and filters and is documented in
-the following section.
-
-#### Pattern Syntax
-
-A mapping / filter is specified as a **YAML dictionary** with patterns as keys and
-results as values.<br />
-The following rules determine which result is chosen for a given filesystem path:
-
-* More specific path patterns win over less specific ones
-* Non-wildcard patterns (full path patterns) win over *subtree wildcards* (`<` at end of pattern)
-
-{{% panel %}}
-The **subtree wildcard** `<` means "*the dataset left of `<` and all its children*".
-{{% / panel %}}
-
-##### Example
-
-```
-# Rule number and its pattern
-1: tank<            # tank and all its children
-2: tank/foo/bar     # full path pattern (no wildcard)
-3: tank/foo<        # tank/foo and all its children
-
-# Which rule applies to given path?
-tank/foo/bar/loo => 3
-tank/bar         => 1
-tank/foo/bar     => 2
-zroot            => NO MATCH
-tank/var/log     => 1
-```
-
-#### Mappings
-
-The example below shows a pull job that would pull remote datasets to the given local paths.<br />
-If there exists no mapping (`NO MATCH`), the filesystem will not be pulled.
-
-```yaml
-pull:
-  app01.example.com: # client identity
-    from: app01.example.com
-    mapping: { 
-      "tank/var/db<":    "zroot/backups/app01/tank/var/db",
-      "tank/var/www<":   "zroot/backups/app01/tank/var/www",
-      "tank/var/log":    "zroot/logbackup/app01",
-    }
-```
-
-```
-tank/var/db`          => zroot/backups/app01/tank/var/db
-tank/var/db/a/child   => zroot/backups/app01/tank/var/db/a/child
-...
-tank/var/www          => zroot/backups/app01/tank/var/www
-tank/var/www/a/child  => zroot/backups/app01/tank/var/www/a/child
-...
-tank/var/log          => zroot/logbackup/app01
-tank/var/log/foo      => NOT MAPPED
-```
-
-#### Filters
-
-Valid filter results: `ok` or `omit`.
-
-The example below shows a pull ACL that allows access to the user homes but not
-to the rest of the system's datasets.
-
-```
-# Example for filter syntax
-pull_acls:
-  backups.example.com: # client identity
-    filter: {
-      "<":              omit,
-      "tank/usr/home<": ok,
-    }
-```

docs/content/configuration/misc.md

@@ -1,52 +0,0 @@
-+++
-title = "Miscellaneous"
-weight = 100
-+++
-
-## Runtime Directories & UNIX Sockets
-
-zrepl daemon creates various UNIX sockets to allow communicating with it:
-
-* the `stdinserver` transport connects to a socket named after `client_identity` parameter
-* the `control` subcommand connects to a defined control socket
-
-There is no further authentication on these sockets.
-Therefore we have to make sure they can only be created and accessed by `zrepl daemon`.
-
-In fact, `zrepl daemon` will not bind a socket to a path in a directory that is world-accessible.
-
-The directories can be configured in the main configuration file:
-
-```yaml
-global:
-  control:
-    sockpath: /var/run/zrepl/control
-  serve:
-    stdinserver:
-      sockdir: /var/run/zrepl/stdinserver
-```
-
-
-## Super-Verbose Job Debugging
-
-You have probably landed here because you opened an issue on GitHub and some developer told you to do this...
-So just read the annotated comments ;)
-
-```yaml
-
-job:
-- name: ...
-  ...
- # JOB DEBUGGING OPTIONS
-  # should be equal for all job types, but each job implements the debugging itself
-  debug:
-    conn: # debug the io.ReadWriteCloser connection
-      read_dump: /tmp/connlog_read   # dump results of Read() invocations to this file
-      write_dump: /tmp/connlog_write # dump results of Write() invocations to this file
-    rpc: # debug the RPC protocol implementation
-      log: true # log output from rpc layer to the job log
-```
-
-{{% notice info %}}
-Connection dumps will almost certainly contain your or other's private data. Do not share it in a bug report.
-{{% /notice %}}

docs/content/configuration/prune.md

@@ -1,9 +0,0 @@
-+++
-title = "Pruning"
-description = "Automated pruning of snapshots"
-weight = 40
-+++
-
-{{% alert theme="warning" %}}Under Construction{{% /alert %}}
-
-## Retention Grid

docs/content/configuration/transports.md

@@ -1,24 +0,0 @@
-+++
-title = "Transports"
-weight = 30
-+++
-
-{{% alert theme="warning" %}}Under Construction{{% /alert %}}
-
-## Stdinserver
-
-
-The SSH transport connects to the remote server using the SSH binary in
-`$PATH` and the parameters specified in the `zrepl` config file.
-
-However, instead of a traditional interactive SSH session, `zrepl` expects
-another instance of `zrepl` on the other side of the connection; You may be
-familiar with this concept from [git shell](https://git-scm.com/docs/git-shell)
-or [Borg Backup](https://borgbackup.readthedocs.io/en/stable/deployment.html).
-
-Check the examples for instructions on how to set this up on your machines!
-
-{{% panel %}}
-The environment variables of the underlying SSH process are cleared. `$SSH_AUTH_SOCK` will not be available. We suggest creating a separate, unencrypted SSH key.
-{{% / panel %}}
-

docs/content/impl/_index.md

@@ -1,46 +0,0 @@
-+++
-title = "Implementation Overview"
-+++
-
-{{% alert theme="warning" %}}Under Construction{{% /alert %}}
-
-The following design aspects may convince you that `zrepl` is superior to a hacked-together shell script solution.
-
-## Language
-
-`zrepl` is written in Go, a real programming language with type safety,
-reasonable performance, testing infrastructure and an (opinionated) idea of
-software engineering.
-
-* key parts & algorithms of `zrepl` are covered by unit tests
-* zrepl is noticably faster than comparable shell scripts
-
-
-## RPC protocol
-
-While it is tempting to just issue a few `ssh remote 'zfs send ...' | zfs recv`, this has a number of drawbacks:
-
-* The snapshot streams need to be compatible.
-* Communication is still unidirectional. Thus, you will most likely
-    * either not take advantage of features such as *compressed send & recv*
-    * or issue additional `ssh` commands in advance to figure out what features are supported on the other side.
-* Advanced logic in shell scripts is ugly to read, poorly testable and a pain to maintain.
-
-`zrepl` takes a different approach:
-
-* Define an RPC protocol.
-* Establish an encrypted, authenticated, bidirectional communication channel...
-* ... with `zrepl` running at both ends of it.
-
- This has several obvious benefits:
-
-* No blank root shell access is given to the other side.
-* Instead, access control lists (ACLs) are used to grant permissions to *authenticated* peers. 
-* The transport mechanism is decoupled from the remaining logic, keeping it extensible (e.g. TCP+TLS)
-
-{{% panel %}}
-Currently, the bidirectional communication channel is multiplexed on top of a single SSH connection.
-Local replication is of course handled efficiently via simple method calls
-See TODO for details.
-{{% / panel %}}
-

docs/content/install/_index.md

@@ -1,85 +0,0 @@
-+++
-title = "Installation"
-weight = 20
-+++
-
-{{% notice note %}}
-Note: check out the [tutorial]({{< relref "tutorial/_index.md" >}}) if you want a first impression of zrepl.
-{{% /notice %}}
-
-## User Privileges
-
-It is possible to run zrepl as an unprivileged user in combination with
-[ZFS delegation](https://www.freebsd.org/doc/handbook/zfs-zfs-allow.html).
-
-Also, there is the possibility to run it in a jail on FreeBSD by delegating a dataset to the jail.
-
-However, until we get around documenting those setups, you will have to run zrepl as root or experiment yourself :)
-
-## Installation
-
-zrepl is currently not packaged on any operating system. Signed & versioned releases are planned but not available yet.
-
-Check out the sources yourself, fetch dependencies using dep, compile and install to the zrepl user's `$PATH`.<br />
-**Note**: if the zrepl binary is not in `$PATH`, you will have to adjust the examples in the [tutorial]({{< relref "tutorial/_index.md" >}}).
-
-```bash
-# NOTE: you may want to checkout & build as an unprivileged user
-cd /root
-git clone https://github.com/zrepl/zrepl.git
-cd zrepl
-dep ensure
-go build -o zrepl
-cp zrepl /usr/local/bin/zrepl
-rehash
-# see if it worked
-zrepl help
-```
-
-## Configuration Files
-
-zrepl searches for its main configuration file in the following locations (in that order):
-
-* `/etc/zrepl/zrepl.yml`
-* `/usr/local/etc/zrepl/zrepl.yml`
-
-Alternatively, use CLI flags to specify a config location.
-
-Copy a config from the [tutorial](/tutorial) or the `cmd/sampleconf` directory to one of these locations and customize it to your setup.
-
-## Runtime Directories
-
-Check the the [configuration documentation]({{< relref "configuration/misc.md#runtime-directories-unix-sockets" >}}) for more information.
-For default settings, the following should to the trick.
-
-```bash
-mkdir -p /var/run/zrepl/stdinserver
-chmod -R 0700 /var/run/zrepl
-```
-
-
-## Running the Daemon
-
-All actual work zrepl does is performed by a daemon process.
-
-Logging is configurable via the config file. Please refer to the [logging documentation]({{< relref "configuration/logging.md" >}}).
-
-```bash
-zrepl daemon
-```
-
-There are no *rc(8)* or *systemd.service(5)* service definitions yet. Note the *daemon(8)* utility on FreeBSD.
-
-{{% notice info %}}
-Make sure to actually monitor the error level output of zrepl: some configuration errors will not make the daemon exit.<br />
-Example: if the daemon cannot create the [stdinserver]({{< relref "configuration/transports.md#stdinserver" >}}) sockets
-in the runtime directory, it will emit an error message but not exit because other tasks such as periodic snapshots & pruning are of equal importance.
-{{% / notice %}}
-
-### Restarting
-
-The daemon handles SIGINT and SIGTERM for graceful shutdown.
-
-Graceful shutdown means at worst that a job will not be rescheduled for the next interval.
-
-The daemon exits as soon as all jobs have reported shut down.

docs/content/tutorial/_index.md

@@ -1,173 +0,0 @@
----
-title: "Tutorial"
-weight: 1
----
-
-This tutorial shows how zrepl can be used to implement a ZFS-based pull backup.
-
-We assume the following scenario
-
-* Production server `prod1` with filesystems to back up
-    * `zroot/var/db`
-    * `zroot/usr/home` and all its child filesystems
-    * **except** `zroot/usr/home/paranoid` belonging to a user doing backups themselves
-* Backup server `backups` with
-    * Filesystem `storage/zrepl/pull/prod1` + children dedicated to backups of `prod1`
-
-Our backup solution should fulfill the following requirements:
-
-* Periodically snapshot the filesystems on `prod1` *every 10 minutes*
-* Incrementally replicate these snapshots to `storage/zrepl/pull/prod1/*` on `backups`
-* Keep only very few snapshots on `prod1` to save disk space
-* Keep a fading history (24 hourly, 30 daily, 6 monthly) of snapshots on `backups`
-
-## Analysis
-
-We can model this situation as two jobs:
-
-* A **source job** on `prod1`
-    * Creates the snapshots
-    * Keeps a few snapshots that are also on `prod1` to enable incremental replication
-* A **pull job** on `prod1`
-    * Pulls the snapshots
-    * Fades out snapshots as they age
-
-{{%expand "Side note: why doesn't `backups` take the snapshots right before replication?" %}}
-After all, a little `ssh prod1 'zfs snapshot...'` wouldn't be so bad, right?
-
-As is the case with all distributed systems, the link between `prod1` and `backups` might be down for an hour or two.
-We do not want to sacrifice our required backup resolution of 10 minute intervals for a temporary connection outage.
-
-When the link comes up again, `backups` will happily catch up the 12 snapshots taken by `prod1` in the meantime, without
-a gap in our backup history.
-{{%/expand%}}
-
-## Install zrepl
-
-Follow the [OS-specific installation instructions](/install/) and come back here.
-
-## Configure `backups`
-
-We define a **pull job** named `pull_prod1` in the [main configuration file](/install/#main-configuration-file):
-
-```yaml
-jobs:
-- name: pull_prod1
-  type: pull
-  connect:
-    type: ssh+stdinserver
-    host: prod1.example.com
-    user: root
-    port: 22
-    identity_file: /etc/zrepl/ssh/prod1
-  interval: 10m
-  mapping: {
-    "<":"storage/zrepl/pull/prod1"
-  }
-  initial_repl_policy: most_recent
-  snapshot_prefix: zrepl_pull_backup_
-  prune:
-    policy: grid
-    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-```
-
-The `connect` section instructs zrepl to use the `stdinserver` transport: instead of directly exposing zrepl on `prod1`
-to the internet, `backups` starts the `zrepl stdinserver` on `prod1` via SSH.
-(You can learn more about what happens [here]({{< relref "configuration/transports.md#stdinserver" >}}), or just continue following this tutorial.)
-
-Thus, we need to create the SSH key pair `/etc/zrepl/ssh/prod1{,.pub}` and later pass the public part to `prod1`
-which will use it to authenticate `backups`. Execute the following commands on `backups` as the root user:
-
-```bash
-cd /etc/zrepl
-mkdir -p ssh
-chmod 0700 ssh
-ssh-keygen -t ed25519 -N '' -f /etc/zrepl/ssh/prod1
-```
-You can learn more about the [**pull job** format here]({{< relref "configuration/jobs.md#pull" >}}) but for now we are good to go.
-
-## Configure `prod1`
-
-We define a corresponding **source job** named `pull_backup` in the [main configuration file](/install/#main-configuration-file)
-`zrepl.yml`:
-
-```yaml
-jobs:
-
-- name: pull_backup
-  type: source
-  serve:
-    type: stdinserver
-    client_identity: backups.example.com
-  datasets: {
-    "zroot/var/db": "ok",
-    "zroot/usr/home<": "ok",
-    "zroot/usr/home/paranoid": "!",
-  }
-  snapshot_prefix: zrepl_pull_backup_
-  interval: 10m
-  prune:
-    policy: grid
-    grid: 1x1d(keep=all)
-
-```
-
-The `serve` section corresponds to the `connect` section in the configuration of `backups`.
-
-We need to allow the SSH key on `backups` to execute `zrepl stdinserver backups.example.com` on
-`prod1`. For good measure, we will in fact enforce that only this command can be executed.
-
-Open `/root/.ssh/authorized_keys` and add either of the the following lines, replacing BACKUPS_SSH_PUBKEY at the end
-of the line with the contents of `/etc/zrepl/ssh/prod1.pub` (note the **.pub** !) from `backups`.
-
-```
-# for OpenSSH >= 7.2
-command="zrepl stdinserver backups.example.com",restrict BACKUPS_SSH_PUBKEY
-# for older OpenSSH versions
-command="zrepl stdinserver backups.example.com",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc  BACKUPS_SSH_PUBKEY
-```
-
-{{% alert theme="info" %}}The entries **must** be on a single line, including the replaced BACKUPS_SSH_PUBKEY{{% /alert %}}
-
-Again, you can learn more about the [**source job** format here]({{< ref "configuration/jobs.md#source" >}}).
-
-## Apply Configuration Changes
-
-We need to restart the zrepl daemon on **both** `prod1` and `backups`.
-
-This is [OS-specific](/install/#restarting).
-
-## Watch it Work
-
-A common setup is to `watch` the log output and `zfs list` of snapshots on both machines.
-
-If you like tmux, here is a handy script that works on FreeBSD:
-
-```bash
-pkg install gnu-watch tmux
-tmux new-window
-tmux split-window "tail -f /var/log/zrepl.log"
-tmux split-window "gnu-watch 'zfs list -t snapshot -o name,creation -s creation | grep zrepl_pull_backup_'"
-tmux select-layout tiled
-```
-
-The Linux equivalent might look like this
-
-```bash
-# make sure tmux is installed & let's assume you use systemd + journald
-tmux new-window
-tmux split-window "journalctl -f -u zrepl.service"
-tmux split-window "watch 'zfs list -t snapshot -o name,creation -s creation | grep zrepl_pull_backup_'"
-tmux select-layout tiled
-```
-
-## Summary
-
-Congratulations, you have a working pull backup. Where to go next?
-
-* Read more about [configuration format, options & job types](/configuration/)
-* Learn about [implementation details](/impl/) of zrepl.
-
-
-
-

docs/global.rst.inc

@@ -0,0 +1,3 @@
+.. global list of substitutions
+
+.. |mainconfig| replace:: :ref:`main configuration file <mainconfigfile>`

docs/implementation.rst

@@ -0,0 +1,62 @@
+.. _implementation_toc:
+
+Implementation Overview
+=======================
+
+.. WARNING::
+
+    Incomplete / under construction
+
+The following design aspects may convince you that ``zrepl`` is superior to a hacked-together shell script solution.
+
+Testability & Performance
+-------------------------
+
+zrepl is written in Go, a real programming language with type safety,
+reasonable performance, testing infrastructure and an (opinionated) idea of
+software engineering.
+
+* key parts & algorithms of zrepl are covered by unit tests (work in progress)
+* zrepl is noticably faster than comparable shell scripts
+
+
+RPC protocol
+------------
+
+While it is tempting to just issue a few ``ssh remote 'zfs send ...' | zfs recv``, this has a number of drawbacks:
+
+* The snapshot streams need to be compatible.
+* Communication is still unidirectional. Thus, you will most likely
+
+  * either not take advantage of features such as *compressed send & recv*
+  * or issue additional ``ssh`` commands in advance to figure out what features are supported on the other side.
+
+* Advanced logic in shell scripts is ugly to read, poorly testable and a pain to maintain.
+
+zrepl takes a different approach:
+
+* Define an RPC protocol.
+* Establish an encrypted, authenticated, bidirectional communication channel...
+* ... with zrepl running at both ends of it.
+
+This has several obvious benefits:
+
+* No blank root shell access is given to the other side.
+* Instead, an *authenticated* peer can *request* filesystem lists, snapshot streams, etc.
+* Requests are then checked against job-specific ACLs, limiting a client to the filesystems it is actually allowed to replicate.
+* The :ref:`transport mechanism <transport>` is decoupled from the remaining logic, keeping it extensible.
+
+Protocol Implementation
+~~~~~~~~~~~~~~~~~~~~~~~
+
+zrepl implements its own RPC protocol.
+This is mostly due to the fact that existing solutions do not provide efficient means to transport large amounts of data.
+
+Package `github.com/zrepl/zrepl/rpc <https://github.com/zrepl/zrepl/tree/master/rpc>`_ builds a special-case handling around returning an ``io.Reader`` as part of a unary RPC call.
+
+Measurements show only a single memory-to-memory copy of a snapshot stream is made using ``github.com/zrepl/zrepl/rpc``, and there is still potential for further optimizations.
+
+Logging & Transparency
+----------------------
+
+zrepl comes with :ref:`rich, structured and configurable logging <logging>`, allowing administators to understand what the software is actually doing.

docs/index.rst

@@ -0,0 +1,89 @@
+.. zrepl documentation master file, created by
+   sphinx-quickstart on Wed Nov  8 22:28:10 2017.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+.. include:: global.rst.inc
+
+zrepl - ZFS replication
+-----------------------
+
+.. ATTENTION::
+    zrepl as well as this documentation is still under active development.
+    It is neither feature complete nor is there a stability guarantee on the configuration format.
+    Use & test at your own risk ;)
+
+Getting started
+~~~~~~~~~~~~~~~
+
+The :ref:`5 minute tutorial setup <tutorial>` gives you a first impression.
+
+Main Features
+~~~~~~~~~~~~~
+
+* Filesystem Replication
+
+  * [x] Local & Remote
+  * [x] Pull mode
+  * [ ] Push mode
+  * [x] Access control checks when pulling datasets
+  * [x] :ref:`Flexible mapping <pattern-mapping>` rules
+  * [x] Bookmarks support
+  * [ ] Feature-negotiation for
+
+    * Resumable `send & receive`
+    * Compressed `send & receive`
+    * Raw encrypted `send & receive` (as soon as it is available)
+
+* Automatic snapshot creation
+
+  * [x] Ensure fixed time interval between snapshots
+
+* Automatic snapshot :ref:`pruning <prune>`
+
+  * [x] Age-based fading (grandfathering scheme)
+
+* Flexible, detailed & structured :ref:`logging <logging>`
+
+  * [x] ``human``, ``logfmt`` and ``json`` formatting
+  * [x] stdout, syslog and TCP (+TLS client auth) outlets
+
+* Maintainable implementation in Go
+
+  * [x] Cross platform
+  * [x] Type safe & testable code
+
+Contributing
+~~~~~~~~~~~~
+
+We are happy about any help we can get!
+
+* Explore the codebase
+
+  * These docs live in the ``docs/`` subdirectory
+
+* Document any non-obvious / confusing / plain broken behavior you encounter when setting up zrepl for the first time
+* Check the *Issues* and *Projects* sections for things to do
+
+.. admonition:: Development Workflow
+    :class: note
+
+    The `GitHub repository <https://github.com/zrepl/zrepl>`_ is where all development happens.
+    Make sure to read the `Developer Documentation section <https://github.com/zrepl/zrepl>`_ and open new issues or pull requests there.
+
+
+Table of Contents
+~~~~~~~~~~~~~~~~~
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   tutorial
+   installation
+   configuration
+   usage
+   implementation
+   changelog
+   GitHub Repository & Issue Tracker <https://github.com/zrepl/zrepl>
+   pr

docs/installation.rst

@@ -0,0 +1,88 @@
+.. _binary releases: https://github.com/zrepl/zrepl/releases
+
+.. _installation:
+
+Installation
+============
+
+.. TIP::
+
+    Note: check out the :ref:`tutorial` if you want a first impression of zrepl.
+
+User Privileges
+---------------
+
+It is possible to run zrepl as an unprivileged user in combination with
+`ZFS delegation <https://www.freebsd.org/doc/handbook/zfs-zfs-allow.html>`_.
+Also, there is the possibility to run it in a jail on FreeBSD by delegating a dataset to the jail.
+However, until we get around documenting those setups, you will have to run zrepl as root or experiment yourself :)
+
+Packages
+--------
+
+zrepl source releases are signed & tagged by the author in the git repository.
+Your OS vendor may provide binary packages of zrepl through the package manager.
+Additionally, `binary releases`_ are provided on GitHub.
+The following list may be incomplete, feel free to submit a PR with an update:
+
+.. list-table::
+    :header-rows: 1
+
+    * - OS / Distro
+      - Install Command
+      - Link
+    * - FreeBSD
+      - ``pkg install zrepl``
+      - `<https://www.freshports.org/sysutils/zrepl/>`_
+    * - Others
+      -
+      - Use `binary releases`_ or build from source.
+
+Compile From Source
+~~~~~~~~~~~~~~~~~~~
+
+Producing a release requires **Go 1.9** or newer and **Python 3** + **pip3** + ``docs/requirements.txt`` for the Sphinx documentation.
+A tutorial to install Go is available over at `golang.org <https://golang.org/doc/install>`_.
+Python and pip3 should probably be installed via your distro's package manager.
+
+Alternatively, you can use the Docker build process:
+it is used to produce the official zrepl `binary releases`_
+and serves as a reference for build dependencies and procedure:
+
+::
+
+    git clone https://github.com/zrepl/zrepl.git
+    cd zrepl
+    sudo docker build -t zrepl_build -f build.Dockerfile .
+    sudo docker run -it --rm \
+        -v "${PWD}:/go/src/github.com/zrepl/zrepl" \
+        --user "$(id -u):$(id -g)" \
+        zrepl_build make vendordeps release
+
+Alternatively, you can install build dependencies on your local system and then build in your ``$GOPATH``:
+
+::
+
+    mkdir -p "${GOPATH}/src/github.com/zrepl/zrepl"
+    git clone https://github.com/zrepl/zrepl.git "${GOPATH}/src/github.com/zrepl/zrepl"
+    cd "${GOPATH}/src/github.com/zrepl/zrepl"
+    ./lazy.sh devsetup
+    make vendordeps release
+
+Build results are located in the ``artifacts/`` directory.
+
+.. NOTE::
+
+    It is your job to install the apropriate binary in the zrepl users's ``$PATH``, e.g. ``/usr/local/bin/zrepl``.
+    Otherwise, the examples in the :ref:`tutorial` may need to be adjusted.
+
+.. _mainconfigfile:
+
+What next?
+----------
+
+Read the :ref:`configuration chapter<configuration_toc>` and optionally create the :ref:`runtime directories <configuration_preface>`.
+Afterwards, continue with the :ref:`usage chapter<usage>`.
+
+**Reminder**: If you want a quick introduction, please read the :ref:`tutorial`.
+

docs/pr.rst

@@ -0,0 +1,8 @@
+Talks & Presentations
+=====================
+
+* Talk at EuroBSDCon2017 FreeBSD DevSummit (
+  `Slides <https://docs.google.com/presentation/d/1EmmeEvOXAWJHCVnOS9-TTsxswbcGKmeLWdY_6BH4w0Q/edit?usp=sharing>`_,
+  `Event <https://wiki.freebsd.org/DevSummit/201709>`_
+  )
+

docs/publish.sh

@@ -3,11 +3,12 @@ set -eo pipefail
 
 GHPAGESREPO="git@github.com:zrepl/zrepl.github.io.git"
 SCRIPTDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-PUBLICDIR="${SCRIPTDIR}/public_git" #use different dir than default dir 'public' (hugo server -D)
+PUBLICDIR="${SCRIPTDIR}/public_git"
 
 checkout_repo_msg() {
-    echo "checkout ${GHPAGESREPO} to ${PUBLICDIR}:"
+    echo "clone ${GHPAGESREPO} to ${PUBLICDIR}:"
     echo "	git clone ${GHPAGESREPO} ${PUBLICDIR}"
+    git clone "${GHPAGESREPO}" "${PUBLICDIR}"
 }
 
 exit_msg() {
@@ -15,6 +16,11 @@ exit_msg() {
 }
 trap exit_msg EXIT
 
+if ! type sphinx-versioning >/dev/null; then
+    echo "install sphinx-versioning and come back"
+    exit 1
+fi
+
 cd "$SCRIPTDIR"
 
 if [ ! -d "$PUBLICDIR" ]; then
@@ -22,7 +28,7 @@ if [ ! -d "$PUBLICDIR" ]; then
     exit 1
 fi
 
-echo -n "PRESS ENTER to confirm you commited the docs changes to the zrepl repo"
+echo -n "PRESS ENTER to confirm you commited and pushed docs changes and tags to the zrepl repo"
 read
 
 pushd "$PUBLICDIR" 
@@ -31,6 +37,7 @@ echo "verify we're in the GitHub pages repo..."
 git remote get-url origin | grep -E "^${GHPAGESREPO}\$"
 if [ "$?" -ne "0" ] ;then 
     checkout_repo_msg
+    echo "finished checkout, please run again"
     exit 1
 fi
 
@@ -44,12 +51,26 @@ git rm -rf .
 popd
 
 echo "building site"
-hugo -d "$PUBLICDIR"
+set -e
+sphinx-versioning build \
+   --show-banner \
+   docs ./public_git \
+   -- -c sphinxconf # older conf.py throw errors because they used
+                    # version = subprocess.show_output(["git", "describe"])
+                    # which fails when building with sphinxcontrib-versioning
+set +e
+
+CURRENT_COMMIT=$(git rev-parse HEAD)
+git status --porcelain
+if [[ "$(git status --porcelain)" != "" ]]; then
+    CURRENT_COMMIT="${CURRENT_COMMIT}(dirty)" 
+fi
+COMMIT_MSG="sphinx-versioning render from publish.sh - `date -u` - ${CURRENT_COMMIT}"
 
 pushd "$PUBLICDIR"
 
 echo "adding and commiting all changes in GitHub pages repo"
 git add -A
-git commit -m "hugo render from publish.sh - `date -u`" #TODO include parent commit id
+git commit -m "$COMMIT_MSG"
 git push origin master
 

docs/requirements.txt

@@ -0,0 +1,28 @@
+Babel==2.5.1
+certifi==2017.11.5
+chardet==3.0.4
+click==6.7
+colorclass==2.2.0
+cryptography==1.7.1
+docutils==0.14
+idna==2.6
+imagesize==0.7.1
+Jinja2==2.10
+keyring==10.1
+keyrings.alt==1.3
+MarkupSafe==1.0
+pyasn1==0.1.9
+pycrypto==2.6.1
+Pygments==2.2.0
+pygobject==3.22.0
+pytz==2017.3
+pyxdg==0.25
+requests==2.18.4
+SecretStorage==2.3.1
+six==1.10.0
+snowballstemmer==1.2.1
+Sphinx==1.6.5
+sphinx-rtd-theme==0.2.4
+sphinxcontrib-versioning==2.2.1
+sphinxcontrib-websupport==1.0.1
+urllib3==1.22

docs/sphinxconf/conf.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# zrepl documentation build configuration file, created by
+# sphinx-quickstart on Wed Nov  8 22:28:10 2017.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['sphinx.ext.todo',
+    'sphinx.ext.githubpages',
+    'sphinx.ext.extlinks']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['../_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'zrepl'
+copyright = '2017, Christian Schwarz'
+author = 'Christian Schwarz'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+#version = set by sphinxcontrib-versioning
+# The full version, including alpha/beta/rc tags.
+#release = version
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['../_static']
+
+
+html_logo = '../_static/zrepl.svg'
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'zrepldoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'zrepl.tex', 'zrepl Documentation',
+     'Christian Schwarz', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'zrepl', 'zrepl Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'zrepl', 'zrepl Documentation',
+     author, 'zrepl', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# -- Options for the extlinks extension -----------------------------------
+# http://www.sphinx-doc.org/en/stable/ext/extlinks.html
+extlinks = {
+    'issue':('https://github.com/zrepl/zrepl/issues/%s', 'issue #'),
+    'sampleconf':('https://github.com/zrepl/zrepl/blob/master/cmd/sampleconf/%s', 'cmd/sampleconf'),
+    'commit':('https://github.com/zrepl/zrepl/commit/%s', 'commit '),
+        }
+

docs/tutorial.rst

@@ -0,0 +1,191 @@
+.. include:: global.rst.inc
+
+.. _tutorial:
+
+Tutorial
+========
+
+
+This tutorial shows how zrepl can be used to implement a ZFS-based pull backup.
+We assume the following scenario:
+
+* Production server ``app-srv`` with filesystems to back up:
+
+  * ``zroot/var/db``
+  * ``zroot/usr/home`` and all its child filesystems
+  * **except** ``zroot/usr/home/paranoid`` belonging to a user doing backups themselves
+
+* Backup server ``backup-srv`` with
+
+  * Filesystem ``storage/zrepl/pull/app-srv`` + children dedicated to backups of ``app-srv``
+
+Our backup solution should fulfill the following requirements:
+
+* Periodically snapshot the filesystems on ``app-srv`` *every 10 minutes*
+* Incrementally replicate these snapshots to ``storage/zrepl/pull/app-srv/*`` on ``backup-srv``
+* Keep only very few snapshots on ``app-srv`` to save disk space
+* Keep a fading history (24 hourly, 30 daily, 6 monthly) of snapshots on ``backup-srv``
+
+Analysis
+--------
+
+We can model this situation as two jobs:
+
+* A **source job** on ``app-srv``
+
+  * Creates the snapshots
+  * Keeps a short history of snapshots to enable incremental replication to ``backup-srv``
+  * Accepts connections from ``backup-srv``
+
+* A **pull job** on ``backup-srv``
+
+  * Connects to the ``zrepl daemon`` process on ``app-srv``
+  * Pulls the snapshots to ``storage/zrepl/pull/app-srv/*``
+  * Fades out snapshots in ``storage/zrepl/pull/app-srv/*`` as they age
+
+
+Why doesn't the **pull job** create the snapshots before pulling?
+
+As is the case with all distributed systems, the link between ``app-srv`` and ``backup-srv`` might be down for an hour or two.
+We do not want to sacrifice our required backup resolution of 10 minute intervals for a temporary connection outage.
+
+When the link comes up again, ``backup-srv`` will happily catch up the 12 snapshots taken by ``app-srv`` in the meantime, without
+a gap in our backup history.
+
+Install zrepl
+-------------
+
+Follow the :ref:`OS-specific installation instructions <installation>` and come back here.
+
+Configure ``backup-srv``
+------------------------
+
+We define a **pull job** named ``pull_app-srv`` in the |mainconfig|: ::
+
+    jobs:
+    - name: pull_app-srv
+      type: pull
+      connect:
+          type: ssh+stdinserver
+          host: app-srv.example.com
+          user: root
+          port: 22
+          identity_file: /etc/zrepl/ssh/identity
+      interval: 10m
+      mapping: {
+          "<":"storage/zrepl/pull/app-srv"
+      }
+      initial_repl_policy: most_recent
+      snapshot_prefix: zrepl_pull_backup_
+      prune:
+          policy: grid
+          grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+
+The ``connect`` section instructs the zrepl daemon to use the ``stdinserver`` transport:
+``backup-srv`` will connect to the specified SSH server and expect ``zrepl stdinserver CLIENT_IDENTITY`` instead of the shell on the other side.
+
+It uses the private key specified at ``connect.identity_file`` which we still need to create: ::
+
+    cd /etc/zrepl
+    mkdir -p ssh
+    chmod 0700 ssh
+    ssh-keygen -t ed25519 -N '' -f /etc/zrepl/ssh/identity
+
+Note that most use cases do not benefit from separate keypairs per remote endpoint.
+Thus, it is sufficient to create one keypair and use it for all ``connect`` directives on one host.
+
+Learn more about :ref:`transport-ssh+stdinserver` transport and the :ref:`pull job <job-pull>` format.
+
+.. _tutorial-configure-app-srv:
+
+Configure ``app-srv``
+---------------------
+
+We define a corresponding **source job** named ``pull_backup`` in the |mainconfig|: ::
+
+    jobs:
+    - name: pull_backup
+      type: source
+      serve:
+          type: stdinserver
+          client_identity: backup-srv.example.com
+      filesystems: {
+          "zroot/var/db": "ok",
+          "zroot/usr/home<": "ok",
+          "zroot/usr/home/paranoid": "!",
+      }
+      snapshot_prefix: zrepl_pull_backup_
+      interval: 10m
+      prune:
+          policy: grid
+          grid: 1x1d(keep=all)
+          keep_bookmarks: 144
+
+
+The ``serve`` section corresponds to the ``connect`` section in the configuration of ``backup-srv``.
+
+We now want to authenticate ``backup-srv`` before allowing it to pull data.
+This is done by limiting SSH connections from ``backup-srv`` to execute the ``stdinserver`` subcommand.
+
+Open ``/root/.ssh/authorized_keys`` and add either of the the following lines.::
+
+    # for OpenSSH >= 7.2
+    command="zrepl stdinserver backup-srv.example.com",restrict CLIENT_SSH_KEY
+    # for older OpenSSH versions
+    command="zrepl stdinserver backup-srv.example.com",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc  CLIENT_SSH_KEY
+
+.. ATTENTION::
+
+    Replace CLIENT_SSH_KEY with the contents of ``/etc/zrepl/ssh/identity.pub`` from ``app-srv``.
+    Mind the trailing ``.pub`` in the filename.
+    The entries **must** be on a single line, including the replaced CLIENT_SSH_KEY.
+
+
+.. HINT::
+
+    You may need to adjust the ``PermitRootLogin`` option in ``/etc/ssh/sshd_config`` to ``forced-commands-only`` or higher for this to work.
+    Refer to sshd_config(5) for details.
+
+The argument ``backup-srv.example.com`` is the client identity of ``backup-srv`` as defined in ``jobs.serve.client_identity``.
+
+Again, both :ref:`transport-ssh+stdinserver` transport and the :ref:`job-source` format are documented.
+
+Apply Configuration Changes
+---------------------------
+
+We need to restart the zrepl daemon on **both** ``app-srv`` and ``backup-srv``.
+This is :ref:`OS-specific <usage-zrepl-daemon-restarting>`.
+
+Watch it Work
+-------------
+
+Run ``zrepl control status`` to view the current activity of the configured jobs.
+If a job encountered problems since it last left idle state, the output contains useful debug log.
+
+Additionally, you can check the detailed structured logs of the `zrepl daemon` process and use GNU *watch* to view the snapshots present on both machines.
+
+If you like tmux, here is a handy script that works on FreeBSD: ::
+
+    pkg install gnu-watch tmux
+    tmux new-window
+    tmux split-window "tail -f /var/log/zrepl.log"
+    tmux split-window "gnu-watch 'zfs list -t snapshot -o name,creation -s creation | grep zrepl_pull_backup_'"
+    tmux select-layout tiled
+
+The Linux equivalent might look like this: ::
+
+    # make sure tmux is installed & let's assume you use systemd + journald
+    tmux new-window
+    tmux split-window "journalctl -f -u zrepl.service"
+    tmux split-window "watch 'zfs list -t snapshot -o name,creation -s creation | grep zrepl_pull_backup_'"
+    tmux select-layout tiled
+
+Summary
+-------
+
+Congratulations, you have a working pull backup. Where to go next?
+
+* Read more about :ref:`configuration format, options & job types <configuration_toc>`
+* Learn about :ref:`implementation details <implementation_toc>` of zrepl.
+
+

docs/usage.rst

@@ -0,0 +1,62 @@
+.. _usage:
+
+*****
+Usage
+*****
+
+============
+CLI Overview
+============
+
+.. NOTE::
+
+    To avoid duplication, the zrepl binary is self-documenting:
+    invoke any subcommand at any level with the ``--help`` flag to get information on the subcommand, available flags, etc.
+
+.. list-table::
+    :widths: 30 70
+    :header-rows: 1
+
+    * - Subcommand
+      - Description
+    * - ``zrepl daemon``
+      - run the daemon, required for all zrepl functionality
+    * - ``zrepl control``
+      - control / query the daemon
+    * - ``zrepl control status``
+      - show job activity / monitoring (``--format raw``)
+    * - ``zrepl test``
+      - test configuration, try pattern syntax, dry run pruning policy, etc.
+    * - ``zrepl stdinserver``
+      - see :ref:`transport-ssh+stdinserver`
+
+.. _usage-zrepl-daemon:
+
+============
+zrepl daemon
+============
+
+All actual work zrepl does is performed by a daemon process.
+Logging is configurable via the config file. Please refer to the :ref:`logging documention <logging>`.
+
+When installating from a package, the package maintainer should have provided an init script / systemd.service file.
+You should thus be able to start zrepl daemon using your init system.
+
+Alternatively, or for running zrepl in the foreground, simply execute ``zrepl daemon``.
+Note that you won't see much output with the :ref:`default logging configuration<logging-default-config>`:
+
+.. ATTENTION::
+
+    Make sure to actually monitor the error level output of zrepl: some configuration errors will not make the daemon exit.
+
+    Example: if the daemon cannot create the :ref:`transport-ssh+stdinserver` sockets in the runtime directory,
+    it will emit an error message but not exit because other tasks such as periodic snapshots & pruning are of equal importance.
+
+.. _usage-zrepl-daemon-restarting:
+
+Restarting
+~~~~~~~~~~
+
+The daemon handles SIGINT and SIGTERM for graceful shutdown.
+Graceful shutdown means at worst that a job will not be rescheduled for the next interval.
+The daemon exits as soon as all jobs have reported shut down.

lazy.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+set -e
+
+if tty -s; then
+    bold=$(tput bold)
+    normal=$(tput sgr0)
+else
+    bold=""
+    normal=""
+fi
+
+step() {
+    echo "${bold}$1${normal}"
+}
+
+if ! type go >/dev/null; then
+    step "go binary not installed or not in \$PATH" 1>&2
+    exit 1
+fi
+
+if [ -z "$GOPATH" ]; then
+    step "Make sure you have your GOPATH configured correctly" 1>&2
+    exit 1
+fi
+
+CHECKOUTPATH="${GOPATH}/src/github.com/zrepl/zrepl"
+
+builddep() {
+    step "Install build depdencies using 'go get' to \$GOPATH/bin"
+    go get -u golang.org/x/tools/cmd/stringer
+    go get -u github.com/golang/dep/cmd/dep
+    if ! type stringer || ! type dep; then
+        echo "Installed dependencies but can't find them in \$PATH, adjust it to contain \$GOPATH/bin" 1>&2
+        exit 1
+    fi
+}
+
+godep() {
+    step "Fetching dependencies using 'dep ensure'"
+    dep ensure
+}
+
+docdep() {
+    if ! type pip3; then
+        step "pip3 binary not installed or not in \$PATH" 1>&2
+        exit 1
+    fi
+    step "Installing doc build dependencies"
+    local reqpath="${CHECKOUTPATH}/docs/requirements.txt"
+    if [ ! -z "$ZREPL_LAZY_DOCS_REQPATH" ]; then
+        reqpath="$ZREPL_LAZY_DOCS_REQPATH"
+    fi
+    pip3 install -r "$reqpath"
+}
+
+release() {
+    step "Making release"
+    make release
+}
+
+for cmd in "$@"; do
+    case "$cmd" in
+        builddep|godep|docdep|release_bins|docs)
+            eval $cmd
+            continue
+            ;;
+        devsetup)
+            step "Installing development dependencies"
+            builddep
+            docdep
+            step "Development dependencies installed"
+            continue
+            ;;
+        *)
+            step "Invalid command ${cmd}, exiting"
+            exit 1
+            ;;
+    esac
+done
+

logger/datastructures.go

@@ -1,10 +1,11 @@
 package logger
 
 import (
-	"context"
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"github.com/pkg/errors"
+	"sync"
 	"time"
 )
 
@@ -14,6 +15,38 @@ func (l Level) MarshalJSON() ([]byte, error) {
 	return json.Marshal(l.String())
 }
 
+func (l *Level) UnmarshalJSON(input []byte) (err error) {
+	var s string
+	if err = json.Unmarshal(input, &s); err != nil {
+		return err
+	}
+	*l, err = ParseLevel(s)
+	return err
+}
+
+// implement flag.Value
+// implement github.com/spf13/pflag.Value
+func (l *Level) Set(s string) error {
+	newl, err := ParseLevel(s)
+	if err != nil {
+		return err
+	}
+	*l = newl
+	return nil
+}
+
+// implement github.com/spf13/pflag.Value
+func (l *Level) Type() string {
+	var buf bytes.Buffer
+	for i, l := range AllLevels {
+		fmt.Fprintf(&buf, "%s", l)
+		if i != len(AllLevels)-1 {
+			fmt.Fprintf(&buf, "|")
+		}
+	}
+	return fmt.Sprintf("(%s)", buf.String())
+}
+
 const (
 	Debug Level = iota
 	Info
@@ -72,20 +105,67 @@ type Entry struct {
 	Fields  Fields
 }
 
+// An outlet receives log entries produced by the Logger and writes them to some destination.
 type Outlet interface {
+	// Write the entry to the destination.
+	//
+	// Logger waits for all outlets to return from WriteEntry() before returning from the log call.
+	// An implementation of Outlet must assert that it does not block in WriteEntry.
+	// Otherwise, it will slow down the program.
+	//
 	// Note: os.Stderr is also used by logger.Logger for reporting errors returned by outlets
 	//       => you probably don't want to log there
-	WriteEntry(ctx context.Context, entry Entry) error
+	WriteEntry(entry Entry) error
 }
 
-type Outlets map[Level][]Outlet
-
-func NewOutlets() Outlets {
-	return make(Outlets, len(AllLevels))
+type Outlets struct {
+	mtx  sync.RWMutex
+	outs map[Level][]Outlet
 }
 
-func (os Outlets) Add(outlet Outlet, minLevel Level) {
-	for _, l := range AllLevels[minLevel:] {
-		os[l] = append(os[l], outlet)
+func NewOutlets() *Outlets {
+	return &Outlets{
+		mtx:  sync.RWMutex{},
+		outs: make(map[Level][]Outlet, len(AllLevels)),
 	}
 }
+
+func (os *Outlets) DeepCopy() (copy *Outlets) {
+	copy = NewOutlets()
+	for level := range os.outs {
+		for i := range os.outs[level] {
+			copy.outs[level] = append(copy.outs[level], os.outs[level][i])
+		}
+	}
+	return copy
+}
+
+func (os *Outlets) Add(outlet Outlet, minLevel Level) {
+	os.mtx.Lock()
+	defer os.mtx.Unlock()
+	for _, l := range AllLevels[minLevel:] {
+		os.outs[l] = append(os.outs[l], outlet)
+	}
+}
+
+func (os *Outlets) Get(level Level) []Outlet {
+	os.mtx.RLock()
+	defer os.mtx.RUnlock()
+	return os.outs[level]
+}
+
+// Return the first outlet added to this Outlets list using Add()
+// with minLevel <= Error.
+// If no such outlet is in this Outlets list, a discarding outlet is returned.
+func (os *Outlets) GetLoggerErrorOutlet() Outlet {
+	os.mtx.RLock()
+	defer os.mtx.RUnlock()
+	if len(os.outs[Error]) < 1 {
+		return nullOutlet{}
+	}
+	return os.outs[Error][0]
+}
+
+type nullOutlet struct{}
+
+func (nullOutlet) WriteEntry(entry Entry) error { return nil }

logger/logger.go

@@ -1,9 +1,7 @@
 package logger
 
 import (
-	"context"
 	"fmt"
-	"os"
 	"runtime/debug"
 	"sync"
 	"time"
@@ -15,17 +13,16 @@ const (
 )
 
 const DefaultUserFieldCapacity = 5
-const InternalErrorPrefix = "github.com/zrepl/zrepl/logger: "
 
 type Logger struct {
 	fields        Fields
-	outlets       Outlets
+	outlets       *Outlets
 	outletTimeout time.Duration
 
 	mtx *sync.Mutex
 }
 
-func NewLogger(outlets Outlets, outletTimeout time.Duration) *Logger {
+func NewLogger(outlets *Outlets, outletTimeout time.Duration) *Logger {
 	return &Logger{
 		make(Fields, DefaultUserFieldCapacity),
 		outlets,
@@ -34,6 +31,29 @@ func NewLogger(outlets Outlets, outletTimeout time.Duration) *Logger {
 	}
 }
 
+type outletResult struct {
+	Outlet Outlet
+	Error  error
+}
+
+func (l *Logger) logInternalError(outlet Outlet, err string) {
+	fields := Fields{}
+	if outlet != nil {
+		if _, ok := outlet.(fmt.Stringer); ok {
+			fields["outlet"] = fmt.Sprintf("%s", outlet)
+		}
+		fields["outlet_type"] = fmt.Sprintf("%T", outlet)
+	}
+	fields[FieldError] = err
+	entry := Entry{
+		Error,
+		"outlet error",
+		time.Now(),
+		fields,
+	}
+	l.outlets.GetLoggerErrorOutlet().WriteEntry(entry)
+}
+
 func (l *Logger) log(level Level, msg string) {
 
 	l.mtx.Lock()
@@ -41,45 +61,43 @@ func (l *Logger) log(level Level, msg string) {
 
 	entry := Entry{level, msg, time.Now(), l.fields}
 
-	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(l.outletTimeout))
-	ech := make(chan error)
-
-	louts := l.outlets[level]
+	louts := l.outlets.Get(level)
+	ech := make(chan outletResult, len(louts))
 	for i := range louts {
-		go func(ctx context.Context, outlet Outlet, entry Entry) {
-			ech <- outlet.WriteEntry(ctx, entry)
-		}(ctx, louts[i], entry)
+		go func(outlet Outlet, entry Entry) {
+			ech <- outletResult{outlet, outlet.WriteEntry(entry)}
+		}(louts[i], entry)
 	}
-
 	for fin := 0; fin < len(louts); fin++ {
-		select {
-		case err := <-ech:
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "%s outlet error: %s\n", InternalErrorPrefix, err)
-			}
-		case <-ctx.Done():
-			if ctx.Err() == context.DeadlineExceeded {
-				fmt.Fprintf(os.Stderr, "%s outlets exceeded deadline, keep waiting anyways", InternalErrorPrefix)
-			}
+		res := <-ech
+		if res.Error != nil {
+			l.logInternalError(res.Outlet, res.Error.Error())
 		}
 	}
-
-	cancel() // make go vet happy
+	close(ech)
 
 }
 
-func (l *Logger) WithField(field string, val interface{}) *Logger {
-
+func (l *Logger) WithOutlet(outlet Outlet, level Level) *Logger {
 	l.mtx.Lock()
 	defer l.mtx.Unlock()
-
-	if _, ok := l.fields[field]; ok {
-		fmt.Fprintf(os.Stderr, "%s caller overwrites field '%s'. Stack:\n%s\n", InternalErrorPrefix, field, string(debug.Stack()))
+	newOutlets := l.outlets.DeepCopy()
+	newOutlets.Add(outlet, level)
+	child := &Logger{
+		fields:        l.fields,
+		outlets:       newOutlets,
+		outletTimeout: l.outletTimeout,
+		mtx:           l.mtx,
 	}
+	return child
+}
+
+// callers must hold l.mtx
+func (l *Logger) forkLogger(field string, val interface{}) *Logger {
 
 	child := &Logger{
 		fields:        make(Fields, len(l.fields)+1),
-		outlets:       l.outlets, // cannot be changed after logger initialized
+		outlets:       l.outlets,
 		outletTimeout: l.outletTimeout,
 		mtx:           l.mtx,
 	}
@@ -89,20 +107,39 @@ func (l *Logger) WithField(field string, val interface{}) *Logger {
 	child.fields[field] = val
 
 	return child
+}
 
+func (l *Logger) ReplaceField(field string, val interface{}) *Logger {
+	l.mtx.Lock()
+	defer l.mtx.Unlock()
+	return l.forkLogger(field, val)
+}
+
+func (l *Logger) WithField(field string, val interface{}) *Logger {
+	l.mtx.Lock()
+	defer l.mtx.Unlock()
+	if val, ok := l.fields[field]; ok && val != nil {
+		l.logInternalError(nil,
+			fmt.Sprintf("caller overwrites field '%s'. Stack: %s", field, string(debug.Stack())))
+	}
+	return l.forkLogger(field, val)
 }
 
 func (l *Logger) WithFields(fields Fields) (ret *Logger) {
 	// TODO optimize
 	ret = l
 	for field, value := range fields {
-		ret = l.WithField(field, value)
+		ret = ret.WithField(field, value)
 	}
 	return ret
 }
 
 func (l *Logger) WithError(err error) *Logger {
-	return l.WithField(FieldError, err.Error())
+	val := interface{}(nil)
+	if err != nil {
+		val = err.Error()
+	}
+	return l.WithField(FieldError, val)
 }
 
 func (l *Logger) Debug(msg string) {

logger/logger_test.go

@@ -1,7 +1,6 @@
 package logger_test
 
 import (
-	"context"
 	"fmt"
 	"github.com/kr/pretty"
 	"github.com/zrepl/zrepl/logger"
@@ -13,7 +12,7 @@ type TestOutlet struct {
 	Record []logger.Entry
 }
 
-func (o *TestOutlet) WriteEntry(ctx context.Context, entry logger.Entry) error {
+func (o *TestOutlet) WriteEntry(entry logger.Entry) error {
 	o.Record = append(o.Record, entry)
 	return nil
 }

rpc/datatype_string.go

@@ -0,0 +1,17 @@
+// Code generated by "stringer -type=DataType"; DO NOT EDIT.
+
+package rpc
+
+import "fmt"
+
+const _DataType_name = "DataTypeNoneDataTypeControlDataTypeMarshaledJSONDataTypeOctets"
+
+var _DataType_index = [...]uint8{0, 12, 27, 48, 62}
+
+func (i DataType) String() string {
+	i -= 1
+	if i >= DataType(len(_DataType_index)-1) {
+		return fmt.Sprintf("DataType(%d)", i+1)
+	}
+	return _DataType_name[_DataType_index[i]:_DataType_index[i+1]]
+}

rpc/frame_layer.go

@@ -57,6 +57,7 @@ func NewErrorHeader(status Status, format string, args ...interface{}) (h *Heade
 	return
 }
 
+//go:generate stringer -type=DataType
 type DataType uint8
 
 const (

sshbytestream/ssh.go

@@ -1,123 +0,0 @@
-package sshbytestream
-
-import (
-	"fmt"
-	"io"
-	"os"
-	"runtime"
-	"syscall"
-
-	"github.com/zrepl/zrepl/util"
-)
-
-type Error struct {
-	Stderr  []byte
-	WaitErr error
-}
-
-func (e Error) Error() string {
-	return fmt.Sprintf("ssh command failed with error: %v. stderr:\n%s\n", e.WaitErr, e.Stderr)
-}
-
-type SSHTransport struct {
-	Host         string
-	User         string
-	Port         uint16
-	IdentityFile string
-	SSHCommand   string
-	Options      []string
-}
-
-var SSHCommand string = "ssh"
-var SSHServerAliveInterval uint = 60
-
-func Incoming() (wc io.ReadWriteCloser, err error) {
-	// derivce ReadWriteCloser from stdin & stdout
-	return IncomingReadWriteCloser{}, nil
-}
-
-type IncomingReadWriteCloser struct{}
-
-func (f IncomingReadWriteCloser) Read(p []byte) (n int, err error) {
-	return os.Stdin.Read(p)
-}
-
-func (f IncomingReadWriteCloser) Write(p []byte) (n int, err error) {
-	return os.Stdout.Write(p)
-}
-
-func (f IncomingReadWriteCloser) Close() (err error) {
-	if err = os.Stdin.Close(); err != nil {
-		return
-	}
-	if err = os.Stdout.Close(); err != nil {
-		return
-	}
-	return
-}
-
-type OutgoingSSHByteStream struct {
-	c *util.IOCommand
-}
-
-func Outgoing(remote SSHTransport) (s OutgoingSSHByteStream, err error) {
-
-	sshArgs := make([]string, 0, 2*len(remote.Options)+4)
-	sshArgs = append(sshArgs,
-		"-p", fmt.Sprintf("%d", remote.Port),
-		"-q",
-		"-i", remote.IdentityFile,
-		"-o", "BatchMode=yes",
-		"-o", fmt.Sprintf("ServerAliveInterval=%d", SSHServerAliveInterval),
-	)
-	for _, option := range remote.Options {
-		sshArgs = append(sshArgs, "-o", option)
-	}
-	sshArgs = append(sshArgs, fmt.Sprintf("%s@%s", remote.User, remote.Host))
-
-	var sshCommand = SSHCommand
-	if len(remote.SSHCommand) > 0 {
-		sshCommand = SSHCommand
-	}
-
-	if s.c, err = util.NewIOCommand(sshCommand, sshArgs, util.IOCommandStderrBufSize); err != nil {
-		return
-	}
-
-	// Clear environment of cmd, ssh shall not rely on SSH_AUTH_SOCK, etc.
-	s.c.Cmd.Env = []string{}
-
-	err = s.c.Start()
-	return
-}
-
-func (s OutgoingSSHByteStream) Read(p []byte) (n int, err error) {
-	return s.c.Read(p)
-}
-
-func (s OutgoingSSHByteStream) Write(p []byte) (n int, err error) {
-	return s.c.Write(p)
-}
-
-func (s OutgoingSSHByteStream) Close() (err error) {
-	err = s.c.Close()
-	if err == nil || s.c.ExitResult == nil {
-		return
-	}
-
-	// SSH catches SIGTERM and has different exit codes on different platforms
-	ws := s.c.ExitResult.WaitStatus
-	switch runtime.GOOS {
-	case "linux":
-		if ws.ExitStatus() == 128+int(syscall.SIGTERM) { // OpenSSH_7.5p1, OpenSSL 1.1.0f  25 May 2017 Arch Linux
-			err = nil
-		}
-	case "freebsd": // OpenSSH_7.2p2, OpenSSL 1.0.2k-freebsd  26 Jan 2017
-		if ws.ExitStatus() == 255 {
-			err = nil
-		}
-	default: // TODO
-	}
-
-	return
-}

util/io.go

@@ -3,7 +3,6 @@ package util
 import (
 	"io"
 	"os"
-	"time"
 )
 
 type ReadWriteCloserLogger struct {
@@ -97,52 +96,3 @@ func (c *ChainedReader) Read(buf []byte) (n int, err error) {
 
 	return
 }
-
-type IOProgress struct {
-	TotalRX uint64
-}
-
-type IOProgressCallback func(progress IOProgress)
-
-type IOProgressWatcher struct {
-	Reader         io.Reader
-	callback       IOProgressCallback
-	callbackTicker *time.Ticker
-	progress       IOProgress
-	updateChannel  chan int
-}
-
-func (w *IOProgressWatcher) KickOff(callbackInterval time.Duration, callback IOProgressCallback) {
-	w.callback = callback
-	w.callbackTicker = time.NewTicker(callbackInterval)
-	w.updateChannel = make(chan int)
-	go func() {
-	outer:
-		for {
-			select {
-			case newBytes, more := <-w.updateChannel:
-				w.progress.TotalRX += uint64(newBytes)
-				if !more {
-					w.callbackTicker.Stop()
-					break outer
-				}
-			case <-w.callbackTicker.C:
-				w.callback(w.progress)
-			}
-		}
-		w.callback(w.progress)
-	}()
-}
-
-func (w *IOProgressWatcher) Progress() IOProgress {
-	return w.progress
-}
-
-func (w *IOProgressWatcher) Read(p []byte) (n int, err error) {
-	n, err = w.Reader.Read(p)
-	w.updateChannel <- n
-	if err != nil {
-		close(w.updateChannel)
-	}
-	return
-}

util/semaphore.go

@@ -0,0 +1,17 @@
+package util
+
+type Semaphore struct {
+	c chan struct{}
+}
+
+func NewSemaphore(cap int) Semaphore {
+	return Semaphore{make(chan struct{}, cap)}
+}
+
+func (s Semaphore) Down() {
+	s.c <- struct{}{}
+}
+
+func (s Semaphore) Up() {
+	<-s.c
+}

zfs/mapping.go

@@ -1,6 +1,9 @@
 package zfs
 
-import "fmt"
+import (
+	"context"
+	"fmt"
+)
 
 type DatasetFilter interface {
 	Filter(p *DatasetPath) (pass bool, err error)
@@ -12,15 +15,21 @@ func ZFSListMapping(filter DatasetFilter) (datasets []*DatasetPath, err error) {
 		panic("filter must not be nil")
 	}
 
-	var lines [][]string
-	lines, err = ZFSList([]string{"name"}, "-r", "-t", "filesystem,volume")
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	rchan := make(chan ZFSListResult)
+	go ZFSListChan(ctx, rchan, []string{"name"}, "-r", "-t", "filesystem,volume")
 
-	datasets = make([]*DatasetPath, 0, len(lines))
+	datasets = make([]*DatasetPath, 0)
+	for r := range rchan {
 
-	for _, line := range lines {
+		if r.err != nil {
+			err = r.err
+			return
+		}
 
 		var path *DatasetPath
-		if path, err = NewDatasetPath(line[0]); err != nil {
+		if path, err = NewDatasetPath(r.fields[0]); err != nil {
 			return
 		}
 

zfs/versions.go

@@ -2,6 +2,7 @@ package zfs
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"strconv"
@@ -61,17 +62,23 @@ type FilesystemVersionFilter interface {
 }
 
 func ZFSListFilesystemVersions(fs *DatasetPath, filter FilesystemVersionFilter) (res []FilesystemVersion, err error) {
-	var fieldLines [][]string
-	fieldLines, err = ZFSList(
+	listResults := make(chan ZFSListResult)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go ZFSListChan(ctx, listResults,
 		[]string{"name", "guid", "createtxg", "creation"},
 		"-r", "-d", "1",
 		"-t", "bookmark,snapshot",
 		"-s", "createtxg", fs.ToString())
-	if err != nil {
-		return
-	}
-	res = make([]FilesystemVersion, 0, len(fieldLines))
-	for _, line := range fieldLines {
+
+	res = make([]FilesystemVersion, 0)
+	for listResult := range listResults {
+		if listResult.err != nil {
+			return nil, listResult.err
+		}
+
+		line := listResult.fields
 
 		if len(line[0]) < 3 {
 			err = errors.New(fmt.Sprintf("snapshot or bookmark name implausibly short: %s", line[0]))

zfs/zfs.go

@@ -10,6 +10,8 @@ import (
 	"os/exec"
 	"strings"
 
+	"context"
+	"github.com/problame/go-rwccmd"
 	"github.com/zrepl/zrepl/util"
 )
 
@@ -185,6 +187,68 @@ func ZFSList(properties []string, zfsArgs ...string) (res [][]string, err error)
 	return
 }
 
+type ZFSListResult struct {
+	fields []string
+	err    error
+}
+
+// ZFSListChan executes `zfs list` and sends the results to the `out` channel.
+// The `out` channel is always closed by ZFSListChan:
+// If an error occurs, it is closed after sending a result with the err field set.
+// If no error occurs, it is just closed.
+// If the operation is cancelled via context, the channel is just closed.
+//
+// However, if callers do not drain `out` or cancel via `ctx`, the process will leak either running because
+// IO is pending or as a zombie.
+func ZFSListChan(ctx context.Context, out chan ZFSListResult, properties []string, zfsArgs ...string) {
+	defer close(out)
+
+	args := make([]string, 0, 4+len(zfsArgs))
+	args = append(args,
+		"list", "-H", "-p",
+		"-o", strings.Join(properties, ","))
+	args = append(args, zfsArgs...)
+
+	sendResult := func(fields []string, err error) (done bool) {
+		select {
+		case <-ctx.Done():
+			return true
+		case out <- ZFSListResult{fields, err}:
+			return false
+		}
+	}
+
+	cmd, err := rwccmd.CommandContext(ctx, ZFS_BINARY, args, []string{})
+	if err != nil {
+		sendResult(nil, err)
+		return
+	}
+	if err = cmd.Start(); err != nil {
+		sendResult(nil, err)
+		return
+	}
+	defer cmd.Close()
+
+	s := bufio.NewScanner(cmd)
+	buf := make([]byte, 1024) // max line length
+	s.Buffer(buf, 0)
+
+	for s.Scan() {
+		fields := strings.SplitN(s.Text(), "\t", len(properties))
+		if len(fields) != len(properties) {
+			sendResult(nil, errors.New("unexpected output"))
+			return
+		}
+		if sendResult(fields, nil) {
+			return
+		}
+	}
+	if s.Err() != nil {
+		sendResult(nil, s.Err())
+	}
+	return
+}
+
 func ZFSSend(fs *DatasetPath, from, to *FilesystemVersion) (stream io.Reader, err error) {
 
 	args := make([]string, 0)
@@ -239,13 +303,39 @@ func ZFSRecv(fs *DatasetPath, stream io.Reader, additionalArgs ...string) (err e
 	return nil
 }
 
-func ZFSSet(fs *DatasetPath, prop, val string) (err error) {
+type ZFSProperties struct {
+	m map[string]string
+}
 
-	if strings.ContainsRune(prop, '=') {
-		panic("prop contains rune '=' which is the delimiter between property name and value")
+func NewZFSProperties() *ZFSProperties {
+	return &ZFSProperties{make(map[string]string, 4)}
+}
+
+func (p *ZFSProperties) Set(key, val string) {
+	p.m[key] = val
+}
+
+func (p *ZFSProperties) appendArgs(args *[]string) (err error) {
+	for prop, val := range p.m {
+		if strings.Contains(prop, "=") {
+			return errors.New("prop contains rune '=' which is the delimiter between property name and value")
+		}
+		*args = append(*args, fmt.Sprintf("%s=%s", prop, val))
 	}
+	return nil
+}
 
-	cmd := exec.Command(ZFS_BINARY, "set", fmt.Sprintf("%s=%s", prop, val), fs.ToString())
+func ZFSSet(fs *DatasetPath, props *ZFSProperties) (err error) {
+
+	args := make([]string, 0)
+	args = append(args, "set")
+	err = props.appendArgs(&args)
+	if err != nil {
+		return err
+	}
+	args = append(args, fs.ToString())
+
+	cmd := exec.Command(ZFS_BINARY, args...)
 
 	stderr := bytes.NewBuffer(make([]byte, 0, 1024))
 	cmd.Stderr = stderr
@@ -286,9 +376,17 @@ func ZFSDestroy(dataset string) (err error) {
 
 }
 
+func zfsBuildSnapName(fs *DatasetPath, name string) string { // TODO defensive
+	return fmt.Sprintf("%s@%s", fs.ToString(), name)
+}
+
+func zfsBuildBookmarkName(fs *DatasetPath, name string) string { // TODO defensive
+	return fmt.Sprintf("%s#%s", fs.ToString(), name)
+}
+
 func ZFSSnapshot(fs *DatasetPath, name string, recursive bool) (err error) {
 
-	snapname := fmt.Sprintf("%s@%s", fs.ToString(), name)
+	snapname := zfsBuildSnapName(fs, name)
 	cmd := exec.Command(ZFS_BINARY, "snapshot", snapname)
 
 	stderr := bytes.NewBuffer(make([]byte, 0, 1024))
@@ -308,3 +406,28 @@ func ZFSSnapshot(fs *DatasetPath, name string, recursive bool) (err error) {
 	return
 
 }
+
+func ZFSBookmark(fs *DatasetPath, snapshot, bookmark string) (err error) {
+
+	snapname := zfsBuildSnapName(fs, snapshot)
+	bookmarkname := zfsBuildBookmarkName(fs, bookmark)
+
+	cmd := exec.Command(ZFS_BINARY, "bookmark", snapname, bookmarkname)
+
+	stderr := bytes.NewBuffer(make([]byte, 0, 1024))
+	cmd.Stderr = stderr
+
+	if err = cmd.Start(); err != nil {
+		return err
+	}
+
+	if err = cmd.Wait(); err != nil {
+		err = ZFSError{
+			Stderr:  stderr.Bytes(),
+			WaitErr: err,
+		}
+	}
+
+	return
+
+}