docs: link to github

refs #26

.gitignore

@@ -1,3 +1,13 @@
+# Build
+artifacts/
+
+# Golang
+vendor/
+
+# IDEs
+.idea/
+.vscode/
+
 # Compiled Object files, Static and Dynamic libs (Shared Objects)
 *.o
 *.a

.gitmodules

@@ -1,4 +0,0 @@
-[submodule "docs/themes/docdock"]
-	path = docs/themes/docdock
-	url = git@github.com:zrepl/hugo-theme-docdock.git
-

Gopkg.lock

@@ -0,0 +1,99 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  name = "github.com/davecgh/go-spew"
+  packages = ["spew"]
+  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
+  version = "v1.1.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/ftrvxmtrx/fd"
+  packages = ["."]
+  revision = "c6d800382fff6dc1412f34269f71b7f83bd059ad"
+
+[[projects]]
+  name = "github.com/go-logfmt/logfmt"
+  packages = ["."]
+  revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5"
+  version = "v0.3.0"
+
+[[projects]]
+  branch = "v2"
+  name = "github.com/go-yaml/yaml"
+  packages = ["."]
+  revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"
+
+[[projects]]
+  name = "github.com/inconshreveable/mousetrap"
+  packages = ["."]
+  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
+  version = "v1.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/jinzhu/copier"
+  packages = ["."]
+  revision = "db4671f3a9b8df855e993f7c94ec5ef1ffb0a23b"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/kr/logfmt"
+  packages = ["."]
+  revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/kr/pretty"
+  packages = ["."]
+  revision = "cfb55aafdaf3ec08f0db22699ab822c50091b1c4"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/kr/text"
+  packages = ["."]
+  revision = "7cafcd837844e784b526369c9bce262804aebc60"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/mitchellh/mapstructure"
+  packages = ["."]
+  revision = "d0303fe809921458f417bcf828397a65db30a7e4"
+
+[[projects]]
+  name = "github.com/pkg/errors"
+  packages = ["."]
+  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
+  version = "v0.8.0"
+
+[[projects]]
+  name = "github.com/pmezard/go-difflib"
+  packages = ["difflib"]
+  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
+  version = "v1.0.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/spf13/cobra"
+  packages = ["."]
+  revision = "b78744579491c1ceeaaa3b40205e56b0591b93a3"
+
+[[projects]]
+  name = "github.com/spf13/pflag"
+  packages = ["."]
+  revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66"
+  version = "v1.0.0"
+
+[[projects]]
+  name = "github.com/stretchr/testify"
+  packages = ["assert"]
+  revision = "69483b4bd14f5845b5a1e55bca19e954e827f1d0"
+  version = "v1.1.4"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  inputs-digest = "d654c5e91ee04baf1a373bb241b94ab41d3838fb69a56e91fe54cc8b962eab56"
+  solver-name = "gps-cdcl"
+  solver-version = 1

Gopkg.toml

@@ -0,0 +1,66 @@
+
+# Gopkg.toml example
+#
+# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#  name = "github.com/x/y"
+#  version = "2.4.0"
+
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/ftrvxmtrx/fd"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/jinzhu/copier"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/kr/pretty"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/mitchellh/go-homedir"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/mitchellh/mapstructure"
+
+[[constraint]]
+  name = "github.com/pkg/errors"
+  version = "0.8.0"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/spf13/cobra"
+
+[[constraint]]
+  name = "github.com/spf13/viper"
+  version = "1.0.0"
+
+[[constraint]]
+  name = "github.com/stretchr/testify"
+  version = "1.1.4"
+
+[[constraint]]
+  branch = "v2"
+  name = "github.com/go-yaml/yaml"
+
+[[constraint]]
+  name = "github.com/go-logfmt/logfmt"
+  version = "*"

Makefile

@@ -0,0 +1,47 @@
+.PHONY: generate build test vet cover release clean
+
+ROOT := github.com/zrepl/zrepl
+SUBPKGS := cmd logger rpc sshbytestream util
+
+_TESTPKGS := $(ROOT) $(foreach p,$(SUBPKGS),$(ROOT)/$(p))
+
+ARTIFACTDIR := artifacts
+
+generate: #not part of the build, must do that manually
+	@for pkg in $(_TESTPKGS); do\
+		go generate "$$pkg" || exit 1; \
+	done;
+
+build:
+		go build -o $(ARTIFACTDIR)/zrepl
+
+test:
+	@for pkg in $(_TESTPKGS); do \
+		echo "Testing $$pkg"; \
+		go test "$$pkg" || exit 1; \
+	done;
+
+vet:
+	@for pkg in $(_TESTPKGS); do \
+		echo "Vetting $$pkg"; \
+		go vet "$$pkg" || exit 1; \
+	done;
+
+cover: artifacts
+	@for pkg in $(_TESTPKGS); do \
+		profile="$(ARTIFACTDIR)/cover-$$(basename $$pkg).out"; \
+		go test -coverprofile "$$profile" $$pkg || exit 1; \
+		if [ -f "$$profile" ]; then \
+   			go tool cover -html="$$profile" -o "$${profile}.html" || exit 2; \
+		fi; \
+	done;
+
+artifacts:
+	mkdir artifacts
+
+release: artifacts vet test
+	GOOS=linux GOARCH=amd64   go build -o "$(ARTIFACTDIR)/zrepl-linux-amd64"
+	GOOS=freebsd GOARCH=amd64 go build -o "$(ARTIFACTDIR)/zrepl-freebsd-amd64"
+
+clean:
+	rm -rf "$(ARTIFACTDIR)"

README.md

@@ -1,2 +1,86 @@
 # zrepl
-ZFS dataset replication. Not a shell script.
+zrepl is a ZFS filesystem backup & replication solution written in Go.
+
+## User Documentation
+
+**User Documentation** cab be found at [zrepl.github.io](https://zrepl.github.io).
+
+## Bug Reports
+
+1. If the issue is reproducible, enable debug logging, reproduce and capture the log.
+2. Open an issue on GitHub, with logs pasted as GitHub gists / inline.
+
+## Feature Requests
+
+1. Does you feature request require default values / some kind of configuration?
+   If so, think of an expressive configuration example.
+2. Think of at least one use case that generalizes from your concrete application.
+3. Open an issue on GitHub with example conf & use case attached.
+
+The above does not apply if you already implemented everything.
+Check out the *Coding Workflow* section below for details.
+
+## Developer Documentation
+
+### Overall Architecture
+
+The application architecture is documented as part of the user docs in the *Implementation* section (`docs/content/impl`).
+Make sure to develop an understanding how zrepl is typically used by studying the user docs first.
+
+### Project Structure
+
+```
+├── cmd
+│   ├── sampleconf          # example configuration
+├── docs                    # sphinx-based documentation
+│   ├── **/*.rst            # documentation in reStructuredText
+│   ├── conf.py             # sphinx configuration
+│   ├── publish.sh          # shell script for automated rendering & deploy to zrepl.github.io repo
+│   ├── public_git          # checkout of zrepl.github.io used by above shell script
+├── jobrun                  # OBSOLETE
+├── logger                  # logger package used by zrepl
+├── rpc                     # rpc protocol implementation
+├── scratchpad              # small example programs demoing some internal packages. probably OBSOLETE
+├── sshbytestream           # io.ReadWriteCloser over SSH
+├── util
+└── zfs                     # ZFS wrappers, filesystemm diffing
+```
+
+### Coding Workflow
+
+* Open an issue when starting to hack on a new feature
+* Commits should reference the issue they are related to
+* Docs improvements not documenting new features do not require an issue.
+
+### Breaking Changes
+
+Backward-incompatible changes must be documented in the git commit message and are listed in `docs/changelog.rst`.
+
+* Config-breaking changes must contain a line `BREAK CONFIG` in the commit message
+* Other breaking changes must contain a line `BREAK` in the commit message
+
+### Glossary & Naming Inconsistencies
+
+In ZFS, *dataset* refers to the objects *filesystem*, *ZVOL* and *snapshot*. <br />
+However, we need a word for *filesystem* & *ZVOL* but not a snapshot, bookmark, etc.
+
+Toward the user, the following terminology is used:
+
+* **filesystem**: a ZFS filesystem or a ZVOL
+* **filesystem version**: a ZFS snapshot or a bookmark
+
+Sadly, the zrepl implementation is inconsistent in its use of these words:
+variables and types are often named *dataset* when they in fact refer to a *filesystem*.
+
+There will not be a big refactoring (an attempt was made, but it's destroying too much history without much gain).
+
+However, new contributions & patches should fix naming without further notice in the commit message.
+
+### Building `docs`
+
+```
+cd docs
+pip install sphinx sphinx-rtd-theme
+make clean html
+xdg-open _build/html/index.html
+```

clone_and_build.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+set -e
+
+bold=$(tput bold)
+normal=$(tput sgr0)
+
+step() {
+    echo "${bold}$1${normal}"
+}
+
+if [ -z "$GOPATH" ]; then
+    step "Make sure you have your GOPATH configured correctly" 1>&2
+    exit 1
+fi
+
+step "Checkout sources to \$GOPATH/github.com/zrepl/zrepl"
+CHECKOUTPATH="${GOPATH}/src/github.com/zrepl/zrepl"
+if [ -e "$CHECKOUTPATH" ]; then
+    echo "${CHECKOUTPATH} already exists"
+    if [ ! -d "$CHECKOUTPATH" ]; then
+        echo "${CHECKOUTPATH} is not a directory, aborting" 1>&2
+    else
+        cd "$CHECKOUTPATH"
+    fi
+else
+    mkdir -p "$GOPATH/src/github.com/zrepl"
+    cd "$GOPATH/src/github.com/zrepl"
+    git clone https://github.com/zrepl/zrepl.git
+    cd zrepl
+fi
+
+step "Install build depdencies using 'go get' to \$GOPATH/bin"
+go get -u golang.org/x/tools/cmd/stringer
+go get -u github.com/golang/dep/cmd/dep
+if ! type stringer || ! type dep; then
+    echo "Installed dependencies but can't find them in \$PATH, adjust it to contain \$GOPATH/bin" 1>&2
+    exit 1
+fi
+
+step "Fetching dependencies using 'dep ensure'"
+dep ensure
+
+step "Making release"
+make release
+
+step "Release artifacts are available in $(pwd)/artifacts"

cmd/autosnap.go

@@ -1,91 +1,148 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/jobrun"
 	"github.com/zrepl/zrepl/zfs"
-	"os"
+	"sort"
-	"sync"
 	"time"
 )
 
-var AutosnapCmd = &cobra.Command{
+type IntervalAutosnap struct {
-	Use:   "autosnap",
+	DatasetFilter    zfs.DatasetFilter
-	Short: "perform automatic snapshotting",
+	Prefix           string
-	Run:   cmdAutosnap,
+	SnapshotInterval time.Duration
+
+	log       Logger
+	snaptimes []snapTime
 }
 
-func init() {
+type snapTime struct {
-	RootCmd.AddCommand(AutosnapCmd)
+	ds   *zfs.DatasetPath
+	time time.Time
 }
 
-func cmdAutosnap(cmd *cobra.Command, args []string) {
+func (a *IntervalAutosnap) Run(ctx context.Context, didSnaps chan struct{}) {
 
-	var wg sync.WaitGroup
+	a.log = ctx.Value(contextKeyLog).(Logger)
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		runner.Start()
-	}()
 
-	if len(args) < 1 {
+	const LOG_TIME_FMT string = time.ANSIC
-		log.Printf("must specify exactly one job as positional argument")
-		os.Exit(1)
-	}
 
-	snap, ok := conf.Autosnaps[args[0]]
+	ds, err := zfs.ZFSListMapping(a.DatasetFilter)
-	if !ok {
-		log.Printf("could not find autosnap job: %s", args[0])
-		os.Exit(1)
-	}
-
-	job := jobrun.Job{
-		Name:           snap.JobName,
-		RepeatStrategy: snap.Interval,
-		RunFunc: func(log jobrun.Logger) error {
-			log.Printf("doing autosnap: %v", snap)
-			ctx := AutosnapContext{snap}
-			return doAutosnap(ctx, log)
-		},
-	}
-	runner.AddJob(job)
-
-	wg.Wait()
-
-}
-
-type AutosnapContext struct {
-	Autosnap *Autosnap
-}
-
-func doAutosnap(ctx AutosnapContext, log Logger) (err error) {
-
-	snap := ctx.Autosnap
-
-	filesystems, err := zfs.ZFSListMapping(snap.DatasetFilter)
 	if err != nil {
-		return fmt.Errorf("cannot filter datasets: %s", err)
+		a.log.WithError(err).Error("cannot list datasets")
+		return
+	}
+	if len(ds) == 0 {
+		a.log.Warn("no filesystem matching filesystem filter")
+		return
 	}
 
-	suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
+	a.snaptimes = make([]snapTime, len(ds))
-	snapname := fmt.Sprintf("%s%s", snap.Prefix, suffix)
 
-	hadError := false
+	now := time.Now()
 
-	for _, fs := range filesystems { // optimization: use recursive snapshots / channel programs here
+	a.log.Debug("examine filesystem state")
-		log.Printf("snapshotting filesystem %s@%s", fs, snapname)
+	for i, d := range ds {
-		err := zfs.ZFSSnapshot(fs, snapname, false)
+
+		l := a.log.WithField(logFSField, d.ToString())
+
+		fsvs, err := zfs.ZFSListFilesystemVersions(d, &PrefixSnapshotFilter{a.Prefix})
 		if err != nil {
-			log.Printf("error snapshotting %s: %s", fs, err)
+			l.WithError(err).Error("cannot list filesystem versions")
-			hadError = true
+			continue
+		}
+		if len(fsvs) <= 0 {
+			l.WithField("prefix", a.Prefix).Info("no filesystem versions with prefix")
+			a.snaptimes[i] = snapTime{d, now}
+			continue
+		}
+
+		// Sort versions by creation
+		sort.SliceStable(fsvs, func(i, j int) bool {
+			return fsvs[i].CreateTXG < fsvs[j].CreateTXG
+		})
+
+		latest := fsvs[len(fsvs)-1]
+		l.WithField("creation", latest.Creation).
+			Debug("found latest snapshot")
+
+		since := now.Sub(latest.Creation)
+		if since < 0 {
+			l.WithField("snapshot", latest.Name).
+				WithField("creation", latest.Creation).
+				Error("snapshot is from the future")
+			continue
+		}
+		next := now
+		if since < a.SnapshotInterval {
+			next = latest.Creation.Add(a.SnapshotInterval)
+		}
+		a.snaptimes[i] = snapTime{d, next}
+	}
+
+	sort.Slice(a.snaptimes, func(i, j int) bool {
+		return a.snaptimes[i].time.Before(a.snaptimes[j].time)
+	})
+
+	syncPoint := a.snaptimes[0]
+	a.log.WithField("sync_point", syncPoint.time.Format(LOG_TIME_FMT)).
+		Info("wait for sync point")
+
+	select {
+	case <-ctx.Done():
+		a.log.WithError(ctx.Err()).Info("context done")
+		return
+
+	case <-time.After(syncPoint.time.Sub(now)):
+		a.log.Debug("snapshot all filesystems to enable further snaps in lockstep")
+		a.doSnapshots(didSnaps)
+	}
+
+	ticker := time.NewTicker(a.SnapshotInterval)
+
+	for {
+		select {
+		case <-ctx.Done():
+			ticker.Stop()
+			a.log.WithError(ctx.Err()).Info("context done")
+			return
+
+		case <-ticker.C:
+			a.doSnapshots(didSnaps)
 		}
 	}
 
-	if hadError {
+}
-		err = fmt.Errorf("errors occurred during autosnap, check logs for details")
+
+func (a *IntervalAutosnap) doSnapshots(didSnaps chan struct{}) {
+
+	// fetch new dataset list in case user added new dataset
+	ds, err := zfs.ZFSListMapping(a.DatasetFilter)
+	if err != nil {
+		a.log.WithError(err).Error("cannot list datasets")
+		return
 	}
 
-	return
+	// TODO channel programs -> allow a little jitter?
+	for _, d := range ds {
+		suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
+		snapname := fmt.Sprintf("%s%s", a.Prefix, suffix)
+
+		a.log.WithField(logFSField, d.ToString()).
+			WithField("snapname", snapname).
+			Info("create snapshot")
+
+		err := zfs.ZFSSnapshot(d, snapname, false)
+		if err != nil {
+			a.log.WithError(err).Error("cannot create snapshot")
+		}
+	}
+
+	select {
+	case didSnaps <- struct{}{}:
+	default:
+		a.log.Error("warning: callback channel is full, discarding")
+	}
 
 }

cmd/config.go

@@ -1,730 +1,100 @@
 package cmd
 
 import (
-	"errors"
-	"fmt"
-	"github.com/jinzhu/copier"
-	"github.com/mitchellh/mapstructure"
-	"github.com/zrepl/zrepl/jobrun"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/sshbytestream"
-	. "github.com/zrepl/zrepl/util"
-	"github.com/zrepl/zrepl/zfs"
-	yaml "gopkg.in/yaml.v2"
 	"io"
-	"io/ioutil"
+
-	"regexp"
+	"fmt"
-	"strconv"
+	"github.com/pkg/errors"
-	"strings"
+	"github.com/zrepl/zrepl/zfs"
-	"time"
 )
 
-var (
-	JobSectionPush     string = "push"
-	JobSectionPull     string = "pull"
-	JobSectionPrune    string = "prune"
-	JobSectionAutosnap string = "autosnap"
-)
-
-type Remote struct {
-	Name      string
-	Transport Transport
-}
-
-type Transport interface {
-	Connect(rpcLog Logger) (rpc.RPCRequester, error)
-}
-type LocalTransport struct {
-	Handler rpc.RPCHandler
-}
-type SSHTransport struct {
-	Host                 string
-	User                 string
-	Port                 uint16
-	IdentityFile         string   `mapstructure:"identity_file"`
-	TransportOpenCommand []string `mapstructure:"transport_open_command"`
-	SSHCommand           string   `mapstructure:"ssh_command"`
-	Options              []string
-	ConnLogReadFile      string `mapstructure:"connlog_read_file"`
-	ConnLogWriteFile     string `mapstructure:"connlog_write_file"`
-}
-
-type Push struct {
-	JobName           string // for use with jobrun package
-	To                *Remote
-	Filter            zfs.DatasetFilter
-	InitialReplPolicy rpc.InitialReplPolicy
-	RepeatStrategy    jobrun.RepeatStrategy
-}
-type Pull struct {
-	JobName           string // for use with jobrun package
-	From              *Remote
-	Mapping           DatasetMapFilter
-	InitialReplPolicy rpc.InitialReplPolicy
-	RepeatStrategy    jobrun.RepeatStrategy
-}
-
-type Prune struct {
-	JobName         string // for use with jobrun package
-	DatasetFilter   zfs.DatasetFilter
-	SnapshotFilter  zfs.FilesystemVersionFilter
-	RetentionPolicy *RetentionGrid // TODO abstract interface to support future policies?
-}
-
-type Autosnap struct {
-	JobName       string // for use with jobrun package
-	Prefix        string
-	Interval      jobrun.RepeatStrategy
-	DatasetFilter zfs.DatasetFilter
-}
-
 type Config struct {
-	Remotes   map[string]*Remote
+	Global Global
-	Pushs     map[string]*Push            // job name -> job
+	Jobs   map[string]Job
-	Pulls     map[string]*Pull            // job name -> job
-	Sinks     map[string]DatasetMapFilter // client identity -> mapping
-	PullACLs  map[string]DatasetMapFilter // client identity -> filter
-	Prunes    map[string]*Prune           // job name -> job
-	Autosnaps map[string]*Autosnap        // job name -> job
 }
 
-func ParseConfig(path string) (config Config, err error) {
+func (c *Config) LookupJob(name string) (j Job, err error) {
-
+	j, ok := c.Jobs[name]
-	c := make(map[string]interface{}, 0)
-
-	var bytes []byte
-
-	if bytes, err = ioutil.ReadFile(path); err != nil {
-		return
-	}
-
-	if err = yaml.Unmarshal(bytes, &c); err != nil {
-		return
-	}
-
-	return parseMain(c)
-}
-
-func parseMain(root map[string]interface{}) (c Config, err error) {
-	if c.Remotes, err = parseRemotes(root["remotes"]); err != nil {
-		return
-	}
-
-	remoteLookup := func(name string) (remote *Remote, err error) {
-		remote = c.Remotes[name]
-		if remote == nil {
-			err = fmt.Errorf("remote '%s' not defined", name)
-		}
-		return
-	}
-
-	if c.Pushs, err = parsePushs(root["pushs"], remoteLookup); err != nil {
-		return
-	}
-	if c.Pulls, err = parsePulls(root["pulls"], remoteLookup); err != nil {
-		return
-	}
-	if c.Sinks, err = parseSinks(root["sinks"]); err != nil {
-		return
-	}
-	if c.PullACLs, err = parsePullACLs(root["pull_acls"]); err != nil {
-		return
-	}
-	if c.Prunes, err = parsePrunes(root["prune"]); err != nil {
-		return
-	}
-	if c.Autosnaps, err = parseAutosnaps(root["autosnap"]); err != nil {
-		return
-	}
-
-	return
-}
-
-func fullJobName(section, name string) (full string, err error) {
-	if len(name) < 1 {
-		err = fmt.Errorf("job name not set")
-		return
-	}
-	full = fmt.Sprintf("%s.%s", section, name)
-	return
-}
-
-func parseRemotes(v interface{}) (remotes map[string]*Remote, err error) {
-
-	asMap := make(map[string]struct {
-		Transport map[string]interface{}
-	}, 0)
-	if err = mapstructure.Decode(v, &asMap); err != nil {
-		return
-	}
-
-	remotes = make(map[string]*Remote, len(asMap))
-	for name, p := range asMap {
-
-		if name == rpc.LOCAL_TRANSPORT_IDENTITY {
-			err = errors.New(fmt.Sprintf("remote name '%s' reserved for local pulls", rpc.LOCAL_TRANSPORT_IDENTITY))
-			return
-		}
-
-		var transport Transport
-		if transport, err = parseTransport(p.Transport); err != nil {
-			return
-		}
-		remotes[name] = &Remote{
-			Name:      name,
-			Transport: transport,
-		}
-	}
-
-	return
-}
-
-func parseTransport(it map[string]interface{}) (t Transport, err error) {
-
-	if len(it) != 1 {
-		err = errors.New("ambiguous transport type")
-		return
-	}
-
-	for key, val := range it {
-		switch key {
-		case "ssh":
-			t := SSHTransport{}
-			if err = mapstructure.Decode(val, &t); err != nil {
-				err = errors.New(fmt.Sprintf("could not parse ssh transport: %s", err))
-				return nil, err
-			}
-			return t, nil
-		default:
-			return nil, errors.New(fmt.Sprintf("unknown transport type '%s'\n", key))
-		}
-	}
-
-	return // unreachable
-
-}
-
-type remoteLookup func(name string) (*Remote, error)
-
-func parsePushs(v interface{}, rl remoteLookup) (p map[string]*Push, err error) {
-
-	asMap := make(map[string]struct {
-		To                string
-		Filter            map[string]string
-		InitialReplPolicy string
-		Repeat            map[string]string
-	}, 0)
-
-	if err = mapstructure.Decode(v, &asMap); err != nil {
-		return
-	}
-
-	p = make(map[string]*Push, len(asMap))
-
-	for name, e := range asMap {
-
-		var toRemote *Remote
-		if toRemote, err = rl(e.To); err != nil {
-			return
-		}
-		push := &Push{
-			To: toRemote,
-		}
-
-		if push.JobName, err = fullJobName(JobSectionPush, name); err != nil {
-			return
-		}
-		if push.Filter, err = parseDatasetMapFilter(e.Filter, true); err != nil {
-			return
-		}
-
-		if push.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, rpc.DEFAULT_INITIAL_REPL_POLICY); err != nil {
-			return
-		}
-
-		if push.RepeatStrategy, err = parseRepeatStrategy(e.Repeat); err != nil {
-			return
-		}
-
-		p[name] = push
-	}
-
-	return
-}
-
-func parsePulls(v interface{}, rl remoteLookup) (p map[string]*Pull, err error) {
-
-	asMap := make(map[string]struct {
-		From              string
-		Mapping           map[string]string
-		InitialReplPolicy string
-		Repeat            map[string]string
-	}, 0)
-
-	if err = mapstructure.Decode(v, &asMap); err != nil {
-		return
-	}
-
-	p = make(map[string]*Pull, len(asMap))
-
-	for name, e := range asMap {
-
-		if len(e.From) < 1 {
-			err = fmt.Errorf("source not set ('from' attribute is empty)")
-			return
-		}
-
-		var fromRemote *Remote
-
-		if e.From == rpc.LOCAL_TRANSPORT_IDENTITY {
-			fromRemote = &Remote{
-				Name:      rpc.LOCAL_TRANSPORT_IDENTITY,
-				Transport: LocalTransport{},
-			}
-		} else {
-			if fromRemote, err = rl(e.From); err != nil {
-				return
-			}
-		}
-
-		pull := &Pull{
-			From: fromRemote,
-		}
-		if pull.JobName, err = fullJobName(JobSectionPull, name); err != nil {
-			return
-		}
-		if pull.Mapping, err = parseDatasetMapFilter(e.Mapping, false); err != nil {
-			return
-		}
-		if pull.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, rpc.DEFAULT_INITIAL_REPL_POLICY); err != nil {
-			return
-		}
-		if pull.RepeatStrategy, err = parseRepeatStrategy(e.Repeat); err != nil {
-			return
-		}
-
-		p[name] = pull
-	}
-
-	return
-}
-
-func parseInitialReplPolicy(v interface{}, defaultPolicy rpc.InitialReplPolicy) (p rpc.InitialReplPolicy, err error) {
-	s, ok := v.(string)
 	if !ok {
-		goto err
+		return nil, errors.Errorf("job '%s' is not defined", name)
 	}
+	return j, nil
+}
 
-	switch {
+type Global struct {
-	case s == "":
+	Serve struct {
-		p = defaultPolicy
+		Stdinserver struct {
-	case s == "most_recent":
+			SockDir string
-		p = rpc.InitialReplPolicyMostRecent
+		}
-	case s == "all":
+	}
-		p = rpc.InitialReplPolicyAll
+	Control struct {
+		Sockpath string
+	}
+	logging *LoggingConfig
+}
+
+type JobDebugSettings struct {
+	Conn struct {
+		ReadDump  string `mapstructure:"read_dump"`
+		WriteDump string `mapstructure:"write_dump"`
+	}
+	RPC struct {
+		Log bool
+	}
+}
+
+type RWCConnecter interface {
+	Connect() (io.ReadWriteCloser, error)
+}
+type AuthenticatedChannelListenerFactory interface {
+	Listen() (AuthenticatedChannelListener, error)
+}
+
+type AuthenticatedChannelListener interface {
+	Accept() (ch io.ReadWriteCloser, err error)
+	Close() (err error)
+}
+
+type SSHStdinServerConnectDescr struct {
+}
+
+type PrunePolicy interface {
+	Prune(fs *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error)
+}
+
+type PruningJob interface {
+	Pruner(side PrunePolicySide, dryRun bool) (Pruner, error)
+}
+
+// A type for constants describing different prune policies of a PruningJob
+// This is mostly a special-case for LocalJob, which is the only job that has two prune policies
+// instead of one.
+// It implements github.com/spf13/pflag.Value to be used as CLI flag for the test subcommand
+type PrunePolicySide string
+
+const (
+	PrunePolicySideDefault PrunePolicySide = ""
+	PrunePolicySideLeft    PrunePolicySide = "left"
+	PrunePolicySideRight   PrunePolicySide = "right"
+)
+
+func (s *PrunePolicySide) String() string {
+	return string(*s)
+}
+
+func (s *PrunePolicySide) Set(news string) error {
+	p := PrunePolicySide(news)
+	switch p {
+	case PrunePolicySideRight:
+		fallthrough
+	case PrunePolicySideLeft:
+		*s = p
 	default:
-		goto err
+		return errors.Errorf("must be either %s or %s", PrunePolicySideLeft, PrunePolicySideRight)
 	}
-
+	return nil
-	return
-
-err:
-	err = errors.New(fmt.Sprintf("expected InitialReplPolicy, got %#v", v))
-	return
 }
 
-func parseRepeatStrategy(r map[string]string) (s jobrun.RepeatStrategy, err error) {
+func (s *PrunePolicySide) Type() string {
-
+	return fmt.Sprintf("%s | %s", PrunePolicySideLeft, PrunePolicySideRight)
-	if r == nil {
-		return jobrun.NoRepeatStrategy{}, nil
-	}
-
-	if repeatStr, ok := r["interval"]; ok {
-		d, err := parseDuration(repeatStr)
-		if err != nil {
-			return nil, err
-		}
-		s = &jobrun.PeriodicRepeatStrategy{d}
-		return s, err
-	} else {
-		return nil, fmt.Errorf("attribute 'interval' not found but required in repeat specification")
-	}
-
-}
-
-func expectList(v interface{}) (asList []interface{}, err error) {
-	var ok bool
-	if asList, ok = v.([]interface{}); !ok {
-		err = errors.New("expected list")
-	}
-	return
-}
-
-func parseSinks(v interface{}) (m map[string]DatasetMapFilter, err error) {
-	var asMap map[string]map[string]interface{}
-	if err = mapstructure.Decode(v, &asMap); err != nil {
-		return
-	}
-
-	m = make(map[string]DatasetMapFilter, len(asMap))
-
-	for identity, entry := range asMap {
-		parseSink := func() (mapping DatasetMapFilter, err error) {
-			mappingMap, ok := entry["mapping"]
-			if !ok {
-				err = fmt.Errorf("no mapping specified")
-				return
-			}
-			mapping, err = parseDatasetMapFilter(mappingMap, false)
-			return
-		}
-		mapping, sinkErr := parseSink()
-		if sinkErr != nil {
-			err = fmt.Errorf("cannot parse sink for identity '%s': %s", identity, sinkErr)
-			return
-		}
-		m[identity] = mapping
-	}
-	return
-}
-
-func parsePullACLs(v interface{}) (m map[string]DatasetMapFilter, err error) {
-	var asMap map[string]map[string]interface{}
-	if err = mapstructure.Decode(v, &asMap); err != nil {
-		return
-	}
-
-	m = make(map[string]DatasetMapFilter, len(asMap))
-
-	for identity, entry := range asMap {
-		parsePullACL := func() (filter DatasetMapFilter, err error) {
-			filterMap, ok := entry["filter"]
-			if !ok {
-				err = fmt.Errorf("no filter specified")
-				return
-			}
-			filter, err = parseDatasetMapFilter(filterMap, true)
-			return
-		}
-		filter, filterErr := parsePullACL()
-		if filterErr != nil {
-			err = fmt.Errorf("cannot parse pull-ACL for identity '%s': %s", identity, filterErr)
-			return
-		}
-		m[identity] = filter
-	}
-	return
-}
-
-func parseDatasetMapFilter(mi interface{}, filterOnly bool) (f DatasetMapFilter, err error) {
-
-	var m map[string]string
-	if err = mapstructure.Decode(mi, &m); err != nil {
-		err = fmt.Errorf("maps / filters must be specified as map[string]string: %s", err)
-		return
-	}
-
-	f = NewDatasetMapFilter(len(m), filterOnly)
-	for pathPattern, mapping := range m {
-		if err = f.Add(pathPattern, mapping); err != nil {
-			err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err)
-			return
-		}
-	}
-	return
-}
-
-func (t SSHTransport) Connect(rpcLog Logger) (r rpc.RPCRequester, err error) {
-	var stream io.ReadWriteCloser
-	var rpcTransport sshbytestream.SSHTransport
-	if err = copier.Copy(&rpcTransport, t); err != nil {
-		return
-	}
-	if stream, err = sshbytestream.Outgoing(rpcTransport); err != nil {
-		return
-	}
-	stream, err = NewReadWriteCloserLogger(stream, t.ConnLogReadFile, t.ConnLogWriteFile)
-	if err != nil {
-		return
-	}
-	return rpc.ConnectByteStreamRPC(stream, rpcLog)
-}
-
-func (t LocalTransport) Connect(rpcLog Logger) (r rpc.RPCRequester, err error) {
-	if t.Handler == nil {
-		panic("local transport with uninitialized handler")
-	}
-	return rpc.ConnectLocalRPC(t.Handler), nil
-}
-
-func (t *LocalTransport) SetHandler(handler rpc.RPCHandler) {
-	t.Handler = handler
-}
-
-func parsePrunes(m interface{}) (rets map[string]*Prune, err error) {
-
-	asList := make(map[string]map[string]interface{}, 0)
-	if err = mapstructure.Decode(m, &asList); err != nil {
-		return
-	}
-
-	rets = make(map[string]*Prune, len(asList))
-
-	for name, e := range asList {
-		var prune *Prune
-		if prune, err = parsePrune(e, name); err != nil {
-			err = fmt.Errorf("cannot parse prune job %s: %s", name, err)
-			return
-		}
-		rets[name] = prune
-	}
-
-	return
-}
-
-func parsePrune(e map[string]interface{}, name string) (prune *Prune, err error) {
-
-	// Only support grid policy for now
-	policyName, ok := e["policy"]
-	if !ok || policyName != "grid" {
-		err = fmt.Errorf("prune job with unimplemented policy '%s'", policyName)
-		return
-	}
-
-	var i struct {
-		Grid           string
-		DatasetFilter  map[string]string `mapstructure:"dataset_filter"`
-		SnapshotFilter map[string]string `mapstructure:"snapshot_filter"`
-	}
-
-	if err = mapstructure.Decode(e, &i); err != nil {
-		return
-	}
-
-	prune = &Prune{}
-
-	if prune.JobName, err = fullJobName(JobSectionPrune, name); err != nil {
-		return
-	}
-
-	// Parse grid policy
-	intervals, err := parseRetentionGridIntervalsString(i.Grid)
-	if err != nil {
-		err = fmt.Errorf("cannot parse retention grid: %s", err)
-		return
-	}
-	// Assert intervals are of increasing length (not necessarily required, but indicates config mistake)
-	lastDuration := time.Duration(0)
-	for i := range intervals {
-		if intervals[i].Length < lastDuration {
-			err = fmt.Errorf("retention grid interval length must be monotonically increasing:"+
-				"interval %d is shorter than %d", i+1, i)
-			return
-		} else {
-			lastDuration = intervals[i].Length
-		}
-	}
-	prune.RetentionPolicy = NewRetentionGrid(intervals)
-
-	// Parse filters
-	if prune.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil {
-		err = fmt.Errorf("cannot parse dataset filter: %s", err)
-		return
-	}
-	if prune.SnapshotFilter, err = parseSnapshotFilter(i.SnapshotFilter); err != nil {
-		err = fmt.Errorf("cannot parse snapshot filter: %s", err)
-		return
-	}
-
-	return
-}
-
-var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
-
-func parseDuration(e string) (d time.Duration, err error) {
-	comps := durationStringRegex.FindStringSubmatch(e)
-	if len(comps) != 3 {
-		err = fmt.Errorf("does not match regex: %s %#v", e, comps)
-		return
-	}
-
-	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
-	if err != nil {
-		return
-	}
-
-	var durationUnit time.Duration
-	switch comps[2] {
-	case "s":
-		durationUnit = time.Second
-	case "m":
-		durationUnit = time.Minute
-	case "h":
-		durationUnit = time.Hour
-	case "d":
-		durationUnit = 24 * time.Hour
-	case "w":
-		durationUnit = 24 * 7 * time.Hour
-	default:
-		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
-		return
-	}
-
-	d = time.Duration(durationFactor) * durationUnit
-	return
-
-}
-
-var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
-
-func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {
-
-	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
-	if comps == nil {
-		err = fmt.Errorf("retention string does not match expected format")
-		return
-	}
-
-	times, err := strconv.Atoi(comps[1])
-	if err != nil {
-		return nil, err
-	} else if times <= 0 {
-		return nil, fmt.Errorf("contains factor <= 0")
-	}
-
-	duration, err := parseDuration(comps[2])
-	if err != nil {
-		return nil, err
-	}
-
-	keepCount := 1
-	if comps[3] != "" {
-		// Decompose key=value, comma separated
-		// For now, only keep_count is supported
-		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
-		res := re.FindStringSubmatch(comps[4])
-		if res == nil || len(res) != 2 {
-			err = fmt.Errorf("interval parameter contains unknown parameters")
-			return
-		}
-		if res[1] == "all" {
-			keepCount = RetentionGridKeepCountAll
-		} else {
-			keepCount, err = strconv.Atoi(res[1])
-			if err != nil {
-				err = fmt.Errorf("cannot parse keep_count value")
-				return
-			}
-		}
-	}
-
-	intervals = make([]RetentionInterval, times)
-	for i := range intervals {
-		intervals[i] = RetentionInterval{
-			Length:    duration,
-			KeepCount: keepCount,
-		}
-	}
-
-	return
-
-}
-
-func parseRetentionGridIntervalsString(s string) (intervals []RetentionInterval, err error) {
-
-	ges := strings.Split(s, "|")
-	intervals = make([]RetentionInterval, 0, 7*len(ges))
-
-	for intervalIdx, e := range ges {
-		parsed, err := parseRetentionGridIntervalString(e)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
-		}
-		intervals = append(intervals, parsed...)
-	}
-
-	return
-}
-
-type prefixSnapshotFilter struct {
-	prefix string
-}
-
-func (f prefixSnapshotFilter) Filter(fsv zfs.FilesystemVersion) (accept bool, err error) {
-	return fsv.Type == zfs.Snapshot && strings.HasPrefix(fsv.Name, f.prefix), nil
-}
-
-func parseSnapshotFilter(fm map[string]string) (snapFilter zfs.FilesystemVersionFilter, err error) {
-	prefix, ok := fm["prefix"]
-	if !ok {
-		err = fmt.Errorf("unsupported snapshot filter")
-		return
-	}
-	snapFilter = prefixSnapshotFilter{prefix}
-	return
-}
-
-func parseAutosnaps(m interface{}) (snaps map[string]*Autosnap, err error) {
-
-	asMap := make(map[string]interface{}, 0)
-	if err = mapstructure.Decode(m, &asMap); err != nil {
-		return
-	}
-
-	snaps = make(map[string]*Autosnap, len(asMap))
-
-	for name, e := range asMap {
-		var snap *Autosnap
-		if snap, err = parseAutosnap(e, name); err != nil {
-			err = fmt.Errorf("cannot parse autonsap job %s: %s", name, err)
-			return
-		}
-		snaps[name] = snap
-	}
-
-	return
-
-}
-
-func parseAutosnap(m interface{}, name string) (a *Autosnap, err error) {
-
-	var i struct {
-		Prefix        string
-		Interval      string
-		DatasetFilter map[string]string `mapstructure:"dataset_filter"`
-	}
-
-	if err = mapstructure.Decode(m, &i); err != nil {
-		err = fmt.Errorf("structure unfit: %s", err)
-		return
-	}
-
-	a = &Autosnap{}
-
-	if a.JobName, err = fullJobName(JobSectionAutosnap, name); err != nil {
-		return
-	}
-
-	if len(i.Prefix) < 1 {
-		err = fmt.Errorf("prefix must not be empty")
-		return
-	}
-	a.Prefix = i.Prefix
-
-	var interval time.Duration
-	if interval, err = parseDuration(i.Interval); err != nil {
-		err = fmt.Errorf("cannot parse interval: %s", err)
-		return
-	}
-	a.Interval = &jobrun.PeriodicRepeatStrategy{interval}
-
-	if len(i.DatasetFilter) == 0 {
-		err = fmt.Errorf("dataset_filter not specified")
-		return
-	}
-	if a.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil {
-		err = fmt.Errorf("cannot parse dataset filter: %s", err)
-	}
-
-	return
 }

cmd/config_connect.go

@@ -0,0 +1,46 @@
+package cmd
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/jinzhu/copier"
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/sshbytestream"
+)
+
+type SSHStdinserverConnecter struct {
+	Host                 string
+	User                 string
+	Port                 uint16
+	IdentityFile         string   `mapstructure:"identity_file"`
+	TransportOpenCommand []string `mapstructure:"transport_open_command"`
+	SSHCommand           string   `mapstructure:"ssh_command"`
+	Options              []string
+}
+
+func parseSSHStdinserverConnecter(i map[string]interface{}) (c *SSHStdinserverConnecter, err error) {
+
+	c = &SSHStdinserverConnecter{}
+	if err = mapstructure.Decode(i, c); err != nil {
+		err = errors.New(fmt.Sprintf("could not parse ssh transport: %s", err))
+		return nil, err
+	}
+
+	// TODO assert fields are filled
+	return
+
+}
+
+func (c *SSHStdinserverConnecter) Connect() (rwc io.ReadWriteCloser, err error) {
+	var rpcTransport sshbytestream.SSHTransport
+	if err = copier.Copy(&rpcTransport, c); err != nil {
+		return
+	}
+	if rwc, err = sshbytestream.Outgoing(rpcTransport); err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	return
+}

cmd/config_fsvfilter.go

@@ -0,0 +1,33 @@
+package cmd
+
+import (
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/zfs"
+	"strings"
+)
+
+type PrefixSnapshotFilter struct {
+	Prefix string
+}
+
+func parseSnapshotPrefix(i string) (p string, err error) {
+	if len(i) <= 0 {
+		err = errors.Errorf("snapshot prefix must not be empty string")
+		return
+	}
+	p = i
+	return
+}
+
+func parsePrefixSnapshotFilter(i string) (f *PrefixSnapshotFilter, err error) {
+	if !(len(i) > 0) {
+		err = errors.Errorf("snapshot prefix must be longer than 0 characters")
+		return
+	}
+	f = &PrefixSnapshotFilter{i}
+	return
+}
+
+func (f *PrefixSnapshotFilter) Filter(fsv zfs.FilesystemVersion) (accept bool, err error) {
+	return fsv.Type == zfs.Snapshot && strings.HasPrefix(fsv.Name, f.Prefix), nil
+}

cmd/config_job_control.go

@@ -0,0 +1,86 @@
+package cmd
+
+import (
+	"context"
+	"github.com/pkg/errors"
+	"net"
+	"net/http"
+	"net/http/pprof"
+)
+
+type ControlJob struct {
+	Name     string
+	sockaddr *net.UnixAddr
+}
+
+func NewControlJob(name, sockpath string) (j *ControlJob, err error) {
+	j = &ControlJob{Name: name}
+
+	j.sockaddr, err = net.ResolveUnixAddr("unix", sockpath)
+	if err != nil {
+		err = errors.Wrap(err, "cannot resolve unix address")
+		return
+	}
+
+	return
+}
+
+func (j *ControlJob) JobName() string {
+	return j.Name
+}
+
+const (
+	ControlJobEndpointProfile string = "/debug/pprof/profile"
+)
+
+func (j *ControlJob) JobStart(ctx context.Context) {
+
+	log := ctx.Value(contextKeyLog).(Logger)
+	defer log.Info("control job finished")
+
+	l, err := ListenUnixPrivate(j.sockaddr)
+	if err != nil {
+		log.WithError(err).Error("error listening")
+		return
+	}
+
+	mux := http.NewServeMux()
+	mux.Handle(ControlJobEndpointProfile, requestLogger{log, pprof.Profile})
+	server := http.Server{Handler: mux}
+
+outer:
+	for {
+
+		served := make(chan error)
+		go func() {
+			served <- server.Serve(l)
+			close(served)
+		}()
+
+		select {
+		case <-ctx.Done():
+			log.WithError(ctx.Err()).Info("context done")
+			server.Shutdown(context.Background())
+			break outer
+		case err = <-served:
+			if err != nil {
+				log.WithError(err).Error("error serving")
+				break outer
+			}
+		}
+
+	}
+
+}
+
+type requestLogger struct {
+	log         Logger
+	handlerFunc func(w http.ResponseWriter, r *http.Request)
+}
+
+func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	log := l.log.WithField("method", r.Method).WithField("url", r.URL)
+	log.Info("start")
+	l.handlerFunc(w, r)
+	log.Info("finish")
+}

cmd/config_job_local.go

@@ -0,0 +1,205 @@
+package cmd
+
+import (
+	"time"
+
+	"context"
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/zfs"
+	"sync"
+)
+
+type LocalJob struct {
+	Name              string
+	Mapping           *DatasetMapFilter
+	SnapshotPrefix    string
+	Interval          time.Duration
+	InitialReplPolicy InitialReplPolicy
+	PruneLHS          PrunePolicy
+	PruneRHS          PrunePolicy
+	Debug             JobDebugSettings
+}
+
+func parseLocalJob(c JobParsingContext, name string, i map[string]interface{}) (j *LocalJob, err error) {
+
+	var asMap struct {
+		Mapping           map[string]string
+		SnapshotPrefix    string `mapstructure:"snapshot_prefix"`
+		Interval          string
+		InitialReplPolicy string                 `mapstructure:"initial_repl_policy"`
+		PruneLHS          map[string]interface{} `mapstructure:"prune_lhs"`
+		PruneRHS          map[string]interface{} `mapstructure:"prune_rhs"`
+		Debug             map[string]interface{}
+	}
+
+	if err = mapstructure.Decode(i, &asMap); err != nil {
+		err = errors.Wrap(err, "mapstructure error")
+		return nil, err
+	}
+
+	j = &LocalJob{Name: name}
+
+	if j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false); err != nil {
+		return
+	}
+
+	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
+		return
+	}
+
+	if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
+		err = errors.Wrap(err, "cannot parse interval")
+		return
+	}
+
+	if j.InitialReplPolicy, err = parseInitialReplPolicy(asMap.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY); err != nil {
+		return
+	}
+
+	if j.PruneLHS, err = parsePrunePolicy(asMap.PruneLHS); err != nil {
+		err = errors.Wrap(err, "cannot parse 'prune_lhs'")
+		return
+	}
+	if j.PruneRHS, err = parsePrunePolicy(asMap.PruneRHS); err != nil {
+		err = errors.Wrap(err, "cannot parse 'prune_rhs'")
+		return
+	}
+
+	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
+		err = errors.Wrap(err, "cannot parse 'debug'")
+		return
+	}
+
+	return
+}
+
+func (j *LocalJob) JobName() string {
+	return j.Name
+}
+
+func (j *LocalJob) JobStart(ctx context.Context) {
+
+	log := ctx.Value(contextKeyLog).(Logger)
+	defer log.Info("exiting")
+
+	local := rpc.NewLocalRPC()
+	// Allow access to any dataset since we control what mapping
+	// is passed to the pull routine.
+	// All local datasets will be passed to its Map() function,
+	// but only those for which a mapping exists will actually be pulled.
+	// We can pay this small performance penalty for now.
+	handler := NewHandler(log.WithField(logTaskField, "handler"), localPullACL{}, &PrefixSnapshotFilter{j.SnapshotPrefix})
+
+	registerEndpoints(local, handler)
+
+	snapper := IntervalAutosnap{
+		DatasetFilter:    j.Mapping.AsFilter(),
+		Prefix:           j.SnapshotPrefix,
+		SnapshotInterval: j.Interval,
+	}
+
+	plhs, err := j.Pruner(PrunePolicySideLeft, false)
+	if err != nil {
+		log.WithError(err).Error("error creating lhs pruner")
+		return
+	}
+	prhs, err := j.Pruner(PrunePolicySideRight, false)
+	if err != nil {
+		log.WithError(err).Error("error creating rhs pruner")
+		return
+	}
+
+	makeCtx := func(parent context.Context, taskName string) (ctx context.Context) {
+		return context.WithValue(parent, contextKeyLog, log.WithField(logTaskField, taskName))
+	}
+	var snapCtx, plCtx, prCtx, pullCtx context.Context
+	snapCtx = makeCtx(ctx, "autosnap")
+	plCtx = makeCtx(ctx, "prune_lhs")
+	prCtx = makeCtx(ctx, "prune_rhs")
+	pullCtx = makeCtx(ctx, "repl")
+
+	didSnaps := make(chan struct{})
+	go snapper.Run(snapCtx, didSnaps)
+
+outer:
+	for {
+
+		select {
+		case <-ctx.Done():
+			break outer
+		case <-didSnaps:
+			log.Debug("finished taking snapshots")
+			log.Info("starting replication procedure")
+		}
+
+		{
+			log := pullCtx.Value(contextKeyLog).(Logger)
+			log.Debug("replicating from lhs to rhs")
+			err := doPull(PullContext{local, log, j.Mapping, j.InitialReplPolicy})
+			if err != nil {
+				log.WithError(err).Error("error replicating lhs to rhs")
+			}
+			// use a ctx as soon as doPull gains ctx support
+			select {
+			case <-ctx.Done():
+				break outer
+			default:
+			}
+		}
+
+		var wg sync.WaitGroup
+
+		log.Info("pruning lhs")
+		wg.Add(1)
+		go func() {
+			plhs.Run(plCtx)
+			wg.Done()
+		}()
+
+		log.Info("pruning rhs")
+		wg.Add(1)
+		go func() {
+			prhs.Run(prCtx)
+			wg.Done()
+		}()
+
+		wg.Wait()
+
+	}
+
+	log.WithError(ctx.Err()).Info("context")
+
+}
+
+func (j *LocalJob) Pruner(side PrunePolicySide, dryRun bool) (p Pruner, err error) {
+
+	var dsfilter zfs.DatasetFilter
+	var pp PrunePolicy
+	switch side {
+	case PrunePolicySideLeft:
+		pp = j.PruneLHS
+		dsfilter = j.Mapping.AsFilter()
+	case PrunePolicySideRight:
+		pp = j.PruneRHS
+		dsfilter, err = j.Mapping.InvertedFilter()
+		if err != nil {
+			err = errors.Wrap(err, "cannot invert mapping for prune_rhs")
+			return
+		}
+	default:
+		err = errors.Errorf("must be either left or right side")
+		return
+	}
+
+	p = Pruner{
+		time.Now(),
+		dryRun,
+		dsfilter,
+		j.SnapshotPrefix,
+		pp,
+	}
+
+	return
+}

cmd/config_job_pull.go

@@ -0,0 +1,165 @@
+package cmd
+
+import (
+	"time"
+
+	"context"
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/util"
+)
+
+type PullJob struct {
+	Name     string
+	Connect  RWCConnecter
+	Interval time.Duration
+	Mapping  *DatasetMapFilter
+	// constructed from mapping during parsing
+	pruneFilter       *DatasetMapFilter
+	SnapshotPrefix    string
+	InitialReplPolicy InitialReplPolicy
+	Prune             PrunePolicy
+	Debug             JobDebugSettings
+}
+
+func parsePullJob(c JobParsingContext, name string, i map[string]interface{}) (j *PullJob, err error) {
+
+	var asMap struct {
+		Connect           map[string]interface{}
+		Interval          string
+		Mapping           map[string]string
+		InitialReplPolicy string `mapstructure:"initial_repl_policy"`
+		Prune             map[string]interface{}
+		SnapshotPrefix    string `mapstructure:"snapshot_prefix"`
+		Debug             map[string]interface{}
+	}
+
+	if err = mapstructure.Decode(i, &asMap); err != nil {
+		err = errors.Wrap(err, "mapstructure error")
+		return nil, err
+	}
+
+	j = &PullJob{Name: name}
+
+	j.Connect, err = parseSSHStdinserverConnecter(asMap.Connect)
+	if err != nil {
+		err = errors.Wrap(err, "cannot parse 'connect'")
+		return nil, err
+	}
+
+	if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
+		err = errors.Wrap(err, "cannot parse 'interval'")
+		return nil, err
+	}
+
+	j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false)
+	if err != nil {
+		err = errors.Wrap(err, "cannot parse 'mapping'")
+		return nil, err
+	}
+
+	if j.pruneFilter, err = j.Mapping.InvertedFilter(); err != nil {
+		err = errors.Wrap(err, "cannot automatically invert 'mapping' for prune job")
+		return nil, err
+	}
+
+	j.InitialReplPolicy, err = parseInitialReplPolicy(asMap.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY)
+	if err != nil {
+		err = errors.Wrap(err, "cannot parse 'initial_repl_policy'")
+		return
+	}
+
+	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
+		return
+	}
+
+	if j.Prune, err = parsePrunePolicy(asMap.Prune); err != nil {
+		err = errors.Wrap(err, "cannot parse prune policy")
+		return
+	}
+
+	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
+		err = errors.Wrap(err, "cannot parse 'debug'")
+		return
+	}
+
+	return
+}
+
+func (j *PullJob) JobName() string {
+	return j.Name
+}
+
+func (j *PullJob) JobStart(ctx context.Context) {
+
+	log := ctx.Value(contextKeyLog).(Logger)
+	defer log.Info("exiting")
+
+	ticker := time.NewTicker(j.Interval)
+
+start:
+
+	log.Info("connecting")
+	rwc, err := j.Connect.Connect()
+	if err != nil {
+		log.WithError(err).Error("error connecting")
+		return
+	}
+
+	rwc, err = util.NewReadWriteCloserLogger(rwc, j.Debug.Conn.ReadDump, j.Debug.Conn.WriteDump)
+	if err != nil {
+		return
+	}
+
+	client := rpc.NewClient(rwc)
+	if j.Debug.RPC.Log {
+		client.SetLogger(log, true)
+	}
+
+	log.Info("starting pull")
+
+	pullLog := log.WithField(logTaskField, "pull")
+	err = doPull(PullContext{client, pullLog, j.Mapping, j.InitialReplPolicy})
+	if err != nil {
+		log.WithError(err).Error("error doing pull")
+	}
+
+	closeRPCWithTimeout(log, client, time.Second*10, "")
+
+	log.Info("starting prune")
+	prunectx := context.WithValue(ctx, contextKeyLog, log.WithField(logTaskField, "prune"))
+	pruner, err := j.Pruner(PrunePolicySideDefault, false)
+	if err != nil {
+		log.WithError(err).Error("error creating pruner")
+		return
+	}
+
+	pruner.Run(prunectx)
+	log.Info("finish prune")
+
+	log.Info("wait for next interval")
+	select {
+	case <-ctx.Done():
+		log.WithError(ctx.Err()).Info("context")
+		return
+	case <-ticker.C:
+		goto start
+	}
+
+}
+
+func (j *PullJob) Pruner(side PrunePolicySide, dryRun bool) (p Pruner, err error) {
+	p = Pruner{
+		time.Now(),
+		dryRun,
+		j.pruneFilter,
+		j.SnapshotPrefix,
+		j.Prune,
+	}
+	return
+}
+
+func (j *PullJob) doRun(ctx context.Context) {
+
+}

cmd/config_job_source.go

@@ -0,0 +1,197 @@
+package cmd
+
+import (
+	"context"
+	"io"
+	"time"
+
+	mapstructure "github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/util"
+)
+
+type SourceJob struct {
+	Name           string
+	Serve          AuthenticatedChannelListenerFactory
+	Filesystems    *DatasetMapFilter
+	SnapshotPrefix string
+	Interval       time.Duration
+	Prune          PrunePolicy
+	Debug          JobDebugSettings
+}
+
+func parseSourceJob(c JobParsingContext, name string, i map[string]interface{}) (j *SourceJob, err error) {
+
+	var asMap struct {
+		Serve          map[string]interface{}
+		Filesystems    map[string]string
+		SnapshotPrefix string `mapstructure:"snapshot_prefix"`
+		Interval       string
+		Prune          map[string]interface{}
+		Debug          map[string]interface{}
+	}
+
+	if err = mapstructure.Decode(i, &asMap); err != nil {
+		err = errors.Wrap(err, "mapstructure error")
+		return nil, err
+	}
+
+	j = &SourceJob{Name: name}
+
+	if j.Serve, err = parseAuthenticatedChannelListenerFactory(c, asMap.Serve); err != nil {
+		return
+	}
+
+	if j.Filesystems, err = parseDatasetMapFilter(asMap.Filesystems, true); err != nil {
+		return
+	}
+
+	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
+		return
+	}
+
+	if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
+		err = errors.Wrap(err, "cannot parse 'interval'")
+		return
+	}
+
+	if j.Prune, err = parsePrunePolicy(asMap.Prune); err != nil {
+		err = errors.Wrap(err, "cannot parse 'prune'")
+		return
+	}
+
+	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
+		err = errors.Wrap(err, "cannot parse 'debug'")
+		return
+	}
+
+	return
+}
+
+func (j *SourceJob) JobName() string {
+	return j.Name
+}
+
+func (j *SourceJob) JobStart(ctx context.Context) {
+
+	log := ctx.Value(contextKeyLog).(Logger)
+	defer log.Info("exiting")
+
+	a := IntervalAutosnap{DatasetFilter: j.Filesystems, Prefix: j.SnapshotPrefix, SnapshotInterval: j.Interval}
+	p, err := j.Pruner(PrunePolicySideDefault, false)
+	if err != nil {
+		log.WithError(err).Error("error creating pruner")
+		return
+	}
+
+	snapContext := context.WithValue(ctx, contextKeyLog, log.WithField(logTaskField, "autosnap"))
+	prunerContext := context.WithValue(ctx, contextKeyLog, log.WithField(logTaskField, "prune"))
+	serveContext := context.WithValue(ctx, contextKeyLog, log.WithField(logTaskField, "serve"))
+	didSnaps := make(chan struct{})
+
+	go j.serve(serveContext)
+	go a.Run(snapContext, didSnaps)
+
+outer:
+	for {
+		select {
+		case <-ctx.Done():
+			break outer
+		case <-didSnaps:
+			log.Info("starting pruner")
+			p.Run(prunerContext)
+			log.Info("pruner done")
+		}
+	}
+	log.WithError(prunerContext.Err()).Info("context")
+
+}
+
+func (j *SourceJob) Pruner(side PrunePolicySide, dryRun bool) (p Pruner, err error) {
+	p = Pruner{
+		time.Now(),
+		dryRun,
+		j.Filesystems,
+		j.SnapshotPrefix,
+		j.Prune,
+	}
+	return
+}
+
+func (j *SourceJob) serve(ctx context.Context) {
+
+	log := ctx.Value(contextKeyLog).(Logger)
+
+	listener, err := j.Serve.Listen()
+	if err != nil {
+		log.WithError(err).Error("error listening")
+		return
+	}
+
+	type rwcChanMsg struct {
+		rwc io.ReadWriteCloser
+		err error
+	}
+	rwcChan := make(chan rwcChanMsg)
+
+	// Serve connections until interrupted or error
+outer:
+	for {
+
+		go func() {
+			rwc, err := listener.Accept()
+			if err != nil {
+				rwcChan <- rwcChanMsg{rwc, err}
+				close(rwcChan)
+				return
+			}
+			rwcChan <- rwcChanMsg{rwc, err}
+		}()
+
+		select {
+
+		case rwcMsg := <-rwcChan:
+
+			if rwcMsg.err != nil {
+				log.WithError(err).Error("error accepting connection")
+				break outer
+			}
+
+			rwc, err := util.NewReadWriteCloserLogger(rwcMsg.rwc, j.Debug.Conn.ReadDump, j.Debug.Conn.WriteDump)
+			if err != nil {
+				panic(err)
+			}
+
+			// construct connection handler
+			handler := NewHandler(log, j.Filesystems, &PrefixSnapshotFilter{j.SnapshotPrefix})
+
+			// handle connection
+			rpcServer := rpc.NewServer(rwc)
+			if j.Debug.RPC.Log {
+				rpclog := log.WithField("subsystem", "rpc")
+				rpcServer.SetLogger(rpclog, true)
+			}
+			registerEndpoints(rpcServer, handler)
+			if err = rpcServer.Serve(); err != nil {
+				log.WithError(err).Error("error serving connection")
+			}
+			rwc.Close()
+
+		case <-ctx.Done():
+			log.WithError(ctx.Err()).Info("context")
+			break outer
+
+		}
+
+	}
+
+	log.Info("closing listener")
+	err = listener.Close()
+	if err != nil {
+		log.WithError(err).Error("error closing listener")
+	}
+
+	return
+
+}

cmd/config_logging.go

@@ -0,0 +1,233 @@
+package cmd
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/logger"
+	"io/ioutil"
+	"os"
+	"time"
+)
+
+type LoggingConfig struct {
+	Outlets logger.Outlets
+}
+
+type SetNoMetadataFormatter interface {
+	SetNoMetadata(noMetadata bool)
+}
+
+type OutletCommon struct {
+	MinLevel  logger.Level
+	Formatter EntryFormatter
+}
+
+func parseLogging(i interface{}) (c *LoggingConfig, err error) {
+
+	c = &LoggingConfig{}
+	c.Outlets = logger.NewOutlets()
+
+	var asList []interface{}
+	if err = mapstructure.Decode(i, &asList); err != nil {
+		return nil, errors.Wrap(err, "mapstructure error")
+	}
+	if len(asList) == 0 {
+		// Default config
+		out := WriterOutlet{&HumanFormatter{}, os.Stdout}
+		c.Outlets.Add(out, logger.Warn)
+		return
+	}
+
+	var syslogOutlets, stdoutOutlets int
+	for lei, le := range asList {
+
+		outlet, minLevel, err := parseOutlet(le)
+		if err != nil {
+			return nil, errors.Wrapf(err, "cannot parse outlet #%d", lei)
+		}
+		var _ logger.Outlet = WriterOutlet{}
+		var _ logger.Outlet = &SyslogOutlet{}
+		switch outlet.(type) {
+		case *SyslogOutlet:
+			syslogOutlets++
+		case WriterOutlet:
+			stdoutOutlets++
+		}
+
+		c.Outlets.Add(outlet, minLevel)
+
+	}
+
+	if syslogOutlets > 1 {
+		return nil, errors.Errorf("can only define one 'syslog' outlet")
+	}
+	if stdoutOutlets > 1 {
+		return nil, errors.Errorf("can only define one 'stdout' outlet")
+	}
+
+	return c, nil
+
+}
+
+func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
+	var is string
+	switch j := i.(type) {
+	case string:
+		is = j
+	default:
+		return nil, errors.Errorf("invalid log format: wrong type: %T", i)
+	}
+
+	switch is {
+	case "human":
+		return &HumanFormatter{}, nil
+	case "logfmt":
+		return &LogfmtFormatter{}, nil
+	case "json":
+		return &JSONFormatter{}, nil
+	default:
+		return nil, errors.Errorf("invalid log format: '%s'", is)
+	}
+
+}
+
+func parseOutlet(i interface{}) (o logger.Outlet, level logger.Level, err error) {
+
+	var in struct {
+		Outlet string
+		Level  string
+		Format string
+	}
+	if err = mapstructure.Decode(i, &in); err != nil {
+		err = errors.Wrap(err, "mapstructure error")
+		return
+	}
+	if in.Outlet == "" || in.Level == "" || in.Format == "" {
+		err = errors.Errorf("must specify 'outlet', 'level' and 'format' field")
+		return
+	}
+
+	common := &OutletCommon{}
+	common.MinLevel, err = logger.ParseLevel(in.Level)
+	if err != nil {
+		err = errors.Wrap(err, "cannot parse 'level' field")
+		return
+	}
+	common.Formatter, err = parseLogFormat(in.Format)
+	if err != nil {
+		err = errors.Wrap(err, "cannot parse")
+		return
+	}
+
+	switch in.Outlet {
+	case "stdout":
+		o, err = parseStdoutOutlet(i, common)
+	case "tcp":
+		o, err = parseTCPOutlet(i, common)
+	case "syslog":
+		o, err = parseSyslogOutlet(i, common)
+	default:
+		err = errors.Errorf("unknown outlet type '%s'", in.Outlet)
+	}
+	return o, common.MinLevel, err
+
+}
+
+func parseStdoutOutlet(i interface{}, common *OutletCommon) (WriterOutlet, error) {
+	return WriterOutlet{
+		common.Formatter,
+		os.Stdout,
+	}, nil
+}
+
+func parseTCPOutlet(i interface{}, common *OutletCommon) (out *TCPOutlet, err error) {
+
+	out = &TCPOutlet{}
+	out.Formatter = common.Formatter
+
+	var in struct {
+		Net           string
+		Address       string
+		RetryInterval string `mapstructure:"retry_interval"`
+		TLS           *struct {
+			CA   string
+			Cert string
+			Key  string
+		}
+	}
+	if err = mapstructure.Decode(i, &in); err != nil {
+		return nil, errors.Wrap(err, "mapstructure error")
+	}
+
+	out.RetryInterval, err = time.ParseDuration(in.RetryInterval)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
+	}
+
+	out.Net, out.Address = in.Net, in.Address
+
+	if in.TLS != nil {
+
+		cert, err := tls.LoadX509KeyPair(in.TLS.Cert, in.TLS.Key)
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot load client cert")
+		}
+
+		var rootCAs *x509.CertPool
+		if in.TLS.CA == "" {
+			if rootCAs, err = x509.SystemCertPool(); err != nil {
+				return nil, errors.Wrap(err, "cannot open system cert pool")
+			}
+		} else {
+			rootCAs = x509.NewCertPool()
+			rootCAPEM, err := ioutil.ReadFile(in.TLS.CA)
+			if err != nil {
+				return nil, errors.Wrap(err, "cannot load CA cert")
+			}
+			if !rootCAs.AppendCertsFromPEM(rootCAPEM) {
+				return nil, errors.New("cannot parse CA cert")
+			}
+		}
+		if err != nil && in.TLS.CA == "" {
+			return nil, errors.Wrap(err, "cannot load root ca pool")
+		}
+
+		out.TLS = &tls.Config{
+			Certificates: []tls.Certificate{cert},
+			RootCAs:      rootCAs,
+		}
+
+		out.TLS.BuildNameToCertificate()
+	}
+
+	return
+
+}
+
+func parseSyslogOutlet(i interface{}, common *OutletCommon) (out *SyslogOutlet, err error) {
+
+	var in struct {
+		RetryInterval string `mapstructure:"retry_interval"`
+	}
+	if err = mapstructure.Decode(i, &in); err != nil {
+		return nil, errors.Wrap(err, "mapstructure error")
+	}
+
+	out = &SyslogOutlet{}
+	out.Formatter = common.Formatter
+	if f, ok := out.Formatter.(SetNoMetadataFormatter); ok {
+		f.SetNoMetadata(true)
+	}
+
+	out.RetryInterval = 0 // default to 0 as we assume local syslog will just work
+	if in.RetryInterval != "" {
+		out.RetryInterval, err = time.ParseDuration(in.RetryInterval)
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
+		}
+	}
+
+	return
+}

cmd/config_mapfilter.go

@@ -1,10 +1,11 @@
 package cmd
 
 import (
-	"errors"
 	"fmt"
 	"strings"
 
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
 	"github.com/zrepl/zrepl/zfs"
 )
 
@@ -13,7 +14,7 @@ type DatasetMapFilter struct {
 
 	// if set, only valid filter entries can be added using Add()
 	// and Map() will always return an error
-	filterOnly bool
+	filterMode bool
 }
 
 type datasetMapFilterEntry struct {
@@ -24,18 +25,17 @@ type datasetMapFilterEntry struct {
 	subtreeMatch bool
 }
 
-var NoMatchError error = errors.New("no match found in mapping")
+func NewDatasetMapFilter(capacity int, filterMode bool) *DatasetMapFilter {
-
+	return &DatasetMapFilter{
-func NewDatasetMapFilter(capacity int, filterOnly bool) DatasetMapFilter {
+		entries:    make([]datasetMapFilterEntry, 0, capacity),
-	return DatasetMapFilter{
+		filterMode: filterMode,
-		entries: make([]datasetMapFilterEntry, 0, capacity),
 	}
 }
 
 func (m *DatasetMapFilter) Add(pathPattern, mapping string) (err error) {
 
-	if m.filterOnly {
+	if m.filterMode {
-		if _, err = parseDatasetFilterResult(mapping); err != nil {
+		if _, err = m.parseDatasetFilterResult(mapping); err != nil {
 			return
 		}
 	}
@@ -103,60 +103,134 @@ func (m DatasetMapFilter) mostSpecificPrefixMapping(path *zfs.DatasetPath) (idx
 
 func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error) {
 
-	if m.filterOnly {
+	if m.filterMode {
 		err = fmt.Errorf("using a filter for mapping simply does not work")
 		return
 	}
 
 	mi, hasMapping := m.mostSpecificPrefixMapping(source)
 	if !hasMapping {
-		err = NoMatchError
+		return nil, nil
-		return
 	}
 	me := m.entries[mi]
 
+	if strings.HasPrefix("!", me.mapping) {
+		// reject mapping
+		return nil, nil
+	}
+
 	target, err = zfs.NewDatasetPath(me.mapping)
 	if err != nil {
 		err = fmt.Errorf("mapping target is not a dataset path: %s", err)
 		return
 	}
-	if m.entries[mi].subtreeMatch {
+	if me.subtreeMatch {
-		// strip common prefix
+		// strip common prefix ('<' wildcards are no special case here)
 		extendComps := source.Copy()
-		if me.path.Empty() {
+		extendComps.TrimPrefix(me.path)
-			// special case: trying to map the root => strip first component
-			extendComps.TrimNPrefixComps(1)
-		} else {
-			extendComps.TrimPrefix(me.path)
-		}
 		target.Extend(extendComps)
 	}
 	return
 }
 
 func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
+
+	if !m.filterMode {
+		err = fmt.Errorf("using a mapping as a filter does not work")
+		return
+	}
+
 	mi, hasMapping := m.mostSpecificPrefixMapping(p)
 	if !hasMapping {
 		pass = false
 		return
 	}
 	me := m.entries[mi]
-	pass, err = parseDatasetFilterResult(me.mapping)
+	pass, err = m.parseDatasetFilterResult(me.mapping)
 	return
 }
 
+// Construct a new filter-only DatasetMapFilter from a mapping
+// The new filter allows excactly those paths that were not forbidden by the mapping.
+func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
+
+	if m.filterMode {
+		err = errors.Errorf("can only invert mappings")
+		return
+	}
+
+	inv = &DatasetMapFilter{
+		make([]datasetMapFilterEntry, len(m.entries)),
+		true,
+	}
+
+	for i, e := range m.entries {
+		inv.entries[i].path, err = zfs.NewDatasetPath(e.mapping)
+		if err != nil {
+			err = errors.Wrapf(err, "mapping cannot be inverted: '%s' is not a dataset path: %s", e.mapping)
+			return
+		}
+		inv.entries[i].mapping = MapFilterResultOk
+		inv.entries[i].subtreeMatch = e.subtreeMatch
+	}
+
+	return inv, nil
+}
+
+// Creates a new DatasetMapFilter in filter mode from a mapping
+// All accepting mapping results are mapped to accepting filter results
+// All rejecting mapping results are mapped to rejecting filter results
+func (m DatasetMapFilter) AsFilter() (f *DatasetMapFilter) {
+
+	f = &DatasetMapFilter{
+		make([]datasetMapFilterEntry, len(m.entries)),
+		true,
+	}
+
+	for i, e := range m.entries {
+		var newe datasetMapFilterEntry = e
+		if strings.HasPrefix(newe.mapping, "!") {
+			newe.mapping = MapFilterResultOmit
+		} else {
+			newe.mapping = MapFilterResultOk
+		}
+		f.entries[i] = newe
+	}
+
+	return f
+}
+
+const (
+	MapFilterResultOk   string = "ok"
+	MapFilterResultOmit string = "!"
+)
+
 // Parse a dataset filter result
-func parseDatasetFilterResult(result string) (pass bool, err error) {
+func (m DatasetMapFilter) parseDatasetFilterResult(result string) (pass bool, err error) {
 	l := strings.ToLower(result)
-	switch strings.ToLower(l) {
+	if l == MapFilterResultOk {
-	case "ok":
+		return true, nil
-		pass = true
+	}
-		return
+	if l == MapFilterResultOmit {
-	case "omit":
+		return false, nil
-		return
+	}
-	default:
+	return false, fmt.Errorf("'%s' is not a valid filter result", result)
-		err = fmt.Errorf("'%s' is not a valid filter result", result)
+}
+
+func parseDatasetMapFilter(mi interface{}, filterMode bool) (f *DatasetMapFilter, err error) {
+
+	var m map[string]string
+	if err = mapstructure.Decode(mi, &m); err != nil {
+		err = fmt.Errorf("maps / filters must be specified as map[string]string: %s", err)
 		return
 	}
+
+	f = NewDatasetMapFilter(len(m), filterMode)
+	for pathPattern, mapping := range m {
+		if err = f.Add(pathPattern, mapping); err != nil {
+			err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err)
+			return
+		}
+	}
 	return
 }

cmd/config_parse.go

@@ -0,0 +1,295 @@
+package cmd
+
+import (
+	"io/ioutil"
+
+	"fmt"
+	yaml "github.com/go-yaml/yaml"
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"os"
+	"regexp"
+	"strconv"
+	"time"
+)
+
+var ConfigFileDefaultLocations []string = []string{
+	"/etc/zrepl/zrepl.yml",
+	"/usr/local/etc/zrepl/zrepl.yml",
+}
+
+const (
+	JobNameControl string = "control"
+)
+
+var ReservedJobNames []string = []string{
+	JobNameControl,
+}
+
+type ConfigParsingContext struct {
+	Global *Global
+}
+
+func ParseConfig(path string) (config *Config, err error) {
+
+	if path == "" {
+		// Try default locations
+		for _, l := range ConfigFileDefaultLocations {
+			stat, err := os.Stat(l)
+			if err != nil {
+				continue
+			}
+			if !stat.Mode().IsRegular() {
+				err = errors.Errorf("file at default location is not a regular file: %s", l)
+				continue
+			}
+			path = l
+			break
+		}
+	}
+
+	var i interface{}
+
+	var bytes []byte
+
+	if bytes, err = ioutil.ReadFile(path); err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+
+	if err = yaml.Unmarshal(bytes, &i); err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+
+	return parseConfig(i)
+}
+
+func parseConfig(i interface{}) (c *Config, err error) {
+
+	var asMap struct {
+		Global map[string]interface{}
+		Jobs   []map[string]interface{}
+	}
+	if err := mapstructure.Decode(i, &asMap); err != nil {
+		return nil, errors.Wrap(err, "config root must be a dict")
+	}
+
+	c = &Config{}
+
+	// Parse global with defaults
+	c.Global.Serve.Stdinserver.SockDir = "/var/run/zrepl/stdinserver"
+	c.Global.Control.Sockpath = "/var/run/zrepl/control"
+
+	err = mapstructure.Decode(asMap.Global, &c.Global)
+	if err != nil {
+		err = errors.Wrap(err, "mapstructure error on 'global' section: %s")
+		return
+	}
+
+	if c.Global.logging, err = parseLogging(asMap.Global["logging"]); err != nil {
+		return nil, errors.Wrap(err, "cannot parse logging section")
+	}
+
+	cpc := ConfigParsingContext{&c.Global}
+	jpc := JobParsingContext{cpc}
+
+	// Jobs
+	c.Jobs = make(map[string]Job, len(asMap.Jobs))
+	for i := range asMap.Jobs {
+		job, err := parseJob(jpc, asMap.Jobs[i])
+		if err != nil {
+			// Try to find its name
+			namei, ok := asMap.Jobs[i]["name"]
+			if !ok {
+				namei = fmt.Sprintf("<no name, entry #%d in list>", i)
+			}
+			err = errors.Wrapf(err, "cannot parse job '%v'", namei)
+			return nil, err
+		}
+		jn := job.JobName()
+		if _, ok := c.Jobs[jn]; ok {
+			err = errors.Errorf("duplicate job name: %s", jn)
+			return nil, err
+		}
+		c.Jobs[job.JobName()] = job
+	}
+
+	cj, err := NewControlJob(JobNameControl, jpc.Global.Control.Sockpath)
+	if err != nil {
+		err = errors.Wrap(err, "cannot create control job")
+		return
+	}
+	c.Jobs[JobNameControl] = cj
+
+	return c, nil
+
+}
+
+func extractStringField(i map[string]interface{}, key string, notempty bool) (field string, err error) {
+	vi, ok := i[key]
+	if !ok {
+		err = errors.Errorf("must have field '%s'", key)
+		return "", err
+	}
+	field, ok = vi.(string)
+	if !ok {
+		err = errors.Errorf("'%s' field must have type string", key)
+		return "", err
+	}
+	if notempty && len(field) <= 0 {
+		err = errors.Errorf("'%s' field must not be empty", key)
+		return "", err
+	}
+	return
+}
+
+type JobParsingContext struct {
+	ConfigParsingContext
+}
+
+func parseJob(c JobParsingContext, i map[string]interface{}) (j Job, err error) {
+
+	name, err := extractStringField(i, "name", true)
+	if err != nil {
+		return
+	}
+
+	for _, r := range ReservedJobNames {
+		if name == r {
+			err = errors.Errorf("job name '%s' is reserved", name)
+			return
+		}
+	}
+
+	jobtype, err := extractStringField(i, "type", true)
+	if err != nil {
+		return
+	}
+
+	switch jobtype {
+	case "pull":
+		return parsePullJob(c, name, i)
+	case "source":
+		return parseSourceJob(c, name, i)
+	case "local":
+		return parseLocalJob(c, name, i)
+	default:
+		return nil, errors.Errorf("unknown job type '%s'", jobtype)
+	}
+
+}
+
+func parseConnect(i map[string]interface{}) (c RWCConnecter, err error) {
+
+	t, err := extractStringField(i, "type", true)
+	if err != nil {
+		return nil, err
+	}
+
+	switch t {
+	case "ssh+stdinserver":
+		return parseSSHStdinserverConnecter(i)
+	default:
+		return nil, errors.Errorf("unknown connection type '%s'", t)
+	}
+
+}
+
+func parseInitialReplPolicy(v interface{}, defaultPolicy InitialReplPolicy) (p InitialReplPolicy, err error) {
+	s, ok := v.(string)
+	if !ok {
+		goto err
+	}
+
+	switch {
+	case s == "":
+		p = defaultPolicy
+	case s == "most_recent":
+		p = InitialReplPolicyMostRecent
+	case s == "all":
+		p = InitialReplPolicyAll
+	default:
+		goto err
+	}
+
+	return
+
+err:
+	err = errors.New(fmt.Sprintf("expected InitialReplPolicy, got %#v", v))
+	return
+}
+
+func parsePrunePolicy(v map[string]interface{}) (p PrunePolicy, err error) {
+
+	policyName, err := extractStringField(v, "policy", true)
+	if err != nil {
+		return
+	}
+
+	switch policyName {
+	case "grid":
+		return parseGridPrunePolicy(v)
+	case "noprune":
+		return NoPrunePolicy{}, nil
+	default:
+		err = errors.Errorf("unknown policy '%s'", policyName)
+		return
+	}
+
+}
+
+func parseAuthenticatedChannelListenerFactory(c JobParsingContext, v map[string]interface{}) (p AuthenticatedChannelListenerFactory, err error) {
+
+	t, err := extractStringField(v, "type", true)
+	if err != nil {
+		return nil, err
+	}
+
+	switch t {
+	case "stdinserver":
+		return parseStdinserverListenerFactory(c, v)
+	default:
+		err = errors.Errorf("unknown type '%s'", t)
+		return
+	}
+
+}
+
+var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
+
+func parsePostitiveDuration(e string) (d time.Duration, err error) {
+	comps := durationStringRegex.FindStringSubmatch(e)
+	if len(comps) != 3 {
+		err = fmt.Errorf("does not match regex: %s %#v", e, comps)
+		return
+	}
+
+	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+	if durationFactor <= 0 {
+		return 0, errors.New("duration must be positive integer")
+	}
+
+	var durationUnit time.Duration
+	switch comps[2] {
+	case "s":
+		durationUnit = time.Second
+	case "m":
+		durationUnit = time.Minute
+	case "h":
+		durationUnit = time.Hour
+	case "d":
+		durationUnit = 24 * time.Hour
+	case "w":
+		durationUnit = 24 * 7 * time.Hour
+	default:
+		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
+		return
+	}
+
+	d = time.Duration(durationFactor) * durationUnit
+	return
+}

cmd/config_prune_grid.go

@@ -0,0 +1,174 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/util"
+	"github.com/zrepl/zrepl/zfs"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type GridPrunePolicy struct {
+	RetentionGrid *util.RetentionGrid
+}
+
+type retentionGridAdaptor struct {
+	zfs.FilesystemVersion
+}
+
+func (a retentionGridAdaptor) Date() time.Time {
+	return a.Creation
+}
+
+func (a retentionGridAdaptor) LessThan(b util.RetentionGridEntry) bool {
+	return a.CreateTXG < b.(retentionGridAdaptor).CreateTXG
+}
+
+func (p *GridPrunePolicy) Prune(_ *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error) {
+
+	// Build adaptors for retention grid
+	adaptors := make([]util.RetentionGridEntry, len(versions))
+	for fsv := range versions {
+		adaptors[fsv] = retentionGridAdaptor{versions[fsv]}
+	}
+
+	sort.SliceStable(adaptors, func(i, j int) bool {
+		return adaptors[i].LessThan(adaptors[j])
+	})
+	now := adaptors[len(adaptors)-1].Date()
+
+	// Evaluate retention grid
+	keepa, removea := p.RetentionGrid.FitEntries(now, adaptors)
+
+	// Revert adaptors
+	keep = make([]zfs.FilesystemVersion, len(keepa))
+	for i := range keepa {
+		keep[i] = keepa[i].(retentionGridAdaptor).FilesystemVersion
+	}
+	remove = make([]zfs.FilesystemVersion, len(removea))
+	for i := range removea {
+		remove[i] = removea[i].(retentionGridAdaptor).FilesystemVersion
+	}
+	return
+
+}
+
+func parseGridPrunePolicy(e map[string]interface{}) (p *GridPrunePolicy, err error) {
+
+	var i struct {
+		Grid string
+	}
+
+	if err = mapstructure.Decode(e, &i); err != nil {
+		err = errors.Wrapf(err, "mapstructure error")
+		return
+	}
+
+	p = &GridPrunePolicy{}
+
+	// Parse grid policy
+	intervals, err := parseRetentionGridIntervalsString(i.Grid)
+	if err != nil {
+		err = fmt.Errorf("cannot parse retention grid: %s", err)
+		return
+	}
+	// Assert intervals are of increasing length (not necessarily required, but indicates config mistake)
+	lastDuration := time.Duration(0)
+	for i := range intervals {
+
+		if intervals[i].Length < lastDuration {
+			// If all intervals before were keep=all, this is ok
+			allPrevKeepCountAll := true
+			for j := i - 1; allPrevKeepCountAll && j >= 0; j-- {
+				allPrevKeepCountAll = intervals[j].KeepCount == util.RetentionGridKeepCountAll
+			}
+			if allPrevKeepCountAll {
+				goto isMonotonicIncrease
+			}
+			err = errors.New("retention grid interval length must be monotonically increasing")
+			return
+		}
+	isMonotonicIncrease:
+		lastDuration = intervals[i].Length
+
+	}
+	p.RetentionGrid = util.NewRetentionGrid(intervals)
+
+	return
+}
+
+var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
+
+func parseRetentionGridIntervalString(e string) (intervals []util.RetentionInterval, err error) {
+
+	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
+	if comps == nil {
+		err = fmt.Errorf("retention string does not match expected format")
+		return
+	}
+
+	times, err := strconv.Atoi(comps[1])
+	if err != nil {
+		return nil, err
+	} else if times <= 0 {
+		return nil, fmt.Errorf("contains factor <= 0")
+	}
+
+	duration, err := parsePostitiveDuration(comps[2])
+	if err != nil {
+		return nil, err
+	}
+
+	keepCount := 1
+	if comps[3] != "" {
+		// Decompose key=value, comma separated
+		// For now, only keep_count is supported
+		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
+		res := re.FindStringSubmatch(comps[4])
+		if res == nil || len(res) != 2 {
+			err = fmt.Errorf("interval parameter contains unknown parameters")
+			return
+		}
+		if res[1] == "all" {
+			keepCount = util.RetentionGridKeepCountAll
+		} else {
+			keepCount, err = strconv.Atoi(res[1])
+			if err != nil {
+				err = fmt.Errorf("cannot parse keep_count value")
+				return
+			}
+		}
+	}
+
+	intervals = make([]util.RetentionInterval, times)
+	for i := range intervals {
+		intervals[i] = util.RetentionInterval{
+			Length:    duration,
+			KeepCount: keepCount,
+		}
+	}
+
+	return
+
+}
+
+func parseRetentionGridIntervalsString(s string) (intervals []util.RetentionInterval, err error) {
+
+	ges := strings.Split(s, "|")
+	intervals = make([]util.RetentionInterval, 0, 7*len(ges))
+
+	for intervalIdx, e := range ges {
+		parsed, err := parseRetentionGridIntervalString(e)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
+		}
+		intervals = append(intervals, parsed...)
+	}
+
+	return
+}

cmd/config_prune_none.go

@@ -0,0 +1,11 @@
+package cmd
+
+import "github.com/zrepl/zrepl/zfs"
+
+type NoPrunePolicy struct{}
+
+func (p NoPrunePolicy) Prune(fs *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error) {
+	keep = versions
+	remove = []zfs.FilesystemVersion{}
+	return
+}

cmd/config_serve_stdinserver.go

@@ -0,0 +1,104 @@
+package cmd
+
+import (
+	"github.com/ftrvxmtrx/fd"
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"io"
+	"net"
+	"os"
+	"path"
+)
+
+type StdinserverListenerFactory struct {
+	ClientIdentity string `mapstructure:"client_identity"`
+	sockaddr       *net.UnixAddr
+}
+
+func parseStdinserverListenerFactory(c JobParsingContext, i map[string]interface{}) (f *StdinserverListenerFactory, err error) {
+
+	f = &StdinserverListenerFactory{}
+
+	if err = mapstructure.Decode(i, f); err != nil {
+		return nil, errors.Wrap(err, "mapstructure error")
+	}
+	if !(len(f.ClientIdentity) > 0) {
+		err = errors.Errorf("must specify 'client_identity'")
+		return
+	}
+
+	f.sockaddr, err = stdinserverListenerSocket(c.Global.Serve.Stdinserver.SockDir, f.ClientIdentity)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func stdinserverListenerSocket(sockdir, clientIdentity string) (addr *net.UnixAddr, err error) {
+	sockpath := path.Join(sockdir, clientIdentity)
+	addr, err = net.ResolveUnixAddr("unix", sockpath)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot resolve unix address")
+	}
+	return addr, nil
+}
+
+func (f *StdinserverListenerFactory) Listen() (al AuthenticatedChannelListener, err error) {
+
+	ul, err := ListenUnixPrivate(f.sockaddr)
+	if err != nil {
+		return nil, errors.Wrapf(err, "cannot listen on unix socket %s", f.sockaddr)
+	}
+
+	l := &StdinserverListener{ul}
+
+	return l, nil
+}
+
+type StdinserverListener struct {
+	l *net.UnixListener
+}
+
+type fdRWC struct {
+	stdin, stdout *os.File
+	control       *net.UnixConn
+}
+
+func (f fdRWC) Read(p []byte) (n int, err error) {
+	return f.stdin.Read(p)
+}
+
+func (f fdRWC) Write(p []byte) (n int, err error) {
+	return f.stdout.Write(p)
+}
+
+func (f fdRWC) Close() (err error) {
+	f.stdin.Close()
+	f.stdout.Close()
+	return f.control.Close()
+}
+
+func (l *StdinserverListener) Accept() (ch io.ReadWriteCloser, err error) {
+	c, err := l.l.Accept()
+	if err != nil {
+		err = errors.Wrap(err, "error accepting on unix listener")
+		return
+	}
+
+	// Read the stdin and stdout of the stdinserver command
+	files, err := fd.Get(c.(*net.UnixConn), 2, []string{"stdin", "stdout"})
+	if err != nil {
+		err = errors.Wrap(err, "error receiving fds from stdinserver command")
+		c.Close()
+	}
+
+	rwc := fdRWC{files[0], files[1], c.(*net.UnixConn)}
+
+	return rwc, nil
+
+}
+
+func (l *StdinserverListener) Close() (err error) {
+	return l.l.Close() // removes socket file automatically
+}

cmd/config_test.go

@@ -4,14 +4,34 @@ import (
 	"testing"
 	"time"
 
+	"github.com/kr/pretty"
 	"github.com/stretchr/testify/assert"
 	"github.com/zrepl/zrepl/util"
 	"github.com/zrepl/zrepl/zfs"
 )
 
-func TestSampleConfigFileIsParsedWithoutErrors(t *testing.T) {
+func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
-	_, err := ParseConfig("./sampleconf/zrepl.yml")
+
-	assert.Nil(t, err)
+	paths := []string{
+		"./sampleconf/localbackup/host1.yml",
+		"./sampleconf/pullbackup/backuphost.yml",
+		"./sampleconf/pullbackup/productionhost.yml",
+		"./sampleconf/random/debugging.yml",
+		"./sampleconf/random/logging.yml",
+	}
+
+	for _, p := range paths {
+
+		c, err := ParseConfig(p)
+		if err != nil {
+			t.Errorf("error parsing %s:\n%+v", p, err)
+		}
+
+		t.Logf("file: %s", p)
+		t.Log(pretty.Sprint(c))
+
+	}
+
 }
 
 func TestParseRetentionGridStringParsing(t *testing.T) {
@@ -61,15 +81,21 @@ func TestDatasetMapFilter(t *testing.T) {
 			t.Logf("expect test from path to be valid: %s", err)
 			t.FailNow()
 		}
+
+		res, err := dmf.Map(fromPath)
+		if to == "" {
+			assert.Nil(t, res)
+			assert.Nil(t, err)
+			t.Logf("%s => NOT MAPPED", fromPath.ToString())
+			return
+		}
+
+		assert.Nil(t, err)
 		toPath, err := zfs.NewDatasetPath(to)
 		if err != nil {
 			t.Logf("expect test to path to be valid: %s", err)
 			t.FailNow()
 		}
-
-		res, err := dmf.Map(fromPath)
-		t.Logf("%s => %s", fromPath.ToString(), res.ToString())
-		assert.Nil(t, err)
 		assert.True(t, res.Equal(toPath))
 	}
 
@@ -90,25 +116,29 @@ func TestDatasetMapFilter(t *testing.T) {
 	}
 
 	map1 := map[string]string{
-		"a/b/c<": "root1",
+		"a/b/c<":     "root1",
-		"a/b<":   "root2",
+		"a/b<":       "root2",
-		"<":      "root3/b/c",
+		"<":          "root3/b/c",
-		"q<":     "root4/1/2",
+		"b":          "!",
+		"a/b/c/d/e<": "!",
+		"q<":         "root4/1/2",
 	}
 
 	expectMapping(map1, "a/b/c", "root1")
 	expectMapping(map1, "a/b/c/d", "root1/d")
+	expectMapping(map1, "a/b/c/d/e", "")
 	expectMapping(map1, "a/b/e", "root2/e")
 	expectMapping(map1, "a/b", "root2")
-	expectMapping(map1, "x", "root3/b/c")
+	expectMapping(map1, "x", "root3/b/c/x")
-	expectMapping(map1, "x/y", "root3/b/c/y")
+	expectMapping(map1, "x/y", "root3/b/c/x/y")
 	expectMapping(map1, "q", "root4/1/2")
+	expectMapping(map1, "b", "")
 	expectMapping(map1, "q/r", "root4/1/2/r")
 
 	filter1 := map[string]string{
-		"<":    "omit",
+		"<":    "!",
 		"a<":   "ok",
-		"a/b<": "omit",
+		"a/b<": "!",
 	}
 
 	expectFilter(filter1, "b", false)
@@ -121,3 +151,72 @@ func TestDatasetMapFilter(t *testing.T) {
 	expectFilter(filter2, "foo", false) // default to omit
 
 }
+
+func TestDatasetMapFilter_AsFilter(t *testing.T) {
+
+	mapspec := map[string]string{
+		"a/b/c<":     "root1",
+		"a/b<":       "root2",
+		"<":          "root3/b/c",
+		"b":          "!",
+		"a/b/c/d/e<": "!",
+		"q<":         "root4/1/2",
+	}
+
+	m, err := parseDatasetMapFilter(mapspec, false)
+	assert.Nil(t, err)
+
+	f := m.AsFilter()
+
+	t.Logf("Mapping:\n%s\nFilter:\n%s", pretty.Sprint(m), pretty.Sprint(f))
+
+	tf := func(f zfs.DatasetFilter, path string, pass bool) {
+		p, err := zfs.NewDatasetPath(path)
+		assert.Nil(t, err)
+		r, err := f.Filter(p)
+		assert.Nil(t, err)
+		assert.Equal(t, pass, r)
+	}
+
+	tf(f, "a/b/c", true)
+	tf(f, "a/b", true)
+	tf(f, "b", false)
+	tf(f, "a/b/c/d/e", false)
+	tf(f, "a/b/c/d/e/f", false)
+	tf(f, "a", true)
+
+}
+
+func TestDatasetMapFilter_InvertedFilter(t *testing.T) {
+	mapspec := map[string]string{
+		"a/b":      "1/2",
+		"a/b/c<":   "3",
+		"a/b/c/d<": "1/2/a",
+		"a/b/d":    "!",
+	}
+
+	m, err := parseDatasetMapFilter(mapspec, false)
+	assert.Nil(t, err)
+
+	inv, err := m.InvertedFilter()
+	assert.Nil(t, err)
+
+	t.Log(pretty.Sprint(inv))
+
+	expectMapping := func(m *DatasetMapFilter, ps string, expRes bool) {
+		p, err := zfs.NewDatasetPath(ps)
+		assert.Nil(t, err)
+		r, err := m.Filter(p)
+		assert.Nil(t, err)
+		assert.Equal(t, expRes, r)
+	}
+
+	expectMapping(inv, "4", false)
+	expectMapping(inv, "3", true)
+	expectMapping(inv, "3/x", true)
+	expectMapping(inv, "1", false)
+	expectMapping(inv, "1/2", true)
+	expectMapping(inv, "1/2/3", false)
+	expectMapping(inv, "1/2/a/b", true)
+
+}

cmd/control.go

@@ -0,0 +1,100 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"github.com/spf13/cobra"
+	"io"
+	golog "log"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+)
+
+var controlCmd = &cobra.Command{
+	Use:   "control",
+	Short: "control zrepl daemon",
+}
+
+var pprofCmd = &cobra.Command{
+	Use:   "pprof cpu OUTFILE",
+	Short: "pprof CPU of daemon to OUTFILE (- for stdout)",
+	Run:   doControlPProf,
+}
+var pprofCmdArgs struct {
+	seconds int64
+}
+
+func init() {
+	RootCmd.AddCommand(controlCmd)
+	controlCmd.AddCommand(pprofCmd)
+	pprofCmd.Flags().Int64Var(&pprofCmdArgs.seconds, "seconds", 30, "seconds to profile")
+}
+
+func doControlPProf(cmd *cobra.Command, args []string) {
+
+	log := golog.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Printf("exiting after error")
+		os.Exit(1)
+	}
+
+	conf, err := ParseConfig(rootArgs.configFile)
+	if err != nil {
+		log.Printf("error parsing config: %s", err)
+		die()
+	}
+
+	if cmd.Flags().Arg(0) != "cpu" {
+		log.Printf("only CPU profiles are supported")
+		log.Printf("%s", cmd.UsageString())
+		die()
+	}
+
+	outfn := cmd.Flags().Arg(1)
+	if outfn == "" {
+		log.Printf("must specify output filename")
+		log.Printf("%s", cmd.UsageString())
+		die()
+	}
+	var out io.Writer
+	if outfn == "-" {
+		out = os.Stdout
+	} else {
+		out, err = os.Create(outfn)
+		if err != nil {
+			log.Printf("error creating output file: %s", err)
+			die()
+		}
+	}
+
+	log.Printf("connecting to daemon")
+	httpc := http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", conf.Global.Control.Sockpath)
+			},
+		},
+	}
+
+	log.Printf("profiling...")
+	v := url.Values{}
+	v.Set("seconds", fmt.Sprintf("%d", pprofCmdArgs.seconds))
+	v.Encode()
+	resp, err := httpc.Get("http://unix" + ControlJobEndpointProfile + "?" + v.Encode())
+	if err != nil {
+		log.Printf("error: %s", err)
+		die()
+	}
+
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		log.Printf("error writing profile: %s", err)
+		die()
+	}
+
+	log.Printf("finished")
+
+}

cmd/daemon.go

@@ -0,0 +1,110 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/logger"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+)
+
+// daemonCmd represents the daemon command
+var daemonCmd = &cobra.Command{
+	Use:   "daemon",
+	Short: "start daemon",
+	Run:   doDaemon,
+}
+
+func init() {
+	RootCmd.AddCommand(daemonCmd)
+}
+
+type Job interface {
+	JobName() string
+	JobStart(ctxt context.Context)
+}
+
+func doDaemon(cmd *cobra.Command, args []string) {
+
+	conf, err := ParseConfig(rootArgs.configFile)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error parsing config: %s", err)
+		os.Exit(1)
+	}
+
+	log := logger.NewLogger(conf.Global.logging.Outlets, 1*time.Second)
+
+	log.Debug("starting daemon")
+	ctx := context.WithValue(context.Background(), contextKeyLog, log)
+	ctx = context.WithValue(ctx, contextKeyLog, log)
+
+	d := NewDaemon(conf)
+	d.Loop(ctx)
+
+}
+
+type contextKey string
+
+const (
+	contextKeyLog contextKey = contextKey("log")
+)
+
+type Daemon struct {
+	conf *Config
+}
+
+func NewDaemon(initialConf *Config) *Daemon {
+	return &Daemon{initialConf}
+}
+
+func (d *Daemon) Loop(ctx context.Context) {
+
+	log := ctx.Value(contextKeyLog).(Logger)
+
+	ctx, cancel := context.WithCancel(ctx)
+
+	sigChan := make(chan os.Signal, 1)
+	finishs := make(chan Job)
+
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+
+	log.Info("starting jobs from config")
+	i := 0
+	for _, job := range d.conf.Jobs {
+		logger := log.WithField(logJobField, job.JobName())
+		logger.Info("starting")
+		i++
+		jobCtx := context.WithValue(ctx, contextKeyLog, logger)
+		go func(j Job) {
+			j.JobStart(jobCtx)
+			finishs <- j
+		}(job)
+	}
+
+	finishCount := 0
+outer:
+	for {
+		select {
+		case <-finishs:
+			finishCount++
+			if finishCount == len(d.conf.Jobs) {
+				log.Info("all jobs finished")
+				break outer
+			}
+
+		case sig := <-sigChan:
+			log.WithField("signal", sig).Info("received signal")
+			log.Info("cancelling all jobs")
+			cancel()
+		}
+	}
+
+	signal.Stop(sigChan)
+	cancel() // make go vet happy
+
+	log.Info("exiting")
+
+}

cmd/handler.go

@@ -2,175 +2,179 @@ package cmd
 
 import (
 	"fmt"
+	"io"
+
+	"github.com/pkg/errors"
 	"github.com/zrepl/zrepl/rpc"
 	"github.com/zrepl/zrepl/zfs"
-	"io"
 )
 
 type DatasetMapping interface {
 	Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error)
 }
 
-type Handler struct {
+type FilesystemRequest struct {
-	Logger          Logger
+	Roots []string // may be nil, indicating interest in all filesystems
-	PullACL         zfs.DatasetFilter
-	SinkMappingFunc func(clientIdentity string) (mapping DatasetMapping, err error)
 }
 
-func (h Handler) HandleFilesystemRequest(r rpc.FilesystemRequest) (roots []*zfs.DatasetPath, err error) {
+type FilesystemVersionsRequest struct {
+	Filesystem *zfs.DatasetPath
+}
 
-	h.Logger.Printf("handling fsr: %#v", r)
+type InitialTransferRequest struct {
+	Filesystem        *zfs.DatasetPath
+	FilesystemVersion zfs.FilesystemVersion
+}
 
-	h.Logger.Printf("using PullACL: %#v", h.PullACL)
+type IncrementalTransferRequest struct {
+	Filesystem *zfs.DatasetPath
+	From       zfs.FilesystemVersion
+	To         zfs.FilesystemVersion
+}
 
-	if roots, err = zfs.ZFSListMapping(h.PullACL); err != nil {
+type Handler struct {
-		h.Logger.Printf("handle fsr err: %v\n", err)
+	logger Logger
+	dsf    zfs.DatasetFilter
+	fsvf   zfs.FilesystemVersionFilter
+}
+
+func NewHandler(logger Logger, dsfilter zfs.DatasetFilter, snapfilter zfs.FilesystemVersionFilter) (h Handler) {
+	return Handler{logger, dsfilter, snapfilter}
+}
+
+func registerEndpoints(server rpc.RPCServer, handler Handler) (err error) {
+	err = server.RegisterEndpoint("FilesystemRequest", handler.HandleFilesystemRequest)
+	if err != nil {
+		panic(err)
+	}
+	err = server.RegisterEndpoint("FilesystemVersionsRequest", handler.HandleFilesystemVersionsRequest)
+	if err != nil {
+		panic(err)
+	}
+	err = server.RegisterEndpoint("InitialTransferRequest", handler.HandleInitialTransferRequest)
+	if err != nil {
+		panic(err)
+	}
+	err = server.RegisterEndpoint("IncrementalTransferRequest", handler.HandleIncrementalTransferRequest)
+	if err != nil {
+		panic(err)
+	}
+	return nil
+}
+
+func (h Handler) HandleFilesystemRequest(r *FilesystemRequest, roots *[]*zfs.DatasetPath) (err error) {
+
+	log := h.logger.WithField("endpoint", "FilesystemRequest")
+
+	log.WithField("request", r).Debug("request")
+	log.WithField("dataset_filter", h.dsf).Debug("dsf")
+
+	allowed, err := zfs.ZFSListMapping(h.dsf)
+	if err != nil {
+		log.WithError(err).Error("error listing filesystems")
 		return
 	}
 
-	h.Logger.Printf("returning: %#v", roots)
+	log.WithField("response", allowed).Debug("response")
-
+	*roots = allowed
 	return
 }
 
-func (h Handler) HandleFilesystemVersionsRequest(r rpc.FilesystemVersionsRequest) (versions []zfs.FilesystemVersion, err error) {
+func (h Handler) HandleFilesystemVersionsRequest(r *FilesystemVersionsRequest, versions *[]zfs.FilesystemVersion) (err error) {
 
-	h.Logger.Printf("handling filesystem versions request: %#v", r)
+	log := h.logger.WithField("endpoint", "FilesystemVersionsRequest")
+
+	log.WithField("request", r).Debug("request")
 
 	// allowed to request that?
-	if h.pullACLCheck(r.Filesystem); err != nil {
+	if h.pullACLCheck(r.Filesystem, nil); err != nil {
+		log.WithError(err).Warn("pull ACL check failed")
 		return
 	}
 
 	// find our versions
-	if versions, err = zfs.ZFSListFilesystemVersions(r.Filesystem, nil); err != nil {
+	vs, err := zfs.ZFSListFilesystemVersions(r.Filesystem, h.fsvf)
-		h.Logger.Printf("our versions error: %#v\n", err)
-		return
-	}
-
-	h.Logger.Printf("our versions: %#v\n", versions)
-	return
-
-}
-
-func (h Handler) HandleInitialTransferRequest(r rpc.InitialTransferRequest) (stream io.Reader, err error) {
-
-	h.Logger.Printf("handling initial transfer request: %#v", r)
-	if err = h.pullACLCheck(r.Filesystem); err != nil {
-		return
-	}
-
-	h.Logger.Printf("invoking zfs send")
-
-	if stream, err = zfs.ZFSSend(r.Filesystem, &r.FilesystemVersion, nil); err != nil {
-		h.Logger.Printf("error sending filesystem: %#v", err)
-	}
-
-	return
-
-}
-
-func (h Handler) HandleIncrementalTransferRequest(r rpc.IncrementalTransferRequest) (stream io.Reader, err error) {
-
-	h.Logger.Printf("handling incremental transfer request: %#v", r)
-	if err = h.pullACLCheck(r.Filesystem); err != nil {
-		return
-	}
-
-	h.Logger.Printf("invoking zfs send")
-
-	if stream, err = zfs.ZFSSend(r.Filesystem, &r.From, &r.To); err != nil {
-		h.Logger.Printf("error sending filesystem: %#v", err)
-	}
-
-	return
-
-}
-
-func (h Handler) HandleResumeTransferRequest(r rpc.ResumeTransferRequest) (stream io.Reader, err error) {
-	// decode receive_resume_token: zfs send -nvP <token>
-	// A) use exit code to determine if could send (exit != 0 means could not send)
-	// B) check ACL if the filesystem in toname field (it is a snapshot so strip its snapshot name first)
-	//    is allowed for the given client
-	/*
-
-			# zfs send -nvt will print nvlist contents on ZoL and FreeBSD, should be good enough
-			# will print regardless of whether it can send or not -> good for us
-			# need clean way to extract that...
-				# expect 'resume token contents:\nnvlist version: 0\n'
-				# parse everything after that is tab-indented as key value pairs separated by = sign
-				# => return dict
-
-			zfs send -nv -t 1-c6491acbe-c8-789c636064000310a500c4ec50360710e72765a526973030b0419460caa7a515a79630c001489e0d493ea9b224b5182451885d7f497e7a69660a0343f79b1b9a8a2b3db65b20c97382e5f312735319188a4bf28b12d353f5931293b34b0b8af5ab8a520b72f4d3f2f31d8a53f35220660300c1091dbe
-		resume token contents:
-		nvlist version: 0
-		        object = 0x6
-		        offset = 0x0
-		        bytes = 0x7100
-		        toguid = 0xb748a92129d8ec8b
-		        toname = storage/backups/zrepl/foo@send
-		cannot resume send: 'storage/backups/zrepl/foo@send' used in the initial send no longer exists
-
-		zfs send -nvt 1-ebbbbea7e-f0-789c636064000310a501c49c50360710a715e5e7a69766a6304041f79b1b9a8a2b3db62b00d9ec48eaf293b252934b181858a0ea30e4d3d28a534b18e00024cf86249f5459925a0ca44fc861d75f920f71c59c5fdf6f7b3eea32b24092e704cbe725e6a632301497e41725a6a7ea27252667971614eb5715a516e4e8a7e5e73b14a7e6a51881cd06005a222749
-		resume token contents:
-		nvlist version: 0
-		        fromguid = 0xb748a92129d8ec8b #NOTE the fromguid field which is only in this one, so don't hardcode
-		        object = 0x4
-		        offset = 0x0
-		        bytes = 0x1ec8
-		        toguid = 0x328ae249dbf7fa9c
-		        toname = storage/backups/zrepl/foo@send2
-		send from storage/backups/zrepl/foo@send to storage/backups/zrepl/foo@send2 estimated size is 1.02M
-
-	*/
-	panic("not implemented")
-	return nil, nil
-}
-
-func (h Handler) HandlePullMeRequest(r rpc.PullMeRequest, clientIdentity string, client rpc.RPCRequester) (err error) {
-
-	// Check if we have a sink for this request
-	// Use that mapping to do what happens in doPull
-
-	h.Logger.Printf("handling PullMeRequest: %#v", r)
-
-	var sinkMapping DatasetMapping
-	sinkMapping, err = h.SinkMappingFunc(clientIdentity)
 	if err != nil {
-		h.Logger.Printf("no sink mapping for client identity '%s', denying PullMeRequest", clientIdentity)
+		log.WithError(err).Error("cannot list filesystem versions")
-		err = fmt.Errorf("no sink for client identity '%s'", clientIdentity)
 		return
 	}
 
-	h.Logger.Printf("doing pull...")
+	log.WithField("resposne", vs).Debug("response")
-
-	err = doPull(PullContext{
-		Remote:            client,
-		Log:               h.Logger,
-		Mapping:           sinkMapping,
-		InitialReplPolicy: r.InitialReplPolicy,
-	})
-	if err != nil {
-		h.Logger.Printf("PullMeRequest failed with error: %s", err)
-		return
-	}
-
-	h.Logger.Printf("finished handling PullMeRequest: %#v", r)
 
+	*versions = vs
 	return
+
 }
 
-func (h Handler) pullACLCheck(p *zfs.DatasetPath) (err error) {
+func (h Handler) HandleInitialTransferRequest(r *InitialTransferRequest, stream *io.Reader) (err error) {
-	var allowed bool
+
-	allowed, err = h.PullACL.Filter(p)
+	log := h.logger.WithField("endpoint", "InitialTransferRequest")
+
+	log.WithField("request", r).Debug("request")
+	if err = h.pullACLCheck(r.Filesystem, &r.FilesystemVersion); err != nil {
+		log.WithError(err).Warn("pull ACL check failed")
+		return
+	}
+
+	log.Debug("invoking zfs send")
+
+	s, err := zfs.ZFSSend(r.Filesystem, &r.FilesystemVersion, nil)
+	if err != nil {
+		log.WithError(err).Error("cannot send filesystem")
+	}
+	*stream = s
+
+	return
+
+}
+
+func (h Handler) HandleIncrementalTransferRequest(r *IncrementalTransferRequest, stream *io.Reader) (err error) {
+
+	log := h.logger.WithField("endpoint", "IncrementalTransferRequest")
+	log.WithField("request", r).Debug("request")
+	if err = h.pullACLCheck(r.Filesystem, &r.From); err != nil {
+		log.WithError(err).Warn("pull ACL check failed")
+		return
+	}
+	if err = h.pullACLCheck(r.Filesystem, &r.To); err != nil {
+		log.WithError(err).Warn("pull ACL check failed")
+		return
+	}
+
+	log.Debug("invoking zfs send")
+
+	s, err := zfs.ZFSSend(r.Filesystem, &r.From, &r.To)
+	if err != nil {
+		log.WithError(err).Error("cannot send filesystem")
+	}
+
+	*stream = s
+	return
+
+}
+
+func (h Handler) pullACLCheck(p *zfs.DatasetPath, v *zfs.FilesystemVersion) (err error) {
+	var fsAllowed, vAllowed bool
+	fsAllowed, err = h.dsf.Filter(p)
 	if err != nil {
 		err = fmt.Errorf("error evaluating ACL: %s", err)
-		h.Logger.Printf(err.Error())
 		return
 	}
-	if !allowed {
+	if !fsAllowed {
 		err = fmt.Errorf("ACL prohibits access to %s", p.ToString())
-		h.Logger.Printf(err.Error())
+		return
+	}
+	if v == nil {
+		return
+	}
+
+	vAllowed, err = h.fsvf.Filter(*v)
+	if err != nil {
+		err = errors.Wrap(err, "error evaluating version filter")
+		return
+	}
+	if !vAllowed {
+		err = fmt.Errorf("ACL prohibits access to %s", v.ToAbsPath(p))
 		return
 	}
 	return

cmd/helpers.go

@@ -0,0 +1,35 @@
+package cmd
+
+import (
+	"github.com/pkg/errors"
+	"net"
+	"os"
+	"path/filepath"
+)
+
+func ListenUnixPrivate(sockaddr *net.UnixAddr) (*net.UnixListener, error) {
+
+	sockdir := filepath.Dir(sockaddr.Name)
+	sdstat, err := os.Stat(sockdir)
+	if err != nil {
+		return nil, errors.Wrapf(err, "cannot stat(2) '%s'", sockdir)
+	}
+	if !sdstat.IsDir() {
+		return nil, errors.Errorf("not a directory: %s", sockdir)
+	}
+	p := sdstat.Mode().Perm()
+	if p&0007 != 0 {
+		return nil, errors.Errorf("socket directory not be world-accessible: %s (permissions are %#o)", sockdir, p)
+	}
+
+	// Maybe things have not been cleaned up before
+	s, err := os.Stat(sockaddr.Name)
+	if err == nil {
+		if s.Mode()&os.ModeSocket != 0 {
+			// opportunistically try to remove it, but if this fails, it is not an error
+			os.Remove(sockaddr.Name)
+		}
+	}
+
+	return net.ListenUnix("unix", sockaddr)
+}

cmd/logging_formatters.go

@@ -0,0 +1,174 @@
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/go-logfmt/logfmt"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/logger"
+	"strings"
+	"time"
+)
+
+type EntryFormatter interface {
+	Format(e *logger.Entry) ([]byte, error)
+}
+
+const (
+	FieldLevel   = "level"
+	FieldMessage = "msg"
+	FieldTime    = "time"
+)
+
+const (
+	logJobField     string = "job"
+	logTaskField    string = "task"
+	logFSField      string = "filesystem"
+	logMapFromField string = "map_from"
+	logMapToField   string = "map_to"
+	logIncFromField string = "inc_from"
+	logIncToField   string = "inc_to"
+)
+
+type NoFormatter struct{}
+
+func (f NoFormatter) Format(e *logger.Entry) ([]byte, error) {
+	return []byte(e.Message), nil
+}
+
+type HumanFormatter struct {
+	NoMetadata bool
+}
+
+var _ SetNoMetadataFormatter = &HumanFormatter{}
+
+func (f *HumanFormatter) SetNoMetadata(noMetadata bool) {
+	f.NoMetadata = noMetadata
+}
+
+func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
+
+	var line bytes.Buffer
+
+	if !f.NoMetadata {
+		fmt.Fprintf(&line, "[%s]", e.Level.Short())
+	}
+
+	prefixFields := []string{logJobField, logTaskField, logFSField}
+	prefixed := make(map[string]bool, len(prefixFields)+2)
+	for _, field := range prefixFields {
+		val, ok := e.Fields[field].(string)
+		if ok {
+			fmt.Fprintf(&line, "[%s]", val)
+			prefixed[field] = true
+		} else {
+			break
+		}
+	}
+	// even more prefix fields
+	mapFrom, mapFromOk := e.Fields[logMapFromField].(string)
+	mapTo, mapToOk := e.Fields[logMapToField].(string)
+	if mapFromOk && mapToOk {
+		fmt.Fprintf(&line, "[%s => %s]", mapFrom, mapTo)
+		prefixed[logMapFromField], prefixed[logMapToField] = true, true
+	}
+	incFrom, incFromOk := e.Fields[logIncFromField].(string)
+	incTo, incToOk := e.Fields[logIncToField].(string)
+	if incFromOk && incToOk {
+		fmt.Fprintf(&line, "[%s => %s]", incFrom, incTo)
+		prefixed[logIncFromField], prefixed[logIncToField] = true, true
+	}
+
+	if line.Len() > 0 {
+		fmt.Fprint(&line, ": ")
+	}
+	fmt.Fprint(&line, e.Message)
+
+	for field, value := range e.Fields {
+
+		if prefixed[field] {
+			continue
+		}
+
+		if strings.ContainsAny(field, " \t") {
+			return nil, errors.Errorf("field must not contain whitespace: '%s'", field)
+		}
+		fmt.Fprintf(&line, " %s=\"%s\"", field, value)
+	}
+
+	return line.Bytes(), nil
+}
+
+type JSONFormatter struct{}
+
+func (f *JSONFormatter) Format(e *logger.Entry) ([]byte, error) {
+	data := make(logger.Fields, len(e.Fields)+3)
+	for k, v := range e.Fields {
+		switch v := v.(type) {
+		case error:
+			// Otherwise errors are ignored by `encoding/json`
+			// https://github.com/sirupsen/logrus/issues/137
+			data[k] = v.Error()
+		default:
+			_, err := json.Marshal(v)
+			if err != nil {
+				return nil, errors.Errorf("field is not JSON encodable: %s", k)
+			}
+			data[k] = v
+		}
+	}
+
+	data[FieldMessage] = e.Message
+	data[FieldTime] = e.Time.Format(time.RFC3339)
+	data[FieldLevel] = e.Level
+
+	return json.Marshal(data)
+
+}
+
+type LogfmtFormatter struct {
+	NoMetadata bool
+}
+
+var _ SetNoMetadataFormatter = &LogfmtFormatter{}
+
+func (f *LogfmtFormatter) SetNoMetadata(noMetadata bool) {
+	f.NoMetadata = noMetadata
+}
+
+func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
+	var buf bytes.Buffer
+	enc := logfmt.NewEncoder(&buf)
+
+	if !f.NoMetadata {
+		enc.EncodeKeyval(FieldTime, e.Time)
+		enc.EncodeKeyval(FieldLevel, e.Level)
+	}
+
+	// at least try and put job and task in front
+	prefixed := make(map[string]bool, 2)
+	prefix := []string{logJobField, logTaskField}
+	for _, pf := range prefix {
+		v, ok := e.Fields[pf]
+		if !ok {
+			break
+		}
+		enc.EncodeKeyval(pf, v)
+		prefixed[pf] = true
+	}
+
+	enc.EncodeKeyval(FieldMessage, e.Message)
+
+	for k, v := range e.Fields {
+		if !prefixed[k] {
+			enc.EncodeKeyval(k, v)
+		}
+	}
+
+	if err := enc.EndRecord(); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}

cmd/logging_outlets.go

@@ -0,0 +1,124 @@
+package cmd
+
+import (
+	"context"
+	"crypto/tls"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/logger"
+	"io"
+	"log/syslog"
+	"net"
+	"time"
+)
+
+type WriterOutlet struct {
+	Formatter EntryFormatter
+	Writer    io.Writer
+}
+
+func (h WriterOutlet) WriteEntry(ctx context.Context, entry logger.Entry) error {
+	bytes, err := h.Formatter.Format(&entry)
+	if err != nil {
+		return err
+	}
+	_, err = h.Writer.Write(bytes)
+	h.Writer.Write([]byte("\n"))
+	return err
+}
+
+type TCPOutlet struct {
+	Formatter    EntryFormatter
+	Net, Address string
+	Dialer       net.Dialer
+	TLS          *tls.Config
+	// Specifies how much time must pass between a connection error and a reconnection attempt
+	// Log entries written to the outlet during this time interval are silently dropped.
+	RetryInterval time.Duration
+	// nil if there was an error sending / connecting to remote server
+	conn net.Conn
+	// Last time an error occurred when sending / connecting to remote server
+	retry time.Time
+}
+
+func (h *TCPOutlet) WriteEntry(ctx context.Context, e logger.Entry) error {
+
+	b, err := h.Formatter.Format(&e)
+	if err != nil {
+		return err
+	}
+
+	if h.conn == nil {
+		if time.Now().Sub(h.retry) < h.RetryInterval {
+			// cool-down phase, drop the log entry
+			return nil
+		}
+
+		if h.TLS != nil {
+			h.conn, err = tls.DialWithDialer(&h.Dialer, h.Net, h.Address, h.TLS)
+		} else {
+			h.conn, err = h.Dialer.DialContext(ctx, h.Net, h.Address)
+		}
+		if err != nil {
+			h.conn = nil
+			h.retry = time.Now()
+			return errors.Wrap(err, "cannot dial")
+		}
+	}
+
+	_, err = h.conn.Write(b)
+	if err == nil {
+		_, err = h.conn.Write([]byte("\n"))
+	}
+	if err != nil {
+		h.conn.Close()
+		h.conn = nil
+		h.retry = time.Now()
+		return errors.Wrap(err, "cannot write")
+	}
+
+	return nil
+}
+
+type SyslogOutlet struct {
+	Formatter          EntryFormatter
+	RetryInterval      time.Duration
+	writer             *syslog.Writer
+	lastConnectAttempt time.Time
+}
+
+func (o *SyslogOutlet) WriteEntry(ctx context.Context, entry logger.Entry) error {
+
+	bytes, err := o.Formatter.Format(&entry)
+	if err != nil {
+		return err
+	}
+
+	s := string(bytes)
+
+	if o.writer == nil {
+		now := time.Now()
+		if now.Sub(o.lastConnectAttempt) < o.RetryInterval {
+			return nil // not an error toward logger
+		}
+		o.writer, err = syslog.New(syslog.LOG_LOCAL0, "zrepl")
+		o.lastConnectAttempt = time.Now()
+		if err != nil {
+			o.writer = nil
+			return err
+		}
+	}
+
+	switch entry.Level {
+	case logger.Debug:
+		return o.writer.Debug(s)
+	case logger.Info:
+		return o.writer.Info(s)
+	case logger.Warn:
+		return o.writer.Warning(s)
+	case logger.Error:
+		return o.writer.Err(s)
+	default:
+		return o.writer.Err(s) // write as error as reaching this case is in fact an error
+	}
+
+}

cmd/main.go

@@ -11,29 +11,16 @@
 package cmd
 
 import (
-	"fmt"
 	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/jobrun"
+	"github.com/zrepl/zrepl/logger"
-	"golang.org/x/sys/unix"
-	"io"
-	golog "log"
-	"net/http"
-	_ "net/http/pprof"
-	"os"
 )
 
-type Logger interface {
+//
-	Printf(format string, v ...interface{})
+//type Logger interface {
-}
+//	Printf(format string, v ...interface{})
+//}
 
-// global state / facilities
+type Logger = *logger.Logger
-var (
-	conf     Config
-	runner   *jobrun.JobRunner
-	logFlags int = golog.LUTC | golog.Ldate | golog.Ltime
-	logOut   io.Writer
-	log      Logger
-)
 
 var RootCmd = &cobra.Command{
 	Use:   "zrepl",
@@ -48,57 +35,9 @@ var RootCmd = &cobra.Command{
 
 var rootArgs struct {
 	configFile string
-	stderrFile string
-	httpPprof  string
 }
 
 func init() {
-	cobra.OnInitialize(initConfig)
+	//cobra.OnInitialize(initConfig)
 	RootCmd.PersistentFlags().StringVar(&rootArgs.configFile, "config", "", "config file path")
-	RootCmd.PersistentFlags().StringVar(&rootArgs.stderrFile, "stderrFile", "-", "redirect stderr to given path")
-	RootCmd.PersistentFlags().StringVar(&rootArgs.httpPprof, "debug.pprof.http", "", "run pprof http server on given port")
-}
-
-func initConfig() {
-
-	// Logging & stderr redirection
-	if rootArgs.stderrFile != "-" {
-		file, err := os.OpenFile(rootArgs.stderrFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
-		if err != nil {
-			return
-		}
-
-		if err = unix.Dup2(int(file.Fd()), int(os.Stderr.Fd())); err != nil {
-			file.WriteString(fmt.Sprintf("error redirecting stderr file %s: %s\n", rootArgs.stderrFile, err))
-			return
-		}
-		logOut = file
-	} else {
-		logOut = os.Stderr
-	}
-
-	log = golog.New(logOut, "", logFlags)
-
-	// CPU profiling
-	if rootArgs.httpPprof != "" {
-		go func() {
-			http.ListenAndServe(rootArgs.httpPprof, nil)
-		}()
-	}
-
-	// Config
-	if rootArgs.configFile == "" {
-		log.Printf("config file not set")
-		os.Exit(1)
-	}
-	var err error
-	if conf, err = ParseConfig(rootArgs.configFile); err != nil {
-		log.Printf("error parsing config: %s", err)
-		os.Exit(1)
-	}
-
-	jobrunLogger := golog.New(os.Stderr, "jobrun ", logFlags)
-	runner = jobrun.NewJobRunner(jobrunLogger)
-	return
-
 }

cmd/prune.go

@@ -1,139 +1,103 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/util"
 	"github.com/zrepl/zrepl/zfs"
-	"os"
-	"sort"
 	"time"
 )
 
-var pruneArgs struct {
+type Pruner struct {
-	job    string
+	Now            time.Time
-	dryRun bool
+	DryRun         bool
+	DatasetFilter  zfs.DatasetFilter
+	SnapshotPrefix string
+	PrunePolicy    PrunePolicy
 }
 
-var PruneCmd = &cobra.Command{
+type PruneResult struct {
-	Use:   "prune",
+	Filesystem *zfs.DatasetPath
-	Short: "perform pruning",
+	All        []zfs.FilesystemVersion
-	Run:   cmdPrune,
+	Keep       []zfs.FilesystemVersion
+	Remove     []zfs.FilesystemVersion
 }
 
-func init() {
+func (p *Pruner) Run(ctx context.Context) (r []PruneResult, err error) {
-	PruneCmd.Flags().StringVar(&pruneArgs.job, "job", "", "job to run")
-	PruneCmd.Flags().BoolVarP(&pruneArgs.dryRun, "dryrun", "n", false, "dry run")
-	RootCmd.AddCommand(PruneCmd)
-}
 
-func cmdPrune(cmd *cobra.Command, args []string) {
+	log := ctx.Value(contextKeyLog).(Logger)
 
-	if len(args) < 1 {
+	if p.DryRun {
-		log.Printf("must specify exactly one job as positional argument")
+		log.Info("doing dry run")
-		os.Exit(1)
 	}
 
-	job, ok := conf.Prunes[args[0]]
+	filesystems, err := zfs.ZFSListMapping(p.DatasetFilter)
-	if !ok {
-		log.Printf("could not find prune job: %s", args[0])
-		os.Exit(1)
-	}
-
-	log.Printf("Beginning prune job:\n%s", job)
-	ctx := PruneContext{job, time.Now(), pruneArgs.dryRun}
-	err := doPrune(ctx, log)
 	if err != nil {
-		log.Printf("Prune job failed with error: %s", err)
+		log.WithError(err).Error("error applying filesystem filter")
-		os.Exit(1)
+		return nil, err
+	}
+	if len(filesystems) <= 0 {
+		log.Info("no filesystems matching filter")
+		return nil, err
 	}
 
-}
+	r = make([]PruneResult, 0, len(filesystems))
-
-type PruneContext struct {
-	Prune  *Prune
-	Now    time.Time
-	DryRun bool
-}
-
-type retentionGridAdaptor struct {
-	zfs.FilesystemVersion
-}
-
-func (a retentionGridAdaptor) Date() time.Time {
-	return a.Creation
-}
-
-func (a retentionGridAdaptor) LessThan(b util.RetentionGridEntry) bool {
-	return a.CreateTXG < b.(retentionGridAdaptor).CreateTXG
-}
-
-func doPrune(ctx PruneContext, log Logger) error {
-
-	if ctx.DryRun {
-		log.Printf("doing dry run")
-	}
-
-	prune := ctx.Prune
-
-	// ZFSListSnapsFiltered --> todo can extend fsfilter or need new? Have already something per fs
-	// Dedicated snapshot object? Adaptor object to FilesystemVersion?
-
-	filesystems, err := zfs.ZFSListMapping(prune.DatasetFilter)
-	if err != nil {
-		return fmt.Errorf("error applying filesystem filter: %s", err)
-	}
 
 	for _, fs := range filesystems {
 
-		fsversions, err := zfs.ZFSListFilesystemVersions(fs, prune.SnapshotFilter)
+		log := log.WithField(logFSField, fs.ToString())
-		if err != nil {
-			return fmt.Errorf("error listing filesytem versions of %s: %s", fs, err)
-		}
 
+		fsversions, err := zfs.ZFSListFilesystemVersions(fs, &PrefixSnapshotFilter{p.SnapshotPrefix})
+		if err != nil {
+			log.WithError(err).Error("error listing filesytem versions")
+			continue
+		}
 		if len(fsversions) == 0 {
+			log.WithField("prefix", p.SnapshotPrefix).Info("no filesystem versions matching prefix")
 			continue
 		}
 
-		adaptors := make([]util.RetentionGridEntry, len(fsversions))
+		keep, remove, err := p.PrunePolicy.Prune(fs, fsversions)
-		for fsv := range fsversions {
+		if err != nil {
-			adaptors[fsv] = retentionGridAdaptor{fsversions[fsv]}
+			log.WithError(err).Error("error evaluating prune policy")
+			continue
 		}
 
-		sort.SliceStable(adaptors, func(i, j int) bool {
+		log.WithField("fsversions", fsversions).
-			return adaptors[i].LessThan(adaptors[j])
+			WithField("keep", keep).
-		})
+			WithField("remove", remove).
-		ctx.Now = adaptors[len(adaptors)-1].Date()
+			Debug("prune policy debug dump")
 
-		describe := func(a retentionGridAdaptor) string {
+		r = append(r, PruneResult{fs, fsversions, keep, remove})
-			timeSince := a.Creation.Sub(ctx.Now)
+
+		makeFields := func(v zfs.FilesystemVersion) (fields map[string]interface{}) {
+			fields = make(map[string]interface{})
+			fields["version"] = v.ToAbsPath(fs)
+			timeSince := v.Creation.Sub(p.Now)
+			fields["age_ns"] = timeSince
 			const day time.Duration = 24 * time.Hour
 			days := timeSince / day
 			remainder := timeSince % day
-			return fmt.Sprintf("%s@%dd%s from now", a.ToAbsPath(fs), days, remainder)
+			fields["age_str"] = fmt.Sprintf("%dd%s", days, remainder)
+			return
 		}
 
-		keep, remove := prune.RetentionPolicy.FitEntries(ctx.Now, adaptors)
+		for _, v := range remove {
-		for _, a := range remove {
+			fields := makeFields(v)
-			r := a.(retentionGridAdaptor)
+			log.WithFields(fields).Info("destroying version")
-			log.Printf("remove %s", describe(r))
+			// echo what we'll do and exec zfs destroy if not dry run
-			// do echo what we'll do and exec zfs destroy if not dry run
+			// TODO special handling for EBUSY (zfs hold)
-			// special handling for EBUSY (zfs hold)
+			// TODO error handling for clones? just echo to cli, skip over, and exit with non-zero status code (we're idempotent)
-			// error handling for clones? just echo to cli, skip over, and exit with non-zero status code (we're idempotent)
+			if !p.DryRun {
-			if !ctx.DryRun {
+				err := zfs.ZFSDestroyFilesystemVersion(fs, v)
-				err := zfs.ZFSDestroy(r.ToAbsPath(fs))
 				if err != nil {
 					// handle
-					log.Printf("error: %s", err)
+					log.WithFields(fields).WithError(err).Error("error destroying version")
 				}
 			}
 		}
-		for _, a := range keep {
+
-			r := a.(retentionGridAdaptor)
-			log.Printf("would keep %s", describe(r))
-		}
 	}
 
-	return nil
+	return
 
 }

cmd/replication.go

@@ -3,221 +3,38 @@ package cmd
 import (
 	"fmt"
 	"io"
-	"os"
-	"sync"
 	"time"
 
-	"github.com/spf13/cobra"
+	"bytes"
-	"github.com/zrepl/zrepl/jobrun"
+	"encoding/json"
 	"github.com/zrepl/zrepl/rpc"
 	"github.com/zrepl/zrepl/util"
 	"github.com/zrepl/zrepl/zfs"
 )
 
-var runArgs struct {
-	job  string
-	once bool
-}
-
-var RunCmd = &cobra.Command{
-	Use:   "run",
-	Short: "run push & pull replication",
-	Run:   cmdRun,
-}
-
-var PushCmd = &cobra.Command{
-	Use:   "push",
-	Short: "run push job (first positional argument)",
-	Run:   cmdPush,
-}
-
-var PullCmd = &cobra.Command{
-	Use:   "pull",
-	Short: "run pull job (first positional argument)",
-	Run:   cmdPull,
-}
-
-func init() {
-	RootCmd.AddCommand(RunCmd)
-	RunCmd.Flags().BoolVar(&runArgs.once, "once", false, "run jobs only once, regardless of configured repeat behavior")
-	RunCmd.Flags().StringVar(&runArgs.job, "job", "", "run only the given job")
-
-	RootCmd.AddCommand(PushCmd)
-	RootCmd.AddCommand(PullCmd)
-}
-
-func cmdPush(cmd *cobra.Command, args []string) {
-
-	if len(args) != 1 {
-		log.Printf("must specify exactly one job as positional argument")
-		os.Exit(1)
-	}
-	job, ok := conf.Pushs[args[0]]
-	if !ok {
-		log.Printf("could not find push job %s", args[0])
-		os.Exit(1)
-	}
-	if err := jobPush(job, log); err != nil {
-		log.Printf("error doing push: %s", err)
-		os.Exit(1)
-	}
-
-}
-
-func cmdPull(cmd *cobra.Command, args []string) {
-
-	if len(args) != 1 {
-		log.Printf("must specify exactly one job as positional argument")
-		os.Exit(1)
-	}
-	job, ok := conf.Pulls[args[0]]
-	if !ok {
-		log.Printf("could not find pull job %s", args[0])
-		os.Exit(1)
-	}
-
-	if err := jobPull(job, log); err != nil {
-		log.Printf("error doing pull: %s", err)
-		os.Exit(1)
-	}
-
-}
-
-func cmdRun(cmd *cobra.Command, args []string) {
-
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		runner.Start()
-	}()
-
-	jobs := make([]jobrun.Job, len(conf.Pulls)+len(conf.Pushs))
-	i := 0
-	for _, pull := range conf.Pulls {
-		jobs[i] = jobrun.Job{
-			Name:           fmt.Sprintf("pull.%d", i),
-			RepeatStrategy: pull.RepeatStrategy,
-			RunFunc: func(log jobrun.Logger) error {
-				log.Printf("doing pull: %v", pull)
-				return jobPull(pull, log)
-			},
-		}
-		i++
-	}
-	for _, push := range conf.Pushs {
-		jobs[i] = jobrun.Job{
-			Name:           fmt.Sprintf("push.%d", i),
-			RepeatStrategy: push.RepeatStrategy,
-			RunFunc: func(log jobrun.Logger) error {
-				log.Printf("doing push: %v", push)
-				return jobPush(push, log)
-			},
-		}
-		i++
-	}
-
-	for _, j := range jobs {
-		if runArgs.once {
-			j.RepeatStrategy = jobrun.NoRepeatStrategy{}
-		}
-		if runArgs.job != "" {
-			if runArgs.job == j.Name {
-				runner.AddJob(j)
-				break
-			}
-			continue
-		}
-		runner.AddJob(j)
-	}
-
-	for {
-		select {
-		case job := <-runner.NotificationChan():
-			log.Printf("job %s reported error: %v\n", job.Name, job.LastError)
-		}
-	}
-
-	wg.Wait()
-
-}
-
 type localPullACL struct{}
 
 func (a localPullACL) Filter(p *zfs.DatasetPath) (pass bool, err error) {
 	return true, nil
 }
 
-func jobPull(pull *Pull, log jobrun.Logger) (err error) {
+const LOCAL_TRANSPORT_IDENTITY string = "local"
 
-	if lt, ok := pull.From.Transport.(LocalTransport); ok {
+const DEFAULT_INITIAL_REPL_POLICY = InitialReplPolicyMostRecent
 
-		lt.SetHandler(Handler{
+type InitialReplPolicy string
-			Logger: log,
-			// Allow access to any dataset since we control what mapping
-			// is passed to the pull routine.
-			// All local datasets will be passed to its Map() function,
-			// but only those for which a mapping exists will actually be pulled.
-			// We can pay this small performance penalty for now.
-			PullACL: localPullACL{},
-		})
-		pull.From.Transport = lt
-		log.Printf("fixing up local transport: %#v", pull.From.Transport)
-	}
 
-	var remote rpc.RPCRequester
+const (
+	InitialReplPolicyMostRecent InitialReplPolicy = "most_recent"
+	InitialReplPolicyAll        InitialReplPolicy = "all"
+)
 
-	if remote, err = pull.From.Transport.Connect(log); err != nil {
+func closeRPCWithTimeout(log Logger, remote rpc.RPCClient, timeout time.Duration, goodbye string) {
-		return
+	log.Info("closing rpc connection")
-	}
-
-	defer closeRPCWithTimeout(log, remote, time.Second*10, "")
-
-	return doPull(PullContext{remote, log, pull.Mapping, pull.InitialReplPolicy})
-}
-
-func jobPush(push *Push, log jobrun.Logger) (err error) {
-
-	if _, ok := push.To.Transport.(LocalTransport); ok {
-		panic("no support for local pushs")
-	}
-
-	var remote rpc.RPCRequester
-	if remote, err = push.To.Transport.Connect(log); err != nil {
-		return err
-	}
-
-	defer closeRPCWithTimeout(log, remote, time.Second*10, "")
-
-	log.Printf("building handler for PullMeRequest")
-	handler := Handler{
-		Logger:          log,
-		PullACL:         push.Filter,
-		SinkMappingFunc: nil, // no need for that in the handler for PullMe
-	}
-	log.Printf("handler: %#v", handler)
-
-	r := rpc.PullMeRequest{
-		InitialReplPolicy: push.InitialReplPolicy,
-	}
-	log.Printf("doing PullMeRequest: %#v", r)
-
-	if err = remote.PullMeRequest(r, handler); err != nil {
-		log.Printf("PullMeRequest failed: %s", err)
-		return
-	}
-
-	log.Printf("push job finished")
-	return
-
-}
-
-func closeRPCWithTimeout(log Logger, remote rpc.RPCRequester, timeout time.Duration, goodbye string) {
-	log.Printf("closing rpc connection")
 
 	ch := make(chan error)
 	go func() {
-		ch <- remote.CloseRequest(rpc.CloseRequest{goodbye})
+		ch <- remote.Close()
 		close(ch)
 	}()
 
@@ -230,20 +47,16 @@ func closeRPCWithTimeout(log Logger, remote rpc.RPCRequester, timeout time.Durat
 	}
 
 	if err != nil {
-		log.Printf("error closing connection: %s", err)
+		log.WithError(err).Error("error closing connection")
-		err = remote.ForceClose()
-		if err != nil {
-			log.Printf("error force-closing connection: %s", err)
-		}
 	}
 	return
 }
 
 type PullContext struct {
-	Remote            rpc.RPCRequester
+	Remote            rpc.RPCClient
 	Log               Logger
 	Mapping           DatasetMapping
-	InitialReplPolicy rpc.InitialReplPolicy
+	InitialReplPolicy InitialReplPolicy
 }
 
 func doPull(pull PullContext) (err error) {
@@ -251,14 +64,14 @@ func doPull(pull PullContext) (err error) {
 	remote := pull.Remote
 	log := pull.Log
 
-	log.Printf("requesting remote filesystem list")
+	log.Info("request remote filesystem list")
-	fsr := rpc.FilesystemRequest{}
+	fsr := FilesystemRequest{}
 	var remoteFilesystems []*zfs.DatasetPath
-	if remoteFilesystems, err = remote.FilesystemRequest(fsr); err != nil {
+	if err = remote.Call("FilesystemRequest", &fsr, &remoteFilesystems); err != nil {
 		return
 	}
 
-	log.Printf("map remote filesystems to local paths and determine order for per-filesystem sync")
+	log.Debug("map remote filesystems to local paths and determine order for per-filesystem sync")
 	type RemoteLocalMapping struct {
 		Remote *zfs.DatasetPath
 		Local  *zfs.DatasetPath
@@ -270,39 +83,42 @@ func doPull(pull PullContext) (err error) {
 		var localFs *zfs.DatasetPath
 		localFs, err = pull.Mapping.Map(remoteFilesystems[fs])
 		if err != nil {
-			if err != NoMatchError {
+			err := fmt.Errorf("error mapping %s: %s", remoteFilesystems[fs], err)
-				err := fmt.Errorf("error mapping %s: %s", remoteFilesystems[fs], err)
+			log.WithError(err).WithField(logMapFromField, remoteFilesystems[fs]).Error("cannot map")
-				log.Printf("%s", err)
+			return err
-				return err
+		}
-			}
+		if localFs == nil {
 			continue
 		}
-		log.Printf("%s => %s", remoteFilesystems[fs].ToString(), localFs.ToString())
+		log.WithField(logMapFromField, remoteFilesystems[fs].ToString()).
+			WithField(logMapToField, localFs.ToString()).Debug("mapping")
 		m := RemoteLocalMapping{remoteFilesystems[fs], localFs}
 		replMapping[m.Local.ToString()] = m
 		localTraversal.Add(m.Local)
 	}
 
-	log.Printf("build cache for already present local filesystem state")
+	log.Debug("build cache for already present local filesystem state")
 	localFilesystemState, err := zfs.ZFSListFilesystemState()
 	if err != nil {
-		log.Printf("error requesting local filesystem state: %s", err)
+		log.WithError(err).Error("cannot request local filesystem state")
 		return err
 	}
 
-	log.Printf("start per-filesystem sync")
+	log.Info("start per-filesystem sync")
 	localTraversal.WalkTopDown(func(v zfs.DatasetPathVisit) bool {
 
+		log := log.WithField(logFSField, v.Path.ToString())
+
 		if v.FilledIn {
 			if _, exists := localFilesystemState[v.Path.ToString()]; exists {
 				// No need to verify if this is a placeholder or not. It is sufficient
 				// to know we can add child filesystems to it
 				return true
 			}
-			log.Printf("creating placeholder filesystem %s", v.Path.ToString())
+			log.Debug("create placeholder filesystem")
 			err = zfs.ZFSCreatePlaceholderFilesystem(v.Path)
 			if err != nil {
-				err = fmt.Errorf("aborting, cannot create placeholder filesystem %s: %s", v.Path, err)
+				log.Error("cannot create placeholder filesystem")
 				return false
 			}
 			return true
@@ -313,122 +129,40 @@ func doPull(pull PullContext) (err error) {
 			panic("internal inconsistency: replMapping should contain mapping for any path that was not filled in by WalkTopDown()")
 		}
 
-		log := func(format string, args ...interface{}) {
+		log = log.WithField(logMapToField, m.Remote.ToString()).
-			log.Printf("[%s => %s]: %s", m.Remote.ToString(), m.Local.ToString(), fmt.Sprintf(format, args...))
+			WithField(logMapFromField, m.Local.ToString())
-		}
 
-		log("examing local filesystem state")
+		log.Debug("examing local filesystem state")
 		localState, localExists := localFilesystemState[m.Local.ToString()]
 		var versions []zfs.FilesystemVersion
 		switch {
 		case !localExists:
-			log("local filesystem does not exist")
+			log.Info("local filesystem does not exist")
 		case localState.Placeholder:
-			log("local filesystem is marked as placeholder")
+			log.Info("local filesystem is marked as placeholder")
-		case localState.ResumeToken != "":
-
-			log("local filesystem has receive_resume_token")
-			if false { // TODO Check if managed resume is disabled (explain information leakage in docs!)
-				log("managed resume tokens are disabled via zrepl config, assuming config change or external administrative action")
-				log("policy forbids aborting the partial recv automatically, skipping this filesystem")
-				log("for zrepl to resume replication for this dataset, administrators should finishing the recv manually or abort the recv via `zfs recv -A %s`", m.Local.ToString())
-				return true // TODO right choice to allow children? ... should be, since the fs exists...
-			}
-
-			log("decoding receive_resume_token")
-			// TODO, see handler comments
-			// TODO override logger with fromguid toguid
-
-			log("requesting resume of transfer")
-			r := rpc.ResumeTransferRequest{m.Remote, localState.ResumeToken}
-			stream, err := remote.ResumeTransferRequest(r)
-			if err != nil {
-
-				log("resume transfer request failed: %s", err)
-				rre, ok := err.(*rpc.ResumeTransferError)
-				if !ok {
-					log("skipping this filesystem, could be temporary issue")
-					return true // TODO right choice to allow children
-				}
-
-				// Determine if we should clear the resume token
-				clearAndUseSnaps := false
-				switch rre.Reason {
-				case rpc.ResumeTransferErrorReasonNotImplemented:
-					fallthrough
-				case rpc.ResumeTransferErrorReasonDisabled:
-					fallthrough
-				case rpc.ResumeTransferErrorReasonZFSErrorPermanent:
-					clearAndUseSnaps = true
-
-				case rpc.ResumeTransferErrorReasonZFSErrorMaybeTemporary:
-					fallthrough
-				default:
-					clearAndUseSnaps = false
-				}
-
-				if !clearAndUseSnaps {
-					log("skipping this filesystem, error identified as temporary")
-					return true // TODO right choice to allow children
-				}
-
-				log("clearing local receive_resume_token")
-				if err := zfs.ZFSRecvAbort(m.Local); err != nil {
-					log("error clearing receive_resume_token: %s", err)
-					return true // TODO right choice to allow children? the filesystem exists, so it should be ok
-				}
-				// TODO go back to top of function (put all this into separate function, then tail recursive call)
-				// TODO return this_function()
-			}
-
-			// TODO warning code duplication, see below. need to unify this
-			log("invoking zfs receive")
-			watcher := util.IOProgressWatcher{Reader: stream}
-			watcher.KickOff(1*time.Second, func(p util.IOProgress) {
-				log("progress on receive operation: %v bytes received", p.TotalRX)
-			})
-
-			recvArgs := []string{"-u"}
-			if localState.Placeholder {
-				log("receive with forced rollback to replace placeholder filesystem")
-				recvArgs = append(recvArgs, "-F")
-			}
-
-			if err = zfs.ZFSRecv(m.Local, &watcher, recvArgs...); err != nil {
-				log("error receiving stream: %s", err)
-				return false
-			}
-			log("finished receiving stream, %v bytes total", watcher.Progress().TotalRX)
-
-			// TODO further problem property handling code must be duplicated here...
-
-			// TODO tail recursive call to this function, we might not be dony syncing yet
-			// TODO return this_function()
-			return true
-
 		default:
-			log("local filesystem exists")
+			log.Debug("local filesystem exists")
-			log("requesting local filesystem versions")
+			log.Debug("requesting local filesystem versions")
 			if versions, err = zfs.ZFSListFilesystemVersions(m.Local, nil); err != nil {
-				log("cannot get local filesystem versions: %s", err)
+				log.WithError(err).Error("cannot get local filesystem versions")
 				return false
 			}
 		}
 
-		log("requesting remote filesystem versions")
+		log.Info("requesting remote filesystem versions")
-		var theirVersions []zfs.FilesystemVersion
+		r := FilesystemVersionsRequest{
-		theirVersions, err = remote.FilesystemVersionsRequest(rpc.FilesystemVersionsRequest{
 			Filesystem: m.Remote,
-		})
+		}
-		if err != nil {
+		var theirVersions []zfs.FilesystemVersion
-			log("error requesting remote filesystem versions: %s", err)
+		if err = remote.Call("FilesystemVersionsRequest", &r, &theirVersions); err != nil {
-			log("stopping replication for all filesystems mapped as children of %s", m.Local.ToString())
+			log.WithError(err).Error("cannot get remote filesystem versions")
+			log.Warn("stopping replication for all filesystems mapped as children of receiving filesystem")
 			return false
 		}
 
-		log("computing diff between remote and local filesystem versions")
+		log.Debug("computing diff between remote and local filesystem versions")
 		diff := zfs.MakeFilesystemDiff(versions, theirVersions)
-		log("%s", diff)
+		log.WithField("diff", diff).Debug("diff between local and remote filesystem")
 
 		if localState.Placeholder && diff.Conflict != zfs.ConflictAllRight {
 			panic("internal inconsistency: local placeholder implies ConflictAllRight")
@@ -437,9 +171,9 @@ func doPull(pull PullContext) (err error) {
 		switch diff.Conflict {
 		case zfs.ConflictAllRight:
 
-			log("performing initial sync, following policy: '%s'", pull.InitialReplPolicy)
+			log.WithField("replication_policy", pull.InitialReplPolicy).Info("performing initial sync, following policy")
 
-			if pull.InitialReplPolicy != rpc.InitialReplPolicyMostRecent {
+			if pull.InitialReplPolicy != InitialReplPolicyMostRecent {
 				panic(fmt.Sprintf("policy '%s' not implemented", pull.InitialReplPolicy))
 			}
 
@@ -451,134 +185,132 @@ func doPull(pull PullContext) (err error) {
 			}
 
 			if len(snapsOnly) < 1 {
-				log("cannot perform initial sync: no remote snapshots. stopping...")
+				log.Warn("cannot perform initial sync: no remote snapshots")
 				return false
 			}
 
-			r := rpc.InitialTransferRequest{
+			r := InitialTransferRequest{
 				Filesystem:        m.Remote,
 				FilesystemVersion: snapsOnly[len(snapsOnly)-1],
 			}
 
-			log("requesting snapshot stream for %s", r.FilesystemVersion)
+			log.WithField("version", r.FilesystemVersion).Debug("requesting snapshot stream")
 
 			var stream io.Reader
-			if stream, err = remote.InitialTransferRequest(r); err != nil {
+
-				log("error requesting initial transfer: %s", err)
+			if err = remote.Call("InitialTransferRequest", &r, &stream); err != nil {
+				log.WithError(err).Error("cannot request initial transfer")
 				return false
 			}
-			log("received initial transfer request response")
+			log.Debug("received initial transfer request response")
 
-			log("invoking zfs receive")
+			log.Debug("invoke zfs receive")
 			watcher := util.IOProgressWatcher{Reader: stream}
 			watcher.KickOff(1*time.Second, func(p util.IOProgress) {
-				log("progress on receive operation: %v bytes received", p.TotalRX)
+				log.WithField("total_rx", p.TotalRX).Info("progress on receive operation")
 			})
 
 			recvArgs := []string{"-u"}
 			if localState.Placeholder {
-				log("receive with forced rollback to replace placeholder filesystem")
+				log.Info("receive with forced rollback to replace placeholder filesystem")
 				recvArgs = append(recvArgs, "-F")
 			}
 
 			if err = zfs.ZFSRecv(m.Local, &watcher, recvArgs...); err != nil {
-				log("error receiving stream: %s", err)
+				log.WithError(err).Error("canot receive stream")
 				return false
 			}
-			log("finished receiving stream, %v bytes total", watcher.Progress().TotalRX)
+			log.WithField("total_rx", watcher.Progress().TotalRX).
+				Info("finished receiving stream")
 
-			log("configuring properties of received filesystem")
+			log.Debug("configuring properties of received filesystem")
 			if err = zfs.ZFSSet(m.Local, "readonly", "on"); err != nil {
-
+				log.WithError(err).Error("cannot set readonly property")
 			}
 
-			log("finished initial transfer")
+			log.Info("finished initial transfer")
 			return true
 
 		case zfs.ConflictIncremental:
 
 			if len(diff.IncrementalPath) < 2 {
-				log("remote and local are in sync")
+				log.Info("remote and local are in sync")
 				return true
 			}
 
-			log("following incremental path from diff")
+			log.Info("following incremental path from diff")
 			var pathRx uint64
 
 			for i := 0; i < len(diff.IncrementalPath)-1; i++ {
 
 				from, to := diff.IncrementalPath[i], diff.IncrementalPath[i+1]
 
-				log := func(format string, args ...interface{}) {
+				log, _ := log.WithField(logIncFromField, from.Name).WithField(logIncToField, to.Name), 0
-					log("[%v/%v][%s => %s]: %s", i+1, len(diff.IncrementalPath)-1,
-						from.Name, to.Name, fmt.Sprintf(format, args...))
-				}
 
-				log("requesting incremental snapshot stream")
+				log.Debug("requesting incremental snapshot stream")
-				r := rpc.IncrementalTransferRequest{
+				r := IncrementalTransferRequest{
 					Filesystem: m.Remote,
 					From:       from,
 					To:         to,
 				}
 				var stream io.Reader
-				if stream, err = remote.IncrementalTransferRequest(r); err != nil {
+				if err = remote.Call("IncrementalTransferRequest", &r, &stream); err != nil {
-					log("error requesting incremental snapshot stream: %s", err)
+					log.WithError(err).Error("cannot request incremental snapshot stream")
 					return false
 				}
 
-				log("invoking zfs receive")
+				log.Debug("invoking zfs receive")
 				watcher := util.IOProgressWatcher{Reader: stream}
 				watcher.KickOff(1*time.Second, func(p util.IOProgress) {
-					log("progress on receive operation: %v bytes received", p.TotalRX)
+					log.WithField("total_rx", p.TotalRX).Info("progress on receive operation")
 				})
 
 				if err = zfs.ZFSRecv(m.Local, &watcher); err != nil {
-					log("error receiving stream: %s", err)
+					log.WithError(err).Error("cannot receive stream")
 					return false
 				}
 
 				totalRx := watcher.Progress().TotalRX
 				pathRx += totalRx
-				log("finished incremental transfer, %v bytes total", totalRx)
+				log.WithField("total_rx", totalRx).Info("finished incremental transfer")
 
 			}
 
-			log("finished following incremental path, %v bytes total", pathRx)
+			log.WithField("total_rx", pathRx).Info("finished following incremental path")
 			return true
 
 		case zfs.ConflictNoCommonAncestor:
-
+			fallthrough
-			log("remote and local filesystem have snapshots, but no common one")
-			log("perform manual replication to establish a common snapshot history")
-			log("remote versions:")
-			for _, v := range diff.MRCAPathRight {
-				log(" %s (GUID %v)", v, v.Guid)
-			}
-			log("local versions:")
-			for _, v := range diff.MRCAPathLeft {
-				log(" %s (GUID %v)", v, v.Guid)
-			}
-			return false
-
 		case zfs.ConflictDiverged:
 
-			log("remote and local filesystem share a history but have diverged")
+			var jsonDiff bytes.Buffer
-			log("perform manual replication or delete snapshots on the receiving" +
+			if err := json.NewEncoder(&jsonDiff).Encode(diff); err != nil {
-				"side  to establish an incremental replication parse")
+				log.WithError(err).Error("cannot JSON-encode diff")
-			log("remote-only versions:")
+				return false
-			for _, v := range diff.MRCAPathRight {
-				log(" %s (GUID %v)", v, v.Guid)
 			}
-			log("local-only versions:")
+
-			for _, v := range diff.MRCAPathLeft {
+			var problem, resolution string
-				log(" %s (GUID %v)", v, v.Guid)
+
+			switch diff.Conflict {
+			case zfs.ConflictNoCommonAncestor:
+				problem = "remote and local filesystem have snapshots, but no common one"
+				resolution = "perform manual establish a common snapshot history"
+			case zfs.ConflictDiverged:
+				problem = "remote and local filesystem share a history but have diverged"
+				resolution = "perform manual replication or delete snapshots on the receiving" +
+					"side  to establish an incremental replication parse"
 			}
+
+			log.WithField("diff", jsonDiff.String()).
+				WithField("problem", problem).
+				WithField("resolution", resolution).
+				Error("manual conflict resolution required")
+
 			return false
 
 		}
 
-		panic("implementation error: this should not be reached")
+		panic("should not be reached")
-		return false
 
 	})
 

cmd/sampleconf/localbackup/host1.yml

@@ -0,0 +1,28 @@
+jobs:
+- name: mirror_local
+  type: local
+
+  # snapshot the filesystems matched by the left-hand-side of the mapping
+  # every 10m with zrepl_ as prefix
+  mapping: {
+    "zroot/var/db<":    "storage/backups/local/zroot/var/db",
+    "zroot/usr/home<":  "storage/backups/local/zroot/usr/home",
+    "zroot/usr/home/paranoid":  "!", #don't backup paranoid user
+    "zroot/poudriere/ports<": "!", #don't backup the ports trees
+  }
+  snapshot_prefix: zrepl_
+  interval: 10m
+  initial_repl_policy: most_recent
+
+  # keep one hour of 10m interval snapshots of filesystems matched by
+  # the left-hand-side of the mapping
+  prune_lhs:
+    policy: grid
+    grid: 1x1h(keep=all)
+
+  # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
+  prune_rhs:
+    policy: grid
+    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+
+

cmd/sampleconf/pullbackup/backuphost.yml

@@ -0,0 +1,27 @@
+jobs:
+- name: fullbackup_prod1
+  type: pull
+  # connect to remote using ssh / stdinserver command
+  connect:
+    type: ssh+stdinserver
+    host: prod1.example.com
+    user: root
+    port: 22
+    identity_file: /root/.ssh/id_ed25519
+
+  # pull (=ask for new snapshots) every 10m, prune afterwards
+  # this will leave us at most 10m behind production
+  interval: 10m
+
+  # pull all offered filesystems to storage/backups/zrepl/pull/prod1.example.com
+  mapping: {
+    "<":"storage/backups/zrepl/pull/prod1.example.com"
+  }
+  initial_repl_policy: most_recent
+
+  # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
+  snapshot_prefix: zrepl_
+  prune:
+    policy: grid
+    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+

cmd/sampleconf/pullbackup/productionhost.yml

@@ -0,0 +1,45 @@
+global:
+  serve:
+    stdinserver:
+      # Directory where AF_UNIX sockets for stdinserver command are placed.
+      #
+      # `zrepl stdinserver CLIENT_IDENTITY`
+      #   * connects to the socket in $sockdir/CLIENT_IDENTITY
+      #   * sends its stdin / stdout file descriptors to the `zrepl daemon` process (see cmsg(3))
+      #   * does nothing more
+      #
+      # This enables a setup where `zrepl daemon` is not directly exposed to the internet
+      # but instead all traffic is tunnelled through SSH.
+      # The server with the source job has an authorized_keys file entry for the public key
+      # used by the corresponding pull job
+      #
+      #   command="/mnt/zrepl stdinserver CLIENT_IDENTITY" ssh-ed25519 AAAAC3NzaC1E... zrepl@pullingserver
+      #
+      # Below is the default value.
+      sockdir: /var/run/zrepl/stdinserver
+
+jobs:
+
+- name: fullbackup_prod1
+  # expect remote to connect via ssh+stdinserver with fullbackup_prod1 as client_identity
+  type: source
+  serve:
+    type: stdinserver # see global.serve.stdinserver for explanation
+    client_identity: fullbackup_prod1
+
+  # snapshot these filesystems every 10m with zrepl_ as prefix
+  filesystems: {
+    "zroot/var/db<": "ok",
+    "zroot/usr/home<": "ok",
+    "zroot/var/tmp": "!", #don't backup /tmp
+  }
+  snapshot_prefix: zrepl_
+  interval: 10m
+
+
+  # keep a one day window 10m interval snapshots in case pull doesn't work (link down, etc)
+  # (we cannot keep more than one day because this host will run out of disk space)
+  prune:
+    policy: grid
+    grid: 1x1d(keep=all)
+

cmd/sampleconf/pushbackup/backuphost.yml

@@ -0,0 +1,20 @@
+jobs:
+- name: fullbackup_prod1
+
+  # expect remote to connect via ssh+stdinserver with fullbackup_prod1 as client_identity
+  type: push-sink
+  serve:
+    type: stdinserver
+    client_identity: fullbackup_prod1
+
+  # map all pushed datasets to storage/backups/zrepl/sink/prod1.example.com
+  mapping: {
+    "<":"storage/backups/zrepl/sink/prod1.example.com"
+  }
+
+  # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
+  prune:
+    policy: grid
+    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+
+

cmd/sampleconf/pushbackup/productionhost.yml

@@ -0,0 +1,26 @@
+jobs:
+- name: fullbackup_prod1
+
+  # connect to remote using ssh / stdinserver command
+  type: push
+  connect:
+      type: ssh+stdinserver
+      host: prod1.example.com
+      user: root
+      port: 22
+      identity_file: /root/.ssh/id_ed25519
+
+  # snapshot these datsets every 10m with zrepl_ as prefix
+  filesystems: {
+    "zroot/var/db<": "ok",
+    "zroot/usr/home<": "!",
+  }
+  snapshot_prefix: zrepl_
+  interval: 10m
+
+  # keep a one day window 10m interval snapshots in case push doesn't work (link down, etc)
+  # (we cannot keep more than one day because this host will run out of disk space)
+  prune:
+    policy: grid
+    grid: 1x1d(keep=all)
+

cmd/sampleconf/random/debugging.yml

@@ -0,0 +1,33 @@
+global:
+  serve:
+    stdinserver:
+      sockdir: /var/run/zrepl/stdinserver
+
+jobs:
+
+- name: debian2_pull
+  # JOB DEBUGGING OPTIONS
+  # should be equal for all job types, but each job implements the debugging itself
+  # => consult job documentation for supported options
+  debug:
+    conn: # debug the io.ReadWriteCloser connection
+      read_dump: /tmp/connlog_read   # dump results of Read() invocations to this file
+      write_dump: /tmp/connlog_write # dump results of Write() invocations to this file
+    rpc: # debug the RPC protocol implementation
+      log: true # log output from rpc layer to the job log
+
+  # ... just to make the unit tests pass.
+  # check other examples, e.g. localbackup or pullbackup for what the sutff below means
+  type: source
+  serve:
+    type: stdinserver
+    client_identity: debian2
+  filesystems: {
+    "pool1/db<": ok
+  }
+  snapshot_prefix: zrepl_
+  interval: 1s
+  prune:
+    policy: grid
+    grid: 1x10s(keep=all)
+

cmd/sampleconf/random/logging.yml

@@ -0,0 +1,24 @@
+global:
+  logging:
+
+    - outlet: stdout
+      level: warn
+      format: human
+
+    - outlet: tcp
+      level: debug
+      format: json
+      net: tcp
+      address: 127.0.0.1:8080
+      retry_interval: 1s
+      tls: # if not specified, use plain TCP
+        ca: sampleconf/random/logging/logserver.crt
+        cert: sampleconf/random/logging/client.crt
+        key: sampleconf/random/logging/client.key
+
+    - outlet: syslog
+      level: debug
+      format: logfmt
+
+
+jobs: []

cmd/sampleconf/random/logging/client.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDIzCCAgsCAQEwDQYJKoZIhvcNAQELBQAwWTELMAkGA1UEBhMCQVUxEzARBgNV
+BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0
+ZDESMBAGA1UEAwwJbG9nc2VydmVyMB4XDTE3MDkyNDEyMzAzNloXDTE3MTAyNDEy
+MzAzNlowVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNV
+BAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xpZW50MIIB
+IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/xJTUlqApeJGzRD+w2J8sZS
+Bo+s+04T987L/M6gaCo8aDSTEb/ZH3XSoU5JEmO6kPpwNNapOsaEhTCjndZQdm5F
+uqiUtAg1uW0HCkBEIDkGr9bFHDKzpewGmmMgfQ2+hfiBR/4ZCrc/vd9P0W9BiWQS
+Dtc7p22XraWPVL8HlSz5K/Ih+V6i8O+kBltZkusiJh2bWPoRp/netiTZuc6du+Wp
+kpWp1OBaTU4GXIAlLj5afF14BBphRQK983Yhaz53BkA7OQ76XxowynMjmuLQVGmK
+f1R9zEJuohTX9XIr1tp/ueRHcS4Awk6LcNZUMCV6270FNSIw2f4hbOZvep+t2wID
+AQABMA0GCSqGSIb3DQEBCwUAA4IBAQACK3OeNzScpiNwz/jpg/usQzvXbZ/wDvml
+YLjtzn/A65ox8a8BhxvH1ydyoCM2YAGYX7+y7qXJnMgRO/v8565CQIVcznHhg9ST
+3828/WqZ3bXf2DV5GxKKQf7hPmBnyVUUhn/Ny91MECED27lZucWiX/bczN8ffDeh
+M3+ngezcJxsOBd4x0gLrqIJCoaFRSeepOaFEW6GHQ8loxE9GmA7FQd2phIpJHFSd
+Z7nQl7X5C1iN2OboEApJHwtmNVC45UlOpg53vo2sDTLhSfdogstiWi8x1HmvhIGM
+j3XHs0Illvo9OwVrmgUph8zQ7pvr/AFrTOIbhgzl/9uVUk5ApwFM
+-----END CERTIFICATE-----

cmd/sampleconf/random/logging/client.csr

@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIICmzCCAYMCAQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx
+ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xp
+ZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/xJTUlqApeJGzRD
++w2J8sZSBo+s+04T987L/M6gaCo8aDSTEb/ZH3XSoU5JEmO6kPpwNNapOsaEhTCj
+ndZQdm5FuqiUtAg1uW0HCkBEIDkGr9bFHDKzpewGmmMgfQ2+hfiBR/4ZCrc/vd9P
+0W9BiWQSDtc7p22XraWPVL8HlSz5K/Ih+V6i8O+kBltZkusiJh2bWPoRp/netiTZ
+uc6du+WpkpWp1OBaTU4GXIAlLj5afF14BBphRQK983Yhaz53BkA7OQ76XxowynMj
+muLQVGmKf1R9zEJuohTX9XIr1tp/ueRHcS4Awk6LcNZUMCV6270FNSIw2f4hbOZv
+ep+t2wIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAKnlr0Qs5KYF85u2YA7DJ5pL
+HwAx+qNoNbox5CS1aynrDBpDTWLaErviUJ+4WxRlRyTMEscMOIOKajbYhqqFmtGZ
+mu3SshZnFihErw8TOQMyU1LGGG+l6r+6ve5TciwJRLla2Y75z7izr6cyvQNRWdLr
+PvxL1/Yqr8LKha12+7o28R4SLf6/GY0GcedqoebRmtuwA/jES0PuGauEUD5lH4cj
+Me8sqRrB+IMHQ5j8hlJX4DbA8UQRUBL64sHkQzeQfWu+qkWmS5I19CFfLNrcH+OV
+yhyjGfN0q0jHyHdpckBhgzS7IIdo6P66AIlm4qpHM7Scra3JaGM7oaZPamJ6f8U=
+-----END CERTIFICATE REQUEST-----

cmd/sampleconf/random/logging/client.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC3/ElNSWoCl4kb
+NEP7DYnyxlIGj6z7ThP3zsv8zqBoKjxoNJMRv9kfddKhTkkSY7qQ+nA01qk6xoSF
+MKOd1lB2bkW6qJS0CDW5bQcKQEQgOQav1sUcMrOl7AaaYyB9Db6F+IFH/hkKtz+9
+30/Rb0GJZBIO1zunbZetpY9UvweVLPkr8iH5XqLw76QGW1mS6yImHZtY+hGn+d62
+JNm5zp275amSlanU4FpNTgZcgCUuPlp8XXgEGmFFAr3zdiFrPncGQDs5DvpfGjDK
+cyOa4tBUaYp/VH3MQm6iFNf1civW2n+55EdxLgDCTotw1lQwJXrbvQU1IjDZ/iFs
+5m96n63bAgMBAAECggEAF4om0sWe06ARwbJJNFjCGpa3LfG5/xk5Qs5pmPnS2iD1
+Q5veaTnzjKvlfA/pF3o9B4mTS59fXY7Cq8vSU0J1XwGy2DPzeqlGPmgtq2kXjkvd
+iCfhZj8ybvsoyR3/rSBSDRADcnOXPqC9fgyRSMmESBDOoql1D3HdIzF4ii46ySIU
+/XQvExS6NWifbP+Ue6DETV8NhreO5PqjeXLITQhhndtc8MDL/8eCNOyN8XjYIWKX
+smlBYtRQYOOY9BHOQgUn6yvPHrtKJNKci+qcQNvWir66mBhY1o40MH5wTIV+8yP2
+Vbm/VzoNKIYgeROsilBW7QTwGvkDn3R11zeTqfUNSQKBgQD0eFzhJAEZi4uBw6Tg
+NKmBC5Y1IHPOsb5gKPNz9Z9j4qYRDySgYl6ISk+2EdhgUCo1NmTk8EIPQjIerUVf
+S+EogFnpsj8U9LR3OM79DaGkNULxrHqhd209/g8DtVgk7yjkxL4vmVOv8qpHMp/7
+eWsylN7AOxj2RB/eXYQBPrw+jQKBgQDAqae9HasLmvpJ9ktTv30yZSKXC+LP4A0D
+RBBmx410VpPd4CvcpCJxXmjer6B7+9L1xHYP2pvsnMBid5i0knuvyK28dYy7fldl
+CzWvb+lqNA5YYPFXQED4oEdihlQczoI1Bm06SFizeAKD1Q9e2c+lgbR/51j8xuXi
+twvhMj/YBwKBgQCZw97/iQrcC2Zq7yiUEOuQjD4lGk1c83U/vGIsTJC9XcCAOFsc
+OeMlrD/oz96d7a4unBDn4qpaOJOXsfpRT0PGmrxy/jcpMiUUW/ntNpa11v5NTeQw
+DRL8DAFbnsNbL8Yz5f+Nps35fBNYBuKTZLJlNTfKByHTO9QjpAQ0WEZEvQKBgQCi
+Ovm83EuYVSKmvxcE6Tyx/8lVqTOO2Vn7wweQlD4/lVujvE0S2L8L+XSS9w5K+GzW
+eFz10p3zarbw80YJ30L5bSEmjVE43BUZR4woMzM4M6dUsiTm1HshIE2b4ALZ0uZ/
+Ye794ceXL9nmSrVLqFsaQZLNFPCwwYb4FiyRry9lZwKBgAO9VbWcN8SEeBDKo3z8
+yRbRTc6sI+AdKY44Dfx0tqOPmTjO3mE4X1GU4sbfD2Bvg3DdjwTuxxC/jHaKu0GG
+dTM0CbrZGbDAj7E87SOcN/PWEeBckSvuQq5H3DQfwIpTmlS1l5oZn9CxRGbLqC2G
+ifnel8XWUG0ROybsr1tk4mzW
+-----END PRIVATE KEY-----

cmd/sampleconf/random/logging/logserver.crt

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDiDCCAnCgAwIBAgIJALhp/WvTQeg/MA0GCSqGSIb3DQEBCwUAMFkxCzAJBgNV
+BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
+aWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMMCWxvZ3NlcnZlcjAeFw0xNzA5MjQxMjI3
+MDRaFw0yNzA5MjIxMjI3MDRaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21l
+LVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNV
+BAMMCWxvZ3NlcnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKs3
+TLYfXhV3hap71tOkhPQlM+m0EKRAo8Nua50Cci5UhDo4JkVpyYok1h+NFkqmjU2b
+IiIuGvsZZPOWYjbWWnSJE4+n5pBFBzcfNQ4d8xVxjANImFn6Tcehhj0WkbDIv/Ge
+364XUgywS7u3EGQj/FO7vZ8KHlUxBHNuPIOPHftwIVRyleh5K32UyBaSpSmnqGos
+rvI1byMuznavcZpOs4vlebZ+Jy6a20iKf9fj/0f0t0O+F5x3JIk07D3zSywhJ4RM
+M0mGIUmYXbh2SMh+f61KDZLDANpz/pMAPbUJe0mxEtBf0tnwK1gEqc3SLwA0EwiM
+8Hnn2iaH5Ln20UE3LOkCAwEAAaNTMFEwHQYDVR0OBBYEFDXoDcwx9SngzZcRYCeP
+BplBecfiMB8GA1UdIwQYMBaAFDXoDcwx9SngzZcRYCePBplBecfiMA8GA1UdEwEB
+/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBADyNvs4AA91x3gurQb1pcPVhK6nR
+mkYSTN1AsDKSRi/X2iCUmR7G7FlF7XW8mntTpHvVzcs+gr94WckH5wqEOA5iZnaw
+PXUWexmdXUge4hmC2q6kBQ5e2ykhSJMRVZXvOLZOZV9qitceamHESV1cKZSNMvZM
+aCSVA1RK61/nUzs04pVp5PFPv9gFxJp9ki39FYFdsgZmM5RZ5I/FqxxvTJzu4RnH
+VPjsMopzARYwJw6dV2bKdFSYOE8B/Vs3Yv0GxjrABw2ko4PkBPTjLIz22x6+Hd9r
+K9BQi4pVmQfvppF5+SORSftlHSS+N47b0DD1rW1f5R6QGi71dFuJGikOwvY=
+-----END CERTIFICATE-----

cmd/sampleconf/random/logging/logserver.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCrN0y2H14Vd4Wq
+e9bTpIT0JTPptBCkQKPDbmudAnIuVIQ6OCZFacmKJNYfjRZKpo1NmyIiLhr7GWTz
+lmI21lp0iROPp+aQRQc3HzUOHfMVcYwDSJhZ+k3HoYY9FpGwyL/xnt+uF1IMsEu7
+txBkI/xTu72fCh5VMQRzbjyDjx37cCFUcpXoeSt9lMgWkqUpp6hqLK7yNW8jLs52
+r3GaTrOL5Xm2ficumttIin/X4/9H9LdDvhecdySJNOw980ssISeETDNJhiFJmF24
+dkjIfn+tSg2SwwDac/6TAD21CXtJsRLQX9LZ8CtYBKnN0i8ANBMIjPB559omh+S5
+9tFBNyzpAgMBAAECggEBAIY8ZwJq+WKvQLb3POjWFf8so9TY/ispGrwAeJKy9j5o
+uPrERw0o8YBDfTVjclS43BQ6Srqtly3DLSjlgL8ps+WmCxYYN2ZpGE0ZRIl65bis
+O2/fnML+wbiAZTTD2xnVatfPDeP6GLQmDFpyHoHEzPIBQZvNXRbBxZGSnhMvQ/x7
+FhqSBQG4kf3b1XDCENIbFEVOBOCg7WtMiIgjEGS7QnW3I65/Zt+Ts1LXRZbz+6na
+Gmi0PGHA/oLUh1NRzsF4zuZn6fFzja5zw4mkt+JvCWEoxg1QhRAxRp6QQwmZ6MIc
+1rw1D4Z+c5UEKyqHeIwZj4M6UNPhCfTXVm47c9eSiGECgYEA4U8pB+7eRo2fqX0C
+nWsWMcmsULJvwplQnUSFenUayPn3E8ammS/ZBHksoKhj82vwIdDbtS1hQZn8Bzsi
+atc8au0wz0YRDcVDzHX4HknXVQayHtP/FTPeSr5hwpoY8vhEbySuxBTBkXCrp4dx
+u5ErfOiYEP3Q1ZvPRywelrATu20CgYEAwonV5dgOcen/4oAirlnvufc2NfqhAQwJ
+FJ/JSVMAcXxPYu3sZMv0dGWrX8mLc+P1+XMCuV/7eBM/vU2LbDzmpeUV8sJfB2jw
+wyKqKXZwBgeq60btriA4f+0ElwRGgU2KSiniUuuTX2JmyftFQx4cVAQRCFk27NY0
+09psSsYyre0CgYBo6unabdtH029EB5iOIW3GZXk+Yrk0TxyA/4WAjsOYTv5FUT4H
+G4bdVGf5sDBLDDpYJOAKsEUXvVLlMx5FzlCuIiGWg7QxS2jU7yJJSG1jhKixPlsM
+Toj3GUyAyC1SB1Ymw1g2qsuwpFzquGG3zFQJ6G3Xi7oRnmqZY+wik3+8yQKBgB11
+SdKYOPe++2SNCrNkIw0CBk9+OEs0S1u4Jn7X9sU4kbzlUlqhF89YZe8HUfqmlmTD
+qbHwet/f6lL8HxSw1Cxi2EP+cu1oUqz53tKQgL4pAxTFlNA9SND2Ty+fEh4aY8p/
+NSphSduzxuTnC8HyGVAPnZSqDcsnVLCP7r4T7TCxAoGAbJygkkk/gZ9pT4fZoIaq
+8CMR8FTfxtkwCuZsWccSMUOWtx9nqet3gbCpKHfyoYZiKB4ke+lnUz4uFS16Y3hG
+kN0hFfvfoNa8eB2Ox7vs60cMMfWJac0H7KSaDDy+EvbhE2KtQADT0eWxMyhzGR8p
+5CbIivB0QCjeQIA8dOQpE8E=
+-----END PRIVATE KEY-----

cmd/sampleconf/zrepl.yml

@@ -1,109 +0,0 @@
-remotes:
-  offsite_backups:
-    transport:
-      ssh:
-        host: 192.168.122.6
-        user: root
-        port: 22
-        identity_file: /etc/zrepl/identities/offsite_backups
-
-pushs:
-
-  offsite:
-    to: offsite_backups
-    filter: {
-      # like in pull_acls
-      "tank/var/db<":   ok,
-      "tank/usr/home<": ok,
-    }
-
-pulls:
-
-  offsite:
-    from: offsite_backups
-    mapping: {
-      # like in sinks
-    }
-
-  # local replication, only allowed in pull mode
-  # the from name 'local' is reserved for this purpose
-  homemirror:
-    from: local
-    repeat:
-      interval: 15m
-    mapping: {
-      "tank/usr/home":"mirrorpool/foo/bar"
-    }
-
-sinks:
-
-  db1:
-    mapping: {
-      # direct mapping
-      "ssdpool/var/db/postgresql9.6":"zroot/backups/db1/pg_data"
-    }
-
-  mirror1:
-    mapping: {
-      # "<" subtree wildcard matches the dataset left of < and all its children
-      "tank/foo/bar<":"zroot/backups/mirror1" 
-    }
-
-  mirror2:
-    # more specific path patterns win over less specific ones
-    # direct mappings win over subtree wildcards
-    # detailed rule precedence: check unit tests & docs for exact behavior
-    # TODO subcommand to test a mapping & filter
-    mapping: {
-      "tank<":                       "zroot/backups/mirror1/tank1",
-      "tank/cdn/root<":              "storage/cdn/root",
-      "tank/legacydb":               "legacypool/backups/legacydb",
-    }
-
-
-pull_acls:
-
-  office_backup:
-    filter: {
-      # valid filter results (right hand side): ok, omit
-      # default is to omit
-      # rule precedence is same as for mappings
-      "tank<":         omit,
-      "tank/usr/home": ok,
-    }
-
-
-prune:
-
-  clean_backups:
-    policy: grid
-    grid: 6x10m | 24x1h | 7x1d | 5 x 1w | 4 x 5w
-    dataset_filter: {
-      "tank/backups/legacyscript<": omit,
-      "tank/backups<":              ok,
-    }
-    snapshot_filter: {
-      prefix: zrepl_
-    }
-
-  hfbak_prune: # cleans up after hfbak autosnap job
-    policy: grid
-    grid: 1x1m(keep=all)
-    dataset_filter: {
-      "pool1<": ok
-    }
-    snapshot_filter: {
-      prefix: zrepl_hfbak_
-    }
-
-autosnap:
-
-  hfbak:
-    prefix: zrepl_hfbak_
-    interval: 1s
-    dataset_filter: {
-      "pool1<": ok
-    }
-    # prune: hfbak_prune
-    # future versions may inline the retention policy here, but for now,
-    # pruning has to be triggered manually (it's safe to run autosnap + prune in parallel)

cmd/stdinserver.go

@@ -2,17 +2,18 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/sshbytestream"
-	"io"
-	golog "log"
 	"os"
+
+	"github.com/ftrvxmtrx/fd"
+	"github.com/spf13/cobra"
+	"io"
+	"log"
+	"net"
 )
 
 var StdinserverCmd = &cobra.Command{
 	Use:   "stdinserver CLIENT_IDENTITY",
-	Short: "start in stdin server mode (from authorized_keys file)",
+	Short: "start in stdinserver mode (from authorized_keys file)",
 	Run:   cmdStdinServer,
 }
 
@@ -22,51 +23,71 @@ func init() {
 
 func cmdStdinServer(cmd *cobra.Command, args []string) {
 
-	var err error
+	log := log.New(os.Stderr, "", log.LUTC|log.Ldate|log.Ltime)
-	defer func() {
+
-		if err != nil {
+	die := func() {
-			log.Printf("stdinserver exiting with error: %s", err)
+		log.Printf("stdinserver exiting after fatal error")
-			os.Exit(1)
+		os.Exit(1)
-		}
+	}
-	}()
+
+	conf, err := ParseConfig(rootArgs.configFile)
+	if err != nil {
+		log.Printf("error parsing config: %s", err)
+		die()
+	}
 
 	if len(args) != 1 || args[0] == "" {
-		err = fmt.Errorf("must specify client identity as positional argument")
+		err = fmt.Errorf("must specify client_identity as positional argument")
-		return
+		die()
 	}
 	identity := args[0]
 
-	pullACL, ok := conf.PullACLs[identity]
+	unixaddr, err := stdinserverListenerSocket(conf.Global.Serve.Stdinserver.SockDir, identity)
-	if !ok {
+	if err != nil {
-		err = fmt.Errorf("could not find PullACL for identity '%s'", identity)
+		log.Printf("%s", err)
-		return
-	}
-
-	var sshByteStream io.ReadWriteCloser
-	if sshByteStream, err = sshbytestream.Incoming(); err != nil {
-		return
-	}
-
-	sinkMapping := func(identity string) (m DatasetMapping, err error) {
-		sink, ok := conf.Sinks[identity]
-		if !ok {
-			return nil, fmt.Errorf("could not find sink for identity '%s'", identity)
-		}
-		return sink, nil
-	}
-
-	sinkLogger := golog.New(logOut, fmt.Sprintf("sink[%s] ", identity), logFlags)
-	handler := Handler{
-		Logger:          sinkLogger,
-		SinkMappingFunc: sinkMapping,
-		PullACL:         pullACL,
-	}
-
-	if err = rpc.ListenByteStreamRPC(sshByteStream, identity, handler, sinkLogger); err != nil {
-		log.Printf("listenbytestreamerror: %#v\n", err)
 		os.Exit(1)
 	}
 
+	log.Printf("opening control connection to zrepld via %s", unixaddr)
+	conn, err := net.DialUnix("unix", nil, unixaddr)
+	if err != nil {
+		log.Printf("error connecting to zrepld: %s", err)
+		die()
+	}
+
+	log.Printf("sending stdin and stdout fds to zrepld")
+	err = fd.Put(conn, os.Stdin, os.Stdout)
+	if err != nil {
+		log.Printf("error: %s", err)
+		die()
+	}
+
+	log.Printf("waiting for zrepld to close control connection")
+	for {
+
+		var buf [64]byte
+		n, err := conn.Read(buf[:])
+		if err == nil && n != 0 {
+			log.Printf("protocol error: read expected to timeout or EOF returned bytes")
+		}
+
+		if err == io.EOF {
+			log.Printf("zrepld closed control connection, terminating")
+			break
+		}
+
+		neterr, ok := err.(net.Error)
+		if !ok {
+			log.Printf("received unexpected error type: %T %s", err, err)
+			die()
+		}
+		if !neterr.Timeout() {
+			log.Printf("receivd unexpected net.Error (not a timeout): %s", neterr)
+			die()
+		}
+		// Read timed out, as expected
+	}
+
 	return
 
 }

cmd/test.go

@@ -0,0 +1,214 @@
+package cmd
+
+import (
+	"os"
+
+	"bytes"
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/kr/pretty"
+	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/zfs"
+	"time"
+)
+
+var testCmd = &cobra.Command{
+	Use:              "test",
+	Short:            "test configuration",
+	PersistentPreRun: testCmdGlobalInit,
+}
+
+var testCmdGlobal struct {
+	log  Logger
+	conf *Config
+}
+
+var testConfigSyntaxCmd = &cobra.Command{
+	Use:   "config",
+	Short: "parse config file and dump parsed datastructure",
+	Run:   doTestConfig,
+}
+
+var testDatasetMapFilter = &cobra.Command{
+	Use:     "pattern jobname test/zfs/dataset/path",
+	Short:   "test dataset mapping / filter specified in config",
+	Example: ` zrepl test pattern prune.clean_backups tank/backups/legacyscript/foo`,
+	Run:     doTestDatasetMapFilter,
+}
+
+var testPrunePolicyArgs struct {
+	side        PrunePolicySide
+	showKept    bool
+	showRemoved bool
+}
+
+var testPrunePolicyCmd = &cobra.Command{
+	Use:   "prune jobname",
+	Short: "do a dry-run of the pruning part of a job",
+	Run:   doTestPrunePolicy,
+}
+
+func init() {
+	RootCmd.AddCommand(testCmd)
+	testCmd.AddCommand(testConfigSyntaxCmd)
+	testCmd.AddCommand(testDatasetMapFilter)
+
+	testPrunePolicyCmd.Flags().VarP(&testPrunePolicyArgs.side, "side", "s", "prune_lhs (left) or prune_rhs (right)")
+	testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showKept, "kept", false, "show kept snapshots")
+	testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showRemoved, "removed", true, "show removed snapshots")
+	testCmd.AddCommand(testPrunePolicyCmd)
+}
+
+func testCmdGlobalInit(cmd *cobra.Command, args []string) {
+
+	out := logger.NewOutlets()
+	out.Add(WriterOutlet{&NoFormatter{}, os.Stdout}, logger.Info)
+	log := logger.NewLogger(out, 1*time.Second)
+	testCmdGlobal.log = log
+
+	var err error
+	if testCmdGlobal.conf, err = ParseConfig(rootArgs.configFile); err != nil {
+		testCmdGlobal.log.Printf("error parsing config file: %s", err)
+		os.Exit(1)
+	}
+
+}
+
+func doTestConfig(cmd *cobra.Command, args []string) {
+
+	log, conf := testCmdGlobal.log, testCmdGlobal.conf
+
+	log.Printf("config ok")
+	log.Printf("%# v", pretty.Formatter(conf))
+	return
+}
+
+func doTestDatasetMapFilter(cmd *cobra.Command, args []string) {
+
+	log, conf := testCmdGlobal.log, testCmdGlobal.conf
+
+	if len(args) != 2 {
+		log.Printf("specify job name as first postitional argument, test input as second")
+		log.Printf(cmd.UsageString())
+		os.Exit(1)
+	}
+	n, i := args[0], args[1]
+
+	jobi, err := conf.LookupJob(n)
+	if err != nil {
+		log.Printf("%s", err)
+		os.Exit(1)
+	}
+
+	var mf *DatasetMapFilter
+	switch j := jobi.(type) {
+	case *PullJob:
+		mf = j.Mapping
+	case *SourceJob:
+		mf = j.Filesystems
+	case *LocalJob:
+		mf = j.Mapping
+	default:
+		panic("incomplete implementation")
+	}
+
+	ip, err := zfs.NewDatasetPath(i)
+	if err != nil {
+		log.Printf("cannot parse test input as ZFS dataset path: %s", err)
+		os.Exit(1)
+	}
+
+	if mf.filterMode {
+		pass, err := mf.Filter(ip)
+		if err != nil {
+			log.Printf("error evaluating filter: %s", err)
+			os.Exit(1)
+		}
+		log.Printf("filter result: %v", pass)
+	} else {
+		res, err := mf.Map(ip)
+		if err != nil {
+			log.Printf("error evaluating mapping: %s", err)
+			os.Exit(1)
+		}
+		toStr := "NO MAPPING"
+		if res != nil {
+			toStr = res.ToString()
+		}
+		log.Printf("%s => %s", ip.ToString(), toStr)
+
+	}
+
+}
+
+func doTestPrunePolicy(cmd *cobra.Command, args []string) {
+
+	log, conf := testCmdGlobal.log, testCmdGlobal.conf
+
+	if cmd.Flags().NArg() != 1 {
+		log.Printf("specify job name as first positional argument")
+		log.Printf(cmd.UsageString())
+		os.Exit(1)
+	}
+
+	jobname := cmd.Flags().Arg(0)
+	jobi, err := conf.LookupJob(jobname)
+	if err != nil {
+		log.Printf("%s", err)
+		os.Exit(1)
+	}
+
+	jobp, ok := jobi.(PruningJob)
+	if !ok {
+		log.Printf("job doesn't do any prunes")
+		os.Exit(0)
+	}
+
+	log.Printf("job dump:\n%s", pretty.Sprint(jobp))
+
+	pruner, err := jobp.Pruner(testPrunePolicyArgs.side, true)
+	if err != nil {
+		log.Printf("cannot create test pruner: %s", err)
+		os.Exit(1)
+	}
+
+	log.Printf("start pruning")
+
+	ctx := context.WithValue(context.Background(), contextKeyLog, log)
+	result, err := pruner.Run(ctx)
+	if err != nil {
+		log.Printf("error running pruner: %s", err)
+		os.Exit(1)
+	}
+
+	sort.Slice(result, func(i, j int) bool {
+		return strings.Compare(result[i].Filesystem.ToString(), result[j].Filesystem.ToString()) == -1
+	})
+
+	var b bytes.Buffer
+	for _, r := range result {
+		fmt.Fprintf(&b, "%s\n", r.Filesystem.ToString())
+
+		if testPrunePolicyArgs.showKept {
+			fmt.Fprintf(&b, "\tkept:\n")
+			for _, v := range r.Keep {
+				fmt.Fprintf(&b, "\t- %s\n", v.Name)
+			}
+		}
+
+		if testPrunePolicyArgs.showRemoved {
+			fmt.Fprintf(&b, "\tremoved:\n")
+			for _, v := range r.Remove {
+				fmt.Fprintf(&b, "\t- %s\n", v.Name)
+			}
+		}
+
+	}
+
+	log.Printf("pruning result:\n%s", b.String())
+
+}

docs/.gitignore

@@ -1 +1,2 @@
-public_git
+_build/
+public_git/

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = zrepl
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/_static/zrepl.svg

@@ -0,0 +1,147 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   id="svg8"
+   version="1.1"
+   viewBox="0 0 70.555558 24.694445"
+   height="70pt"
+   width="200pt"
+   sodipodi:docname="zrepl.svg"
+   inkscape:version="0.92.2 5c3e80d, 2017-08-06">
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="2560"
+     inkscape:window-height="1405"
+     id="namedview10"
+     showgrid="false"
+     inkscape:zoom="3.1350001"
+     inkscape:cx="6.2268563"
+     inkscape:cy="-2.6953765"
+     inkscape:window-x="0"
+     inkscape:window-y="35"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="layer1" />
+  <defs
+     id="defs2" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     transform="translate(0,-272.30556)"
+     id="layer1">
+    <g
+       aria-label="zrepl"
+       transform="matrix(0.26458334,0,0,0.26458334,0.52916655,273.36389)"
+       style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:40px;line-height:25px;font-family:monospace;-inkscape-font-specification:'monospace, Bold';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:center;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:middle;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       id="flowRoot829">
+      <path
+         d="m 74.504883,26.943359 11.523437,-13.125 H 74.973633 V 9.5410156 h 17.480469 v 4.4726564 l -11.503907,13.125 h 11.503907 v 4.277344 H 74.504883 Z"
+         style="stroke:#f8f8f8;stroke-opacity:1;fill:#f8f8f8;fill-opacity:1"
+         id="path873" />
+      <path
+         d="m 100.11035,9.5410156 h 5.15625 l 0.56641,3.3984374 q 0.83984,-1.875 2.57812,-2.910156 1.73828,-1.0156251 4.00391,-1.0156251 1.26953,0 2.36328,0.2734375 1.25,0.3515625 2.12891,0.8398436 v 5.449219 q -0.9375,-0.859375 -2.1875,-1.289063 -1.23047,-0.429687 -2.75391,-0.429687 -1.81641,0 -3.18359,0.625 -1.36719,0.605469 -2.10938,1.835937 -0.19531,0.3125 -0.37109,0.761719 -0.17578,0.449219 -0.27344,1.054688 -0.0977,0.566406 -0.15625,1.308593 -0.0391,0.742188 -0.0391,1.894532 v 10.078125 h -5.72266 z"
+         style="stroke:#f8f8f8;stroke-opacity:1;fill:#f8f8f8;fill-opacity:1"
+         id="path875" />
+      <path
+         d="m 132.4541,31.982422 q -2.85156,0 -4.9414,-0.722656 -2.07032,-0.742188 -3.53516,-2.226563 -2.91016,-2.96875 -2.91016,-8.4375 0,-5.371094 2.79297,-8.476562 2.79297,-3.1054691 7.65625,-3.1054691 3.10547,-0.097656 5.42969,1.2500001 2.34375,1.347656 3.59375,3.867187 1.26953,2.519532 1.17187,5.878907 v 2.324218 H 126.8877 q 0.0195,2.578125 1.52343,3.847657 1.50391,1.269531 4.47266,1.269531 1.93359,0 3.90625,-0.566406 0.89844,-0.273438 1.89453,-0.703125 0.99609,-0.429688 2.10938,-1.09375 v 5.253906 q -1.99219,0.820312 -4.0625,1.230469 -1.05469,0.214843 -2.14844,0.3125 -1.07422,0.09766 -2.12891,0.09766 z m 3.4961,-13.945313 q -0.0195,-2.285156 -1.15235,-3.4375 -1.11328,-1.152343 -3.28125,-1.152343 -2.01172,0 -3.18359,1.191406 -1.17188,1.210937 -1.38672,3.417969 z"
+         style="stroke:#f8f8f8;stroke-opacity:1;fill:#f8f8f8;fill-opacity:1"
+         id="path877" />
+      <path
+         d="m 146.45801,9.5410156 h 5.13672 l 0.5664,3.2812504 q 0.89844,-1.894532 2.36328,-2.8515629 1.42579,-0.9570312 3.45704,-0.9570312 3.71093,0 5.80078,2.9882811 1.05468,1.523438 1.5625,3.691406 0.50781,2.148438 0.50781,4.824219 0,2.851563 -0.54688,4.980469 -0.54687,2.109375 -1.54296,3.515625 -1.07422,1.523437 -2.59766,2.246094 -1.50391,0.722656 -3.28125,0.722656 -0.91797,0 -1.73828,-0.214844 -0.80078,-0.195312 -1.54297,-0.703125 -0.625,-0.410156 -1.28906,-1.09375 -0.64453,-0.703125 -1.15235,-1.640625 v 11.40625 h -5.70312 z m 9.70703,17.6757814 q 1.85547,0 2.91016,-1.796875 1.05468,-1.777344 1.05468,-4.960938 0,-3.164062 -1.05468,-4.960937 -1.05469,-1.796875 -2.91016,-1.796875 -1.83594,0 -2.92969,1.796875 -1.07422,1.777344 -1.07422,4.960937 0,3.183594 1.07422,4.960938 1.09375,1.796875 2.92969,1.796875 z"
+         style="stroke:#f8f8f8;stroke-opacity:1;fill:#f8f8f8;fill-opacity:1"
+         id="path879" />
+      <path
+         d="m 182.98145,31.572266 q -4.0625,0 -5.85938,-2.109375 -1.77734,-2.070313 -1.77734,-7.011719 V 5.4199219 h -5.80078 V 1.0253906 h 11.52343 V 22.451172 q 0,2.558594 0.80078,3.652344 0.78125,1.074218 2.71485,1.074218 h 4.57031 v 4.394532 z"
+         style="stroke:#f8f8f8;stroke-opacity:1;fill:#f8f8f8;fill-opacity:1"
+         id="path881" />
+    </g>
+    <path
+       inkscape:connector-curvature="0"
+       id="path837"
+       d="M 26.170418,286.30982 H 43.35751"
+       style="font-style:normal;font-weight:normal;font-size:medium;line-height:25px;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#f8f8f8;fill-opacity:1;fill-rule:evenodd;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />
+    <g
+       aria-label="ZFS replication"
+       style="font-style:normal;font-weight:normal;font-size:10.58333397px;line-height:6.61458349px;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
+       id="text820">
+      <path
+         d="m 14.758548,294.66467 h -3.493327 v -0.47887 l 2.652724,-3.99286 h -2.570041 v -0.565 h 3.338297 v 0.47887 l -2.652723,3.99287 h 2.72507 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path844" />
+      <path
+         d="M 16.353627,294.66467 H 15.71973 v -5.03673 h 2.814643 v 0.55811 h -2.180746 v 1.78456 h 2.046387 v 0.55466 h -2.046387 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path846" />
+      <path
+         d="m 22.241296,293.31764 q 0,0.65801 -0.475423,1.03697 -0.475424,0.37896 -1.322917,0.37896 -0.895725,0 -1.384929,-0.23771 v -0.60633 q 0.320394,0.1378 0.699355,0.21704 0.382406,0.0792 0.713135,0.0792 0.564995,0 0.850938,-0.21704 0.285943,-0.21704 0.285943,-0.596 0,-0.25149 -0.103353,-0.41341 -0.103352,-0.16192 -0.347954,-0.30317 -0.241157,-0.14125 -0.733806,-0.32039 -0.699354,-0.25494 -0.999077,-0.59945 -0.299724,-0.34795 -0.299724,-0.89572 0,-0.58911 0.440973,-0.93707 0.444417,-0.34795 1.167887,-0.34795 0.754476,0 1.388373,0.28249 l -0.19637,0.54433 q -0.647678,-0.26872 -1.205783,-0.26872 -0.447863,0 -0.7028,0.19293 -0.251492,0.19292 -0.251492,0.54088 0,0.24804 0.09646,0.41341 0.09991,0.16192 0.316949,0.29628 0.220486,0.13436 0.689019,0.30661 0.56155,0.20671 0.837158,0.40308 0.275608,0.19292 0.406521,0.44786 0.130914,0.25149 0.130914,0.60289 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path848" />
+      <path
+         d="m 26.768151,290.81306 q 0.251492,0 0.444418,0.0413 l -0.07579,0.57188 q -0.21015,-0.0482 -0.409966,-0.0482 -0.299723,0 -0.558105,0.16537 -0.254937,0.16536 -0.399631,0.45819 -0.144694,0.28939 -0.144694,0.64424 v 2.01882 h -0.620118 v -3.78271 h 0.509874 l 0.0689,0.69246 h 0.02756 q 0.217041,-0.37551 0.513319,-0.56844 0.296279,-0.19292 0.644233,-0.19292 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path850" />
+      <path
+         d="m 29.465661,294.73357 q -0.850939,0 -1.336697,-0.51331 -0.482313,-0.51677 -0.482313,-1.41938 0,-0.90951 0.451307,-1.44694 0.451308,-0.54088 1.219564,-0.54088 0.713135,0 1.136881,0.46164 0.423747,0.4582 0.423747,1.24368 v 0.37551 h -2.590712 q 0.01723,0.64424 0.327284,0.97841 0.310059,0.33418 0.8785,0.33418 0.303168,0 0.575331,-0.0517 0.272162,-0.0551 0.637342,-0.21015 v 0.54432 q -0.313503,0.13436 -0.589111,0.18948 -0.275608,0.0551 -0.651123,0.0551 z m -0.155029,-3.41408 q -0.444418,0 -0.7028,0.28594 -0.258382,0.28594 -0.306613,0.79582 h 1.925808 q -0.0069,-0.53055 -0.244602,-0.80616 -0.237711,-0.2756 -0.671793,-0.2756 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path852" />
+      <path
+         d="m 33.630782,294.73357 q -0.761366,0 -1.171333,-0.54777 h -0.04134 l 0.01378,0.14125 q 0.02756,0.26872 0.02756,0.47887 v 1.55374 h -0.620117 v -5.4777 h 0.509874 l 0.08268,0.51676 h 0.02756 q 0.220486,-0.31006 0.509874,-0.44786 0.289388,-0.1378 0.668349,-0.1378 0.73725,0 1.143772,0.51332 0.409966,0.50987 0.409966,1.44005 0,0.92673 -0.409966,1.44694 -0.409967,0.5202 -1.150662,0.5202 z m -0.103353,-3.4003 q -0.561551,0 -0.813043,0.31694 -0.248047,0.31695 -0.254937,0.99564 v 0.12058 q 0,0.76825 0.254937,1.11276 0.254937,0.34107 0.826823,0.34107 0.475423,0 0.740696,-0.38586 0.268717,-0.38585 0.268717,-1.07487 0,-0.69246 -0.268717,-1.05764 -0.265273,-0.36862 -0.754476,-0.36862 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path854" />
+      <path
+         d="m 36.800269,294.66467 h -0.620117 v -5.36057 h 0.620117 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path856" />
+      <path
+         d="m 38.619281,294.66467 h -0.620117 v -3.78271 h 0.620117 z m -0.668348,-4.80591 q 0,-0.2067 0.103352,-0.29972 0.106798,-0.093 0.261828,-0.093 0.144694,0 0.251492,0.093 0.110243,0.093 0.110243,0.29972 0,0.20327 -0.110243,0.29973 -0.106798,0.0965 -0.251492,0.0965 -0.15503,0 -0.261828,-0.0965 -0.103352,-0.0965 -0.103352,-0.29973 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path858" />
+      <path
+         d="m 41.334016,294.73357 q -0.816488,0 -1.27124,-0.49953 -0.454753,-0.50299 -0.454753,-1.44005 0,-0.95085 0.458198,-1.46417 0.461642,-0.51676 1.312581,-0.51676 0.275608,0 0.544325,0.0586 0.272163,0.0551 0.447863,0.1447 l -0.189481,0.51676 q -0.478868,-0.17914 -0.816487,-0.17914 -0.571886,0 -0.844049,0.36173 -0.272162,0.35829 -0.272162,1.07143 0,0.68557 0.272162,1.05075 0.272163,0.36173 0.806153,0.36173 0.499538,0 0.981852,-0.22048 v 0.55121 q -0.392741,0.20326 -0.974962,0.20326 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path860" />
+      <path
+         d="m 45.543923,294.66467 -0.120578,-0.53743 h -0.02756 q -0.282498,0.35484 -0.564996,0.48231 -0.279052,0.12402 -0.706244,0.12402 -0.558106,0 -0.875055,-0.29283 -0.316948,-0.29283 -0.316948,-0.82682 0,-1.15066 1.815565,-1.20578 l 0.640788,-0.0241 v -0.22393 q 0,-0.43064 -0.186035,-0.6339 -0.186035,-0.2067 -0.596002,-0.2067 -0.299723,0 -0.568441,0.0896 -0.265272,0.0896 -0.499538,0.19981 l -0.189481,-0.46508 q 0.285943,-0.15159 0.623563,-0.23772 0.337619,-0.0861 0.668348,-0.0861 0.685574,0 1.019748,0.30316 0.334175,0.30317 0.334175,0.96463 v 2.57693 z m -1.27813,-0.43063 q 0.520209,0 0.816487,-0.27906 0.299724,-0.28249 0.299724,-0.79926 v -0.34106 l -0.558106,0.0241 q -0.651123,0.0241 -0.950846,0.20671 -0.296278,0.18259 -0.296278,0.57533 0,0.29628 0.179145,0.45475 0.18259,0.15848 0.509874,0.15848 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path862" />
+      <path
+         d="m 48.427466,294.22715 q 0.120578,0 0.282498,-0.0241 0.161919,-0.0241 0.244602,-0.0551 v 0.47542 q -0.08613,0.0379 -0.268718,0.0723 -0.179145,0.0379 -0.36518,0.0379 -1.109321,0 -1.109321,-1.16788 v -2.20142 h -0.53399 v -0.29628 l 0.54088,-0.24804 0.248047,-0.80616 h 0.368625 v 0.86817 h 1.092096 v 0.48231 h -1.092096 v 2.18419 q 0,0.32729 0.15503,0.50299 0.158474,0.1757 0.437527,0.1757 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path864" />
+      <path
+         d="m 50.332607,294.66467 h -0.620118 v -3.78271 h 0.620118 z m -0.668349,-4.80591 q 0,-0.2067 0.103353,-0.29972 0.106798,-0.093 0.261827,-0.093 0.144694,0 0.251492,0.093 0.110243,0.093 0.110243,0.29972 0,0.20327 -0.110243,0.29973 -0.106798,0.0965 -0.251492,0.0965 -0.155029,0 -0.261827,-0.0965 -0.103353,-0.0965 -0.103353,-0.29973 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path866" />
+      <path
+         d="m 54.818119,292.76643 q 0,0.92673 -0.468533,1.44694 -0.468533,0.5202 -1.295356,0.5202 -0.51332,0 -0.909506,-0.23771 -0.396186,-0.24115 -0.609782,-0.68902 -0.213595,-0.44786 -0.213595,-1.04041 0,-0.92329 0.461642,-1.43661 0.461643,-0.51676 1.295356,-0.51676 0.799262,0 1.267795,0.5271 0.471979,0.52365 0.471979,1.42627 z m -2.855985,0 q 0,0.71313 0.279053,1.0852 0.282498,0.36863 0.830268,0.36863 0.54088,0 0.823378,-0.36863 0.282498,-0.37207 0.282498,-1.0852 0,-0.71314 -0.285943,-1.07143 -0.282498,-0.36173 -0.830268,-0.36173 -1.098986,0 -1.098986,1.43316 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path868" />
+      <path
+         d="m 58.380348,294.66467 v -2.41845 q 0,-0.46165 -0.206706,-0.68558 -0.20326,-0.22737 -0.640787,-0.22737 -0.582222,0 -0.847494,0.32383 -0.261827,0.3204 -0.261827,1.04731 v 1.96026 h -0.620117 v -3.78271 h 0.499538 l 0.09302,0.51676 h 0.03445 q 0.1757,-0.27905 0.492649,-0.43063 0.316948,-0.15503 0.699354,-0.15503 0.695909,0 1.033529,0.33762 0.337619,0.33761 0.337619,1.05075 v 2.46324 z"
+         style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:7.05555582px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#f8f8f8;fill-opacity:1;stroke:#f8f8f8;stroke-width:0.26458335px;stroke-opacity:1"
+         id="path870" />
+    </g>
+  </g>
+</svg>

docs/archetypes/default.md

@@ -1,6 +0,0 @@
----
-title: "{{ replace .TranslationBaseName "-" " " | title }}"
-date: {{ .Date }}
-draft: true
----
-

docs/changelog.rst

@@ -0,0 +1,43 @@
+.. |break_config| replace:: **[BREAK]**
+.. |bugfix| replace:: [BUG]
+
+Changelog
+=========
+
+The changelog summarized bugfixes that are deemed relevant for users.
+Developers should consult the git commit log or GitHub issue tracker.
+
+0.0.2
+-----
+
+Breaking
+~~~~~~~~
+
+* |break_config| :commit:`b95260f`: ``global.logging`` is no longer a dictionary but a list
+
+* |break_config| :commit:`3e647c1`: ``source`` job field ``datasets`` renamed to ``filesystems``
+
+  * **NOTE**: zrepl will parse missing ``filesystems`` field as an empty filter,
+    i.e. no filesystems are presented to the other side.
+
+* |bugfix| :commit:`72d2885` fix aliasing bug with root `<` subtree wildcard
+
+  * Filesystems paths with final match at blank `s` subtree wildcard are now appended to the target path
+  * Non-root subtree wildcards, e.g. `zroot/foo/bar<` still map directrly onto the target path
+
+* Support days (``d``) and weeks (``w``) in durations
+
+* Docs
+
+  * Ditch Hugo, move to Python Sphinx
+  * Improve & simplify tutorial (single SSH key per installation)
+  * Document pruning policies
+  * Document job types
+  * Document logging
+  * Start updating implementation overview
+
+
+0.0.1
+-----
+
+* Initial release

docs/conf.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# zrepl documentation build configuration file, created by
+# sphinx-quickstart on Wed Nov  8 22:28:10 2017.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['sphinx.ext.todo',
+    'sphinx.ext.githubpages',
+    'sphinx.ext.extlinks']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'zrepl'
+copyright = '2017, Christian Schwarz'
+author = 'Christian Schwarz'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+import subprocess
+version = subprocess.check_output(["git", "describe"]).decode("utf-8")
+# The full version, including alpha/beta/rc tags.
+release = version
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+
+html_logo = '_static/zrepl.svg'
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'zrepldoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'zrepl.tex', 'zrepl Documentation',
+     'Christian Schwarz', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'zrepl', 'zrepl Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'zrepl', 'zrepl Documentation',
+     author, 'zrepl', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# -- Options for the extlinks extension -----------------------------------
+# http://www.sphinx-doc.org/en/stable/ext/extlinks.html
+extlinks = {
+    'issue':('https://github.com/zrepl/zrepl/issues/%s', 'issue #'),
+    'sampleconf':('https://github.com/zrepl/zrepl/blob/master/cmd/sampleconf/%s', 'cmd/sampleconf'),
+    'commit':('https://github.com/zrepl/zrepl/commit/%s', 'commit '),
+        }
+

docs/config.toml

@@ -1,17 +0,0 @@
-baseURL = "https://zrepl.github.io"
-languageCode = "en-us"
-title = "zrepl - ZFS replication"
-theme = "docdock"
-
-[[menu.shortcuts]]
-pre = "<h3>More</h3>"
-name = "<i class='fa fa-github'></i> GitHub repo"
-identifier = "ds"
-url = "https://github.com/zrepl/zrepl"
-weight = 10
-
-[[menu.shortcuts]]
-name = "<i class='fa fa-twitter'></i> Primary Author on Twitter"
-url = "https://twitter.com/problame"
-weight = 11
-

docs/configuration.rst

@@ -0,0 +1,15 @@
+
+.. _configuration_toc:
+
+*************
+Configuration
+*************
+
+.. toctree::
+
+    configuration/jobs
+    configuration/transports
+    configuration/map_filter_syntax
+    configuration/prune
+    configuration/logging
+    configuration/misc

docs/configuration/jobs.rst

@@ -0,0 +1,184 @@
+.. include:: ../global.rst.inc
+
+.. |patient| replace:: :ref:`patient <job-term-patient>`
+.. |serve-transport| replace:: :ref:`serve transport<transport-ssh+stdinserver-serve>`
+.. |connect-transport| replace:: :ref:`connect transport<transport-ssh+stdinserver-connect>`
+.. |mapping| replace:: :ref:`mapping <pattern-mapping>`
+.. |filter| replace:: :ref:`filter <pattern-filter>`
+.. |prune| replace:: :ref:`prune <prune>`
+
+.. _job:
+
+Job Types
+=========
+
+A *job* is the unit of activity tracked by the zrepl daemon and configured in the |mainconfig|.
+Every job has a unique ``name``, a ``type`` and type-dependent fields which are documented on this page.
+Check out the :ref:`tutorial` and :sampleconf:`/` for examples on how job types are actually used.
+
+.. ATTENTION::
+
+    Currently, zrepl does not replicate filesystem properties.
+    Whe receiving a filesystem, it is never mounted (`-u` flag)  and `mountpoint=none` is set.
+    This is temporary and being worked on :issue:`24`.
+
+.. _job-source:
+
+Source Job
+----------
+
+Example: :sampleconf:`pullbackup/productionhost.yml`.
+
+.. list-table::
+    :widths: 20 80
+    :header-rows: 1
+
+    * - Parameter
+      - Comment
+    * - ``type``
+      - = ``source``
+    * - ``name``
+      - unique name of the job
+    * - ``serve``
+      - |serve-transport|
+    * - ``filesystems``
+      - |filter| for filesystems to expose to client
+    * - ``snapshot_prefix``
+      - prefix for ZFS snapshots taken by this job
+    * - ``interval``
+      - snapshotting interval
+    * - ``prune``
+      - |prune| policy for filesytems in ``filesystems`` with prefix ``snapshot_prefix``
+
+
+- Snapshotting Task (every ``interval``, |patient|)
+
+  - A snapshot of filesystems matched by ``filesystems`` is taken every ``interval`` with prefix ``snapshot_prefix``.
+  - The ``prune`` policy is triggered on filesystems matched by ``filesystems`` with snapshots matched by ``snapshot_prefix``.
+
+- Serve Task
+
+  - Wait for connections from pull job using ``serve``.
+
+A source job is the counterpart to a :ref:`job-pull`.
+
+Note that the prune policy determines the maximum replication lag:
+a pull job may stop replication due to link failure, misconfiguration or administrative action.
+The source prune policy will eventually destroy the last common snapshot between source and pull job, requiring full replication.
+Make sure you read the |prune| policy documentation.
+
+
+.. _job-pull:
+
+Pull Job
+--------
+
+Example: :sampleconf:`pullbackup/backuphost.yml`
+
+.. list-table::
+    :widths: 20 80
+    :header-rows: 1
+
+    * - Parameter
+      - Comment
+    * - ``type``
+      - = ``pull``
+    * - ``name``
+      - unqiue name of the job
+    * - ``connect``
+      - |connect-transport|
+    * - ``interval``
+      - Interval between pull attempts
+    * - ``mapping``
+      - |mapping| for remote to local filesystems
+    * - ``initial_repl_policy``
+      - default = ``most_recent``, initial replication policy
+    * - ``snapshot_prefix``
+      - prefix snapshots must match to be considered for replication & pruning
+    * - ``prune``
+      - |prune| policy for local filesystems reachable by ``mapping``
+
+* Main Task (every ``interval``, |patient|)
+ 
+  #. A connection to the remote source job is established using the strategy in ``connect``
+  #. ``mapping`` maps filesystems presented by the remote side to local *target filesystems*
+  #. Those remote filesystems with a local *target filesystem* are replicated
+ 
+     #. Only snapshots with prefix ``snapshot_prefix`` are replicated.
+     #. If possible, incremental replication takes place.
+     #. If the local target filesystem does not exist, ``initial_repl_policy`` is used.
+     #. On conflicts, an error is logged but replication of other filesystems with mapping continues.
+  
+  #. The ``prune`` policy is triggered for all *target filesystems*
+
+A pull job is the counterpart to a :ref:`job-source`.
+
+
+.. _job-local:
+
+Local Job
+---------
+
+Example: :sampleconf:`localbackup/host1.yml`
+
+.. list-table::
+    :widths: 20 80
+    :header-rows: 1
+
+    * - Parameter
+      - Comment
+    * - ``type``
+      - = ``local``
+    * - ``name``
+      - unqiue name of the job
+    * - ``mapping``
+      - |mapping| from source to target filesystem (both local)
+    * - ``snapshot_prefix``
+      - prefix for ZFS snapshots taken by this job
+    * - ``interval``
+      - snapshotting & replication interval
+    * - ``initial_repl_policy``
+      - default = ``most_recent``, initial replication policy
+    * - ``prune_lhs``
+      -  pruning policy on left-hand-side (source)
+    * - ``prune_rhs``
+      - pruning policy on right-hand-side (target)
+
+* Main Task (every ``interval``, |patient|)
+
+  #. Evaluate ``mapping`` for local filesystems, those with a *target filesystem* are called *mapped filesystems*.
+  #. Snapshot *mapped filesystems* with ``snapshot_prefix``.
+  #. Replicate *mapped filesystems* to their respective *target filesystems*:
+
+     #. Only snapshots with prefix ``snapshot_prefix`` are replicated.
+     #. If possible, incremental replication takes place.
+     #. If the *target filesystem* does not exist, ``initial_repl_policy`` is used.
+     #. On conflicts, an error is logged but replication of other *mapped filesystems* continues.
+
+  #. The ``prune_lhs`` policy is triggered for all *mapped filesystems*
+  #. The ``prune_rhs`` policy is triggered for all *target filesystems*
+
+A local job is combination of source & pull job executed on the same machine.
+
+Terminology
+-----------
+
+task
+
+    A job consists of one or more tasks and a task consists of one or more steps.
+    Some tasks may be periodic while others wait for an event to occur.
+
+
+patient task
+
+    .. _job-term-patient:
+
+    A patient task is supposed to execute some task every `interval`.
+    We call the start of the task an *invocation*.
+
+    * If the task completes in less than `interval`, the task is restarted at `last_invocation + interval`.
+    * Otherwise, a patient job
+        * logs a warning as soon as a task exceeds its configured `interval`
+        * waits for the last invocation to finish
+        * logs a warning with the effective task duration
+        * immediately starts a new invocation of the task

docs/configuration/logging.rst

@@ -0,0 +1,198 @@
+.. include:: ../global.rst.inc
+
+.. _logging:
+
+Logging
+=======
+
+zrepl uses structured logging to provide users with easily processable log messages.
+
+Logging outlets are configured in the ``global`` section of the |mainconfig|.
+Check out :sampleconf:`random/logging.yml` for an example on how to configure multiple outlets:
+
+::
+
+    global:
+      logging:
+    
+        - outlet: OUTLET_TYPE
+          level: MINIMUM_LEVEL
+          format: FORMAT
+    
+        - outlet: OUTLET_TYPE
+          level: MINIMUM_LEVEL
+          format: FORMAT
+    
+        ...
+    
+    jobs: ...
+
+Default Configuration
+---------------------
+
+By default, the following logging configuration is used
+
+::
+
+    global:
+      logging:
+    
+        - outlet: "stdout"
+          level:  "warn"
+          format: "human"
+
+.. ATTENTION::
+    Output to **stderr** should always be considered a **critical error**.
+    Only errors in the logging infrastructure itself, e.g. IO errors when writing to an outlet, are sent to stderr.
+
+Building Blocks
+---------------
+
+The following sections document the semantics of the different log levels, formats and outlet types.
+
+.. _logging-levels:
+
+Levels
+~~~~~~
+
+.. list-table::
+    :widths: 10 10 80
+    :header-rows: 1
+
+    * - Level
+      - SHORT
+      - Description
+    * - ``error``
+      - ``ERRO``
+      - immediate action required
+    * - ``warn``
+      - ``WARN``
+      - symptoms for misconfiguration, soon expected failure, etc.
+    * - ``info``
+      - ``INFO``
+      - explains what happens without too much detail
+    * - ``debug``
+      - ``DEBG``
+      - tracing information, state dumps, etc. useful for debugging.
+
+Incorrectly classified messages are considered a bug and should be reported.
+
+.. _logging-formats:
+
+Formats
+~~~~~~~
+
+.. list-table::
+    :widths: 10 90
+    :header-rows: 1
+
+    * - Format
+      - Description
+    * - ``human``
+      - emphasizes context by putting job, task, step and other context variables into brackets
+        before the actual message, followed by remaining fields in logfmt style|
+    * - ``logfmt``
+      - `logfmt <https://brandur.org/logfmt>`_ output. zrepl uses `this Go package <https://github.com/go-logfmt/logfmt>`_.
+    * - ``json``
+      - JSON formatted output. Each line is a valid JSON document. Fields are marshaled by
+        ``encoding/json.Marshal()``, which is particularly useful for processing in
+        log aggregation or when processing state dumps.
+
+.. _logging-outlet-stdout:
+
+``stdout`` Outlet
+-----------------
+
+.. list-table::
+    :widths: 10 90
+    :header-rows: 1
+
+    * - Parameter
+      - Comment
+    * - ``outlet``
+      - ``stdout``
+    * - ``level``
+      -  minimum  :ref:`log level <logging-levels>`
+    * - ``format``
+      - output :ref:`format <logging-formats>`
+
+Writes all log entries with minimum level ``level`` formatted by ``format`` to stdout.
+
+Can only be specified once.
+
+``syslog`` Outlet
+-----------------
+.. list-table::
+    :widths: 10 90
+    :header-rows: 1
+
+    * - Parameter
+      - Comment
+    * - ``outlet``
+      - ``syslog``
+    * - ``level``
+      -  minimum  :ref:`log level <logging-levels>`
+    * - ``format``
+      - output :ref:`format <logging-formats>`
+    * - ``retry_interval``
+      - Interval between reconnection attempts to syslog (default = 0)
+
+Writes all log entries formatted by ``format`` to syslog.
+On normal setups, you should not need to change the ``retry_interval``.
+
+Can only be specified once.
+
+``tcp`` Outlet
+--------------
+
+.. WARNING::
+
+    The TCP outlet is not fully asynchronous and blocks the calling goroutine when it cannot connect.
+    Currently it should only be used for local connections that are guaranteed to not fail / be slow.
+    This issue is tracked in :issue:`26`
+
+.. list-table::
+    :widths: 10 90
+    :header-rows: 1
+
+    * - Parameter
+      - Comment
+    * - ``outlet``
+      - ``tcp``
+    * - ``level``
+      -  minimum  :ref:`log level <logging-levels>`
+    * - ``format``
+      - output :ref:`format <logging-formats>`
+    * - ``net``
+      - ``tcp`` in most cases
+    * - ``address``
+      - remote network, e.g. ``logs.example.com:10202``
+    * - ``retry_interval``
+      - Interval between reconnection attempts to ``address``
+    * - ``tls``
+      - TLS config (see below)
+
+Establishes a TCP connection to ``address`` and sends log messages with minimum level ``level`` formatted by ``format``.
+
+If ``tls`` is not specified, an unencrypted connection is established.
+
+If ``tls`` is specified, the TCP connection is secured with TLS + Client Authentication.
+This is particularly useful in combination with log aggregation services that run on an other machine.
+
+.. list-table::
+    :widths: 10 90
+    :header-rows: 1
+
+    * - Parameter
+      - Description
+    * - ``ca``
+      - PEM-encoded certificate authority that signed the remote server's TLS certificate
+    * - ``cert``
+      - PEM-encoded client certificate identifying this zrepl daemon toward the remote server
+    * - ``key``
+      - PEM-encoded, unencrypted client private key identifying this zrepl daemon toward the remote server
+
+.. NOTE::
+
+    zrepl uses Go's ``crypto/tls`` and ``crypto/x509`` packages and leaves all but the required fields in ``tls.Config`` at their default values.
+    In case of a security defect in these packages, zrepl has to be rebuilt because Go binaries are statically linked.

docs/configuration/map_filter_syntax.rst

@@ -0,0 +1,104 @@
+.. include:: ../global.rst.inc
+
+Mapping & Filter Syntax
+=======================
+
+For various job types, a filesystem ``mapping`` or ``filter`` needs to be
+specified.
+
+Both have in common that they take a filesystem path (in the ZFS filesystem hierarchy)as parameters and return something.
+Mappings return a *target filesystem* and filters return a *filter result*.
+
+The pattern syntax is the same for mappings and filters and is documented in the following section.
+
+Common Pattern Syntax
+---------------------
+
+A mapping / filter is specified as a **YAML dictionary** with patterns as keys and
+results as values.
+The following rules determine which result is chosen for a given filesystem path:
+
+* More specific path patterns win over less specific ones
+* Non-wildcard patterns (full path patterns) win over *subtree wildcards* (`<` at end of pattern)
+
+The **subtree wildcard** ``<`` means "the dataset left of ``<`` and all its children".
+
+Example
+~~~~~~~
+
+::
+
+    # Rule number and its pattern
+    1: tank<            # tank and all its children
+    2: tank/foo/bar     # full path pattern (no wildcard)
+    3: tank/foo<        # tank/foo and all its children
+    
+    # Which rule applies to given path?
+    tank/foo/bar/loo => 3
+    tank/bar         => 1
+    tank/foo/bar     => 2
+    zroot            => NO MATCH
+    tank/var/log     => 1
+
+.. _pattern-mapping:
+
+Mappings
+--------
+
+Mappings map a *source filesystem path* to a *target filesystem path*.
+Per pattern, either a target filesystem path or ``"!"`` is specified as a result.
+
+* If no pattern matches, there exists no target filesystem (``NO MATCH``).
+* If the result is a ``"!"``, there exists no target filesystem (``NO MATCH``).
+* If the pattern is a non-wildcard pattern, the source path is mapped to the target path on the right.
+* If the pattern ends with a *subtree wildcard* (``<``), the source path is **prefix-trimmed** with the path specified left of ``<``.
+
+  * Note: this means that only for *wildcard-only* patterns (pattern= ``<`` ) is the source path simply appended to the target path.
+
+The example is from the :sampleconf:`localbackup/host1.yml` example config.
+
+::
+
+    jobs:
+    - name: mirror_local
+      type: local
+      mapping: {
+        "zroot/var/db<":    "storage/backups/local/zroot/var/db",
+        "zroot/usr/home<":  "storage/backups/local/zroot/usr/home",
+        "zroot/usr/home/paranoid":  "!", #don't backup paranoid user
+        "zroot/poudriere/ports<": "!", #don't backup the ports trees
+      }
+      ...
+
+
+::
+
+    zroot/var/db                        => storage/backups/local/zroot/var/db
+    zroot/var/db/a/child                => storage/backups/local/zroot/var/db/a/child
+    zroot/usr/home                      => storage/backups/local/zroot/usr/home
+    zroot/usr/home/paranoid             => NOT MAPPED
+    zroot/usr/home/bob                  => storage/backups/local/zroot/usr/home/bob
+    zroot/usr/src                       => NOT MAPPED
+    zroot/poudriere/ports/2017Q3        => NOT MAPPED
+    zroot/poudriere/ports/HEAD          => NOT MAPPED
+
+.. _pattern-filter:
+
+Filters
+-------
+
+Valid filter results: ``ok`` or ``!``.
+
+The example below show the source job from the :ref:`tutorial <tutorial-configure-app-srv>`:
+The corresponding pull job is allowed access to ``zroot/var/db``, ``zroot/usr/home`` + children except ``zroot/usr/home/paranoid``::
+
+    jobs:
+    - name: pull_backup
+      type: source
+      ...
+      filesystems: {
+        "zroot/var/db": "ok",
+        "zroot/usr/home<": "ok",
+        "zroot/usr/home/paranoid": "!",
+      }
+      ...

docs/configuration/misc.rst

@@ -0,0 +1,63 @@
+Miscellaneous
+=============
+
+.. _conf-runtime-directories:
+
+Runtime Directories & UNIX Sockets
+----------------------------------
+
+zrepl daemon creates various UNIX sockets to allow communicating with it:
+
+* the :ref:`transport-ssh+stdinserver` transport connects to a socket named after ``client_identity`` parameter
+* the ``control`` CLI subcommand connects to a defined control socket
+
+There is no further authentication on these sockets.
+Therefore we have to make sure they can only be created and accessed by ``zrepl daemon``.
+
+In fact, ``zrepl daemon`` will not bind a socket to a path in a directory that is world-accessible.
+
+The directories can be configured in the main configuration file:
+
+::
+
+    global:
+      control:
+        sockpath: /var/run/zrepl/control
+      serve:
+        stdinserver:
+          sockdir: /var/run/zrepl/stdinserver
+
+
+Durations & Intervals
+---------------------
+
+Interval & duration fields in job definitions, pruning configurations, etc. must match the following regex:
+
+::
+
+    var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
+    // s = second, m = minute, h = hour, d = day, w = week (7 days)
+
+Super-Verbose Job Debugging
+---------------------------
+
+You have probably landed here because you opened an issue on GitHub and some developer told you to do this...
+So just read the annotated comments ;)
+
+::
+
+    job:
+    - name: ...
+      ...
+     # JOB DEBUGGING OPTIONS
+      # should be equal for all job types, but each job implements the debugging itself
+      debug:
+        conn: # debug the io.ReadWriteCloser connection
+          read_dump: /tmp/connlog_read   # dump results of Read() invocations to this file
+          write_dump: /tmp/connlog_write # dump results of Write() invocations to this file
+        rpc: # debug the RPC protocol implementation
+          log: true # log output from rpc layer to the job log
+
+.. ATTENTION::
+
+    Connection dumps will almost certainly contain your or other's private data. Do not share it in a bug report.

docs/configuration/prune.rst

@@ -0,0 +1,64 @@
+.. _prune:
+
+Pruning Policies
+================
+
+In zrepl, *pruning* means *destroying snapshots by some policy*.
+
+A *pruning policy* takes a list of snapshots and -- for each snapshot -- decides whether it should be kept or destroyed.
+
+The job context defines which snapshots are even considered for pruning, for example through the ``snapshot_prefix`` variable.
+Check the respective :ref:`job definition <job>` for details.
+
+Currently, the :ref:`prune-retention-grid` is the only supported pruning policy.
+
+.. _prune-retention-grid:
+
+Retention Grid
+--------------
+
+::
+
+    jobs:
+    - name: pull_app-srv
+      ...
+      prune:
+        policy: grid
+        grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+                │               │
+                └─ one hour interval
+                                │
+                                └─ 24 adjacent one-hour intervals
+
+The retention grid can be thought of as a time-based sieve:
+The ``grid`` field specifies a list of adjacent time intervals:
+the left edge of the leftmost (first) interval is the ``creation`` date of the youngest snapshot.
+All intervals to its right describe time intervals further in the past.
+
+Each interval carries a maximum number of snapshots to keep.
+It is secified via ``(keep=N)``, where ``N`` is either ``all`` (all snapshots are kept) or a positive integer.
+The default value is **1**.
+
+The following procedure happens during pruning:
+
+#. The list of snapshots eligible for pruning is sorted by ``creation``
+#. The left edge of the first interval is aligned to the ``creation`` date of the youngest snapshot
+#. A list of buckets is created, one for each interval
+#. The list of snapshots is split up into the buckets.
+#. For each bucket
+
+   #. the contained snapshot list is sorted by creation.
+   #. snapshots from the list, oldest first, are destroyed until the specified ``keep`` count is reached.
+   #. all remaining snapshots on the list are kept.
+
+.. ATTENTION::
+
+    .. TODO this is obsolete as soon as bookmarks are created during snapshotting
+
+    The configuration of the first interval (``1x1h(keep=all)`` in the example) determines the **maximum allowable replication lag** between source and destination.
+    After the first interval, source and destination likely have different retention settings.
+    This means source and destination may prune different snapshots, prohibiting incremental replication froms snapshots that are not in the first interval.
+
+    **Always** configure the first interval to ``1x?(keep=all)``, substituting ``?`` with the maximum time replication may fail due to downtimes, maintenance, connectivity issues, etc.
+    After outages longer than ``?`` you may be required to perform **full replication** again.
+

docs/configuration/transports.rst

@@ -0,0 +1,112 @@
+.. highlight:: bash
+
+.. _transport:
+
+Transports
+==========
+
+A transport provides an authenticated `io.ReadWriteCloser <https://golang.org/pkg/io/#ReadWriteCloser>`_ to the RPC layer.
+(An ``io.ReadWriteCloser`` is essentially a bidirectional reliable communication channel.)
+
+Currently, only the ``ssh+stdinserver`` transport is supported.
+
+.. _transport-ssh+stdinserver:
+
+``ssh+stdinserver`` Transport
+-----------------------------
+
+The way the ``ssh+stdinserver`` transport works is inspired by `git shell <https://git-scm.com/docs/git-shell>`_ and `Borg Backup <https://borgbackup.readthedocs.io/en/stable/deployment.html>`_.
+It is implemented in the Go package ``github.com/zrepl/zrepl/sshbytestream``.
+The config excerpts are taken from the :ref:`tutorial` which you should complete before reading further.
+
+.. _transport-ssh+stdinserver-serve:
+
+Serve Mode
+~~~~~~~~~~
+
+::
+
+    jobs:
+    - name: pull_backup
+      type: source
+      serve:
+        type: stdinserver
+        client_identity: backup-srv.example.com
+      ...
+
+The serving job opens a UNIX socket named after ``client_identity`` in the runtime directory, e.g. ``/var/run/zrepl/stdinserver/backup-srv.example.com``.
+
+On the same machine, the ``zrepl stdinserver $client_identity`` command connects to that socket.
+For example, ``zrepl stdinserver backup-srv.example.com`` connects to the UNIX socket ``/var/run/zrepl/stdinserver/backup-srv.example.com``.
+
+It then passes its stdin and stdout file descriptors to the zrepl daemon via *cmsg(3)*.
+zrepl daemon in turn combines them into an ``io.ReadWriteCloser``:
+a ``Write()`` turns into a write to stdout, a ``Read()`` turns into a read from stdin.
+
+Interactive use of the ``stdinserver`` subcommand does not make much sense.
+However, we can force its execution when a user with a particular SSH pubkey connects via SSH.
+This can be achieved with an entry in the ``authorized_keys`` file of the serving zrepl daemon.
+
+::
+
+    # for OpenSSH >= 7.2
+    command="zrepl stdinserver CLIENT_IDENTITY",restrict CLIENT_SSH_KEY
+    # for older OpenSSH versions
+    command="zrepl stdinserver CLIENT_IDENTITY",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc CLIENT_SSH_KEY
+
+* CLIENT_IDENTITY is substituted with ``backup-srv.example.com`` in our example
+* CLIENT_SSH_KEY is substituted with the public part of the SSH keypair specified in the ``connect`` directive on the connecting host.
+
+.. NOTE::
+
+    You may need to adjust the ``PermitRootLogin`` option in ``/etc/ssh/sshd_config`` to ``forced-commands-only`` or higher for this to work.
+    Refer to sshd_config(5) for details.
+
+To recap, this is of how client authentication works with the ``ssh+stdinserver`` transport:
+
+* Connections to the ``client_identity`` UNIX socket are blindly trusted by zrepl daemon.
+* Thus, the runtime directory must be private to the zrepl user (checked by zrepl daemon)
+* The admin of the host with the serving zrepl daemon controls the ``authorized_keys`` file.
+* Thus, the administrator controls the mapping ``PUBKEY -> CLIENT_IDENTITY``.
+
+.. _transport-ssh+stdinserver-connect:
+
+Connect Mode
+~~~~~~~~~~~~
+
+::
+
+    jobs:
+    - name: pull_app-srv
+      type: pull
+      connect:
+        type: ssh+stdinserver
+        host: app-srv.example.com
+        user: root
+        port: 22
+        identity_file: /etc/zrepl/ssh/identity
+        options: # optional
+        - "Compression=on"
+
+The connecting zrepl daemon
+
+#. Creates a pipe
+#. Forks
+#. In the forked process
+
+   #. Replaces forked stdin and stdout with the corresponding pipe ends
+   #. Executes the ``ssh`` binary found in ``$PATH``.
+
+      #. The identity file (``-i``) is set to ``$identity_file``.
+      #. The remote user, host and port correspond to those configured.
+      #. Further options can be specified using the ``options`` field, which appends each entry in the list to the command line using ``-o $entry``.
+
+1. Wraps the pipe ends in an ``io.ReadWriteCloser`` and uses it for RPC.
+
+As discussed in the section above, the connecting zrepl daemon expects that ``zrepl stdinserver $client_identity`` is  executed automatically via an ``authorized_keys`` file entry.
+
+.. NOTE::
+
+    The environment variables of the underlying SSH process are cleared. ``$SSH_AUTH_SOCK`` will not be available.
+    It is suggested to create a separate, unencrypted SSH key solely for that purpose.
+

docs/content/_header.md

@@ -1 +0,0 @@
-zrepl Documentation

docs/content/_index.md

@@ -1,45 +0,0 @@
-+++
-title = "zrepl - ZFS replication"
-+++
-
-# zrepl - ZFS replication
-
-`zrepl` is a tool for replicating ZFS filesystems.
-
-{{% panel theme="danger" header="Important" %}}
-`zrepl` as well as this documentation is still under active
-development. Some of the features below are not implemented yet. Use & test at your own risk ;)
-{{% / panel  %}}
-
-## Main Features
-
-* filesystem replication
-    * local & over network (SSH)
-    * push & pull mode
-    * snapshots & bookmarks support
-    * feature-negotiation for
-        * resumable `send & receive`
-        * compressed `send & receive`
-        * raw encrypted `send & receive` (as soon as it is available)
-    * access control checks when pulling datasets
-    * [flexible mappings]({{< ref "configuration/overview.md#mapping-filter-syntax" >}}) for filesystems
-* automatic snapshot creation
-    * periodic interval
-* automatic snapshot pruning
-    * [Retention Grid]({{< ref "configuration/snapshots.md#retention-grid" >}})
-   
-## Contributing
-
-`zrepl` is usable but nowhere near a stable release right now -  we are happy
-about contributors!
-
-* Explore the codebase
-    * These docs live in the `docs/` subdirectory
-* Document non-obvious / confusing / plain broken things you encounter when using `zrepl` for the first time
-* Check the *Issues* and *Projects* sections for things to do ;)
-
-{{% panel header="<i class='fa fa-github'></i> Getting your code merged"%}}
-[The <i class='fa fa-github'></i> GitHub repository](https://github.com/zrepl/zrepl) is where all development happens.
-Open your issue / PR there.
-{{% /panel %}}
-

docs/content/configuration/_index.md

@@ -1,6 +0,0 @@
-+++
-title = "Configuration"
-alwaysopen = true
-+++
-
-{{% children description="true" %}}

docs/content/configuration/example_pull_backup.md

@@ -1,9 +0,0 @@
-+++
-title = "Example: Pull Backup"
-description = "Example configuration for a typical pull-backup scenario, e.g. server to server"
-+++
-
-Example configuration for a typical pull-backup scenario, e.g. server to server
-
-{{% alert theme="warning"%}}TBD{{% /alert %}}
-

docs/content/configuration/example_push_backup.md

@@ -1,8 +0,0 @@
-+++
-title = "Example: Push Backup"
-description = "Example configuration for a typical push-backup scenario, e.g. laptop to NAS"
-+++
-
-Example configuration for a typical push-backup scenario, e.g. laptop to NAS
-
-{{% alert theme="warning"%}}TBD{{% /alert %}}

docs/content/configuration/overview.md

@@ -1,149 +0,0 @@
-+++
-title = "Overview"
-weight = 100
-description = "Configuration format, SSH authentication, etc."
-+++
-
-{{% panel header="Recommendation" %}}
-Keep the [sample configuration file](https://github.com/zrepl/zrepl/blob/master/cmd/sampleconf/zrepl.yml) open on the side while reading this document!
-{{% / panel %}}
-
-All configuration is managed in a single YAML file.<br />
-It is structured by sections roughly corresponding to `zrepl` subcommands:
-
-```yaml
-# REPLICATION
-# Remote zrepl instances where pull and push jobs connect to
-remotes: 
-  name_of_remote: #...
-# Push jobs (replication from local to remote)
-pushs: 
-  name_of_push_job: #...
-  name_of_other_push_job: #...
-# pull jobs (replication from remote to local & local to local)
-pulls: 
-  name_of_pull_job: #...
-# mapping incoming pushs to local datasets
-sinks: 
-  client_identity: #...
-# access control for remote pull jobs
-pull_acls: 
-  client_identity: #...
-
-# SNAPSHOT MANAGEMENT
-# Automatic snapshotting of filesystems
-autosnap:
-  name_of_autosnap_job: #... 
-# Automatic pruning of snapshots based on creation date
-prune:
-  name_of_prune_job: #... 
-```
-
-When using `zrepl(8)`, a *subcommand* is passed the *job name* as a positional argument:
-
-```yaml
-autosnap: # subcommand
-  db: # job name
-    prefix: zrepl_ 
-    interval: 10m
-    dataset_filter: {
-      "tank/db<": ok
-    }
-```
-```bash
-$ zrepl autosnap --config zrepl.yml db
-```
-
-Run `zrepl --help` for a list of subcommands and options.
-
-## Mapping & Filter Syntax
-
-For various job types, a filesystem `mapping` or `filter` needs to be
-specified.
-
-Both have in common that they return a result for a given filesystem path (in
-the ZFS filesystem hierarchy): mappings return a *target filesystem*, filters
-return a *filter result* (`omit`, `ok`).
-
-The pattern syntax is the same for mappings and filters and is documented in
-the following section.
-
-#### Pattern Syntax
-
-A mapping / filter is specified as a **YAML dictionary** with patterns as keys and
-results as values.<br />
-The following rules determine which result is chosen for a given filesystem path:
-
-* More specific path patterns win over less specific ones
-* Non-wildcard patterns (full path patterns) win over *subtree wildcards* (`<` at end of pattern)
-
-{{% panel %}}
-The **subtree wildcard** `<` means "*the dataset left of `<` and all its children*".
-{{% / panel %}}
-
-##### Example
-
-```
-# Rule number and its pattern
-1: tank<            # tank and all its children
-2: tank/foo/bar     # full path pattern (no wildcard)
-3: tank/foo<        # tank/foo and all its children
-
-# Which rule applies to given path?
-tank/foo/bar/loo => 3
-tank/bar         => 1
-tank/foo/bar     => 2
-zroot            => NO MATCH
-tank/var/log     => 1
-```
-
-#### Mappings
-
-The example below shows a pull job that would pull remote datasets to the given local paths.<br />
-If there exists no mapping (`NO MATCH`), the filesystem will not be pulled.
-
-```yaml
-pull:
-  app01.example.com: # client identity
-    from: app01.example.com
-    mapping: { 
-      "tank/var/db<":    "zroot/backups/app01/tank/var/db",
-      "tank/var/www<":   "zroot/backups/app01/tank/var/www",
-      "tank/var/log":    "zroot/logbackup/app01",
-    }
-```
-
-```
-tank/var/db`          => zroot/backups/app01/tank/var/db
-tank/var/db/a/child   => zroot/backups/app01/tank/var/db/a/child
-...
-tank/var/www          => zroot/backups/app01/tank/var/www
-tank/var/www/a/child  => zroot/backups/app01/tank/var/www/a/child
-...
-tank/var/log          => zroot/logbackup/app01
-tank/var/log/foo      => NOT MAPPED
-```
-
-#### Filters
-
-Valid filter results: `ok` or `omit`.
-
-The example below shows a pull ACL that allows access to the user homes but not
-to the rest of the system's datasets.
-
-```
-# Example for filter syntax
-pull_acls:
-  backups.example.com: # client identity
-    filter: {
-      "<":              omit,
-      "tank/usr/home<": ok,
-    }
-```
-
-## Next up
-
-* [Automating snapshot creation & pruning]({{< ref "configuration/snapshots.md" >}})
-
-* [Replicating filesystems]({{< ref "configuration/replication.md" >}})
-

docs/content/configuration/replication.md

@@ -1,40 +0,0 @@
-+++
-title = "Filesystem Replication"
-description = "Replicating filesystems with existing bookmarks & snapshots"
-weight = 300
-+++
-
-{{% alert theme="warning"%}}Under Construction{{% /alert %}}
-
-### Remotes
-
-The `remotes` section specifies remote `zrepl` instances from which to pull from / push backups to:
-
-```yaml
-remotes:
-  offsite_backups:
-    transport:
-      ssh:
-        host: 192.168.122.6
-        user: root
-        port: 22
-        identity_file: /etc/zrepl/identities/offsite_backups
-```
-
-#### SSH Transport
-
-The SSH transport connects to the remote server using the SSH binary in
-`$PATH` and the parameters specified in the `zrepl` config file.
-
-However, instead of a traditional interactive SSH session, `zrepl` expects
-another instance of `zrepl` on the other side of the connection; You may be
-familiar with this concept from [git shell](https://git-scm.com/docs/git-shell)
-or [Borg Backup](https://borgbackup.readthedocs.io/en/stable/deployment.html).
-
-Check the examples for instructions on how to set this up on your machines! 
-
-{{% panel %}}
-The environment variables of the underlying SSH process are cleared. `$SSH_AUTH_SOCK` will not be available. We suggest creating a separate, unencrypted SSH key.
-{{% / panel %}}
-
-

docs/content/configuration/snapshots.md

@@ -1,9 +0,0 @@
-+++
-title = "Snapshot Management"
-description = "Automated snapshot creation & pruning"
-weight = 200
-+++
-
-{{% alert theme="warning"%}}TBD{{% /alert %}}
-
-## Retention Grid

docs/content/impl/_index.md

@@ -1,44 +0,0 @@
-+++
-title = "Implementation Overview"
-+++
-
-The following design aspects may convince you that `zrepl` is superior to a hacked-together shell script solution.
-
-## Language
-
-`zrepl` is written in Go, a real programming language with type safety,
-reasonable performance, testing infrastructure and an (opinionated) idea of
-software engineering.
-
-* key parts & algorithms of `zrepl` are covered by unit tests
-* zrepl is noticably faster than comparable shell scripts
-
-
-## RPC protocol
-
-While it is tempting to just issue a few `ssh remote 'zfs send ...' | zfs recv`, this has a number of drawbacks:
-
-* The snapshot streams need to be compatible.
-* Communication is still unidirectional. Thus, you will most likely
-    * either not take advantage of features such as *compressed send & recv*
-    * or issue additional `ssh` commands in advance to figure out what features are supported on the other side.
-* Advanced logic in shell scripts is ugly to read, poorly testable and a pain to maintain.
-
-`zrepl` takes a different approach:
-
-* Define an RPC protocol.
-* Establish an encrypted, authenticated, bidirectional communication channel...
-* ... with `zrepl` running at both ends of it.
-
- This has several obvious benefits:
-
-* No blank root shell access is given to the other side.
-* Instead, access control lists (ACLs) are used to grant permissions to *authenticated* peers. 
-* The transport mechanism is decoupled from the remaining logic, keeping it extensible (e.g. TCP+TLS)
-
-{{% panel %}}
-Currently, the bidirectional communication channel is multiplexed on top of a single SSH connection.
-Local replication is of course handled efficiently via simple method calls
-See TODO for details.
-{{% / panel %}}
-

docs/global.rst.inc

@@ -0,0 +1,3 @@
+.. global list of substitutions
+
+.. |mainconfig| replace:: :ref:`main configuration file <mainconfigfile>`

docs/implementation.rst

@@ -0,0 +1,62 @@
+.. _implementation_toc:
+
+Implementation Overview
+=======================
+
+.. WARNING::
+
+    Incomplete / under construction
+
+The following design aspects may convince you that ``zrepl`` is superior to a hacked-together shell script solution.
+
+Testability & Performance
+-------------------------
+
+zrepl is written in Go, a real programming language with type safety,
+reasonable performance, testing infrastructure and an (opinionated) idea of
+software engineering.
+
+* key parts & algorithms of zrepl are covered by unit tests (work in progress)
+* zrepl is noticably faster than comparable shell scripts
+
+
+RPC protocol
+------------
+
+While it is tempting to just issue a few ``ssh remote 'zfs send ...' | zfs recv``, this has a number of drawbacks:
+
+* The snapshot streams need to be compatible.
+* Communication is still unidirectional. Thus, you will most likely
+
+  * either not take advantage of features such as *compressed send & recv*
+  * or issue additional ``ssh`` commands in advance to figure out what features are supported on the other side.
+
+* Advanced logic in shell scripts is ugly to read, poorly testable and a pain to maintain.
+
+zrepl takes a different approach:
+
+* Define an RPC protocol.
+* Establish an encrypted, authenticated, bidirectional communication channel...
+* ... with zrepl running at both ends of it.
+
+This has several obvious benefits:
+
+* No blank root shell access is given to the other side.
+* Instead, an *authenticated* peer can *request* filesystem lists, snapshot streams, etc.
+* Requests are then checked against job-specific ACLs, limiting a client to the filesystems it is actually allowed to replicate.
+* The :ref:`transport mechanism <transport>` is decoupled from the remaining logic, keeping it extensible.
+
+Protocol Implementation
+~~~~~~~~~~~~~~~~~~~~~~~
+
+zrepl implements its own RPC protocol.
+This is mostly due to the fact that existing solutions do not provide efficient means to transport large amounts of data.
+
+Package `github.com/zrepl/zrepl/rpc <https://github.com/zrepl/zrepl/tree/master/rpc>`_ builds a special-case handling around returning an ``io.Reader`` as part of a unary RPC call.
+
+Measurements show only a single memory-to-memory copy of a snapshot stream is made using ``github.com/zrepl/zrepl/rpc``, and there is still potential for further optimizations.
+
+Logging & Transparency
+----------------------
+
+zrepl comes with :ref:`rich, structured and configurable logging <logging>`, allowing administators to understand what the software is actually doing.

docs/index.rst

@@ -0,0 +1,88 @@
+.. zrepl documentation master file, created by
+   sphinx-quickstart on Wed Nov  8 22:28:10 2017.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+.. include:: global.rst.inc
+
+zrepl - ZFS replication
+-----------------------
+
+.. ATTENTION::
+    zrepl as well as this documentation is still under active development.
+    It is neither feature complete nor is there a stability guarantee on the configuration format.
+    Use & test at your own risk ;)
+
+Getting started
+~~~~~~~~~~~~~~~
+
+The :ref:`5 minute tutorial setup <tutorial>` gives you a first impression.
+
+Main Features
+~~~~~~~~~~~~~
+
+* Filesystem Replication
+
+  * [x] Local & Remote
+  * [x] Pull mode
+  * [ ] Push mode
+  * [x] Access control checks when pulling datasets
+  * [x] :ref:`Flexible mapping <pattern-mapping>` rules
+  * [x] Bookmarks support
+  * [ ] Feature-negotiation for
+
+    * Resumable `send & receive`
+    * Compressed `send & receive`
+    * Raw encrypted `send & receive` (as soon as it is available)
+
+* Automatic snapshot creation
+
+  * [x] Ensure fixed time interval between snapshots
+
+* Automatic snapshot :ref:`pruning <prune>`
+
+  * [x] Age-based fading (grandfathering scheme)
+
+* Flexible, detailed & structured :ref:`logging <logging>`
+
+  * [x] ``human``, ``logfmt`` and ``json`` formatting
+  * [x] stdout, syslog and TCP (+TLS client auth) outlets
+
+* Maintainable implementation in Go
+
+  * [x] Cross platform
+  * [x] Type safe & testable code
+
+Contributing
+~~~~~~~~~~~~
+
+We are happy about any help we can get!
+
+* Explore the codebase
+
+  * These docs live in the ``docs/`` subdirectory
+
+* Document any non-obvious / confusing / plain broken behavior you encounter when setting up zrepl for the first time
+* Check the *Issues* and *Projects* sections for things to do
+
+.. admonition:: Development Workflow
+    :class: note
+
+    The `GitHub repository <https://github.com/zrepl/zrepl>`_ is where all development happens.
+    Make sure to read the `Developer Documentation section <https://github.com/zrepl/zrepl>`_ and open new issues or pull requests there.
+
+
+Table of Contents
+~~~~~~~~~~~~~~~~~
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   tutorial
+   installation
+   configuration
+   implementation
+   changelog
+   GitHub Repository & Issue Tracker <https://github.com/zrepl/zrepl>
+   pr

docs/installation.rst

@@ -0,0 +1,130 @@
+.. _installation:
+
+Installation
+============
+
+.. TIP::
+
+    Note: check out the :ref:`tutorial` if you want a first impression of zrepl.
+
+User Privileges
+---------------
+
+It is possible to run zrepl as an unprivileged user in combination with
+`ZFS delegation <https://www.freebsd.org/doc/handbook/zfs-zfs-allow.html>`_.
+Also, there is the possibility to run it in a jail on FreeBSD by delegating a dataset to the jail.
+However, until we get around documenting those setups, you will have to run zrepl as root or experiment yourself :)
+
+Packages
+--------
+
+zrepl releases are signed & tagged by the author in the git repository.
+Your OS vendor may provide binary packages of zrepl through the package manager.
+The following list may be incomplete, feel free to submit a PR with an update:
+
+.. list-table::
+    :header-rows: 1
+
+    * - OS / Distro
+      - Install Command
+      - Link
+    * - FreeBSD
+      - ``pkg install zrepl``
+      - `<https://www.freshports.org/sysutils/zrepl/>`_
+    * - Others
+      -
+      - Install from source, see below
+
+Compile From Source
+~~~~~~~~~~~~~~~~~~~
+
+Go 1.9 or newer and a configured ``$GOPATH`` environment variable and a few build dependencies are required to build zrepl.
+A tutorial is available over at `golang.org <https://golang.org/doc/install>`_.
+If Go 1.9 is not available on your distro consider build in Docker (see below).
+
+The following shell script checks out the zrepl project into your ``$GOPATH``,
+installs the build dependencies, installs dependencies using ``dep ensure`` and does a ``make release``.
+Build artifacts are placed into ``$GOPATH/github.com/zrepl/zrepl/artifacts/``.
+
+When doing builds afterwards, it should be sufficient to checkout the new revision, run ``dep ensure`` and ``make release``.
+You may want to switch to a tagged commit (we use `semver <http://semver.org>`_) but master should generally be considered stable.
+
+**Note**: it is your job to install the apropriate binary in the zrepl users's ``$PATH``, e.g. ``/usr/local/bin/zrepl``.
+Otherwise, the examples in the :ref:`tutorial` may need to be adjusted.
+
+**You are encouraged to understand what happens by auditing the script.**
+
+::
+
+    curl 'https://raw.githubusercontent.com/zrepl/zrepl/master/clone_and_build.sh' | sh
+
+You can also build in a Docker container if you want an isolated build environment or don't have a compatible Go version.
+
+::
+
+    git clone https://github.com/zrepl/zrepl.git
+    cd zrepl
+    sudo docker run -it --rm \
+        -v "${PWD}:/zrepl" \
+        --user "$(id -u):$(id -g)" \
+        golang:latest bash -c 'export CLONEPATH=/go/src/github.com/zrepl; mkdir -p "$CLONEPATH" && ln -s /zrepl $CLONEPATH/zrepl && ${CLONEPATH}/zrepl/clone_and_build.sh'
+
+.. literalinclude:: ../clone_and_build.sh
+    :language: sh
+
+
+.. _mainconfigfile:
+
+Configuration Files
+-------------------
+
+zrepl searches for its main configuration file in the following locations (in that order):
+
+* ``/etc/zrepl/zrepl.yml``
+* ``/usr/local/etc/zrepl/zrepl.yml``
+
+Alternatively, use CLI flags to specify a config location.
+Copy a config from the :ref:`tutorial` or the ``cmd/sampleconf`` directory to one of these locations and customize it to your setup.
+
+Runtime Directories
+-------------------
+
+zrepl requires ephemeral runtime directories where control sockets, etc are placed.
+Refer to the :ref:`configuration documentation <conf-runtime-directories>` for more information.
+
+When installing from a package, the package maintainer should have taken care of setting them up through the init system.
+Alternatively, for default settings, the following should to the trick.
+
+::
+
+    mkdir -p /var/run/zrepl/stdinserver
+    chmod -R 0700 /var/run/zrepl
+
+
+Running the Daemon
+------------------
+
+All actual work zrepl does is performed by a daemon process.
+Logging is configurable via the config file. Please refer to the :ref:`logging documention <logging>`.
+
+When installating from a package, the package maintainer should have provided an init script / systemd.service file.
+You should thus be able to start zrepl daemon using your init system.
+
+Alternatively, or for running zrepl in the foreground, simply execute ``zrepl daemon``.
+Note that you won't see any output unless you configure :ref:`stdout logging outlet <logging-outlet-stdout>`.
+
+.. ATTENTION::
+
+    Make sure to actually monitor the error level output of zrepl: some configuration errors will not make the daemon exit.
+
+    Example: if the daemon cannot create the :ref:`transport-ssh+stdinserver` sockets in the runtime directory,
+    it will emit an error message but not exit because other tasks such as periodic snapshots & pruning are of equal importance.
+
+.. _install-restarting:
+
+Restarting
+~~~~~~~~~~
+
+The daemon handles SIGINT and SIGTERM for graceful shutdown.
+Graceful shutdown means at worst that a job will not be rescheduled for the next interval.
+The daemon exits as soon as all jobs have reported shut down.

docs/pr.rst

@@ -0,0 +1,8 @@
+Talks & Presentations
+=====================
+
+* Talk at EuroBSDCon2017 FreeBSD DevSummit (
+  `Slides <https://docs.google.com/presentation/d/1EmmeEvOXAWJHCVnOS9-TTsxswbcGKmeLWdY_6BH4w0Q/edit?usp=sharing>`_,
+  `Event <https://wiki.freebsd.org/DevSummit/201709>`_
+  )
+

docs/publish.sh

@@ -3,11 +3,12 @@ set -eo pipefail
 
 GHPAGESREPO="git@github.com:zrepl/zrepl.github.io.git"
 SCRIPTDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-PUBLICDIR="${SCRIPTDIR}/public_git" #use different dir than default dir 'public' (hugo server -D)
+PUBLICDIR="${SCRIPTDIR}/public_git"
 
 checkout_repo_msg() {
-    echo "checkout ${GHPAGESREPO} to ${PUBLICDIR}:"
+    echo "clone ${GHPAGESREPO} to ${PUBLICDIR}:"
     echo "	git clone ${GHPAGESREPO} ${PUBLICDIR}"
+    git clone "${GHPAGESREPO}" "${PUBLICDIR}"
 }
 
 exit_msg() {
@@ -31,6 +32,7 @@ echo "verify we're in the GitHub pages repo..."
 git remote get-url origin | grep -E "^${GHPAGESREPO}\$"
 if [ "$?" -ne "0" ] ;then 
     checkout_repo_msg
+    echo "finished checkout, please run again"
     exit 1
 fi
 
@@ -44,12 +46,23 @@ git rm -rf .
 popd
 
 echo "building site"
-hugo -d "$PUBLICDIR"
+set -e
+make clean
+make html
+rsync -a _build/html/ public_git/
+set +e
+
+CURRENT_COMMIT=$(git rev-parse HEAD)
+git status --porcelain
+if [[ "$(git status --porcelain)" != "" ]]; then
+    CURRENT_COMMIT="${CURRENT_COMMIT}(dirty)" 
+fi
+COMMIT_MSG="sphinx render from publish.sh - `date -u` - ${CURRENT_COMMIT}"
 
 pushd "$PUBLICDIR"
 
 echo "adding and commiting all changes in GitHub pages repo"
 git add -A
-git commit -m "hugo render from publish.sh - `date -u`" #TODO include parent commit id
+git commit -m "$COMMIT_MSG"
 git push origin master
 

docs/tutorial.rst

@@ -0,0 +1,187 @@
+.. include:: global.rst.inc
+
+.. _tutorial:
+
+Tutorial
+========
+
+
+This tutorial shows how zrepl can be used to implement a ZFS-based pull backup.
+We assume the following scenario:
+
+* Production server ``app-srv`` with filesystems to back up:
+
+  * ``zroot/var/db``
+  * ``zroot/usr/home`` and all its child filesystems
+  * **except** ``zroot/usr/home/paranoid`` belonging to a user doing backups themselves
+
+* Backup server ``backup-srv`` with
+
+  * Filesystem ``storage/zrepl/pull/app-srv`` + children dedicated to backups of ``app-srv``
+
+Our backup solution should fulfill the following requirements:
+
+* Periodically snapshot the filesystems on ``app-srv`` *every 10 minutes*
+* Incrementally replicate these snapshots to ``storage/zrepl/pull/app-srv/*`` on ``backup-srv``
+* Keep only very few snapshots on ``app-srv`` to save disk space
+* Keep a fading history (24 hourly, 30 daily, 6 monthly) of snapshots on ``backup-srv``
+
+Analysis
+--------
+
+We can model this situation as two jobs:
+
+* A **source job** on ``app-srv``
+
+  * Creates the snapshots
+  * Keeps a short history of snapshots to enable incremental replication to ``backup-srv``
+  * Accepts connections from ``backup-srv``
+
+* A **pull job** on ``backup-srv``
+
+  * Connects to the ``zrepl daemon`` process on ``app-srv``
+  * Pulls the snapshots to ``storage/zrepl/pull/app-srv/*``
+  * Fades out snapshots in ``storage/zrepl/pull/app-srv/*`` as they age
+
+
+Why doesn't the **pull job** create the snapshots before pulling?
+
+As is the case with all distributed systems, the link between ``app-srv`` and ``backup-srv`` might be down for an hour or two.
+We do not want to sacrifice our required backup resolution of 10 minute intervals for a temporary connection outage.
+
+When the link comes up again, ``backup-srv`` will happily catch up the 12 snapshots taken by ``app-srv`` in the meantime, without
+a gap in our backup history.
+
+Install zrepl
+-------------
+
+Follow the :ref:`OS-specific installation instructions <installation>` and come back here.
+
+Configure ``backup-srv``
+------------------------
+
+We define a **pull job** named ``pull_app-srv`` in the |mainconfig|: ::
+
+    jobs:
+    - name: pull_app-srv
+    type: pull
+    connect:
+        type: ssh+stdinserver
+        host: app-srv.example.com
+        user: root
+        port: 22
+        identity_file: /etc/zrepl/ssh/identity
+    interval: 10m
+    mapping: {
+        "<":"storage/zrepl/pull/app-srv"
+    }
+    initial_repl_policy: most_recent
+    snapshot_prefix: zrepl_pull_backup_
+    prune:
+        policy: grid
+        grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+
+The ``connect`` section instructs the zrepl daemon to use the ``stdinserver`` transport:
+``backup-srv`` will connect to the specified SSH server and expect ``zrepl stdinserver CLIENT_IDENTITY`` instead of the shell on the other side.
+
+It uses the private key specified at ``connect.identity_file`` which we still need to create: ::
+
+    cd /etc/zrepl
+    mkdir -p ssh
+    chmod 0700 ssh
+    ssh-keygen -t ed25519 -N '' -f /etc/zrepl/ssh/identity
+
+Note that most use cases do not benefit from separate keypairs per remote endpoint.
+Thus, it is sufficient to create one keypair and use it for all ``connect`` directives on one host.
+
+Learn more about :ref:`transport-ssh+stdinserver` transport and the :ref:`pull job <job-pull>` format.
+
+.. _tutorial-configure-app-srv:
+
+Configure ``app-srv``
+---------------------
+
+We define a corresponding **source job** named ``pull_backup`` in the |mainconfig|: ::
+
+    jobs:
+    - name: pull_backup
+    type: source
+    serve:
+        type: stdinserver
+        client_identity: backup-srv.example.com
+    filesystems: {
+        "zroot/var/db": "ok",
+        "zroot/usr/home<": "ok",
+        "zroot/usr/home/paranoid": "!",
+    }
+    snapshot_prefix: zrepl_pull_backup_
+    interval: 10m
+    prune:
+        policy: grid
+        grid: 1x1d(keep=all)
+
+
+The ``serve`` section corresponds to the ``connect`` section in the configuration of ``backup-srv``.
+
+We now want to authenticate ``backup-srv`` before allowing it to pull data.
+This is done by limiting SSH connections from ``backup-srv`` to execute the ``stdinserver`` subcommand.
+
+Open ``/root/.ssh/authorized_keys`` and add either of the the following lines.::
+
+    # for OpenSSH >= 7.2
+    command="zrepl stdinserver backup-srv.example.com",restrict CLIENT_SSH_KEY
+    # for older OpenSSH versions
+    command="zrepl stdinserver backup-srv.example.com",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc  CLIENT_SSH_KEY
+
+.. ATTENTION::
+
+    Replace CLIENT_SSH_KEY with the contents of ``/etc/zrepl/ssh/identity.pub`` from ``app-srv``.
+    Mind the trailing ``.pub`` in the filename.
+    The entries **must** be on a single line, including the replaced CLIENT_SSH_KEY.
+
+
+.. HINT::
+
+    You may need to adjust the ``PermitRootLogin`` option in ``/etc/ssh/sshd_config`` to ``forced-commands-only`` or higher for this to work.
+    Refer to sshd_config(5) for details.
+
+The argument ``backup-srv.example.com`` is the client identity of ``backup-srv`` as defined in ``jobs.serve.client_identity``.
+
+Again, both :ref:`transport-ssh+stdinserver` transport and the :ref:`job-source` format are documented.
+
+Apply Configuration Changes
+---------------------------
+
+We need to restart the zrepl daemon on **both** ``app-srv`` and ``backup-srv``.
+This is :ref:`OS-specific <install-restarting>`.
+
+Watch it Work
+-------------
+
+A common setup is to ``watch`` the log output and ``zfs list`` of snapshots on both machines.
+
+If you like tmux, here is a handy script that works on FreeBSD: ::
+
+    pkg install gnu-watch tmux
+    tmux new-window
+    tmux split-window "tail -f /var/log/zrepl.log"
+    tmux split-window "gnu-watch 'zfs list -t snapshot -o name,creation -s creation | grep zrepl_pull_backup_'"
+    tmux select-layout tiled
+
+The Linux equivalent might look like this: ::
+
+    # make sure tmux is installed & let's assume you use systemd + journald
+    tmux new-window
+    tmux split-window "journalctl -f -u zrepl.service"
+    tmux split-window "watch 'zfs list -t snapshot -o name,creation -s creation | grep zrepl_pull_backup_'"
+    tmux select-layout tiled
+
+Summary
+-------
+
+Congratulations, you have a working pull backup. Where to go next?
+
+* Read more about :ref:`configuration format, options & job types <configuration_toc>`
+* Learn about :ref:`implementation details <implementation_toc>` of zrepl.
+
+

jobrun/jobrun.go

@@ -1,158 +0,0 @@
-package jobrun
-
-import (
-	"fmt"
-	"time"
-)
-
-type Logger interface {
-	Printf(format string, v ...interface{})
-}
-
-type jobLogger struct {
-	MainLog Logger
-	JobName string
-}
-
-func (l jobLogger) Printf(format string, v ...interface{}) {
-	l.MainLog.Printf(fmt.Sprintf("job[%s]: %s", l.JobName, format), v...)
-}
-
-type Job struct {
-	Name           string
-	RunFunc        func(log Logger) (err error)
-	LastStart      time.Time
-	LastError      error
-	DueAt          time.Time
-	RepeatStrategy RepeatStrategy
-}
-
-type JobRunResult struct {
-	Start  time.Time
-	Finish time.Time
-	Error  error
-}
-
-func (r JobRunResult) RunTime() time.Duration { return r.Finish.Sub(r.Start) }
-
-type RepeatStrategy interface {
-	ShouldReschedule(lastResult JobRunResult) (nextDue time.Time, reschedule bool)
-}
-
-type JobRunner struct {
-	logger           Logger
-	notificationChan chan Job
-	newJobChan       chan Job
-	finishedJobChan  chan Job
-	scheduleTimer    <-chan time.Time
-	pending          map[string]Job
-	running          map[string]Job
-}
-
-func NewJobRunner(logger Logger) *JobRunner {
-	return &JobRunner{
-		logger:           logger,
-		notificationChan: make(chan Job),
-		newJobChan:       make(chan Job),
-		finishedJobChan:  make(chan Job),
-		pending:          make(map[string]Job),
-		running:          make(map[string]Job),
-	}
-}
-
-func (r *JobRunner) AddJobChan() chan<- Job {
-	return r.newJobChan
-}
-
-func (r *JobRunner) AddJob(j Job) {
-	r.newJobChan <- j
-}
-
-func (r *JobRunner) NotificationChan() <-chan Job {
-	return r.notificationChan
-}
-
-func (r *JobRunner) Start() {
-
-loop:
-	select {
-
-	case newJob := <-r.newJobChan:
-
-		_, jobPending := r.pending[newJob.Name]
-		_, jobRunning := r.running[newJob.Name]
-
-		if jobPending || jobRunning {
-			panic("job already in runner")
-		}
-
-		r.pending[newJob.Name] = newJob
-
-	case finishedJob := <-r.finishedJobChan:
-
-		delete(r.running, finishedJob.Name)
-
-		res := JobRunResult{
-			Start:  finishedJob.LastStart,
-			Finish: time.Now(),
-			Error:  finishedJob.LastError,
-		}
-
-		r.logger.Printf("[%s] finished after %s\n", finishedJob.Name, res.RunTime())
-
-		dueTime, resched := finishedJob.RepeatStrategy.ShouldReschedule(res)
-		if resched {
-			r.logger.Printf("[%s] rescheduling to %s", finishedJob.Name, dueTime)
-			finishedJob.DueAt = dueTime
-			r.pending[finishedJob.Name] = finishedJob
-		}
-
-	case <-r.scheduleTimer:
-	}
-
-	if len(r.pending) == 0 && len(r.running) == 0 {
-		return
-	}
-
-	// Find jobs to run
-	var now time.Time
-	var jobPending bool
-
-	now = time.Now()
-	jobPending = false
-
-	nextJobDue := now.Add(time.Minute) // max(pending.Interval)
-
-	for jobName, job := range r.pending {
-
-		if job.DueAt.After(now) {
-			if job.DueAt.Before(nextJobDue) {
-				nextJobDue = job.DueAt
-			}
-			jobPending = true
-			continue
-		}
-		// This job is due, run it
-
-		delete(r.pending, jobName)
-		r.running[jobName] = job
-		job.LastStart = now
-
-		go func(job Job) {
-			jobLog := jobLogger{r.logger, job.Name}
-			if err := job.RunFunc(jobLog); err != nil {
-				job.LastError = err
-				r.notificationChan <- job
-			}
-			r.finishedJobChan <- job
-		}(job)
-
-	}
-
-	if jobPending || len(r.running) > 0 {
-		r.logger.Printf("waiting until %v\n", nextJobDue)
-		r.scheduleTimer = time.After(nextJobDue.Sub(now))
-		goto loop
-	}
-
-}

jobrun/repeat_strategies.go

@@ -1,25 +0,0 @@
-package jobrun
-
-import (
-	"time"
-)
-
-type NoRepeatStrategy struct{}
-
-func (s NoRepeatStrategy) ShouldReschedule(lastResult JobRunResult) (time.Time, bool) {
-	return time.Time{}, false
-}
-
-type PeriodicRepeatStrategy struct {
-	Interval time.Duration
-}
-
-func (s *PeriodicRepeatStrategy) ShouldReschedule(lastResult JobRunResult) (next time.Time, shouldRun bool) {
-	// Don't care about the result
-	shouldRun = true
-	next = lastResult.Start.Add(s.Interval)
-	if next.Before(time.Now()) {
-		next = time.Now()
-	}
-	return
-}

logger/datastructures.go

@@ -0,0 +1,91 @@
+package logger
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/pkg/errors"
+	"time"
+)
+
+type Level int
+
+func (l Level) MarshalJSON() ([]byte, error) {
+	return json.Marshal(l.String())
+}
+
+const (
+	Debug Level = iota
+	Info
+	Warn
+	Error
+)
+
+func (l Level) Short() string {
+	switch l {
+	case Debug:
+		return "DEBG"
+	case Info:
+		return "INFO"
+	case Warn:
+		return "WARN"
+	case Error:
+		return "ERRO"
+	default:
+		return fmt.Sprintf("%s", l)
+	}
+}
+
+func (l Level) String() string {
+	switch l {
+	case Debug:
+		return "debug"
+	case Info:
+		return "info"
+	case Warn:
+		return "warn"
+	case Error:
+		return "error"
+	default:
+		return fmt.Sprintf("%s", string(l))
+	}
+}
+
+func ParseLevel(s string) (l Level, err error) {
+	for _, l := range AllLevels {
+		if s == l.String() {
+			return l, nil
+		}
+	}
+	return -1, errors.Errorf("unknown level '%s'", s)
+}
+
+// Levels ordered least severe to most severe
+var AllLevels []Level = []Level{Debug, Info, Warn, Error}
+
+type Fields map[string]interface{}
+
+type Entry struct {
+	Level   Level
+	Message string
+	Time    time.Time
+	Fields  Fields
+}
+
+type Outlet interface {
+	// Note: os.Stderr is also used by logger.Logger for reporting errors returned by outlets
+	//       => you probably don't want to log there
+	WriteEntry(ctx context.Context, entry Entry) error
+}
+
+type Outlets map[Level][]Outlet
+
+func NewOutlets() Outlets {
+	return make(Outlets, len(AllLevels))
+}
+
+func (os Outlets) Add(outlet Outlet, minLevel Level) {
+	for _, l := range AllLevels[minLevel:] {
+		os[l] = append(os[l], outlet)
+	}
+}

logger/logger.go

@@ -0,0 +1,130 @@
+package logger
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+const (
+	// The field set by WithError function
+	FieldError = "err"
+)
+
+const DefaultUserFieldCapacity = 5
+const InternalErrorPrefix = "github.com/zrepl/zrepl/logger: "
+
+type Logger struct {
+	fields        Fields
+	outlets       Outlets
+	outletTimeout time.Duration
+
+	mtx *sync.Mutex
+}
+
+func NewLogger(outlets Outlets, outletTimeout time.Duration) *Logger {
+	return &Logger{
+		make(Fields, DefaultUserFieldCapacity),
+		outlets,
+		outletTimeout,
+		&sync.Mutex{},
+	}
+}
+
+func (l *Logger) log(level Level, msg string) {
+
+	l.mtx.Lock()
+	defer l.mtx.Unlock()
+
+	entry := Entry{level, msg, time.Now(), l.fields}
+
+	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(l.outletTimeout))
+	ech := make(chan error)
+
+	louts := l.outlets[level]
+	for i := range louts {
+		go func(ctx context.Context, outlet Outlet, entry Entry) {
+			ech <- outlet.WriteEntry(ctx, entry)
+		}(ctx, louts[i], entry)
+	}
+
+	for fin := 0; fin < len(louts); fin++ {
+		select {
+		case err := <-ech:
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "%s outlet error: %s\n", InternalErrorPrefix, err)
+			}
+		case <-ctx.Done():
+			if ctx.Err() == context.DeadlineExceeded {
+				fmt.Fprintf(os.Stderr, "%s outlets exceeded deadline, keep waiting anyways", InternalErrorPrefix)
+			}
+		}
+	}
+
+	cancel() // make go vet happy
+
+}
+
+func (l *Logger) WithField(field string, val interface{}) *Logger {
+
+	l.mtx.Lock()
+	defer l.mtx.Unlock()
+
+	if _, ok := l.fields[field]; ok {
+		fmt.Fprintf(os.Stderr, "%s caller overwrites field '%s'. Stack:\n%s\n", InternalErrorPrefix, field, string(debug.Stack()))
+	}
+
+	child := &Logger{
+		fields:        make(Fields, len(l.fields)+1),
+		outlets:       l.outlets, // cannot be changed after logger initialized
+		outletTimeout: l.outletTimeout,
+		mtx:           l.mtx,
+	}
+	for k, v := range l.fields {
+		child.fields[k] = v
+	}
+	child.fields[field] = val
+
+	return child
+
+}
+
+func (l *Logger) WithFields(fields Fields) (ret *Logger) {
+	// TODO optimize
+	ret = l
+	for field, value := range fields {
+		ret = l.WithField(field, value)
+	}
+	return ret
+}
+
+func (l *Logger) WithError(err error) *Logger {
+	val := interface{}(nil)
+	if err != nil {
+		val = err.Error()
+	}
+	return l.WithField(FieldError, val)
+}
+
+func (l *Logger) Debug(msg string) {
+	l.log(Debug, msg)
+}
+
+func (l *Logger) Info(msg string) {
+	l.log(Info, msg)
+}
+
+func (l *Logger) Warn(msg string) {
+	l.log(Warn, msg)
+}
+
+func (l *Logger) Error(msg string) {
+	l.log(Error, msg)
+}
+
+func (l *Logger) Printf(format string, args ...interface{}) {
+	l.log(Error, fmt.Sprintf(format, args...))
+}

logger/logger_test.go

@@ -0,0 +1,47 @@
+package logger_test
+
+import (
+	"context"
+	"fmt"
+	"github.com/kr/pretty"
+	"github.com/zrepl/zrepl/logger"
+	"testing"
+	"time"
+)
+
+type TestOutlet struct {
+	Record []logger.Entry
+}
+
+func (o *TestOutlet) WriteEntry(ctx context.Context, entry logger.Entry) error {
+	o.Record = append(o.Record, entry)
+	return nil
+}
+
+func NewTestOutlet() *TestOutlet {
+	return &TestOutlet{make([]logger.Entry, 0)}
+}
+
+func TestLogger_Basic(t *testing.T) {
+
+	outlet_arr := []logger.Outlet{
+		NewTestOutlet(),
+		NewTestOutlet(),
+	}
+
+	outlets := logger.NewOutlets()
+	for _, o := range outlet_arr {
+		outlets.Add(o, logger.Debug)
+	}
+
+	l := logger.NewLogger(outlets, 1*time.Second)
+
+	l.Info("foobar")
+
+	l.WithField("fieldname", "fieldval").Info("log with field")
+
+	l.WithError(fmt.Errorf("fooerror")).Error("error")
+
+	t.Log(pretty.Sprint(outlet_arr))
+
+}

rpc/client.go

@@ -0,0 +1,136 @@
+package rpc
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"reflect"
+
+	"github.com/pkg/errors"
+)
+
+type Client struct {
+	ml     *MessageLayer
+	logger Logger
+}
+
+func NewClient(rwc io.ReadWriteCloser) *Client {
+	return &Client{NewMessageLayer(rwc), noLogger{}}
+}
+
+func (c *Client) SetLogger(logger Logger, logMessageLayer bool) {
+	c.logger = logger
+	if logMessageLayer {
+		c.ml.logger = logger
+	} else {
+		c.ml.logger = noLogger{}
+	}
+}
+
+func (c *Client) Close() (err error) {
+
+	c.logger.Printf("sending Close request")
+	header := Header{
+		DataType: DataTypeControl,
+		Endpoint: ControlEndpointClose,
+		Accept:   DataTypeControl,
+	}
+	err = c.ml.WriteHeader(&header)
+	if err != nil {
+		return
+	}
+
+	c.logger.Printf("reading Close ACK")
+	ack, err := c.ml.ReadHeader()
+	if err != nil {
+		return err
+	}
+	c.logger.Printf("received Close ACK: %#v", ack)
+	if ack.Error != StatusOK {
+		err = errors.Errorf("error hanging up: remote error (%s) %s", ack.Error, ack.ErrorMessage)
+		return
+	}
+
+	c.logger.Printf("closing MessageLayer")
+	if err = c.ml.Close(); err != nil {
+		c.logger.Printf("error closing RWC: %+v", err)
+		return
+	}
+
+	return err
+}
+
+func (c *Client) recvResponse() (h *Header, err error) {
+	h, err = c.ml.ReadHeader()
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot read header")
+	}
+	// TODO validate
+	return
+}
+
+func (c *Client) writeRequest(h *Header) (err error) {
+	// TODO validate
+	err = c.ml.WriteHeader(h)
+	if err != nil {
+		return errors.Wrap(err, "cannot write header")
+	}
+	return
+}
+
+func (c *Client) Call(endpoint string, in, out interface{}) (err error) {
+
+	var accept DataType
+	{
+		outType := reflect.TypeOf(out)
+		if typeIsIOReaderPtr(outType) {
+			accept = DataTypeOctets
+		} else {
+			accept = DataTypeMarshaledJSON
+		}
+	}
+
+	h := Header{
+		Endpoint: endpoint,
+		DataType: DataTypeMarshaledJSON,
+		Accept:   accept,
+	}
+
+	if err = c.writeRequest(&h); err != nil {
+		return err
+	}
+
+	var buf bytes.Buffer
+	if err = json.NewEncoder(&buf).Encode(in); err != nil {
+		panic("cannot encode 'in' parameter")
+	}
+	if err = c.ml.WriteData(&buf); err != nil {
+		return err
+	}
+
+	rh, err := c.recvResponse()
+	if err != nil {
+		return err
+	}
+	if rh.Error != StatusOK {
+		return &RPCError{rh}
+	}
+
+	rd := c.ml.ReadData()
+
+	switch accept {
+	case DataTypeOctets:
+		c.logger.Printf("setting out to ML data reader")
+		outPtr := out.(*io.Reader) // we checked that above
+		*outPtr = rd
+	case DataTypeMarshaledJSON:
+		c.logger.Printf("decoding marshaled json")
+		if err = json.NewDecoder(c.ml.ReadData()).Decode(out); err != nil {
+			return errors.Wrap(err, "cannot decode marshaled reply")
+		}
+	default:
+		panic("implementation error") // accept is controlled by us
+	}
+
+	return
+}

rpc/frame_layer.go

@@ -0,0 +1,301 @@
+package rpc
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"github.com/pkg/errors"
+)
+
+type Frame struct {
+	Type          FrameType
+	NoMoreFrames  bool
+	PayloadLength uint32
+}
+
+//go:generate stringer -type=FrameType
+type FrameType uint8
+
+const (
+	FrameTypeHeader  FrameType = 0x01
+	FrameTypeData    FrameType = 0x02
+	FrameTypeTrailer FrameType = 0x03
+	FrameTypeRST     FrameType = 0xff
+)
+
+//go:generate stringer -type=Status
+type Status uint64
+
+const (
+	StatusOK Status = 1 + iota
+	StatusRequestError
+	StatusServerError
+	// Returned when an error occurred but the side at fault cannot be determined
+	StatusError
+)
+
+type Header struct {
+	// Request-only
+	Endpoint string
+	// Data type of body (request & reply)
+	DataType DataType
+	// Request-only
+	Accept DataType
+	// Reply-only
+	Error Status
+	// Reply-only
+	ErrorMessage string
+}
+
+func NewErrorHeader(status Status, format string, args ...interface{}) (h *Header) {
+	h = &Header{}
+	h.Error = status
+	h.ErrorMessage = fmt.Sprintf(format, args...)
+	return
+}
+
+type DataType uint8
+
+const (
+	DataTypeNone DataType = 1 + iota
+	DataTypeControl
+	DataTypeMarshaledJSON
+	DataTypeOctets
+)
+
+const (
+	MAX_PAYLOAD_LENGTH = 4 * 1024 * 1024
+	MAX_HEADER_LENGTH  = 4 * 1024
+)
+
+type frameBridgingReader struct {
+	l         *MessageLayer
+	frameType FrameType
+	// < 0 means no limit
+	bytesLeftToLimit int
+	f                Frame
+}
+
+func NewFrameBridgingReader(l *MessageLayer, frameType FrameType, totalLimit int) *frameBridgingReader {
+	return &frameBridgingReader{l, frameType, totalLimit, Frame{}}
+}
+
+func (r *frameBridgingReader) Read(b []byte) (n int, err error) {
+	if r.bytesLeftToLimit == 0 {
+		r.l.logger.Printf("limit reached, returning EOF")
+		return 0, io.EOF
+	}
+	log := r.l.logger
+	if r.f.PayloadLength == 0 {
+
+		if r.f.NoMoreFrames {
+			r.l.logger.Printf("no more frames flag set, returning EOF")
+			err = io.EOF
+			return
+		}
+
+		log.Printf("reading frame")
+		r.f, err = r.l.readFrame()
+		if err != nil {
+			log.Printf("error reading frame: %+v", err)
+			return 0, err
+		}
+		log.Printf("read frame: %#v", r.f)
+		if r.f.Type != r.frameType {
+			err = errors.Wrapf(err, "expected frame of type %s", r.frameType)
+			return 0, err
+		}
+	}
+	maxread := len(b)
+	if maxread > int(r.f.PayloadLength) {
+		maxread = int(r.f.PayloadLength)
+	}
+	if r.bytesLeftToLimit > 0 && maxread > r.bytesLeftToLimit {
+		maxread = r.bytesLeftToLimit
+	}
+	nb, err := r.l.rwc.Read(b[:maxread])
+	log.Printf("read  %v from rwc\n", nb)
+	if nb < 0 {
+		panic("should not return negative number of bytes")
+	}
+	r.f.PayloadLength -= uint32(nb)
+	r.bytesLeftToLimit -= nb
+	return nb, err // TODO io.EOF for maxread = r.f.PayloadLength ?
+}
+
+type frameBridgingWriter struct {
+	l         *MessageLayer
+	frameType FrameType
+	// < 0 means no limit
+	bytesLeftToLimit int
+	payloadLength    int
+	buffer           *bytes.Buffer
+}
+
+func NewFrameBridgingWriter(l *MessageLayer, frameType FrameType, totalLimit int) *frameBridgingWriter {
+	return &frameBridgingWriter{l, frameType, totalLimit, MAX_PAYLOAD_LENGTH, bytes.NewBuffer(make([]byte, 0, MAX_PAYLOAD_LENGTH))}
+}
+
+func (w *frameBridgingWriter) Write(b []byte) (n int, err error) {
+	for n = 0; n < len(b); {
+		i, err := w.writeUntilFrameFull(b[n:])
+		n += i
+		if err != nil {
+			return n, errors.WithStack(err)
+		}
+	}
+	return
+}
+
+func (w *frameBridgingWriter) writeUntilFrameFull(b []byte) (n int, err error) {
+	if len(b) <= 0 {
+		return
+	}
+	if w.bytesLeftToLimit == 0 {
+		err = errors.Errorf("message exceeds max number of allowed bytes")
+		return
+	}
+	maxwrite := len(b)
+	remainingInFrame := w.payloadLength - w.buffer.Len()
+
+	if maxwrite > remainingInFrame {
+		maxwrite = remainingInFrame
+	}
+	if w.bytesLeftToLimit > 0 && maxwrite > w.bytesLeftToLimit {
+		maxwrite = w.bytesLeftToLimit
+	}
+	w.buffer.Write(b[:maxwrite])
+	w.bytesLeftToLimit -= maxwrite
+	n = maxwrite
+	if w.bytesLeftToLimit == 0 {
+		err = w.flush(true)
+	} else if w.buffer.Len() == w.payloadLength {
+		err = w.flush(false)
+	}
+	return
+}
+
+func (w *frameBridgingWriter) flush(nomore bool) (err error) {
+
+	f := Frame{w.frameType, nomore, uint32(w.buffer.Len())}
+	err = w.l.writeFrame(f)
+	if err != nil {
+		errors.WithStack(err)
+	}
+	_, err = w.buffer.WriteTo(w.l.rwc)
+	return
+}
+
+func (w *frameBridgingWriter) Close() (err error) {
+	return w.flush(true)
+}
+
+type MessageLayer struct {
+	rwc    io.ReadWriteCloser
+	logger Logger
+}
+
+func NewMessageLayer(rwc io.ReadWriteCloser) *MessageLayer {
+	return &MessageLayer{rwc, noLogger{}}
+}
+
+func (l *MessageLayer) Close() (err error) {
+	f := Frame{
+		Type:         FrameTypeRST,
+		NoMoreFrames: true,
+	}
+	if err = l.writeFrame(f); err != nil {
+		l.logger.Printf("error sending RST frame: %s", err)
+		return errors.WithStack(err)
+	}
+	return nil
+}
+
+var RST error = fmt.Errorf("reset frame observed on connection")
+
+func (l *MessageLayer) readFrame() (f Frame, err error) {
+	err = binary.Read(l.rwc, binary.LittleEndian, &f.Type)
+	if err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	err = binary.Read(l.rwc, binary.LittleEndian, &f.NoMoreFrames)
+	if err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	err = binary.Read(l.rwc, binary.LittleEndian, &f.PayloadLength)
+	if err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	if f.Type == FrameTypeRST {
+		l.logger.Printf("read RST frame")
+		err = RST
+		return
+	}
+	if f.PayloadLength > MAX_PAYLOAD_LENGTH {
+		err = errors.Errorf("frame exceeds max payload length")
+		return
+	}
+	return
+}
+
+func (l *MessageLayer) writeFrame(f Frame) (err error) {
+	err = binary.Write(l.rwc, binary.LittleEndian, &f.Type)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	err = binary.Write(l.rwc, binary.LittleEndian, &f.NoMoreFrames)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	err = binary.Write(l.rwc, binary.LittleEndian, &f.PayloadLength)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	if f.PayloadLength > MAX_PAYLOAD_LENGTH {
+		err = errors.Errorf("frame exceeds max payload length")
+		return
+	}
+	return
+}
+
+func (l *MessageLayer) ReadHeader() (h *Header, err error) {
+
+	r := NewFrameBridgingReader(l, FrameTypeHeader, MAX_HEADER_LENGTH)
+	h = &Header{}
+	if err = json.NewDecoder(r).Decode(&h); err != nil {
+		l.logger.Printf("cannot decode marshaled header: %s", err)
+		return nil, err
+	}
+	return h, nil
+}
+
+func (l *MessageLayer) WriteHeader(h *Header) (err error) {
+	w := NewFrameBridgingWriter(l, FrameTypeHeader, MAX_HEADER_LENGTH)
+	err = json.NewEncoder(w).Encode(h)
+	if err != nil {
+		return errors.Wrap(err, "cannot encode header, probably fatal")
+	}
+	w.Close()
+	return
+}
+
+func (l *MessageLayer) ReadData() (reader io.Reader) {
+	r := NewFrameBridgingReader(l, FrameTypeData, -1)
+	return r
+}
+
+func (l *MessageLayer) WriteData(source io.Reader) (err error) {
+	w := NewFrameBridgingWriter(l, FrameTypeData, -1)
+	_, err = io.Copy(w, source)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	err = w.Close()
+	return
+}

rpc/frametype_string.go

@@ -0,0 +1,27 @@
+// Code generated by "stringer -type=FrameType"; DO NOT EDIT.
+
+package rpc
+
+import "fmt"
+
+const (
+	_FrameType_name_0 = "FrameTypeHeaderFrameTypeDataFrameTypeTrailer"
+	_FrameType_name_1 = "FrameTypeRST"
+)
+
+var (
+	_FrameType_index_0 = [...]uint8{0, 15, 28, 44}
+	_FrameType_index_1 = [...]uint8{0, 12}
+)
+
+func (i FrameType) String() string {
+	switch {
+	case 1 <= i && i <= 3:
+		i -= 1
+		return _FrameType_name_0[_FrameType_index_0[i]:_FrameType_index_0[i+1]]
+	case i == 255:
+		return _FrameType_name_1
+	default:
+		return fmt.Sprintf("FrameType(%d)", i)
+	}
+}

rpc/local.go

@@ -0,0 +1,63 @@
+package rpc
+
+import (
+	"github.com/pkg/errors"
+	"reflect"
+)
+
+type LocalRPC struct {
+	endpoints map[string]reflect.Value
+}
+
+func NewLocalRPC() *LocalRPC {
+	return &LocalRPC{make(map[string]reflect.Value, 0)}
+}
+
+func (s *LocalRPC) RegisterEndpoint(name string, handler interface{}) (err error) {
+	_, ok := s.endpoints[name]
+	if ok {
+		return errors.Errorf("already set up an endpoint for '%s'", name)
+	}
+	ep, err := makeEndpointDescr(handler)
+	if err != nil {
+		return err
+	}
+	s.endpoints[name] = ep.handler
+	return nil
+}
+
+func (s *LocalRPC) Serve() (err error) {
+	panic("local cannot serve")
+}
+
+func (c *LocalRPC) Call(endpoint string, in, out interface{}) (err error) {
+	ep, ok := c.endpoints[endpoint]
+	if !ok {
+		panic("implementation error: implementation should not call local RPC without knowing which endpoints exist")
+	}
+
+	args := []reflect.Value{reflect.ValueOf(in), reflect.ValueOf(out)}
+
+	if err = checkRPCParamTypes(args[0].Type(), args[1].Type()); err != nil {
+		return
+	}
+
+	rets := ep.Call(args)
+
+	if len(rets) != 1 {
+		panic("implementation error: endpoints must have one error ")
+	}
+	if err = checkRPCReturnType(rets[0].Type()); err != nil {
+		panic(err)
+	}
+
+	err = nil
+	if !rets[0].IsNil() {
+		err = rets[0].Interface().(error) // we checked that above
+	}
+	return
+}
+
+func (c *LocalRPC) Close() (err error) {
+	return nil
+}

rpc/resumetransfererrorreason_string.go

@@ -1,16 +0,0 @@
-// Code generated by "stringer -type ResumeTransferErrorReason"; DO NOT EDIT.
-
-package rpc
-
-import "fmt"
-
-const _ResumeTransferErrorReason_name = "ResumeTransferErrorReasonNotImplementedResumeTransferErrorReasonDisabledResumeTransferErrorReasonZFSErrorPermanentResumeTransferErrorReasonZFSErrorMaybeTemporary"
-
-var _ResumeTransferErrorReason_index = [...]uint8{0, 39, 72, 114, 161}
-
-func (i ResumeTransferErrorReason) String() string {
-	if i >= ResumeTransferErrorReason(len(_ResumeTransferErrorReason_index)-1) {
-		return fmt.Sprintf("ResumeTransferErrorReason(%d)", i)
-	}
-	return _ResumeTransferErrorReason_name[_ResumeTransferErrorReason_index[i]:_ResumeTransferErrorReason_index[i+1]]
-}

rpc/rpc.go

@@ -1,592 +0,0 @@
-package rpc
-
-import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"fmt"
-	. "github.com/zrepl/zrepl/util"
-	"github.com/zrepl/zrepl/zfs"
-	"io"
-	"reflect"
-	"time"
-)
-
-type RPCRequester interface {
-	FilesystemRequest(r FilesystemRequest) (roots []*zfs.DatasetPath, err error)
-	FilesystemVersionsRequest(r FilesystemVersionsRequest) (versions []zfs.FilesystemVersion, err error)
-	InitialTransferRequest(r InitialTransferRequest) (io.Reader, error)
-	IncrementalTransferRequest(r IncrementalTransferRequest) (io.Reader, error)
-	ResumeTransferRequest(r ResumeTransferRequest) (io.Reader, error)
-	PullMeRequest(r PullMeRequest, handler RPCHandler) (err error)
-	CloseRequest(r CloseRequest) (err error)
-	ForceClose() (err error)
-}
-
-type RPCHandler interface {
-	HandleFilesystemRequest(r FilesystemRequest) (roots []*zfs.DatasetPath, err error)
-
-	// returned versions ordered by birthtime, oldest first
-	HandleFilesystemVersionsRequest(r FilesystemVersionsRequest) (versions []zfs.FilesystemVersion, err error)
-
-	HandleInitialTransferRequest(r InitialTransferRequest) (io.Reader, error)
-	HandleIncrementalTransferRequest(r IncrementalTransferRequest) (io.Reader, error)
-	HandleResumeTransferRequest(r ResumeTransferRequest) (io.Reader, error)
-
-	// invert roles, i.e. handler becomes server and performs the requested pull using the client connection
-	HandlePullMeRequest(r PullMeRequest, clientIdentity string, client RPCRequester) (err error)
-}
-
-type Logger interface {
-	Printf(format string, args ...interface{})
-}
-
-const ByteStreamRPCProtocolVersion = 1
-
-type ByteStream interface {
-	io.ReadWriteCloser
-}
-
-type ByteStreamRPC struct {
-	conn           ByteStream
-	log            Logger
-	clientIdentity string
-}
-
-func ConnectByteStreamRPC(conn ByteStream, log Logger) (RPCRequester, error) {
-
-	rpc := ByteStreamRPC{
-		conn: conn,
-		log:  log,
-	}
-
-	// Assert protocol versions are equal
-	err := rpc.ProtocolVersionRequest()
-	if err != nil {
-		return nil, err
-	}
-
-	return rpc, nil
-}
-
-type ByteStreamRPCDecodeJSONError struct {
-	Type       reflect.Type
-	DecoderErr error
-}
-
-func (e ByteStreamRPCDecodeJSONError) Error() string {
-	return fmt.Sprintf("cannot decode %s: %s", e.Type, e.DecoderErr)
-}
-
-func ListenByteStreamRPC(conn ByteStream, clientIdentity string, handler RPCHandler, log Logger) error {
-
-	c := ByteStreamRPC{
-		conn:           conn,
-		log:            log,
-		clientIdentity: clientIdentity,
-	}
-	return c.serverLoop(handler)
-
-}
-
-func (c ByteStreamRPC) serverLoop(handler RPCHandler) error {
-
-	// A request consists of two subsequent chunked JSON objects
-	// Object 1: RequestHeader => contains type of Request Body
-	// Object 2: RequestBody, e.g. IncrementalTransferRequest
-	// A response is always a ResponseHeader followed by bytes to be interpreted
-	// 	as indicated by the ResponseHeader.ResponseType, e.g.
-	// 	 a) a chunked response
-	//   b) or another JSON object
-
-	conn := c.conn
-	log := c.log
-	defer func() {
-		panicObj := recover()
-		// if we just exited, we don't want to close the connection (PullMeRequest depends on this)
-		log.Printf("exiting server loop, panic object %#v", panicObj)
-		if panicObj != nil {
-			conn.Close()
-		}
-	}()
-
-	send := func(r interface{}) {
-		if err := writeChunkedJSON(conn, r); err != nil {
-			panic(err)
-		}
-	}
-
-	sendError := func(id ErrorId, msg string) {
-		r := ResponseHeader{
-			ErrorId:      id,
-			ResponseType: RNONE,
-			Message:      msg,
-		}
-		log.Printf("sending error response: %#v", r)
-		if err := writeChunkedJSON(conn, r); err != nil {
-			log.Printf("error sending error response: %#v", err)
-			panic(err)
-		}
-	}
-
-	recv := func(r interface{}) (err error) {
-		return readChunkedJSON(conn, r)
-	}
-
-	for {
-
-		var header RequestHeader = RequestHeader{}
-		if err := recv(&header); err != nil {
-			sendError(EDecodeHeader, err.Error())
-			return conn.Close()
-		}
-
-		switch header.Type {
-		case RTProtocolVersionRequest:
-			var rq ByteStreamRPCProtocolVersionRequest
-			if err := recv(&rq); err != nil {
-				sendError(EDecodeRequestBody, err.Error())
-				return conn.Close()
-			}
-
-			if rq.ClientVersion != ByteStreamRPCProtocolVersion {
-				sendError(EProtocolVersionMismatch, "")
-				return conn.Close()
-			}
-
-			r := ResponseHeader{
-				RequestId:    header.Id,
-				ResponseType: ROK,
-			}
-			send(&r)
-
-		case RTCloseRequest:
-
-			var rq CloseRequest
-			if err := recv(&rq); err != nil {
-				sendError(EDecodeRequestBody, err.Error())
-				return conn.Close()
-			}
-
-			log.Printf("close request with goodbye: %s", rq.Goodbye)
-
-			send(&ResponseHeader{
-				RequestId:    header.Id,
-				ResponseType: ROK,
-			})
-
-			return conn.Close()
-
-		case RTFilesystemRequest:
-
-			var rq FilesystemRequest
-			if err := recv(&rq); err != nil {
-				sendError(EDecodeRequestBody, "")
-				return conn.Close()
-			}
-
-			roots, err := handler.HandleFilesystemRequest(rq)
-			if err != nil {
-				sendError(EHandler, err.Error())
-				return conn.Close()
-			} else {
-				r := ResponseHeader{
-					RequestId:    header.Id,
-					ResponseType: RFilesystems,
-				}
-				send(&r)
-				send(&roots)
-			}
-
-		case RTFilesystemVersionsRequest:
-
-			var rq FilesystemVersionsRequest
-			if err := recv(&rq); err != nil {
-				sendError(EDecodeRequestBody, err.Error())
-				return err
-			}
-
-			diff, err := handler.HandleFilesystemVersionsRequest(rq)
-			if err != nil {
-				sendError(EHandler, err.Error())
-				return err
-			} else {
-				r := ResponseHeader{
-					RequestId:    header.Id,
-					ResponseType: RFilesystemDiff,
-				}
-				send(&r)
-				send(&diff)
-			}
-
-			log.Printf("finished FilesystemVersionReqeust")
-
-		case RTInitialTransferRequest:
-			var rq InitialTransferRequest
-			if err := recv(&rq); err != nil {
-				sendError(EDecodeRequestBody, "")
-				return conn.Close()
-			}
-			log.Printf("initial transfer request: %#v", rq)
-
-			snapReader, err := handler.HandleInitialTransferRequest(rq)
-			if err != nil {
-				sendError(EHandler, err.Error())
-				return conn.Close()
-			} else {
-
-				r := ResponseHeader{
-					RequestId:    header.Id,
-					ResponseType: RChunkedStream,
-				}
-				send(&r)
-
-				chunker := NewChunker(snapReader)
-				watcher := IOProgressWatcher{Reader: &chunker}
-				watcher.KickOff(1*time.Second, func(p IOProgress) {
-					log.Printf("progress sending initial snapshot stream: %v bytes sent", p.TotalRX)
-				})
-				_, err := io.Copy(conn, &watcher)
-				if err != nil {
-					log.Printf("error sending initial snapshot stream: %s", err)
-					panic(err)
-				}
-				log.Printf("finished sending initial snapshot stream: total %v bytes sent", watcher.Progress().TotalRX)
-			}
-
-		case RTIncrementalTransferRequest:
-
-			var rq IncrementalTransferRequest
-			if err := recv(&rq); err != nil {
-				sendError(EDecodeRequestBody, "")
-				return conn.Close()
-			}
-
-			snapReader, err := handler.HandleIncrementalTransferRequest(rq)
-			if err != nil {
-				sendError(EHandler, err.Error())
-			} else {
-
-				r := ResponseHeader{
-					RequestId:    header.Id,
-					ResponseType: RChunkedStream,
-				}
-				send(&r)
-
-				chunker := NewChunker(snapReader)
-
-				watcher := IOProgressWatcher{Reader: &chunker}
-				watcher.KickOff(1*time.Second, func(p IOProgress) {
-					log.Printf("progress sending incremental snapshot stream: %v bytes sent", p.TotalRX)
-				})
-				_, err := io.Copy(conn, &watcher)
-				if err != nil {
-					panic(err)
-				}
-				log.Printf("finished sending incremental snapshot stream: total %v bytes sent", watcher.Progress().TotalRX)
-			}
-
-		case RTResumeTransferRequest:
-
-			var rq ResumeTransferRequest
-			if err := recv(&rq); err != nil {
-				sendError(EDecodeRequestBody, "")
-				return conn.Close()
-			}
-
-			stream, err := handler.HandleResumeTransferRequest(rq)
-			if err != nil {
-				sendError(EHandler, err.Error())
-			} else {
-				r := ResponseHeader{
-					RequestId:    header.Id,
-					ResponseType: RChunkedStream,
-				}
-				send(&r)
-
-				chunker := NewChunker(stream)
-
-				watcher := IOProgressWatcher{Reader: &chunker}
-				watcher.KickOff(1*time.Second, func(p IOProgress) {
-					log.Printf("progress sending resumed stream: %v bytes sent", p.TotalRX)
-				})
-				_, err := io.Copy(conn, &watcher)
-				if err != nil {
-					panic(err)
-				}
-				log.Printf("finished sending resumed stream: total %v bytes sent", watcher.Progress().TotalRX)
-			}
-
-		case RTPullMeRequest:
-
-			var rq PullMeRequest
-			if err := recv(&rq); err != nil {
-				sendError(EDecodeRequestBody, err.Error())
-
-				return conn.Close()
-			}
-
-			if rq.Finished {
-				// we are the client that sent a PullMeRequest with Finished = false
-				// and then entered this server loop
-				log.Printf("PullMeRequest.Finished == true, exiting server loop")
-				send(ResponseHeader{
-					RequestId:    header.Id,
-					ResponseType: ROK,
-				})
-				return nil
-			}
-
-			// We are a server receiving a PullMeRequest from a client
-
-			log.Printf("confirming PullMeRequest")
-			send(ResponseHeader{
-				RequestId:    header.Id,
-				ResponseType: ROK,
-			})
-
-			log.Printf("pulling from client '%s', expecting client is in server loop", c.clientIdentity)
-			if c.clientIdentity == "" || c.clientIdentity == LOCAL_TRANSPORT_IDENTITY {
-				err := fmt.Errorf("client has bad name: '%s'", c.clientIdentity)
-				log.Printf(err.Error())
-				panic(err)
-			}
-			pullErr := handler.HandlePullMeRequest(rq, c.clientIdentity, c)
-			if pullErr != nil {
-				log.Printf("pulling failed with error: %s", pullErr)
-				panic(pullErr)
-			}
-
-			log.Printf("finished handling PullMeRequest, sending Finished = true")
-			req := PullMeRequest{Finished: true}
-			c.sendRequestReceiveHeader(req, ROK)
-
-		default:
-			sendError(EUnknownRequestType, "")
-			return conn.Close()
-		}
-	}
-
-	return nil
-}
-
-func writeChunkedJSON(conn io.Writer, r interface{}) (err error) {
-	var buf bytes.Buffer
-	encoder := json.NewEncoder(&buf)
-	encoder.Encode(r)
-	ch := NewChunker(&buf)
-	_, err = io.Copy(conn, &ch)
-	return
-}
-
-func readChunkedJSON(conn io.ReadWriter, r interface{}) (err error) {
-	unch := NewUnchunker(conn)
-	dec := json.NewDecoder(unch)
-	err = dec.Decode(r)
-	if err != nil {
-		err = ByteStreamRPCDecodeJSONError{
-			Type:       reflect.TypeOf(r),
-			DecoderErr: err,
-		}
-	}
-	closeErr := unch.Close()
-	if err == nil && closeErr != nil {
-		err = closeErr
-	}
-	return
-}
-
-func inferRequestType(v interface{}) (RequestType, error) {
-	switch v.(type) {
-	case ByteStreamRPCProtocolVersionRequest:
-		return RTProtocolVersionRequest, nil
-	case FilesystemRequest:
-		return RTFilesystemRequest, nil
-	case FilesystemVersionsRequest:
-		return RTFilesystemVersionsRequest, nil
-	case InitialTransferRequest:
-		return RTInitialTransferRequest, nil
-	case IncrementalTransferRequest:
-		return RTIncrementalTransferRequest, nil
-	case ResumeTransferRequest:
-		return RTResumeTransferRequest, nil
-	case PullMeRequest:
-		return RTPullMeRequest, nil
-	case CloseRequest:
-		return RTCloseRequest, nil
-	default:
-		return 0, errors.New(fmt.Sprintf("cannot infer request type for type '%v'",
-			reflect.TypeOf(v)))
-	}
-}
-
-func genUUID() [16]byte {
-	return [16]byte{} // TODO
-}
-
-func (c ByteStreamRPC) sendRequest(v interface{}) (err error) {
-
-	var rt RequestType
-
-	if rt, err = inferRequestType(v); err != nil {
-		return
-	}
-
-	h := RequestHeader{
-		Type: rt,
-		Id:   genUUID(),
-	}
-
-	if err = writeChunkedJSON(c.conn, h); err != nil {
-		return
-	}
-	if err = writeChunkedJSON(c.conn, v); err != nil {
-		return
-	}
-
-	return
-}
-
-func (c ByteStreamRPC) expectResponseType(rt ResponseType) (err error) {
-
-	var h ResponseHeader
-	if err = readChunkedJSON(c.conn, &h); err != nil {
-		return
-	}
-
-	if h.ResponseType != rt {
-		return errors.New(fmt.Sprintf("unexpected response type in response header: got %#v, expected %#v. response header: %#v",
-			h.ResponseType, rt, h))
-	}
-	return
-}
-
-func (c ByteStreamRPC) sendRequestReceiveHeader(request interface{}, rt ResponseType) (err error) {
-
-	if err = c.sendRequest(request); err != nil {
-		return err
-	}
-
-	if err = c.expectResponseType(rt); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (c ByteStreamRPC) ProtocolVersionRequest() (err error) {
-	b := ByteStreamRPCProtocolVersionRequest{
-		ClientVersion: ByteStreamRPCProtocolVersion,
-	}
-
-	// OK response means the remote side can cope with our protocol version
-	return c.sendRequestReceiveHeader(b, ROK)
-}
-
-func (c ByteStreamRPC) FilesystemRequest(r FilesystemRequest) (roots []*zfs.DatasetPath, err error) {
-
-	if err = c.sendRequestReceiveHeader(r, RFilesystems); err != nil {
-		return
-	}
-
-	roots = make([]*zfs.DatasetPath, 0)
-
-	if err = readChunkedJSON(c.conn, &roots); err != nil {
-		return
-	}
-
-	return
-}
-
-func (c ByteStreamRPC) FilesystemVersionsRequest(r FilesystemVersionsRequest) (versions []zfs.FilesystemVersion, err error) {
-
-	if err = c.sendRequestReceiveHeader(r, RFilesystemDiff); err != nil {
-		return
-	}
-
-	err = readChunkedJSON(c.conn, &versions)
-	return
-}
-
-func (c ByteStreamRPC) InitialTransferRequest(r InitialTransferRequest) (unchunker io.Reader, err error) {
-
-	if err = c.sendRequestReceiveHeader(r, RChunkedStream); err != nil {
-		return
-	}
-
-	unchunker = NewUnchunker(c.conn)
-	return
-}
-
-func (c ByteStreamRPC) IncrementalTransferRequest(r IncrementalTransferRequest) (unchunker io.Reader, err error) {
-	if err = c.sendRequestReceiveHeader(r, RChunkedStream); err != nil {
-		return
-	}
-	unchunker = NewUnchunker(c.conn)
-	return
-}
-
-func (c ByteStreamRPC) ResumeTransferRequest(r ResumeTransferRequest) (io.Reader, error) {
-	// TODO reconstruct ResumeTransferError
-	if err := c.sendRequestReceiveHeader(r, RChunkedStream); err != nil {
-		return nil, err
-	}
-	return NewUnchunker(c.conn), nil
-}
-
-func (c ByteStreamRPC) PullMeRequest(r PullMeRequest, handler RPCHandler) (err error) {
-	err = c.sendRequestReceiveHeader(r, ROK)
-	return c.serverLoop(handler)
-}
-
-func (c ByteStreamRPC) CloseRequest(r CloseRequest) (err error) {
-	if err = c.sendRequestReceiveHeader(r, ROK); err != nil {
-		return
-	}
-	err = c.conn.Close()
-	return
-}
-
-func (c ByteStreamRPC) ForceClose() (err error) {
-	return c.conn.Close()
-}
-
-type LocalRPC struct {
-	handler RPCHandler
-}
-
-func ConnectLocalRPC(handler RPCHandler) RPCRequester {
-	return LocalRPC{handler}
-}
-
-func (c LocalRPC) FilesystemRequest(r FilesystemRequest) (roots []*zfs.DatasetPath, err error) {
-	return c.handler.HandleFilesystemRequest(r)
-}
-
-func (c LocalRPC) FilesystemVersionsRequest(r FilesystemVersionsRequest) (versions []zfs.FilesystemVersion, err error) {
-	return c.handler.HandleFilesystemVersionsRequest(r)
-}
-
-func (c LocalRPC) InitialTransferRequest(r InitialTransferRequest) (io.Reader, error) {
-	return c.handler.HandleInitialTransferRequest(r)
-}
-
-func (c LocalRPC) IncrementalTransferRequest(r IncrementalTransferRequest) (reader io.Reader, err error) {
-	reader, err = c.handler.HandleIncrementalTransferRequest(r)
-	return
-}
-
-func (c LocalRPC) ResumeTransferRequest(r ResumeTransferRequest) (io.Reader, error) {
-	return c.handler.HandleResumeTransferRequest(r)
-}
-
-func (c LocalRPC) PullMeRequest(r PullMeRequest, handler RPCHandler) (err error) {
-	// The config syntactically only allows local Pulls, hence this is never called
-	// In theory, the following line should work:
-	// return handler.HandlePullMeRequest(r, LOCAL_TRANSPORT_IDENTITY, c)
-	panic("internal inconsistency: local pull me request unsupported")
-}
-
-func (c LocalRPC) CloseRequest(r CloseRequest) error { return nil }
-
-func (c LocalRPC) ForceClose() error { return nil }

rpc/rpc_test.go

@@ -1,33 +0,0 @@
-package rpc
-
-import (
-	"bytes"
-	"github.com/stretchr/testify/assert"
-	"github.com/zrepl/zrepl/util"
-	"io"
-	"strings"
-	"testing"
-)
-
-func TestByteStreamRPCDecodeJSONError(t *testing.T) {
-
-	r := strings.NewReader("{'a':'aber'}")
-
-	var chunked bytes.Buffer
-	ch := util.NewChunker(r)
-	io.Copy(&chunked, &ch)
-
-	type SampleType struct {
-		A uint
-	}
-	var s SampleType
-	err := readChunkedJSON(&chunked, &s)
-	assert.NotNil(t, err)
-
-	_, ok := err.(ByteStreamRPCDecodeJSONError)
-	if !ok {
-		t.Errorf("expected ByteStreamRPCDecodeJSONError, got %t\n", err)
-		t.Errorf("%s\n", err)
-	}
-
-}

rpc/server.go

@@ -0,0 +1,259 @@
+package rpc
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"reflect"
+
+	"github.com/pkg/errors"
+)
+
+type Server struct {
+	ml        *MessageLayer
+	logger    Logger
+	endpoints map[string]endpointDescr
+}
+
+type typeMap struct {
+	local reflect.Type
+	proto DataType
+}
+type endpointDescr struct {
+	inType  typeMap
+	outType typeMap
+	handler reflect.Value
+}
+
+type MarshaledJSONEndpoint func(bodyJSON interface{})
+
+func NewServer(rwc io.ReadWriteCloser) *Server {
+	ml := NewMessageLayer(rwc)
+	return &Server{
+		ml, noLogger{}, make(map[string]endpointDescr),
+	}
+}
+
+func (s *Server) SetLogger(logger Logger, logMessageLayer bool) {
+	s.logger = logger
+	if logMessageLayer {
+		s.ml.logger = logger
+	} else {
+		s.ml.logger = noLogger{}
+	}
+}
+
+func (s *Server) RegisterEndpoint(name string, handler interface{}) (err error) {
+	_, ok := s.endpoints[name]
+	if ok {
+		return errors.Errorf("already set up an endpoint for '%s'", name)
+	}
+	s.endpoints[name], err = makeEndpointDescr(handler)
+	return
+}
+
+func checkResponseHeader(h *Header) (err error) {
+	var statusNotSet Status
+	if h.Error == statusNotSet {
+		return errors.Errorf("status has zero-value")
+	}
+	return nil
+}
+
+func (s *Server) writeResponse(h *Header) (err error) {
+	// TODO validate
+	return s.ml.WriteHeader(h)
+}
+
+func (s *Server) recvRequest() (h *Header, err error) {
+	h, err = s.ml.ReadHeader()
+	if err != nil {
+		s.logger.Printf("error reading header: %s", err)
+		return nil, err
+	}
+
+	s.logger.Printf("validating request")
+	err = nil // TODO validate
+	if err == nil {
+		return h, nil
+	}
+	s.logger.Printf("request validation error: %s", err)
+
+	r := NewErrorHeader(StatusRequestError, "%s", err)
+	return nil, s.writeResponse(r)
+}
+
+var doneServeNext error = errors.New("this should not cause a HangUp() in the server")
+var doneStopServing error = errors.New("this should cause the server to close the connection")
+
+var ProtocolError error = errors.New("protocol error, server should hang up")
+
+const ControlEndpointClose string = "Close"
+
+// Serve the connection until failure or the client hangs up
+func (s *Server) Serve() (err error) {
+	for {
+
+		err = s.ServeRequest()
+
+		if err == nil {
+			continue
+		}
+		if err == doneServeNext {
+			s.logger.Printf("subroutine returned pseudo-error indicating early-exit")
+			err = nil
+			continue
+		}
+
+		if err == doneStopServing {
+			s.logger.Printf("subroutine returned pseudo-error indicating close request")
+			err = nil
+			break
+		}
+
+		break
+	}
+
+	if err != nil {
+		s.logger.Printf("an error occurred that could not be handled on PRC protocol level: %+v", err)
+	}
+
+	s.logger.Printf("cloing MessageLayer")
+	if mlErr := s.ml.Close(); mlErr != nil {
+		s.logger.Printf("error closing MessageLayer: %+v", mlErr)
+	}
+
+	return err
+}
+
+// Serve a single request
+// * wait for request to come in
+// * call handler
+// * reply
+//
+// The connection is left open, the next bytes on the conn should be
+// the next request header.
+//
+// Returns an err != nil if the error is bad enough to hang up on the client.
+// Examples: 		protocol version mismatches, protocol errors in general, ...
+// Non-Examples:	a handler error
+func (s *Server) ServeRequest() (err error) {
+
+	ml := s.ml
+
+	s.logger.Printf("reading header")
+	h, err := s.recvRequest()
+	if err != nil {
+		return err
+	}
+
+	if h.DataType == DataTypeControl {
+		switch h.Endpoint {
+		case ControlEndpointClose:
+			ack := Header{Error: StatusOK, DataType: DataTypeControl}
+			err = s.writeResponse(&ack)
+			if err != nil {
+				return err
+			}
+			return doneStopServing
+		default:
+			r := NewErrorHeader(StatusRequestError, "unregistered control endpoint %s", h.Endpoint)
+			return s.writeResponse(r)
+		}
+	}
+
+	ep, ok := s.endpoints[h.Endpoint]
+	if !ok {
+		r := NewErrorHeader(StatusRequestError, "unregistered endpoint %s", h.Endpoint)
+		return s.writeResponse(r)
+	}
+
+	if ep.inType.proto != h.DataType {
+		r := NewErrorHeader(StatusRequestError, "wrong DataType for endpoint %s (has %s, you provided %s)", h.Endpoint, ep.inType.proto, h.DataType)
+		return s.writeResponse(r)
+	}
+
+	if ep.outType.proto != h.Accept {
+		r := NewErrorHeader(StatusRequestError, "wrong Accept for endpoint %s (has %s, you provided %s)", h.Endpoint, ep.outType.proto, h.Accept)
+		return s.writeResponse(r)
+	}
+
+	dr := ml.ReadData()
+
+	// Determine inval
+	var inval reflect.Value
+	switch ep.inType.proto {
+	case DataTypeMarshaledJSON:
+		// Unmarshal input
+		inval = reflect.New(ep.inType.local.Elem())
+		invalIface := inval.Interface()
+		err = json.NewDecoder(dr).Decode(invalIface)
+		if err != nil {
+			r := NewErrorHeader(StatusRequestError, "cannot decode marshaled JSON: %s", err)
+			return s.writeResponse(r)
+		}
+	case DataTypeOctets:
+		// Take data as is
+		inval = reflect.ValueOf(dr)
+	default:
+		panic("not implemented")
+	}
+
+	outval := reflect.New(ep.outType.local.Elem()) // outval is a double pointer
+
+	s.logger.Printf("before handler, inval=%v outval=%v", inval, outval)
+
+	// Call the handler
+	errs := ep.handler.Call([]reflect.Value{inval, outval})
+
+	if !errs[0].IsNil() {
+		he := errs[0].Interface().(error) // we checked that before...
+		s.logger.Printf("handler returned error: %s", err)
+		r := NewErrorHeader(StatusError, "%s", he.Error())
+		return s.writeResponse(r)
+	}
+
+	switch ep.outType.proto {
+
+	case DataTypeMarshaledJSON:
+
+		var dataBuf bytes.Buffer
+		// Marshal output
+		err = json.NewEncoder(&dataBuf).Encode(outval.Interface())
+		if err != nil {
+			r := NewErrorHeader(StatusServerError, "cannot marshal response: %s", err)
+			return s.writeResponse(r)
+		}
+
+		replyHeader := Header{
+			Error:    StatusOK,
+			DataType: ep.outType.proto,
+		}
+		if err = s.writeResponse(&replyHeader); err != nil {
+			return err
+		}
+
+		if err = ml.WriteData(&dataBuf); err != nil {
+			return
+		}
+
+	case DataTypeOctets:
+
+		h := Header{
+			Error:    StatusOK,
+			DataType: DataTypeOctets,
+		}
+		if err = s.writeResponse(&h); err != nil {
+			return
+		}
+
+		reader := outval.Interface().(*io.Reader) // we checked that when adding the endpoint
+		err = ml.WriteData(*reader)
+		if err != nil {
+			return err
+		}
+
+	}
+
+	return nil
+}

rpc/shared.go

@@ -0,0 +1,111 @@
+package rpc
+
+import (
+	"fmt"
+	"github.com/pkg/errors"
+	"io"
+	"reflect"
+)
+
+type RPCServer interface {
+	Serve() (err error)
+	RegisterEndpoint(name string, handler interface{}) (err error)
+}
+
+type RPCClient interface {
+	Call(endpoint string, in, out interface{}) (err error)
+	Close() (err error)
+}
+
+type Logger interface {
+	Printf(format string, args ...interface{})
+}
+
+type noLogger struct{}
+
+func (l noLogger) Printf(format string, args ...interface{}) {}
+func typeIsIOReader(t reflect.Type) bool {
+	return t == reflect.TypeOf((*io.Reader)(nil)).Elem()
+}
+
+func typeIsIOReaderPtr(t reflect.Type) bool {
+	return t == reflect.TypeOf((*io.Reader)(nil))
+}
+
+// An error returned by the Client if the response indicated a status code other than StatusOK
+type RPCError struct {
+	ResponseHeader *Header
+}
+
+func (e *RPCError) Error() string {
+	return fmt.Sprintf("%s: %s", e.ResponseHeader.Error, e.ResponseHeader.ErrorMessage)
+}
+
+type RPCProtoError struct {
+	Message         string
+	UnderlyingError error
+}
+
+func (e *RPCProtoError) Error() string {
+	return e.Message
+}
+
+func checkRPCParamTypes(in, out reflect.Type) (err error) {
+	if !(in.Kind() == reflect.Ptr || typeIsIOReader(in)) {
+		err = errors.Errorf("input parameter must be a pointer or an io.Reader, is of kind %s, type %s", in.Kind(), in)
+		return
+	}
+	if !(out.Kind() == reflect.Ptr) {
+		err = errors.Errorf("second input parameter (the non-error output parameter) must be a pointer or an *io.Reader")
+		return
+	}
+	return nil
+}
+
+func checkRPCReturnType(rt reflect.Type) (err error) {
+	errInterfaceType := reflect.TypeOf((*error)(nil)).Elem()
+	if !rt.Implements(errInterfaceType) {
+		err = errors.Errorf("handler must return an error")
+		return
+	}
+	return nil
+}
+
+func makeEndpointDescr(handler interface{}) (descr endpointDescr, err error) {
+
+	ht := reflect.TypeOf(handler)
+
+	if ht.Kind() != reflect.Func {
+		err = errors.Errorf("handler must be of kind reflect.Func")
+		return
+	}
+
+	if ht.NumIn() != 2 || ht.NumOut() != 1 {
+		err = errors.Errorf("handler must have exactly two input parameters and one output parameter")
+		return
+	}
+	if err = checkRPCParamTypes(ht.In(0), ht.In(1)); err != nil {
+		return
+	}
+	if err = checkRPCReturnType(ht.Out(0)); err != nil {
+		return
+	}
+
+	descr.handler = reflect.ValueOf(handler)
+	descr.inType.local = ht.In(0)
+	descr.outType.local = ht.In(1)
+
+	if typeIsIOReader(ht.In(0)) {
+		descr.inType.proto = DataTypeOctets
+	} else {
+		descr.inType.proto = DataTypeMarshaledJSON
+	}
+
+	if typeIsIOReaderPtr(ht.In(1)) {
+		descr.outType.proto = DataTypeOctets
+	} else {
+		descr.outType.proto = DataTypeMarshaledJSON
+	}
+
+	return
+}

rpc/status_string.go

@@ -0,0 +1,17 @@
+// Code generated by "stringer -type=Status"; DO NOT EDIT.
+
+package rpc
+
+import "fmt"
+
+const _Status_name = "StatusOKStatusRequestErrorStatusServerErrorStatusError"
+
+var _Status_index = [...]uint8{0, 8, 26, 43, 54}
+
+func (i Status) String() string {
+	i -= 1
+	if i >= Status(len(_Status_index)-1) {
+		return fmt.Sprintf("Status(%d)", i+1)
+	}
+	return _Status_name[_Status_index[i]:_Status_index[i+1]]
+}

rpc/structs.go

@@ -1,144 +0,0 @@
-package rpc
-
-import (
-	"encoding/json"
-	"io"
-
-	"github.com/zrepl/zrepl/zfs"
-)
-
-var _ json.Marshaler = &zfs.DatasetPath{}
-var _ json.Unmarshaler = &zfs.DatasetPath{}
-
-type RequestId [16]byte
-type RequestType uint8
-
-const (
-	RTProtocolVersionRequest     RequestType = 0x01
-	RTFilesystemRequest                      = 0x10
-	RTFilesystemVersionsRequest              = 0x11
-	RTInitialTransferRequest                 = 0x12
-	RTIncrementalTransferRequest             = 0x13
-	RTResumeTransferRequest                  = 0x14
-	RTPullMeRequest                          = 0x20
-	RTCloseRequest                           = 0xf0
-)
-
-type RequestHeader struct {
-	Type RequestType
-	Id   [16]byte // UUID
-}
-
-type FilesystemRequest struct {
-	Roots []string // may be nil, indicating interest in all filesystems
-}
-
-type FilesystemVersionsRequest struct {
-	Filesystem *zfs.DatasetPath
-}
-
-type InitialTransferRequest struct {
-	Filesystem        *zfs.DatasetPath
-	FilesystemVersion zfs.FilesystemVersion
-}
-
-func (r InitialTransferRequest) Respond(snapshotReader io.Reader) {
-
-}
-
-type IncrementalTransferRequest struct {
-	Filesystem *zfs.DatasetPath
-	From       zfs.FilesystemVersion
-	To         zfs.FilesystemVersion
-}
-
-type ResumeTransferRequest struct {
-	Filesystem *zfs.DatasetPath
-	Token      string
-}
-
-type ResumeTransferError struct {
-	Reason   ResumeTransferErrorReason
-	ZFSError string
-}
-
-func (e *ResumeTransferError) Error() string {
-	return e.Reason.String()
-}
-
-//go:generate stringer -type ResumeTransferErrorReason
-type ResumeTransferErrorReason uint8
-
-const (
-	ResumeTransferErrorReasonNotImplemented ResumeTransferErrorReason = iota
-	ResumeTransferErrorReasonDisabled
-	ResumeTransferErrorReasonZFSErrorPermanent
-	ResumeTransferErrorReasonZFSErrorMaybeTemporary
-)
-
-func (r IncrementalTransferRequest) Respond(snapshotReader io.Reader) {
-
-}
-
-type ByteStreamRPCProtocolVersionRequest struct {
-	ClientVersion uint8
-}
-
-const LOCAL_TRANSPORT_IDENTITY string = "local"
-
-const DEFAULT_INITIAL_REPL_POLICY = InitialReplPolicyMostRecent
-
-type InitialReplPolicy string
-
-const (
-	InitialReplPolicyMostRecent InitialReplPolicy = "most_recent"
-	InitialReplPolicyAll        InitialReplPolicy = "all"
-)
-
-type PullMeRequest struct {
-	// if true, the other fields are undefined
-	Finished          bool
-	InitialReplPolicy InitialReplPolicy
-}
-
-type CloseRequest struct {
-	Goodbye string
-}
-
-type ErrorId uint8
-
-const (
-	ENoError                 ErrorId = 0
-	EDecodeHeader                    = 1
-	EUnknownRequestType              = 2
-	EDecodeRequestBody               = 3
-	EProtocolVersionMismatch         = 4
-	EHandler                         = 5
-)
-
-type ResponseType uint8
-
-const (
-	RNONE           ResponseType = 0x0
-	ROK                          = 0x1
-	RFilesystems                 = 0x10
-	RFilesystemDiff              = 0x11
-	RChunkedStream               = 0x20
-)
-
-type ResponseHeader struct {
-	RequestId    RequestId
-	ErrorId      ErrorId
-	Message      string
-	ResponseType ResponseType
-}
-
-func NewByteStreamRPCProtocolVersionRequest() ByteStreamRPCProtocolVersionRequest {
-	return ByteStreamRPCProtocolVersionRequest{
-		ClientVersion: ByteStreamRPCProtocolVersion,
-	}
-}
-
-func newUUID() [16]byte {
-	return [16]byte{}
-}

scratchpad/chunker/.gitignore

@@ -1 +0,0 @@
-chunker

scratchpad/chunker/main.go

@@ -1,84 +0,0 @@
-package main
-
-import (
-	"flag"
-	"github.com/zrepl/zrepl/sshbytestream"
-	. "github.com/zrepl/zrepl/util"
-	// "bytes"
-	_ "bufio"
-	// "strings"
-	"io"
-	"log"
-	"os"
-	_ "time"
-)
-
-func main() {
-
-	mode := flag.String("mode", "", "incoming|outgoing")
-	incomingFile := flag.String("incoming.file", "", "file to deliver to callers")
-	outgoingHost := flag.String("outgoing.sshHost", "", "ssh host")
-	outgoingUser := flag.String("outgoing.sshUser", "", "ssh user")
-	outgoingIdentity := flag.String("outgoing.sshIdentity", "", "ssh private key")
-	outgoingPort := flag.Uint("outgoing.sshPort", 22, "ssh port")
-	outgoingFile := flag.String("outgoing.File", "", "")
-	flag.Parse()
-
-	switch {
-	case (*mode == "incoming"):
-
-		conn, err := sshbytestream.Incoming()
-		if err != nil {
-			panic(err)
-		}
-
-		file, err := os.Open(*incomingFile)
-		if err != nil {
-			panic(err)
-		}
-
-		chunker := NewChunker(file)
-
-		_, err = io.Copy(conn, &chunker)
-		if err != nil && err != io.EOF {
-			panic(err)
-		}
-
-		log.Printf("Chunk Count: %d\n", chunker.ChunkCount)
-
-	case *mode == "outgoing":
-
-		conn, err := sshbytestream.Outgoing(sshbytestream.SSHTransport{
-			Host:         *outgoingHost,
-			User:         *outgoingUser,
-			IdentityFile: *outgoingIdentity,
-			Port:         uint16(*outgoingPort),
-		})
-		if err != nil {
-			panic(err)
-		}
-
-		f, err := os.OpenFile(*outgoingFile, os.O_CREATE|os.O_WRONLY, 0600)
-		if err != nil {
-			panic(err)
-		}
-
-		unchunker := NewUnchunker(conn)
-
-		_, err = io.Copy(f, unchunker)
-		if err != nil {
-			panic(err)
-		}
-
-		conn.Close()
-
-		log.Printf("Chunk Count: %d\n", unchunker.ChunkCount)
-
-		os.Exit(0)
-
-	default:
-		panic("unsupported mode!")
-
-	}
-
-}

scratchpad/repeat/main.go

@@ -1,46 +0,0 @@
-package main
-
-import (
-	"log"
-	"os"
-	"os/exec"
-	"time"
-)
-
-var config struct {
-	duration time.Duration
-}
-
-func usage() {
-	log.Printf("usage: repeat repeatInterval command [args...]")
-}
-
-func main() {
-
-	args := os.Args
-
-	if len(args) < 3 {
-		usage()
-		os.Exit(1)
-	}
-	repeatInterval, err := time.ParseDuration(args[1])
-	if err != nil {
-		log.Printf("cannot parse interval: %s", err)
-		usage()
-		os.Exit(1)
-	}
-
-	var lastStart time.Time
-
-	for {
-		cmd := exec.Command(args[2], args[3:]...)
-		cmd.Stdout = os.Stdout
-		cmd.Stderr = os.Stderr
-		lastStart = time.Now()
-		if err := cmd.Run(); err != nil {
-			panic(err)
-		}
-		time.Sleep(lastStart.Add(repeatInterval).Sub(time.Now()))
-	}
-
-}

scratchpad/repeat/sleepx.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-echo -n ' start '
-date
-sleep $1
-echo -n 'done '
-date 

sshbytestream/ssh.go

@@ -77,7 +77,7 @@ func Outgoing(remote SSHTransport) (s OutgoingSSHByteStream, err error) {
 
 	var sshCommand = SSHCommand
 	if len(remote.SSHCommand) > 0 {
-		sshCommand = SSHCommand
+		sshCommand = remote.SSHCommand
 	}
 
 	if s.c, err = util.NewIOCommand(sshCommand, sshArgs, util.IOCommandStderrBufSize); err != nil {

zfs/diff.go

@@ -189,21 +189,15 @@ const ZREPL_PLACEHOLDER_PROPERTY_NAME string = "zrepl:placeholder"
 
 type FilesystemState struct {
 	Placeholder bool
-	// If != "", the receive_resume_token found on the receiving side of a resumable send & recv
+	// TODO extend with resume token when that feature is finally added
-	ResumeToken string
 }
 
 // A somewhat efficient way to determine if a filesystem exists on this host.
 // Particularly useful if exists is called more than once (will only fork exec once and cache the result)
 func ZFSListFilesystemState() (localState map[string]FilesystemState, err error) {
 
-	properties := []string{"name", ZREPL_PLACEHOLDER_PROPERTY_NAME}
-	if CLICompat.ResumableSendRecv {
-		properties = append(properties, "receive_resume_token")
-	}
-
 	var actual [][]string
-	if actual, err = ZFSList(properties, "-t", "filesystem,volume"); err != nil {
+	if actual, err = ZFSList([]string{"name", ZREPL_PLACEHOLDER_PROPERTY_NAME}, "-t", "filesystem,volume"); err != nil {
 		return
 	}
 
@@ -211,15 +205,11 @@ func ZFSListFilesystemState() (localState map[string]FilesystemState, err error)
 	for _, e := range actual {
 		dp, err := NewDatasetPath(e[0])
 		if err != nil {
-			fmt.Errorf("ZFS does not return parseable dataset path: %s", e[0])
+			return nil, fmt.Errorf("ZFS does not return parseable dataset path: %s", e[0])
 		}
 		placeholder, _ := IsPlaceholder(dp, e[1])
-		receive_resume_token := ""
-		if CLICompat.ResumableSendRecv {
-			receive_resume_token = e[2]
-		}
 		localState[e[0]] = FilesystemState{
-			placeholder, receive_resume_token,
+			placeholder,
 		}
 	}
 	return

zfs/versions.go

@@ -25,7 +25,6 @@ func (t VersionType) DelimiterChar() string {
 	default:
 		panic(fmt.Sprintf("unexpected VersionType %#v", t))
 	}
-	return ""
 }
 
 type FilesystemVersion struct {