prototype resumable send & recv

.gitignore

@@ -1,13 +1,3 @@
-# Build
-artifacts/
-
-# Golang
-vendor/
-
-# IDEs
-.idea/
-.vscode/
-
 # Compiled Object files, Static and Dynamic libs (Shared Objects)
 *.o
 *.a

Gopkg.lock

@@ -1,177 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/fsnotify/fsnotify"
-  packages = ["."]
-  revision = "629574ca2a5df945712d3079857300b5e4da0236"
-  version = "v1.4.2"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/ftrvxmtrx/fd"
-  packages = ["."]
-  revision = "c6d800382fff6dc1412f34269f71b7f83bd059ad"
-
-[[projects]]
-  name = "github.com/go-logfmt/logfmt"
-  packages = ["."]
-  revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5"
-  version = "v0.3.0"
-
-[[projects]]
-  branch = "v2"
-  name = "github.com/go-yaml/yaml"
-  packages = ["."]
-  revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/hcl"
-  packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"]
-  revision = "68e816d1c783414e79bc65b3994d9ab6b0a722ab"
-
-[[projects]]
-  name = "github.com/inconshreveable/mousetrap"
-  packages = ["."]
-  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
-  version = "v1.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/jinzhu/copier"
-  packages = ["."]
-  revision = "db4671f3a9b8df855e993f7c94ec5ef1ffb0a23b"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/kr/logfmt"
-  packages = ["."]
-  revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/kr/pretty"
-  packages = ["."]
-  revision = "cfb55aafdaf3ec08f0db22699ab822c50091b1c4"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/kr/text"
-  packages = ["."]
-  revision = "7cafcd837844e784b526369c9bce262804aebc60"
-
-[[projects]]
-  name = "github.com/magiconair/properties"
-  packages = ["."]
-  revision = "be5ece7dd465ab0765a9682137865547526d1dfb"
-  version = "v1.7.3"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/mitchellh/go-homedir"
-  packages = ["."]
-  revision = "b8bc1bf767474819792c23f32d8286a45736f1c6"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/mitchellh/mapstructure"
-  packages = ["."]
-  revision = "d0303fe809921458f417bcf828397a65db30a7e4"
-
-[[projects]]
-  name = "github.com/pelletier/go-buffruneio"
-  packages = ["."]
-  revision = "c37440a7cf42ac63b919c752ca73a85067e05992"
-  version = "v0.2.0"
-
-[[projects]]
-  name = "github.com/pelletier/go-toml"
-  packages = ["."]
-  revision = "5ccdfb18c776b740aecaf085c4d9a2779199c279"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  name = "github.com/pmezard/go-difflib"
-  packages = ["difflib"]
-  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/spf13/afero"
-  packages = [".","mem"]
-  revision = "ee1bd8ee15a1306d1f9201acc41ef39cd9f99a1b"
-
-[[projects]]
-  name = "github.com/spf13/cast"
-  packages = ["."]
-  revision = "acbeb36b902d72a7a4c18e8f3241075e7ab763e4"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/spf13/cobra"
-  packages = ["."]
-  revision = "b78744579491c1ceeaaa3b40205e56b0591b93a3"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/spf13/jwalterweatherman"
-  packages = ["."]
-  revision = "12bd96e66386c1960ab0f74ced1362f66f552f7b"
-
-[[projects]]
-  name = "github.com/spf13/pflag"
-  packages = ["."]
-  revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/spf13/viper"
-  packages = ["."]
-  revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/stretchr/testify"
-  packages = ["assert"]
-  revision = "69483b4bd14f5845b5a1e55bca19e954e827f1d0"
-  version = "v1.1.4"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sys"
-  packages = ["unix"]
-  revision = "429f518978ab01db8bb6f44b66785088e7fba58b"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/text"
-  packages = ["internal/gen","internal/triegen","internal/ucd","transform","unicode/cldr","unicode/norm"]
-  revision = "1cbadb444a806fd9430d14ad08967ed91da4fa0a"
-
-[[projects]]
-  branch = "v2"
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  inputs-digest = "7ce2ead5225e4bb72a34132538171a649b26de574596e909730658ddbc904cd5"
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,66 +0,0 @@
-
-# Gopkg.toml example
-#
-# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
-# for detailed Gopkg.toml documentation.
-#
-# required = ["github.com/user/thing/cmd/thing"]
-# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
-#
-# [[constraint]]
-#   name = "github.com/user/project"
-#   version = "1.0.0"
-#
-# [[constraint]]
-#   name = "github.com/user/project2"
-#   branch = "dev"
-#   source = "github.com/myfork/project2"
-#
-# [[override]]
-#  name = "github.com/x/y"
-#  version = "2.4.0"
-
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/ftrvxmtrx/fd"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/jinzhu/copier"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/kr/pretty"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/mitchellh/go-homedir"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/mitchellh/mapstructure"
-
-[[constraint]]
-  name = "github.com/pkg/errors"
-  version = "0.8.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/spf13/cobra"
-
-[[constraint]]
-  name = "github.com/spf13/viper"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "github.com/stretchr/testify"
-  version = "1.1.4"
-
-[[constraint]]
-  branch = "v2"
-  name = "github.com/go-yaml/yaml"
-
-[[constraint]]
-  name = "github.com/go-logfmt/logfmt"
-  version = "*"

Makefile

@@ -1,47 +0,0 @@
-.PHONY: generate build test vet cover release clean
-
-ROOT := github.com/zrepl/zrepl
-SUBPKGS := cmd logger rpc sshbytestream util
-
-_TESTPKGS := $(ROOT) $(foreach p,$(SUBPKGS),$(ROOT)/$(p))
-
-ARTIFACTDIR := artifacts
-
-build: generate
-		go build -o $(ARTIFACTDIR)/zrepl
-
-generate:
-	@for pkg in $(_TESTPKGS); do\
-		go generate "$$pkg" || exit 1; \
-	done;
-
-test:
-	@for pkg in $(_TESTPKGS); do \
-		echo "Testing $$pkg"; \
-		go test "$$pkg" || exit 1; \
-	done;
-
-vet:
-	@for pkg in $(_TESTPKGS); do \
-		echo "Vetting $$pkg"; \
-		go vet "$$pkg" || exit 1; \
-	done;
-
-cover: artifacts
-	@for pkg in $(_TESTPKGS); do \
-		profile="$(ARTIFACTDIR)/cover-$$(basename $$pkg).out"; \
-		go test -coverprofile "$$profile" $$pkg || exit 1; \
-		if [ -f "$$profile" ]; then \
-   			go tool cover -html="$$profile" -o "$${profile}.html" || exit 2; \
-		fi; \
-	done;
-
-artifacts:
-	mkdir artifacts
-
-release: generate artifacts vet test
-	GOOS=linux GOARCH=amd64   go build -o "$(ARTIFACTDIR)/zrepl-linux-amd64"
-	GOOS=freebsd GOARCH=amd64 go build -o "$(ARTIFACTDIR)/zrepl-freebsd-amd64"
-
-clean:
-	rm -rf "$(ARTIFACTDIR)"

README.md

@@ -1,54 +1,2 @@
 # zrepl
-zrepl is a ZFS filesystem backup & replication solution written in Go.
+ZFS dataset replication. Not a shell script.
-
-## User Documentation
-
-**User Documentation** cab be found at [zrepl.github.io](https://zrepl.github.io).
-
-## Bug Reports
-
-1. If the issue is reproducible, enable debug logging, reproduce and capture the log.
-2. Open an issue on GitHub, with logs pasted as GitHub gists / inline.
-
-## Feature Requests
-
-1. Does you feature request require default values / some kind of configuration?
-   If so, think of an expressive configuration example.
-2. Think of at least one use case that generalizes from your concrete application.
-3. Open an issue on GitHub with example conf & use case attached.
-
-The above does not apply if you already implemented everything.
-Check out the *Coding Workflow* section below for details.
-
-## Developer Documentation
-
-### Overall Architecture
-
-The application architecture is documented as part of the user docs in the *Implementation* section (`docs/content/impl`).
-Make sure to develop an understanding how zrepl is typically used by studying the user docs first.
-
-### Project Structure
-
-```
-├── cmd
-│   ├── sampleconf          # example configuration
-├── docs
-│   ├── content             # hugo-based documentation -> sources for ./public_git
-│   ├── deploy.sh           # shell script for automated rendering & deploy to zrepl.github.io repo
-│   ├── public_git          # used by above shell script
-│   └── themes
-│       └── docdock         # submodule of our docdock theme fork
-├── jobrun                  # OBSOLETE
-├── logger                  # logger package used by zrepl
-├── rpc                     # rpc protocol implementation
-├── scratchpad              # small example programs demoing some internal packages. probably OBSOLETE
-├── sshbytestream           # io.ReadWriteCloser over SSH
-├── util
-└── zfs                     # ZFS wrappers, filesystemm diffing
-```
-
-### Coding Workflow
-
-* Open an issue when starting to hack on a new feature
-* Commits should reference the issue they are related to
-* Docs improvements not documenting new features do not require an issue.

cmd/autosnap.go

@@ -1,148 +1,91 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/jobrun"
 	"github.com/zrepl/zrepl/zfs"
-	"sort"
+	"os"
+	"sync"
 	"time"
 )
 
-type IntervalAutosnap struct {
+var AutosnapCmd = &cobra.Command{
-	DatasetFilter    zfs.DatasetFilter
+	Use:   "autosnap",
-	Prefix           string
+	Short: "perform automatic snapshotting",
-	SnapshotInterval time.Duration
+	Run:   cmdAutosnap,
-
-	log       Logger
-	snaptimes []snapTime
 }
 
-type snapTime struct {
+func init() {
-	ds   *zfs.DatasetPath
+	RootCmd.AddCommand(AutosnapCmd)
-	time time.Time
 }
 
-func (a *IntervalAutosnap) Run(ctx context.Context, didSnaps chan struct{}) {
+func cmdAutosnap(cmd *cobra.Command, args []string) {
 
-	a.log = ctx.Value(contextKeyLog).(Logger)
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		runner.Start()
+	}()
 
-	const LOG_TIME_FMT string = time.ANSIC
+	if len(args) < 1 {
+		log.Printf("must specify exactly one job as positional argument")
+		os.Exit(1)
+	}
 
-	ds, err := zfs.ZFSListMapping(a.DatasetFilter)
+	snap, ok := conf.Autosnaps[args[0]]
+	if !ok {
+		log.Printf("could not find autosnap job: %s", args[0])
+		os.Exit(1)
+	}
+
+	job := jobrun.Job{
+		Name:           snap.JobName,
+		RepeatStrategy: snap.Interval,
+		RunFunc: func(log jobrun.Logger) error {
+			log.Printf("doing autosnap: %v", snap)
+			ctx := AutosnapContext{snap}
+			return doAutosnap(ctx, log)
+		},
+	}
+	runner.AddJob(job)
+
+	wg.Wait()
+
+}
+
+type AutosnapContext struct {
+	Autosnap *Autosnap
+}
+
+func doAutosnap(ctx AutosnapContext, log Logger) (err error) {
+
+	snap := ctx.Autosnap
+
+	filesystems, err := zfs.ZFSListMapping(snap.DatasetFilter)
 	if err != nil {
-		a.log.WithError(err).Error("cannot list datasets")
+		return fmt.Errorf("cannot filter datasets: %s", err)
-		return
-	}
-	if len(ds) == 0 {
-		a.log.WithError(err).Error("no datasets matching dataset filter")
-		return
 	}
 
-	a.snaptimes = make([]snapTime, len(ds))
+	suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
+	snapname := fmt.Sprintf("%s%s", snap.Prefix, suffix)
 
-	now := time.Now()
+	hadError := false
 
-	a.log.Debug("examine filesystem state")
+	for _, fs := range filesystems { // optimization: use recursive snapshots / channel programs here
-	for i, d := range ds {
+		log.Printf("snapshotting filesystem %s@%s", fs, snapname)
-
+		err := zfs.ZFSSnapshot(fs, snapname, false)
-		l := a.log.WithField(logFSField, d.ToString())
-
-		fsvs, err := zfs.ZFSListFilesystemVersions(d, &PrefixSnapshotFilter{a.Prefix})
 		if err != nil {
-			l.WithError(err).Error("cannot list filesystem versions")
+			log.Printf("error snapshotting %s: %s", fs, err)
-			continue
+			hadError = true
-		}
-		if len(fsvs) <= 0 {
-			l.WithField("prefix", a.Prefix).Info("no filesystem versions with prefix")
-			a.snaptimes[i] = snapTime{d, now}
-			continue
-		}
-
-		// Sort versions by creation
-		sort.SliceStable(fsvs, func(i, j int) bool {
-			return fsvs[i].CreateTXG < fsvs[j].CreateTXG
-		})
-
-		latest := fsvs[len(fsvs)-1]
-		l.WithField("creation", latest.Creation).
-			Debug("found latest snapshot")
-
-		since := now.Sub(latest.Creation)
-		if since < 0 {
-			l.WithField("snapshot", latest.Name).
-				WithField("creation", latest.Creation).
-				Error("snapshot is from the future")
-			continue
-		}
-		next := now
-		if since < a.SnapshotInterval {
-			next = latest.Creation.Add(a.SnapshotInterval)
-		}
-		a.snaptimes[i] = snapTime{d, next}
-	}
-
-	sort.Slice(a.snaptimes, func(i, j int) bool {
-		return a.snaptimes[i].time.Before(a.snaptimes[j].time)
-	})
-
-	syncPoint := a.snaptimes[0]
-	a.log.WithField("sync_point", syncPoint.time.Format(LOG_TIME_FMT)).
-		Info("wait for sync point")
-
-	select {
-	case <-ctx.Done():
-		a.log.WithError(ctx.Err()).Info("context done")
-		return
-
-	case <-time.After(syncPoint.time.Sub(now)):
-		a.log.Debug("snapshot all filesystems to enable further snaps in lockstep")
-		a.doSnapshots(didSnaps)
-	}
-
-	ticker := time.NewTicker(a.SnapshotInterval)
-
-	for {
-		select {
-		case <-ctx.Done():
-			ticker.Stop()
-			a.log.WithError(ctx.Err()).Info("context done")
-			return
-
-		case <-ticker.C:
-			a.doSnapshots(didSnaps)
 		}
 	}
-
+
-}
+	if hadError {
-
+		err = fmt.Errorf("errors occurred during autosnap, check logs for details")
-func (a *IntervalAutosnap) doSnapshots(didSnaps chan struct{}) {
+	}
-
+
-	// fetch new dataset list in case user added new dataset
+	return
-	ds, err := zfs.ZFSListMapping(a.DatasetFilter)
-	if err != nil {
-		a.log.WithError(err).Error("cannot list datasets")
-		return
-	}
-
-	// TODO channel programs -> allow a little jitter?
-	for _, d := range ds {
-		suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
-		snapname := fmt.Sprintf("%s%s", a.Prefix, suffix)
-
-		a.log.WithField(logFSField, d.ToString()).
-			WithField("snapname", snapname).
-			Info("create snapshot")
-
-		err := zfs.ZFSSnapshot(d, snapname, false)
-		if err != nil {
-			a.log.WithError(err).Error("cannot create snapshot")
-		}
-	}
-
-	select {
-	case didSnaps <- struct{}{}:
-	default:
-		a.log.Error("warning: callback channel is full, discarding")
-	}
 
 }

cmd/config.go

@@ -1,100 +1,730 @@
 package cmd
 
 import (
-	"io"
+	"errors"
-
 	"fmt"
-	"github.com/pkg/errors"
+	"github.com/jinzhu/copier"
+	"github.com/mitchellh/mapstructure"
+	"github.com/zrepl/zrepl/jobrun"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/sshbytestream"
+	. "github.com/zrepl/zrepl/util"
 	"github.com/zrepl/zrepl/zfs"
+	yaml "gopkg.in/yaml.v2"
+	"io"
+	"io/ioutil"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
 )
 
+var (
+	JobSectionPush     string = "push"
+	JobSectionPull     string = "pull"
+	JobSectionPrune    string = "prune"
+	JobSectionAutosnap string = "autosnap"
+)
+
+type Remote struct {
+	Name      string
+	Transport Transport
+}
+
+type Transport interface {
+	Connect(rpcLog Logger) (rpc.RPCRequester, error)
+}
+type LocalTransport struct {
+	Handler rpc.RPCHandler
+}
+type SSHTransport struct {
+	Host                 string
+	User                 string
+	Port                 uint16
+	IdentityFile         string   `mapstructure:"identity_file"`
+	TransportOpenCommand []string `mapstructure:"transport_open_command"`
+	SSHCommand           string   `mapstructure:"ssh_command"`
+	Options              []string
+	ConnLogReadFile      string `mapstructure:"connlog_read_file"`
+	ConnLogWriteFile     string `mapstructure:"connlog_write_file"`
+}
+
+type Push struct {
+	JobName           string // for use with jobrun package
+	To                *Remote
+	Filter            zfs.DatasetFilter
+	InitialReplPolicy rpc.InitialReplPolicy
+	RepeatStrategy    jobrun.RepeatStrategy
+}
+type Pull struct {
+	JobName           string // for use with jobrun package
+	From              *Remote
+	Mapping           DatasetMapFilter
+	InitialReplPolicy rpc.InitialReplPolicy
+	RepeatStrategy    jobrun.RepeatStrategy
+}
+
+type Prune struct {
+	JobName         string // for use with jobrun package
+	DatasetFilter   zfs.DatasetFilter
+	SnapshotFilter  zfs.FilesystemVersionFilter
+	RetentionPolicy *RetentionGrid // TODO abstract interface to support future policies?
+}
+
+type Autosnap struct {
+	JobName       string // for use with jobrun package
+	Prefix        string
+	Interval      jobrun.RepeatStrategy
+	DatasetFilter zfs.DatasetFilter
+}
+
 type Config struct {
-	Global Global
+	Remotes   map[string]*Remote
-	Jobs   map[string]Job
+	Pushs     map[string]*Push            // job name -> job
+	Pulls     map[string]*Pull            // job name -> job
+	Sinks     map[string]DatasetMapFilter // client identity -> mapping
+	PullACLs  map[string]DatasetMapFilter // client identity -> filter
+	Prunes    map[string]*Prune           // job name -> job
+	Autosnaps map[string]*Autosnap        // job name -> job
 }
 
-func (c *Config) LookupJob(name string) (j Job, err error) {
+func ParseConfig(path string) (config Config, err error) {
-	j, ok := c.Jobs[name]
+
-	if !ok {
+	c := make(map[string]interface{}, 0)
-		return nil, errors.Errorf("job '%s' is not defined", name)
+
+	var bytes []byte
+
+	if bytes, err = ioutil.ReadFile(path); err != nil {
+		return
 	}
-	return j, nil
+
+	if err = yaml.Unmarshal(bytes, &c); err != nil {
+		return
+	}
+
+	return parseMain(c)
 }
 
-type Global struct {
+func parseMain(root map[string]interface{}) (c Config, err error) {
-	Serve struct {
+	if c.Remotes, err = parseRemotes(root["remotes"]); err != nil {
-		Stdinserver struct {
+		return
-			SockDir string
+	}
+
+	remoteLookup := func(name string) (remote *Remote, err error) {
+		remote = c.Remotes[name]
+		if remote == nil {
+			err = fmt.Errorf("remote '%s' not defined", name)
+		}
+		return
+	}
+
+	if c.Pushs, err = parsePushs(root["pushs"], remoteLookup); err != nil {
+		return
+	}
+	if c.Pulls, err = parsePulls(root["pulls"], remoteLookup); err != nil {
+		return
+	}
+	if c.Sinks, err = parseSinks(root["sinks"]); err != nil {
+		return
+	}
+	if c.PullACLs, err = parsePullACLs(root["pull_acls"]); err != nil {
+		return
+	}
+	if c.Prunes, err = parsePrunes(root["prune"]); err != nil {
+		return
+	}
+	if c.Autosnaps, err = parseAutosnaps(root["autosnap"]); err != nil {
+		return
+	}
+
+	return
+}
+
+func fullJobName(section, name string) (full string, err error) {
+	if len(name) < 1 {
+		err = fmt.Errorf("job name not set")
+		return
+	}
+	full = fmt.Sprintf("%s.%s", section, name)
+	return
+}
+
+func parseRemotes(v interface{}) (remotes map[string]*Remote, err error) {
+
+	asMap := make(map[string]struct {
+		Transport map[string]interface{}
+	}, 0)
+	if err = mapstructure.Decode(v, &asMap); err != nil {
+		return
+	}
+
+	remotes = make(map[string]*Remote, len(asMap))
+	for name, p := range asMap {
+
+		if name == rpc.LOCAL_TRANSPORT_IDENTITY {
+			err = errors.New(fmt.Sprintf("remote name '%s' reserved for local pulls", rpc.LOCAL_TRANSPORT_IDENTITY))
+			return
+		}
+
+		var transport Transport
+		if transport, err = parseTransport(p.Transport); err != nil {
+			return
+		}
+		remotes[name] = &Remote{
+			Name:      name,
+			Transport: transport,
 		}
 	}
-	Control struct {
+
-		Sockpath string
+	return
+}
+
+func parseTransport(it map[string]interface{}) (t Transport, err error) {
+
+	if len(it) != 1 {
+		err = errors.New("ambiguous transport type")
+		return
 	}
-	logging *LoggingConfig
-}
 
-type JobDebugSettings struct {
+	for key, val := range it {
-	Conn struct {
+		switch key {
-		ReadDump  string `mapstructure:"read_dump"`
+		case "ssh":
-		WriteDump string `mapstructure:"write_dump"`
+			t := SSHTransport{}
+			if err = mapstructure.Decode(val, &t); err != nil {
+				err = errors.New(fmt.Sprintf("could not parse ssh transport: %s", err))
+				return nil, err
+			}
+			return t, nil
+		default:
+			return nil, errors.New(fmt.Sprintf("unknown transport type '%s'\n", key))
+		}
 	}
-	RPC struct {
+
-		Log bool
+	return // unreachable
+
+}
+
+type remoteLookup func(name string) (*Remote, error)
+
+func parsePushs(v interface{}, rl remoteLookup) (p map[string]*Push, err error) {
+
+	asMap := make(map[string]struct {
+		To                string
+		Filter            map[string]string
+		InitialReplPolicy string
+		Repeat            map[string]string
+	}, 0)
+
+	if err = mapstructure.Decode(v, &asMap); err != nil {
+		return
 	}
+
+	p = make(map[string]*Push, len(asMap))
+
+	for name, e := range asMap {
+
+		var toRemote *Remote
+		if toRemote, err = rl(e.To); err != nil {
+			return
+		}
+		push := &Push{
+			To: toRemote,
+		}
+
+		if push.JobName, err = fullJobName(JobSectionPush, name); err != nil {
+			return
+		}
+		if push.Filter, err = parseDatasetMapFilter(e.Filter, true); err != nil {
+			return
+		}
+
+		if push.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, rpc.DEFAULT_INITIAL_REPL_POLICY); err != nil {
+			return
+		}
+
+		if push.RepeatStrategy, err = parseRepeatStrategy(e.Repeat); err != nil {
+			return
+		}
+
+		p[name] = push
+	}
+
+	return
 }
 
-type RWCConnecter interface {
+func parsePulls(v interface{}, rl remoteLookup) (p map[string]*Pull, err error) {
-	Connect() (io.ReadWriteCloser, error)
+
-}
+	asMap := make(map[string]struct {
-type AuthenticatedChannelListenerFactory interface {
+		From              string
-	Listen() (AuthenticatedChannelListener, error)
+		Mapping           map[string]string
+		InitialReplPolicy string
+		Repeat            map[string]string
+	}, 0)
+
+	if err = mapstructure.Decode(v, &asMap); err != nil {
+		return
+	}
+
+	p = make(map[string]*Pull, len(asMap))
+
+	for name, e := range asMap {
+
+		if len(e.From) < 1 {
+			err = fmt.Errorf("source not set ('from' attribute is empty)")
+			return
+		}
+
+		var fromRemote *Remote
+
+		if e.From == rpc.LOCAL_TRANSPORT_IDENTITY {
+			fromRemote = &Remote{
+				Name:      rpc.LOCAL_TRANSPORT_IDENTITY,
+				Transport: LocalTransport{},
+			}
+		} else {
+			if fromRemote, err = rl(e.From); err != nil {
+				return
+			}
+		}
+
+		pull := &Pull{
+			From: fromRemote,
+		}
+		if pull.JobName, err = fullJobName(JobSectionPull, name); err != nil {
+			return
+		}
+		if pull.Mapping, err = parseDatasetMapFilter(e.Mapping, false); err != nil {
+			return
+		}
+		if pull.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, rpc.DEFAULT_INITIAL_REPL_POLICY); err != nil {
+			return
+		}
+		if pull.RepeatStrategy, err = parseRepeatStrategy(e.Repeat); err != nil {
+			return
+		}
+
+		p[name] = pull
+	}
+
+	return
 }
 
-type AuthenticatedChannelListener interface {
+func parseInitialReplPolicy(v interface{}, defaultPolicy rpc.InitialReplPolicy) (p rpc.InitialReplPolicy, err error) {
-	Accept() (ch io.ReadWriteCloser, err error)
+	s, ok := v.(string)
-	Close() (err error)
+	if !ok {
-}
+		goto err
+	}
 
-type SSHStdinServerConnectDescr struct {
+	switch {
-}
+	case s == "":
-
+		p = defaultPolicy
-type PrunePolicy interface {
+	case s == "most_recent":
-	Prune(fs *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error)
+		p = rpc.InitialReplPolicyMostRecent
-}
+	case s == "all":
-
+		p = rpc.InitialReplPolicyAll
-type PruningJob interface {
-	Pruner(side PrunePolicySide, dryRun bool) (Pruner, error)
-}
-
-// A type for constants describing different prune policies of a PruningJob
-// This is mostly a special-case for LocalJob, which is the only job that has two prune policies
-// instead of one.
-// It implements github.com/spf13/pflag.Value to be used as CLI flag for the test subcommand
-type PrunePolicySide string
-
-const (
-	PrunePolicySideDefault PrunePolicySide = ""
-	PrunePolicySideLeft    PrunePolicySide = "left"
-	PrunePolicySideRight   PrunePolicySide = "right"
-)
-
-func (s *PrunePolicySide) String() string {
-	return string(*s)
-}
-
-func (s *PrunePolicySide) Set(news string) error {
-	p := PrunePolicySide(news)
-	switch p {
-	case PrunePolicySideRight:
-		fallthrough
-	case PrunePolicySideLeft:
-		*s = p
 	default:
-		return errors.Errorf("must be either %s or %s", PrunePolicySideLeft, PrunePolicySideRight)
+		goto err
 	}
-	return nil
+
+	return
+
+err:
+	err = errors.New(fmt.Sprintf("expected InitialReplPolicy, got %#v", v))
+	return
 }
 
-func (s *PrunePolicySide) Type() string {
+func parseRepeatStrategy(r map[string]string) (s jobrun.RepeatStrategy, err error) {
-	return fmt.Sprintf("%s | %s", PrunePolicySideLeft, PrunePolicySideRight)
+
+	if r == nil {
+		return jobrun.NoRepeatStrategy{}, nil
+	}
+
+	if repeatStr, ok := r["interval"]; ok {
+		d, err := parseDuration(repeatStr)
+		if err != nil {
+			return nil, err
+		}
+		s = &jobrun.PeriodicRepeatStrategy{d}
+		return s, err
+	} else {
+		return nil, fmt.Errorf("attribute 'interval' not found but required in repeat specification")
+	}
+
+}
+
+func expectList(v interface{}) (asList []interface{}, err error) {
+	var ok bool
+	if asList, ok = v.([]interface{}); !ok {
+		err = errors.New("expected list")
+	}
+	return
+}
+
+func parseSinks(v interface{}) (m map[string]DatasetMapFilter, err error) {
+	var asMap map[string]map[string]interface{}
+	if err = mapstructure.Decode(v, &asMap); err != nil {
+		return
+	}
+
+	m = make(map[string]DatasetMapFilter, len(asMap))
+
+	for identity, entry := range asMap {
+		parseSink := func() (mapping DatasetMapFilter, err error) {
+			mappingMap, ok := entry["mapping"]
+			if !ok {
+				err = fmt.Errorf("no mapping specified")
+				return
+			}
+			mapping, err = parseDatasetMapFilter(mappingMap, false)
+			return
+		}
+		mapping, sinkErr := parseSink()
+		if sinkErr != nil {
+			err = fmt.Errorf("cannot parse sink for identity '%s': %s", identity, sinkErr)
+			return
+		}
+		m[identity] = mapping
+	}
+	return
+}
+
+func parsePullACLs(v interface{}) (m map[string]DatasetMapFilter, err error) {
+	var asMap map[string]map[string]interface{}
+	if err = mapstructure.Decode(v, &asMap); err != nil {
+		return
+	}
+
+	m = make(map[string]DatasetMapFilter, len(asMap))
+
+	for identity, entry := range asMap {
+		parsePullACL := func() (filter DatasetMapFilter, err error) {
+			filterMap, ok := entry["filter"]
+			if !ok {
+				err = fmt.Errorf("no filter specified")
+				return
+			}
+			filter, err = parseDatasetMapFilter(filterMap, true)
+			return
+		}
+		filter, filterErr := parsePullACL()
+		if filterErr != nil {
+			err = fmt.Errorf("cannot parse pull-ACL for identity '%s': %s", identity, filterErr)
+			return
+		}
+		m[identity] = filter
+	}
+	return
+}
+
+func parseDatasetMapFilter(mi interface{}, filterOnly bool) (f DatasetMapFilter, err error) {
+
+	var m map[string]string
+	if err = mapstructure.Decode(mi, &m); err != nil {
+		err = fmt.Errorf("maps / filters must be specified as map[string]string: %s", err)
+		return
+	}
+
+	f = NewDatasetMapFilter(len(m), filterOnly)
+	for pathPattern, mapping := range m {
+		if err = f.Add(pathPattern, mapping); err != nil {
+			err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err)
+			return
+		}
+	}
+	return
+}
+
+func (t SSHTransport) Connect(rpcLog Logger) (r rpc.RPCRequester, err error) {
+	var stream io.ReadWriteCloser
+	var rpcTransport sshbytestream.SSHTransport
+	if err = copier.Copy(&rpcTransport, t); err != nil {
+		return
+	}
+	if stream, err = sshbytestream.Outgoing(rpcTransport); err != nil {
+		return
+	}
+	stream, err = NewReadWriteCloserLogger(stream, t.ConnLogReadFile, t.ConnLogWriteFile)
+	if err != nil {
+		return
+	}
+	return rpc.ConnectByteStreamRPC(stream, rpcLog)
+}
+
+func (t LocalTransport) Connect(rpcLog Logger) (r rpc.RPCRequester, err error) {
+	if t.Handler == nil {
+		panic("local transport with uninitialized handler")
+	}
+	return rpc.ConnectLocalRPC(t.Handler), nil
+}
+
+func (t *LocalTransport) SetHandler(handler rpc.RPCHandler) {
+	t.Handler = handler
+}
+
+func parsePrunes(m interface{}) (rets map[string]*Prune, err error) {
+
+	asList := make(map[string]map[string]interface{}, 0)
+	if err = mapstructure.Decode(m, &asList); err != nil {
+		return
+	}
+
+	rets = make(map[string]*Prune, len(asList))
+
+	for name, e := range asList {
+		var prune *Prune
+		if prune, err = parsePrune(e, name); err != nil {
+			err = fmt.Errorf("cannot parse prune job %s: %s", name, err)
+			return
+		}
+		rets[name] = prune
+	}
+
+	return
+}
+
+func parsePrune(e map[string]interface{}, name string) (prune *Prune, err error) {
+
+	// Only support grid policy for now
+	policyName, ok := e["policy"]
+	if !ok || policyName != "grid" {
+		err = fmt.Errorf("prune job with unimplemented policy '%s'", policyName)
+		return
+	}
+
+	var i struct {
+		Grid           string
+		DatasetFilter  map[string]string `mapstructure:"dataset_filter"`
+		SnapshotFilter map[string]string `mapstructure:"snapshot_filter"`
+	}
+
+	if err = mapstructure.Decode(e, &i); err != nil {
+		return
+	}
+
+	prune = &Prune{}
+
+	if prune.JobName, err = fullJobName(JobSectionPrune, name); err != nil {
+		return
+	}
+
+	// Parse grid policy
+	intervals, err := parseRetentionGridIntervalsString(i.Grid)
+	if err != nil {
+		err = fmt.Errorf("cannot parse retention grid: %s", err)
+		return
+	}
+	// Assert intervals are of increasing length (not necessarily required, but indicates config mistake)
+	lastDuration := time.Duration(0)
+	for i := range intervals {
+		if intervals[i].Length < lastDuration {
+			err = fmt.Errorf("retention grid interval length must be monotonically increasing:"+
+				"interval %d is shorter than %d", i+1, i)
+			return
+		} else {
+			lastDuration = intervals[i].Length
+		}
+	}
+	prune.RetentionPolicy = NewRetentionGrid(intervals)
+
+	// Parse filters
+	if prune.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil {
+		err = fmt.Errorf("cannot parse dataset filter: %s", err)
+		return
+	}
+	if prune.SnapshotFilter, err = parseSnapshotFilter(i.SnapshotFilter); err != nil {
+		err = fmt.Errorf("cannot parse snapshot filter: %s", err)
+		return
+	}
+
+	return
+}
+
+var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
+
+func parseDuration(e string) (d time.Duration, err error) {
+	comps := durationStringRegex.FindStringSubmatch(e)
+	if len(comps) != 3 {
+		err = fmt.Errorf("does not match regex: %s %#v", e, comps)
+		return
+	}
+
+	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
+	if err != nil {
+		return
+	}
+
+	var durationUnit time.Duration
+	switch comps[2] {
+	case "s":
+		durationUnit = time.Second
+	case "m":
+		durationUnit = time.Minute
+	case "h":
+		durationUnit = time.Hour
+	case "d":
+		durationUnit = 24 * time.Hour
+	case "w":
+		durationUnit = 24 * 7 * time.Hour
+	default:
+		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
+		return
+	}
+
+	d = time.Duration(durationFactor) * durationUnit
+	return
+
+}
+
+var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
+
+func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {
+
+	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
+	if comps == nil {
+		err = fmt.Errorf("retention string does not match expected format")
+		return
+	}
+
+	times, err := strconv.Atoi(comps[1])
+	if err != nil {
+		return nil, err
+	} else if times <= 0 {
+		return nil, fmt.Errorf("contains factor <= 0")
+	}
+
+	duration, err := parseDuration(comps[2])
+	if err != nil {
+		return nil, err
+	}
+
+	keepCount := 1
+	if comps[3] != "" {
+		// Decompose key=value, comma separated
+		// For now, only keep_count is supported
+		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
+		res := re.FindStringSubmatch(comps[4])
+		if res == nil || len(res) != 2 {
+			err = fmt.Errorf("interval parameter contains unknown parameters")
+			return
+		}
+		if res[1] == "all" {
+			keepCount = RetentionGridKeepCountAll
+		} else {
+			keepCount, err = strconv.Atoi(res[1])
+			if err != nil {
+				err = fmt.Errorf("cannot parse keep_count value")
+				return
+			}
+		}
+	}
+
+	intervals = make([]RetentionInterval, times)
+	for i := range intervals {
+		intervals[i] = RetentionInterval{
+			Length:    duration,
+			KeepCount: keepCount,
+		}
+	}
+
+	return
+
+}
+
+func parseRetentionGridIntervalsString(s string) (intervals []RetentionInterval, err error) {
+
+	ges := strings.Split(s, "|")
+	intervals = make([]RetentionInterval, 0, 7*len(ges))
+
+	for intervalIdx, e := range ges {
+		parsed, err := parseRetentionGridIntervalString(e)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
+		}
+		intervals = append(intervals, parsed...)
+	}
+
+	return
+}
+
+type prefixSnapshotFilter struct {
+	prefix string
+}
+
+func (f prefixSnapshotFilter) Filter(fsv zfs.FilesystemVersion) (accept bool, err error) {
+	return fsv.Type == zfs.Snapshot && strings.HasPrefix(fsv.Name, f.prefix), nil
+}
+
+func parseSnapshotFilter(fm map[string]string) (snapFilter zfs.FilesystemVersionFilter, err error) {
+	prefix, ok := fm["prefix"]
+	if !ok {
+		err = fmt.Errorf("unsupported snapshot filter")
+		return
+	}
+	snapFilter = prefixSnapshotFilter{prefix}
+	return
+}
+
+func parseAutosnaps(m interface{}) (snaps map[string]*Autosnap, err error) {
+
+	asMap := make(map[string]interface{}, 0)
+	if err = mapstructure.Decode(m, &asMap); err != nil {
+		return
+	}
+
+	snaps = make(map[string]*Autosnap, len(asMap))
+
+	for name, e := range asMap {
+		var snap *Autosnap
+		if snap, err = parseAutosnap(e, name); err != nil {
+			err = fmt.Errorf("cannot parse autonsap job %s: %s", name, err)
+			return
+		}
+		snaps[name] = snap
+	}
+
+	return
+
+}
+
+func parseAutosnap(m interface{}, name string) (a *Autosnap, err error) {
+
+	var i struct {
+		Prefix        string
+		Interval      string
+		DatasetFilter map[string]string `mapstructure:"dataset_filter"`
+	}
+
+	if err = mapstructure.Decode(m, &i); err != nil {
+		err = fmt.Errorf("structure unfit: %s", err)
+		return
+	}
+
+	a = &Autosnap{}
+
+	if a.JobName, err = fullJobName(JobSectionAutosnap, name); err != nil {
+		return
+	}
+
+	if len(i.Prefix) < 1 {
+		err = fmt.Errorf("prefix must not be empty")
+		return
+	}
+	a.Prefix = i.Prefix
+
+	var interval time.Duration
+	if interval, err = parseDuration(i.Interval); err != nil {
+		err = fmt.Errorf("cannot parse interval: %s", err)
+		return
+	}
+	a.Interval = &jobrun.PeriodicRepeatStrategy{interval}
+
+	if len(i.DatasetFilter) == 0 {
+		err = fmt.Errorf("dataset_filter not specified")
+		return
+	}
+	if a.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil {
+		err = fmt.Errorf("cannot parse dataset filter: %s", err)
+	}
+
+	return
 }

cmd/config_connect.go

@@ -1,46 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"io"
-
-	"github.com/jinzhu/copier"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/sshbytestream"
-)
-
-type SSHStdinserverConnecter struct {
-	Host                 string
-	User                 string
-	Port                 uint16
-	IdentityFile         string   `mapstructure:"identity_file"`
-	TransportOpenCommand []string `mapstructure:"transport_open_command"`
-	SSHCommand           string   `mapstructure:"ssh_command"`
-	Options              []string
-}
-
-func parseSSHStdinserverConnecter(i map[string]interface{}) (c *SSHStdinserverConnecter, err error) {
-
-	c = &SSHStdinserverConnecter{}
-	if err = mapstructure.Decode(i, c); err != nil {
-		err = errors.New(fmt.Sprintf("could not parse ssh transport: %s", err))
-		return nil, err
-	}
-
-	// TODO assert fields are filled
-	return
-
-}
-
-func (c *SSHStdinserverConnecter) Connect() (rwc io.ReadWriteCloser, err error) {
-	var rpcTransport sshbytestream.SSHTransport
-	if err = copier.Copy(&rpcTransport, c); err != nil {
-		return
-	}
-	if rwc, err = sshbytestream.Outgoing(rpcTransport); err != nil {
-		err = errors.WithStack(err)
-		return
-	}
-	return
-}

cmd/config_fsvfilter.go

@@ -1,33 +0,0 @@
-package cmd
-
-import (
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/zfs"
-	"strings"
-)
-
-type PrefixSnapshotFilter struct {
-	Prefix string
-}
-
-func parseSnapshotPrefix(i string) (p string, err error) {
-	if len(i) <= 0 {
-		err = errors.Errorf("snapshot prefix must not be empty string")
-		return
-	}
-	p = i
-	return
-}
-
-func parsePrefixSnapshotFilter(i string) (f *PrefixSnapshotFilter, err error) {
-	if !(len(i) > 0) {
-		err = errors.Errorf("snapshot prefix must be longer than 0 characters")
-		return
-	}
-	f = &PrefixSnapshotFilter{i}
-	return
-}
-
-func (f *PrefixSnapshotFilter) Filter(fsv zfs.FilesystemVersion) (accept bool, err error) {
-	return fsv.Type == zfs.Snapshot && strings.HasPrefix(fsv.Name, f.Prefix), nil
-}

cmd/config_job_control.go

@@ -1,86 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"github.com/pkg/errors"
-	"net"
-	"net/http"
-	"net/http/pprof"
-)
-
-type ControlJob struct {
-	Name     string
-	sockaddr *net.UnixAddr
-}
-
-func NewControlJob(name, sockpath string) (j *ControlJob, err error) {
-	j = &ControlJob{Name: name}
-
-	j.sockaddr, err = net.ResolveUnixAddr("unix", sockpath)
-	if err != nil {
-		err = errors.Wrap(err, "cannot resolve unix address")
-		return
-	}
-
-	return
-}
-
-func (j *ControlJob) JobName() string {
-	return j.Name
-}
-
-const (
-	ControlJobEndpointProfile string = "/debug/pprof/profile"
-)
-
-func (j *ControlJob) JobStart(ctx context.Context) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-	defer log.Info("control job finished")
-
-	l, err := ListenUnixPrivate(j.sockaddr)
-	if err != nil {
-		log.WithError(err).Error("error listening")
-		return
-	}
-
-	mux := http.NewServeMux()
-	mux.Handle(ControlJobEndpointProfile, requestLogger{log, pprof.Profile})
-	server := http.Server{Handler: mux}
-
-outer:
-	for {
-
-		served := make(chan error)
-		go func() {
-			served <- server.Serve(l)
-			close(served)
-		}()
-
-		select {
-		case <-ctx.Done():
-			log.WithError(err).Info("contex done")
-			server.Shutdown(context.Background())
-			break outer
-		case err = <-served:
-			if err != nil {
-				log.WithError(err).Error("error serving")
-				break outer
-			}
-		}
-
-	}
-
-}
-
-type requestLogger struct {
-	log         Logger
-	handlerFunc func(w http.ResponseWriter, r *http.Request)
-}
-
-func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	log := l.log.WithField("method", r.Method).WithField("url", r.URL)
-	log.Info("start")
-	l.handlerFunc(w, r)
-	log.Info("finish")
-}

cmd/config_job_local.go

@@ -1,205 +0,0 @@
-package cmd
-
-import (
-	"time"
-
-	"context"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/zfs"
-	"sync"
-)
-
-type LocalJob struct {
-	Name              string
-	Mapping           *DatasetMapFilter
-	SnapshotPrefix    string
-	Interval          time.Duration
-	InitialReplPolicy InitialReplPolicy
-	PruneLHS          PrunePolicy
-	PruneRHS          PrunePolicy
-	Debug             JobDebugSettings
-}
-
-func parseLocalJob(c JobParsingContext, name string, i map[string]interface{}) (j *LocalJob, err error) {
-
-	var asMap struct {
-		Mapping           map[string]string
-		SnapshotPrefix    string `mapstructure:"snapshot_prefix"`
-		Interval          string
-		InitialReplPolicy string                 `mapstructure:"initial_repl_policy"`
-		PruneLHS          map[string]interface{} `mapstructure:"prune_lhs"`
-		PruneRHS          map[string]interface{} `mapstructure:"prune_rhs"`
-		Debug             map[string]interface{}
-	}
-
-	if err = mapstructure.Decode(i, &asMap); err != nil {
-		err = errors.Wrap(err, "mapstructure error")
-		return nil, err
-	}
-
-	j = &LocalJob{Name: name}
-
-	if j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false); err != nil {
-		return
-	}
-
-	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
-		return
-	}
-
-	if j.Interval, err = time.ParseDuration(asMap.Interval); err != nil {
-		err = errors.Wrap(err, "cannot parse interval")
-		return
-	}
-
-	if j.InitialReplPolicy, err = parseInitialReplPolicy(asMap.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY); err != nil {
-		return
-	}
-
-	if j.PruneLHS, err = parsePrunePolicy(asMap.PruneLHS); err != nil {
-		err = errors.Wrap(err, "cannot parse 'prune_lhs'")
-		return
-	}
-	if j.PruneRHS, err = parsePrunePolicy(asMap.PruneRHS); err != nil {
-		err = errors.Wrap(err, "cannot parse 'prune_rhs'")
-		return
-	}
-
-	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
-		err = errors.Wrap(err, "cannot parse 'debug'")
-		return
-	}
-
-	return
-}
-
-func (j *LocalJob) JobName() string {
-	return j.Name
-}
-
-func (j *LocalJob) JobStart(ctx context.Context) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-	defer log.Info("exiting")
-
-	local := rpc.NewLocalRPC()
-	// Allow access to any dataset since we control what mapping
-	// is passed to the pull routine.
-	// All local datasets will be passed to its Map() function,
-	// but only those for which a mapping exists will actually be pulled.
-	// We can pay this small performance penalty for now.
-	handler := NewHandler(log.WithField(logTaskField, "handler"), localPullACL{}, &PrefixSnapshotFilter{j.SnapshotPrefix})
-
-	registerEndpoints(local, handler)
-
-	snapper := IntervalAutosnap{
-		DatasetFilter:    j.Mapping.AsFilter(),
-		Prefix:           j.SnapshotPrefix,
-		SnapshotInterval: j.Interval,
-	}
-
-	plhs, err := j.Pruner(PrunePolicySideLeft, false)
-	if err != nil {
-		log.WithError(err).Error("error creating lhs pruner")
-		return
-	}
-	prhs, err := j.Pruner(PrunePolicySideRight, false)
-	if err != nil {
-		log.WithError(err).Error("error creating rhs pruner")
-		return
-	}
-
-	makeCtx := func(parent context.Context, taskName string) (ctx context.Context) {
-		return context.WithValue(parent, contextKeyLog, log.WithField(logTaskField, taskName))
-	}
-	var snapCtx, plCtx, prCtx, pullCtx context.Context
-	snapCtx = makeCtx(ctx, "autosnap")
-	plCtx = makeCtx(ctx, "prune_lhs")
-	prCtx = makeCtx(ctx, "prune_rhs")
-	pullCtx = makeCtx(ctx, "repl")
-
-	didSnaps := make(chan struct{})
-	go snapper.Run(snapCtx, didSnaps)
-
-outer:
-	for {
-
-		select {
-		case <-ctx.Done():
-			break outer
-		case <-didSnaps:
-			log.Debug("finished taking snapshots")
-			log.Info("starting replication procedure")
-		}
-
-		{
-			log := pullCtx.Value(contextKeyLog).(Logger)
-			log.Debug("replicating from lhs to rhs")
-			err := doPull(PullContext{local, log, j.Mapping, j.InitialReplPolicy})
-			if err != nil {
-				log.WithError(err).Error("error replicating lhs to rhs")
-			}
-			// use a ctx as soon as doPull gains ctx support
-			select {
-			case <-ctx.Done():
-				break outer
-			default:
-			}
-		}
-
-		var wg sync.WaitGroup
-
-		log.Info("pruning lhs")
-		wg.Add(1)
-		go func() {
-			plhs.Run(plCtx)
-			wg.Done()
-		}()
-
-		log.Info("pruning rhs")
-		wg.Add(1)
-		go func() {
-			prhs.Run(prCtx)
-			wg.Done()
-		}()
-
-		wg.Wait()
-
-	}
-
-	log.WithError(ctx.Err()).Info("context")
-
-}
-
-func (j *LocalJob) Pruner(side PrunePolicySide, dryRun bool) (p Pruner, err error) {
-
-	var dsfilter zfs.DatasetFilter
-	var pp PrunePolicy
-	switch side {
-	case PrunePolicySideLeft:
-		pp = j.PruneLHS
-		dsfilter = j.Mapping.AsFilter()
-	case PrunePolicySideRight:
-		pp = j.PruneRHS
-		dsfilter, err = j.Mapping.InvertedFilter()
-		if err != nil {
-			err = errors.Wrap(err, "cannot invert mapping for prune_rhs")
-			return
-		}
-	default:
-		err = errors.Errorf("must be either left or right side")
-		return
-	}
-
-	p = Pruner{
-		time.Now(),
-		dryRun,
-		dsfilter,
-		j.SnapshotPrefix,
-		pp,
-	}
-
-	return
-}

cmd/config_job_pull.go

@@ -1,165 +0,0 @@
-package cmd
-
-import (
-	"time"
-
-	"context"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/util"
-)
-
-type PullJob struct {
-	Name     string
-	Connect  RWCConnecter
-	Interval time.Duration
-	Mapping  *DatasetMapFilter
-	// constructed from mapping during parsing
-	pruneFilter       *DatasetMapFilter
-	SnapshotPrefix    string
-	InitialReplPolicy InitialReplPolicy
-	Prune             PrunePolicy
-	Debug             JobDebugSettings
-}
-
-func parsePullJob(c JobParsingContext, name string, i map[string]interface{}) (j *PullJob, err error) {
-
-	var asMap struct {
-		Connect           map[string]interface{}
-		Interval          string
-		Mapping           map[string]string
-		InitialReplPolicy string `mapstructure:"initial_repl_policy"`
-		Prune             map[string]interface{}
-		SnapshotPrefix    string `mapstructure:"snapshot_prefix"`
-		Debug             map[string]interface{}
-	}
-
-	if err = mapstructure.Decode(i, &asMap); err != nil {
-		err = errors.Wrap(err, "mapstructure error")
-		return nil, err
-	}
-
-	j = &PullJob{Name: name}
-
-	j.Connect, err = parseSSHStdinserverConnecter(asMap.Connect)
-	if err != nil {
-		err = errors.Wrap(err, "cannot parse 'connect'")
-		return nil, err
-	}
-
-	if j.Interval, err = time.ParseDuration(asMap.Interval); err != nil {
-		err = errors.Wrap(err, "cannot parse 'interval'")
-		return nil, err
-	}
-
-	j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false)
-	if err != nil {
-		err = errors.Wrap(err, "cannot parse 'mapping'")
-		return nil, err
-	}
-
-	if j.pruneFilter, err = j.Mapping.InvertedFilter(); err != nil {
-		err = errors.Wrap(err, "cannot automatically invert 'mapping' for prune job")
-		return nil, err
-	}
-
-	j.InitialReplPolicy, err = parseInitialReplPolicy(asMap.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY)
-	if err != nil {
-		err = errors.Wrap(err, "cannot parse 'initial_repl_policy'")
-		return
-	}
-
-	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
-		return
-	}
-
-	if j.Prune, err = parsePrunePolicy(asMap.Prune); err != nil {
-		err = errors.Wrap(err, "cannot parse prune policy")
-		return
-	}
-
-	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
-		err = errors.Wrap(err, "cannot parse 'debug'")
-		return
-	}
-
-	return
-}
-
-func (j *PullJob) JobName() string {
-	return j.Name
-}
-
-func (j *PullJob) JobStart(ctx context.Context) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-	defer log.Info("exiting")
-
-	ticker := time.NewTicker(j.Interval)
-
-start:
-
-	log.Info("connecting")
-	rwc, err := j.Connect.Connect()
-	if err != nil {
-		log.WithError(err).Error("error connecting")
-		return
-	}
-
-	rwc, err = util.NewReadWriteCloserLogger(rwc, j.Debug.Conn.ReadDump, j.Debug.Conn.WriteDump)
-	if err != nil {
-		return
-	}
-
-	client := rpc.NewClient(rwc)
-	if j.Debug.RPC.Log {
-		client.SetLogger(log, true)
-	}
-
-	log.Info("starting pull")
-
-	pullLog := log.WithField(logTaskField, "pull")
-	err = doPull(PullContext{client, pullLog, j.Mapping, j.InitialReplPolicy})
-	if err != nil {
-		log.WithError(err).Error("error doing pull")
-	}
-
-	closeRPCWithTimeout(log, client, time.Second*10, "")
-
-	log.Info("starting prune")
-	prunectx := context.WithValue(ctx, contextKeyLog, log.WithField(logTaskField, "prune"))
-	pruner, err := j.Pruner(PrunePolicySideDefault, false)
-	if err != nil {
-		log.WithError(err).Error("error creating pruner")
-		return
-	}
-
-	pruner.Run(prunectx)
-	log.Info("finish prune")
-
-	log.Info("wait for next interval")
-	select {
-	case <-ctx.Done():
-		log.WithError(ctx.Err()).Info("context")
-		return
-	case <-ticker.C:
-		goto start
-	}
-
-}
-
-func (j *PullJob) Pruner(side PrunePolicySide, dryRun bool) (p Pruner, err error) {
-	p = Pruner{
-		time.Now(),
-		dryRun,
-		j.pruneFilter,
-		j.SnapshotPrefix,
-		j.Prune,
-	}
-	return
-}
-
-func (j *PullJob) doRun(ctx context.Context) {
-
-}

cmd/config_job_source.go

@@ -1,191 +0,0 @@
-package cmd
-
-import (
-	"context"
-	mapstructure "github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/util"
-	"io"
-	"time"
-)
-
-type SourceJob struct {
-	Name           string
-	Serve          AuthenticatedChannelListenerFactory
-	Datasets       *DatasetMapFilter
-	SnapshotPrefix string
-	Interval       time.Duration
-	Prune          PrunePolicy
-	Debug          JobDebugSettings
-}
-
-func parseSourceJob(c JobParsingContext, name string, i map[string]interface{}) (j *SourceJob, err error) {
-
-	var asMap struct {
-		Serve          map[string]interface{}
-		Datasets       map[string]string
-		SnapshotPrefix string `mapstructure:"snapshot_prefix"`
-		Interval       string
-		Prune          map[string]interface{}
-		Debug          map[string]interface{}
-	}
-
-	if err = mapstructure.Decode(i, &asMap); err != nil {
-		err = errors.Wrap(err, "mapstructure error")
-		return nil, err
-	}
-
-	j = &SourceJob{Name: name}
-
-	if j.Serve, err = parseAuthenticatedChannelListenerFactory(c, asMap.Serve); err != nil {
-		return
-	}
-
-	if j.Datasets, err = parseDatasetMapFilter(asMap.Datasets, true); err != nil {
-		return
-	}
-
-	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
-		return
-	}
-
-	if j.Interval, err = time.ParseDuration(asMap.Interval); err != nil {
-		err = errors.Wrap(err, "cannot parse 'interval'")
-		return
-	}
-
-	if j.Prune, err = parsePrunePolicy(asMap.Prune); err != nil {
-		err = errors.Wrap(err, "cannot parse 'prune'")
-		return
-	}
-
-	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
-		err = errors.Wrap(err, "cannot parse 'debug'")
-		return
-	}
-
-	return
-}
-
-func (j *SourceJob) JobName() string {
-	return j.Name
-}
-
-func (j *SourceJob) JobStart(ctx context.Context) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-	defer log.Info("exiting")
-
-	a := IntervalAutosnap{DatasetFilter: j.Datasets, Prefix: j.SnapshotPrefix, SnapshotInterval: j.Interval}
-	p, err := j.Pruner(PrunePolicySideDefault, false)
-	if err != nil {
-		log.WithError(err).Error("error creating pruner")
-		return
-	}
-
-	snapContext := context.WithValue(ctx, contextKeyLog, log.WithField(logTaskField, "autosnap"))
-	prunerContext := context.WithValue(ctx, contextKeyLog, log.WithField(logTaskField, "prune"))
-	serveContext := context.WithValue(ctx, contextKeyLog, log.WithField(logTaskField, "serve"))
-	didSnaps := make(chan struct{})
-
-	go j.serve(serveContext)
-	go a.Run(snapContext, didSnaps)
-
-outer:
-	for {
-		select {
-		case <-ctx.Done():
-			break outer
-		case <-didSnaps:
-			log.Info("starting pruner")
-			p.Run(prunerContext)
-			log.Info("pruner done")
-		}
-	}
-	log.WithError(prunerContext.Err()).Info("context")
-
-}
-
-func (j *SourceJob) Pruner(side PrunePolicySide, dryRun bool) (p Pruner, err error) {
-	p = Pruner{
-		time.Now(),
-		dryRun,
-		j.Datasets,
-		j.SnapshotPrefix,
-		j.Prune,
-	}
-	return
-}
-
-func (j *SourceJob) serve(ctx context.Context) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-
-	listener, err := j.Serve.Listen()
-	if err != nil {
-		log.WithError(err).Error("error listening")
-		return
-	}
-
-	rwcChan := make(chan io.ReadWriteCloser)
-
-	// Serve connections until interrupted or error
-outer:
-	for {
-
-		go func() {
-			rwc, err := listener.Accept()
-			if err != nil {
-				log.WithError(err).Error("error accepting connection")
-				close(rwcChan)
-				return
-			}
-			rwcChan <- rwc
-		}()
-
-		select {
-
-		case rwc, notClosed := <-rwcChan:
-
-			if !notClosed {
-				break outer // closed because of accept error
-			}
-
-			rwc, err := util.NewReadWriteCloserLogger(rwc, j.Debug.Conn.ReadDump, j.Debug.Conn.WriteDump)
-			if err != nil {
-				panic(err)
-			}
-
-			// construct connection handler
-			handler := NewHandler(log, j.Datasets, &PrefixSnapshotFilter{j.SnapshotPrefix})
-
-			// handle connection
-			rpcServer := rpc.NewServer(rwc)
-			if j.Debug.RPC.Log {
-				rpclog := log.WithField("subsystem", "rpc")
-				rpcServer.SetLogger(rpclog, true)
-			}
-			registerEndpoints(rpcServer, handler)
-			if err = rpcServer.Serve(); err != nil {
-				log.WithError(err).Error("error serving connection")
-			}
-			rwc.Close()
-
-		case <-ctx.Done():
-			log.WithError(ctx.Err()).Info("context")
-			break outer
-
-		}
-
-	}
-
-	log.Info("closing listener")
-	err = listener.Close()
-	if err != nil {
-		log.WithError(err).Error("error closing listener")
-	}
-
-	return
-
-}

cmd/config_logging.go

@@ -1,191 +0,0 @@
-package cmd
-
-import (
-	"crypto/tls"
-	"crypto/x509"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/logger"
-	"io/ioutil"
-	"os"
-	"time"
-)
-
-type LoggingConfig struct {
-	Outlets logger.Outlets
-}
-
-type SetNoMetadataFormatter interface {
-	SetNoMetadata(noMetadata bool)
-}
-
-func parseLogging(i interface{}) (c *LoggingConfig, err error) {
-
-	c = &LoggingConfig{}
-	if i == nil {
-		return c, nil
-	}
-
-	var asMap struct {
-		Stdout struct {
-			Level  string
-			Format string
-		}
-		TCP struct {
-			Level         string
-			Format        string
-			Net           string
-			Address       string
-			RetryInterval string `mapstructure:"retry_interval"`
-			TLS           *struct {
-				CA   string
-				Cert string
-				Key  string
-			}
-		}
-		Syslog struct {
-			Enable        bool
-			Format        string
-			RetryInterval string `mapstructure:"retry_interval"`
-		}
-	}
-	if err = mapstructure.Decode(i, &asMap); err != nil {
-		return nil, errors.Wrap(err, "mapstructure error")
-	}
-
-	c.Outlets = logger.NewOutlets()
-
-	if asMap.Stdout.Level != "" {
-
-		out := WriterOutlet{
-			&HumanFormatter{},
-			os.Stdout,
-		}
-
-		level, err := logger.ParseLevel(asMap.Stdout.Level)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot parse 'level'")
-		}
-
-		if asMap.Stdout.Format != "" {
-			out.Formatter, err = parseLogFormat(asMap.Stdout.Format)
-			if err != nil {
-				return nil, errors.Wrap(err, "cannot parse 'format'")
-			}
-		}
-
-		c.Outlets.Add(out, level)
-
-	}
-
-	if asMap.TCP.Address != "" {
-
-		out := &TCPOutlet{}
-
-		out.Formatter, err = parseLogFormat(asMap.TCP.Format)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot parse 'format'")
-		}
-
-		lvl, err := logger.ParseLevel(asMap.TCP.Level)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot parse 'level'")
-		}
-
-		out.RetryInterval, err = time.ParseDuration(asMap.TCP.RetryInterval)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
-		}
-
-		out.Net, out.Address = asMap.TCP.Net, asMap.TCP.Address
-
-		if asMap.TCP.TLS != nil {
-
-			cert, err := tls.LoadX509KeyPair(asMap.TCP.TLS.Cert, asMap.TCP.TLS.Key)
-			if err != nil {
-				return nil, errors.Wrap(err, "cannot load client cert")
-			}
-
-			var rootCAs *x509.CertPool
-			if asMap.TCP.TLS.CA == "" {
-				if rootCAs, err = x509.SystemCertPool(); err != nil {
-					return nil, errors.Wrap(err, "cannot open system cert pool")
-				}
-			} else {
-				rootCAs = x509.NewCertPool()
-				rootCAPEM, err := ioutil.ReadFile(asMap.TCP.TLS.CA)
-				if err != nil {
-					return nil, errors.Wrap(err, "cannot load CA cert")
-				}
-				if !rootCAs.AppendCertsFromPEM(rootCAPEM) {
-					return nil, errors.New("cannot parse CA cert")
-				}
-			}
-			if err != nil && asMap.TCP.TLS.CA == "" {
-				return nil, errors.Wrap(err, "cannot load root ca pool")
-			}
-
-			out.TLS = &tls.Config{
-				Certificates: []tls.Certificate{cert},
-				RootCAs:      rootCAs,
-			}
-
-			out.TLS.BuildNameToCertificate()
-		}
-
-		c.Outlets.Add(out, lvl)
-	}
-
-	if asMap.Syslog.Enable {
-
-		out := &SyslogOutlet{}
-
-		out.Formatter = &HumanFormatter{}
-		if asMap.Syslog.Format != "" {
-			out.Formatter, err = parseLogFormat(asMap.Syslog.Format)
-			if err != nil {
-				return nil, errors.Wrap(err, "cannot parse 'format'")
-			}
-		}
-
-		if f, ok := out.Formatter.(SetNoMetadataFormatter); ok {
-			f.SetNoMetadata(true)
-		}
-
-		out.RetryInterval = 0 // default to 0 as we assume local syslog will just work
-		if asMap.Syslog.RetryInterval != "" {
-			out.RetryInterval, err = time.ParseDuration(asMap.Syslog.RetryInterval)
-			if err != nil {
-				return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
-			}
-		}
-
-		c.Outlets.Add(out, logger.Debug)
-
-	}
-
-	return c, nil
-
-}
-
-func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
-	var is string
-	switch j := i.(type) {
-	case string:
-		is = j
-	default:
-		return nil, errors.Errorf("invalid log format: wrong type: %T", i)
-	}
-
-	switch is {
-	case "human":
-		return &HumanFormatter{}, nil
-	case "logfmt":
-		return &LogfmtFormatter{}, nil
-	case "json":
-		return &JSONFormatter{}, nil
-	default:
-		return nil, errors.Errorf("invalid log format: '%s'", is)
-	}
-
-}

cmd/config_mapfilter.go

@@ -1,11 +1,10 @@
 package cmd
 
 import (
+	"errors"
 	"fmt"
 	"strings"
 
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
 	"github.com/zrepl/zrepl/zfs"
 )
 
@@ -14,7 +13,7 @@ type DatasetMapFilter struct {
 
 	// if set, only valid filter entries can be added using Add()
 	// and Map() will always return an error
-	filterMode bool
+	filterOnly bool
 }
 
 type datasetMapFilterEntry struct {
@@ -25,17 +24,18 @@ type datasetMapFilterEntry struct {
 	subtreeMatch bool
 }
 
-func NewDatasetMapFilter(capacity int, filterMode bool) *DatasetMapFilter {
+var NoMatchError error = errors.New("no match found in mapping")
-	return &DatasetMapFilter{
+
-		entries:    make([]datasetMapFilterEntry, 0, capacity),
+func NewDatasetMapFilter(capacity int, filterOnly bool) DatasetMapFilter {
-		filterMode: filterMode,
+	return DatasetMapFilter{
+		entries: make([]datasetMapFilterEntry, 0, capacity),
 	}
 }
 
 func (m *DatasetMapFilter) Add(pathPattern, mapping string) (err error) {
 
-	if m.filterMode {
+	if m.filterOnly {
-		if _, err = m.parseDatasetFilterResult(mapping); err != nil {
+		if _, err = parseDatasetFilterResult(mapping); err != nil {
 			return
 		}
 	}
@@ -103,22 +103,18 @@ func (m DatasetMapFilter) mostSpecificPrefixMapping(path *zfs.DatasetPath) (idx
 
 func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error) {
 
-	if m.filterMode {
+	if m.filterOnly {
 		err = fmt.Errorf("using a filter for mapping simply does not work")
 		return
 	}
 
 	mi, hasMapping := m.mostSpecificPrefixMapping(source)
 	if !hasMapping {
-		return nil, nil
+		err = NoMatchError
+		return
 	}
 	me := m.entries[mi]
 
-	if strings.HasPrefix("!", me.mapping) {
-		// reject mapping
-		return nil, nil
-	}
-
 	target, err = zfs.NewDatasetPath(me.mapping)
 	if err != nil {
 		err = fmt.Errorf("mapping target is not a dataset path: %s", err)
@@ -139,103 +135,28 @@ func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath,
 }
 
 func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
-
-	if !m.filterMode {
-		err = fmt.Errorf("using a mapping as a filter does not work")
-		return
-	}
-
 	mi, hasMapping := m.mostSpecificPrefixMapping(p)
 	if !hasMapping {
 		pass = false
 		return
 	}
 	me := m.entries[mi]
-	pass, err = m.parseDatasetFilterResult(me.mapping)
+	pass, err = parseDatasetFilterResult(me.mapping)
 	return
 }
 
-// Construct a new filter-only DatasetMapFilter from a mapping
-// The new filter allows excactly those paths that were not forbidden by the mapping.
-func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
-
-	if m.filterMode {
-		err = errors.Errorf("can only invert mappings")
-		return
-	}
-
-	inv = &DatasetMapFilter{
-		make([]datasetMapFilterEntry, len(m.entries)),
-		true,
-	}
-
-	for i, e := range m.entries {
-		inv.entries[i].path, err = zfs.NewDatasetPath(e.mapping)
-		if err != nil {
-			err = errors.Wrapf(err, "mapping cannot be inverted: '%s' is not a dataset path: %s", e.mapping)
-			return
-		}
-		inv.entries[i].mapping = MapFilterResultOk
-		inv.entries[i].subtreeMatch = e.subtreeMatch
-	}
-
-	return inv, nil
-}
-
-// Creates a new DatasetMapFilter in filter mode from a mapping
-// All accepting mapping results are mapped to accepting filter results
-// All rejecting mapping results are mapped to rejecting filter results
-func (m DatasetMapFilter) AsFilter() (f *DatasetMapFilter) {
-
-	f = &DatasetMapFilter{
-		make([]datasetMapFilterEntry, len(m.entries)),
-		true,
-	}
-
-	for i, e := range m.entries {
-		var newe datasetMapFilterEntry = e
-		if strings.HasPrefix(newe.mapping, "!") {
-			newe.mapping = MapFilterResultOmit
-		} else {
-			newe.mapping = MapFilterResultOk
-		}
-		f.entries[i] = newe
-	}
-
-	return f
-}
-
-const (
-	MapFilterResultOk   string = "ok"
-	MapFilterResultOmit string = "!"
-)
-
 // Parse a dataset filter result
-func (m DatasetMapFilter) parseDatasetFilterResult(result string) (pass bool, err error) {
+func parseDatasetFilterResult(result string) (pass bool, err error) {
 	l := strings.ToLower(result)
-	if l == MapFilterResultOk {
+	switch strings.ToLower(l) {
-		return true, nil
+	case "ok":
-	}
+		pass = true
-	if l == MapFilterResultOmit {
+		return
-		return false, nil
+	case "omit":
-	}
+		return
-	return false, fmt.Errorf("'%s' is not a valid filter result", result)
+	default:
-}
+		err = fmt.Errorf("'%s' is not a valid filter result", result)
-
-func parseDatasetMapFilter(mi interface{}, filterMode bool) (f *DatasetMapFilter, err error) {
-
-	var m map[string]string
-	if err = mapstructure.Decode(mi, &m); err != nil {
-		err = fmt.Errorf("maps / filters must be specified as map[string]string: %s", err)
 		return
-	}
-
-	f = NewDatasetMapFilter(len(m), filterMode)
-	for pathPattern, mapping := range m {
-		if err = f.Add(pathPattern, mapping); err != nil {
-			err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err)
-			return
-		}
 	}
 	return
 }

cmd/config_parse.go

@@ -1,254 +0,0 @@
-package cmd
-
-import (
-	"io/ioutil"
-
-	"fmt"
-	yaml "github.com/go-yaml/yaml"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"os"
-)
-
-var ConfigFileDefaultLocations []string = []string{
-	"/etc/zrepl/zrepl.yml",
-	"/usr/local/etc/zrepl/zrepl.yml",
-}
-
-const (
-	JobNameControl string = "control"
-)
-
-var ReservedJobNames []string = []string{
-	JobNameControl,
-}
-
-type ConfigParsingContext struct {
-	Global *Global
-}
-
-func ParseConfig(path string) (config *Config, err error) {
-
-	if path == "" {
-		// Try default locations
-		for _, l := range ConfigFileDefaultLocations {
-			stat, err := os.Stat(l)
-			if err != nil {
-				continue
-			}
-			if !stat.Mode().IsRegular() {
-				err = errors.Errorf("file at default location is not a regular file: %s", l)
-				continue
-			}
-			path = l
-			break
-		}
-	}
-
-	var i interface{}
-
-	var bytes []byte
-
-	if bytes, err = ioutil.ReadFile(path); err != nil {
-		err = errors.WithStack(err)
-		return
-	}
-
-	if err = yaml.Unmarshal(bytes, &i); err != nil {
-		err = errors.WithStack(err)
-		return
-	}
-
-	return parseConfig(i)
-}
-
-func parseConfig(i interface{}) (c *Config, err error) {
-
-	var asMap struct {
-		Global map[string]interface{}
-		Jobs   []map[string]interface{}
-	}
-	if err := mapstructure.Decode(i, &asMap); err != nil {
-		return nil, errors.Wrap(err, "config root must be a dict")
-	}
-
-	c = &Config{}
-
-	// Parse global with defaults
-	c.Global.Serve.Stdinserver.SockDir = "/var/run/zrepl/stdinserver"
-	c.Global.Control.Sockpath = "/var/run/zrepl/control"
-
-	err = mapstructure.Decode(asMap.Global, &c.Global)
-	if err != nil {
-		err = errors.Wrap(err, "mapstructure error on 'global' section: %s")
-		return
-	}
-
-	if c.Global.logging, err = parseLogging(asMap.Global["logging"]); err != nil {
-		return nil, errors.Wrap(err, "cannot parse logging section")
-	}
-
-	cpc := ConfigParsingContext{&c.Global}
-	jpc := JobParsingContext{cpc}
-
-	// Jobs
-	c.Jobs = make(map[string]Job, len(asMap.Jobs))
-	for i := range asMap.Jobs {
-		job, err := parseJob(jpc, asMap.Jobs[i])
-		if err != nil {
-			// Try to find its name
-			namei, ok := asMap.Jobs[i]["name"]
-			if !ok {
-				namei = fmt.Sprintf("<no name, entry #%d in list>", i)
-			}
-			err = errors.Wrapf(err, "cannot parse job '%v'", namei)
-			return nil, err
-		}
-		jn := job.JobName()
-		if _, ok := c.Jobs[jn]; ok {
-			err = errors.Errorf("duplicate job name: %s", jn)
-			return nil, err
-		}
-		c.Jobs[job.JobName()] = job
-	}
-
-	cj, err := NewControlJob(JobNameControl, jpc.Global.Control.Sockpath)
-	if err != nil {
-		err = errors.Wrap(err, "cannot create control job")
-		return
-	}
-	c.Jobs[JobNameControl] = cj
-
-	return c, nil
-
-}
-
-func extractStringField(i map[string]interface{}, key string, notempty bool) (field string, err error) {
-	vi, ok := i[key]
-	if !ok {
-		err = errors.Errorf("must have field '%s'", key)
-		return "", err
-	}
-	field, ok = vi.(string)
-	if !ok {
-		err = errors.Errorf("'%s' field must have type string", key)
-		return "", err
-	}
-	if notempty && len(field) <= 0 {
-		err = errors.Errorf("'%s' field must not be empty", key)
-		return "", err
-	}
-	return
-}
-
-type JobParsingContext struct {
-	ConfigParsingContext
-}
-
-func parseJob(c JobParsingContext, i map[string]interface{}) (j Job, err error) {
-
-	name, err := extractStringField(i, "name", true)
-	if err != nil {
-		return
-	}
-
-	for _, r := range ReservedJobNames {
-		if name == r {
-			err = errors.Errorf("job name '%s' is reserved", name)
-			return
-		}
-	}
-
-	jobtype, err := extractStringField(i, "type", true)
-	if err != nil {
-		return
-	}
-
-	switch jobtype {
-	case "pull":
-		return parsePullJob(c, name, i)
-	case "source":
-		return parseSourceJob(c, name, i)
-	case "local":
-		return parseLocalJob(c, name, i)
-	default:
-		return nil, errors.Errorf("unknown job type '%s'", jobtype)
-	}
-
-}
-
-func parseConnect(i map[string]interface{}) (c RWCConnecter, err error) {
-
-	t, err := extractStringField(i, "type", true)
-	if err != nil {
-		return nil, err
-	}
-
-	switch t {
-	case "ssh+stdinserver":
-		return parseSSHStdinserverConnecter(i)
-	default:
-		return nil, errors.Errorf("unknown connection type '%s'", t)
-	}
-
-}
-
-func parseInitialReplPolicy(v interface{}, defaultPolicy InitialReplPolicy) (p InitialReplPolicy, err error) {
-	s, ok := v.(string)
-	if !ok {
-		goto err
-	}
-
-	switch {
-	case s == "":
-		p = defaultPolicy
-	case s == "most_recent":
-		p = InitialReplPolicyMostRecent
-	case s == "all":
-		p = InitialReplPolicyAll
-	default:
-		goto err
-	}
-
-	return
-
-err:
-	err = errors.New(fmt.Sprintf("expected InitialReplPolicy, got %#v", v))
-	return
-}
-
-func parsePrunePolicy(v map[string]interface{}) (p PrunePolicy, err error) {
-
-	policyName, err := extractStringField(v, "policy", true)
-	if err != nil {
-		return
-	}
-
-	switch policyName {
-	case "grid":
-		return parseGridPrunePolicy(v)
-	case "noprune":
-		return NoPrunePolicy{}, nil
-	default:
-		err = errors.Errorf("unknown policy '%s'", policyName)
-		return
-	}
-
-}
-
-func parseAuthenticatedChannelListenerFactory(c JobParsingContext, v map[string]interface{}) (p AuthenticatedChannelListenerFactory, err error) {
-
-	t, err := extractStringField(v, "type", true)
-	if err != nil {
-		return nil, err
-	}
-
-	switch t {
-	case "stdinserver":
-		return parseStdinserverListenerFactory(c, v)
-	default:
-		err = errors.Errorf("unknown type '%s'", t)
-		return
-	}
-
-}

cmd/config_prune_grid.go

@@ -1,201 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/util"
-	"github.com/zrepl/zrepl/zfs"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-	"time"
-)
-
-type GridPrunePolicy struct {
-	RetentionGrid *util.RetentionGrid
-}
-
-type retentionGridAdaptor struct {
-	zfs.FilesystemVersion
-}
-
-func (a retentionGridAdaptor) Date() time.Time {
-	return a.Creation
-}
-
-func (a retentionGridAdaptor) LessThan(b util.RetentionGridEntry) bool {
-	return a.CreateTXG < b.(retentionGridAdaptor).CreateTXG
-}
-
-func (p *GridPrunePolicy) Prune(_ *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error) {
-
-	// Build adaptors for retention grid
-	adaptors := make([]util.RetentionGridEntry, len(versions))
-	for fsv := range versions {
-		adaptors[fsv] = retentionGridAdaptor{versions[fsv]}
-	}
-
-	sort.SliceStable(adaptors, func(i, j int) bool {
-		return adaptors[i].LessThan(adaptors[j])
-	})
-	now := adaptors[len(adaptors)-1].Date()
-
-	// Evaluate retention grid
-	keepa, removea := p.RetentionGrid.FitEntries(now, adaptors)
-
-	// Revert adaptors
-	keep = make([]zfs.FilesystemVersion, len(keepa))
-	for i := range keepa {
-		keep[i] = keepa[i].(retentionGridAdaptor).FilesystemVersion
-	}
-	remove = make([]zfs.FilesystemVersion, len(removea))
-	for i := range removea {
-		remove[i] = removea[i].(retentionGridAdaptor).FilesystemVersion
-	}
-	return
-
-}
-
-func parseGridPrunePolicy(e map[string]interface{}) (p *GridPrunePolicy, err error) {
-
-	var i struct {
-		Grid string
-	}
-
-	if err = mapstructure.Decode(e, &i); err != nil {
-		err = errors.Wrapf(err, "mapstructure error")
-		return
-	}
-
-	p = &GridPrunePolicy{}
-
-	// Parse grid policy
-	intervals, err := parseRetentionGridIntervalsString(i.Grid)
-	if err != nil {
-		err = fmt.Errorf("cannot parse retention grid: %s", err)
-		return
-	}
-	// Assert intervals are of increasing length (not necessarily required, but indicates config mistake)
-	lastDuration := time.Duration(0)
-	for i := range intervals {
-		if intervals[i].Length < lastDuration {
-			err = fmt.Errorf("retention grid interval length must be monotonically increasing:"+
-				"interval %d is shorter than %d", i+1, i)
-			return
-		} else {
-			lastDuration = intervals[i].Length
-		}
-	}
-	p.RetentionGrid = util.NewRetentionGrid(intervals)
-
-	return
-}
-
-var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
-
-func parseDuration(e string) (d time.Duration, err error) {
-	comps := durationStringRegex.FindStringSubmatch(e)
-	if len(comps) != 3 {
-		err = fmt.Errorf("does not match regex: %s %#v", e, comps)
-		return
-	}
-
-	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
-	if err != nil {
-		return
-	}
-
-	var durationUnit time.Duration
-	switch comps[2] {
-	case "s":
-		durationUnit = time.Second
-	case "m":
-		durationUnit = time.Minute
-	case "h":
-		durationUnit = time.Hour
-	case "d":
-		durationUnit = 24 * time.Hour
-	case "w":
-		durationUnit = 24 * 7 * time.Hour
-	default:
-		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
-		return
-	}
-
-	d = time.Duration(durationFactor) * durationUnit
-	return
-
-}
-
-var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
-
-func parseRetentionGridIntervalString(e string) (intervals []util.RetentionInterval, err error) {
-
-	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
-	if comps == nil {
-		err = fmt.Errorf("retention string does not match expected format")
-		return
-	}
-
-	times, err := strconv.Atoi(comps[1])
-	if err != nil {
-		return nil, err
-	} else if times <= 0 {
-		return nil, fmt.Errorf("contains factor <= 0")
-	}
-
-	duration, err := parseDuration(comps[2])
-	if err != nil {
-		return nil, err
-	}
-
-	keepCount := 1
-	if comps[3] != "" {
-		// Decompose key=value, comma separated
-		// For now, only keep_count is supported
-		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
-		res := re.FindStringSubmatch(comps[4])
-		if res == nil || len(res) != 2 {
-			err = fmt.Errorf("interval parameter contains unknown parameters")
-			return
-		}
-		if res[1] == "all" {
-			keepCount = util.RetentionGridKeepCountAll
-		} else {
-			keepCount, err = strconv.Atoi(res[1])
-			if err != nil {
-				err = fmt.Errorf("cannot parse keep_count value")
-				return
-			}
-		}
-	}
-
-	intervals = make([]util.RetentionInterval, times)
-	for i := range intervals {
-		intervals[i] = util.RetentionInterval{
-			Length:    duration,
-			KeepCount: keepCount,
-		}
-	}
-
-	return
-
-}
-
-func parseRetentionGridIntervalsString(s string) (intervals []util.RetentionInterval, err error) {
-
-	ges := strings.Split(s, "|")
-	intervals = make([]util.RetentionInterval, 0, 7*len(ges))
-
-	for intervalIdx, e := range ges {
-		parsed, err := parseRetentionGridIntervalString(e)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
-		}
-		intervals = append(intervals, parsed...)
-	}
-
-	return
-}

cmd/config_prune_none.go

@@ -1,11 +0,0 @@
-package cmd
-
-import "github.com/zrepl/zrepl/zfs"
-
-type NoPrunePolicy struct{}
-
-func (p NoPrunePolicy) Prune(fs *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error) {
-	keep = versions
-	remove = []zfs.FilesystemVersion{}
-	return
-}

cmd/config_serve_stdinserver.go

@@ -1,104 +0,0 @@
-package cmd
-
-import (
-	"github.com/ftrvxmtrx/fd"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"io"
-	"net"
-	"os"
-	"path"
-)
-
-type StdinserverListenerFactory struct {
-	ClientIdentity string `mapstructure:"client_identity"`
-	sockaddr       *net.UnixAddr
-}
-
-func parseStdinserverListenerFactory(c JobParsingContext, i map[string]interface{}) (f *StdinserverListenerFactory, err error) {
-
-	f = &StdinserverListenerFactory{}
-
-	if err = mapstructure.Decode(i, f); err != nil {
-		return nil, errors.Wrap(err, "mapstructure error")
-	}
-	if !(len(f.ClientIdentity) > 0) {
-		err = errors.Errorf("must specify 'client_identity'")
-		return
-	}
-
-	f.sockaddr, err = stdinserverListenerSocket(c.Global.Serve.Stdinserver.SockDir, f.ClientIdentity)
-	if err != nil {
-		return
-	}
-
-	return
-}
-
-func stdinserverListenerSocket(sockdir, clientIdentity string) (addr *net.UnixAddr, err error) {
-	sockpath := path.Join(sockdir, clientIdentity)
-	addr, err = net.ResolveUnixAddr("unix", sockpath)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot resolve unix address")
-	}
-	return addr, nil
-}
-
-func (f *StdinserverListenerFactory) Listen() (al AuthenticatedChannelListener, err error) {
-
-	ul, err := ListenUnixPrivate(f.sockaddr)
-	if err != nil {
-		return nil, errors.Wrapf(err, "cannot listen on unix socket %s", f.sockaddr)
-	}
-
-	l := &StdinserverListener{ul}
-
-	return l, nil
-}
-
-type StdinserverListener struct {
-	l *net.UnixListener
-}
-
-type fdRWC struct {
-	stdin, stdout *os.File
-	control       *net.UnixConn
-}
-
-func (f fdRWC) Read(p []byte) (n int, err error) {
-	return f.stdin.Read(p)
-}
-
-func (f fdRWC) Write(p []byte) (n int, err error) {
-	return f.stdout.Write(p)
-}
-
-func (f fdRWC) Close() (err error) {
-	f.stdin.Close()
-	f.stdout.Close()
-	return f.control.Close()
-}
-
-func (l *StdinserverListener) Accept() (ch io.ReadWriteCloser, err error) {
-	c, err := l.l.Accept()
-	if err != nil {
-		err = errors.Wrap(err, "error accepting on unix listener")
-		return
-	}
-
-	// Read the stdin and stdout of the stdinserver command
-	files, err := fd.Get(c.(*net.UnixConn), 2, []string{"stdin", "stdout"})
-	if err != nil {
-		err = errors.Wrap(err, "error receiving fds from stdinserver command")
-		c.Close()
-	}
-
-	rwc := fdRWC{files[0], files[1], c.(*net.UnixConn)}
-
-	return rwc, nil
-
-}
-
-func (l *StdinserverListener) Close() (err error) {
-	return l.l.Close() // removes socket file automatically
-}

cmd/config_test.go

@@ -4,34 +4,14 @@ import (
 	"testing"
 	"time"
 
-	"github.com/kr/pretty"
 	"github.com/stretchr/testify/assert"
 	"github.com/zrepl/zrepl/util"
 	"github.com/zrepl/zrepl/zfs"
 )
 
-func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
+func TestSampleConfigFileIsParsedWithoutErrors(t *testing.T) {
-
+	_, err := ParseConfig("./sampleconf/zrepl.yml")
-	paths := []string{
+	assert.Nil(t, err)
-		"./sampleconf/localbackup/host1.yml",
-		"./sampleconf/pullbackup/backuphost.yml",
-		"./sampleconf/pullbackup/productionhost.yml",
-		"./sampleconf/random/debugging.yml",
-		"./sampleconf/random/logging.yml",
-	}
-
-	for _, p := range paths {
-
-		c, err := ParseConfig(p)
-		if err != nil {
-			t.Errorf("error parsing %s:\n%+v", p, err)
-		}
-
-		t.Logf("file: %s", p)
-		t.Log(pretty.Sprint(c))
-
-	}
-
 }
 
 func TestParseRetentionGridStringParsing(t *testing.T) {
@@ -81,21 +61,15 @@ func TestDatasetMapFilter(t *testing.T) {
 			t.Logf("expect test from path to be valid: %s", err)
 			t.FailNow()
 		}
-
-		res, err := dmf.Map(fromPath)
-		if to == "" {
-			assert.Nil(t, res)
-			assert.Nil(t, err)
-			t.Logf("%s => NOT MAPPED", fromPath.ToString())
-			return
-		}
-
-		assert.Nil(t, err)
 		toPath, err := zfs.NewDatasetPath(to)
 		if err != nil {
 			t.Logf("expect test to path to be valid: %s", err)
 			t.FailNow()
 		}
+
+		res, err := dmf.Map(fromPath)
+		t.Logf("%s => %s", fromPath.ToString(), res.ToString())
+		assert.Nil(t, err)
 		assert.True(t, res.Equal(toPath))
 	}
 
@@ -116,29 +90,25 @@ func TestDatasetMapFilter(t *testing.T) {
 	}
 
 	map1 := map[string]string{
-		"a/b/c<":     "root1",
+		"a/b/c<": "root1",
-		"a/b<":       "root2",
+		"a/b<":   "root2",
-		"<":          "root3/b/c",
+		"<":      "root3/b/c",
-		"b":          "!",
+		"q<":     "root4/1/2",
-		"a/b/c/d/e<": "!",
-		"q<":         "root4/1/2",
 	}
 
 	expectMapping(map1, "a/b/c", "root1")
 	expectMapping(map1, "a/b/c/d", "root1/d")
-	expectMapping(map1, "a/b/c/d/e", "")
 	expectMapping(map1, "a/b/e", "root2/e")
 	expectMapping(map1, "a/b", "root2")
 	expectMapping(map1, "x", "root3/b/c")
 	expectMapping(map1, "x/y", "root3/b/c/y")
 	expectMapping(map1, "q", "root4/1/2")
-	expectMapping(map1, "b", "")
 	expectMapping(map1, "q/r", "root4/1/2/r")
 
 	filter1 := map[string]string{
-		"<":    "!",
+		"<":    "omit",
 		"a<":   "ok",
-		"a/b<": "!",
+		"a/b<": "omit",
 	}
 
 	expectFilter(filter1, "b", false)
@@ -151,72 +121,3 @@ func TestDatasetMapFilter(t *testing.T) {
 	expectFilter(filter2, "foo", false) // default to omit
 
 }
-
-func TestDatasetMapFilter_AsFilter(t *testing.T) {
-
-	mapspec := map[string]string{
-		"a/b/c<":     "root1",
-		"a/b<":       "root2",
-		"<":          "root3/b/c",
-		"b":          "!",
-		"a/b/c/d/e<": "!",
-		"q<":         "root4/1/2",
-	}
-
-	m, err := parseDatasetMapFilter(mapspec, false)
-	assert.Nil(t, err)
-
-	f := m.AsFilter()
-
-	t.Logf("Mapping:\n%s\nFilter:\n%s", pretty.Sprint(m), pretty.Sprint(f))
-
-	tf := func(f zfs.DatasetFilter, path string, pass bool) {
-		p, err := zfs.NewDatasetPath(path)
-		assert.Nil(t, err)
-		r, err := f.Filter(p)
-		assert.Nil(t, err)
-		assert.Equal(t, pass, r)
-	}
-
-	tf(f, "a/b/c", true)
-	tf(f, "a/b", true)
-	tf(f, "b", false)
-	tf(f, "a/b/c/d/e", false)
-	tf(f, "a/b/c/d/e/f", false)
-	tf(f, "a", true)
-
-}
-
-func TestDatasetMapFilter_InvertedFilter(t *testing.T) {
-	mapspec := map[string]string{
-		"a/b":      "1/2",
-		"a/b/c<":   "3",
-		"a/b/c/d<": "1/2/a",
-		"a/b/d":    "!",
-	}
-
-	m, err := parseDatasetMapFilter(mapspec, false)
-	assert.Nil(t, err)
-
-	inv, err := m.InvertedFilter()
-	assert.Nil(t, err)
-
-	t.Log(pretty.Sprint(inv))
-
-	expectMapping := func(m *DatasetMapFilter, ps string, expRes bool) {
-		p, err := zfs.NewDatasetPath(ps)
-		assert.Nil(t, err)
-		r, err := m.Filter(p)
-		assert.Nil(t, err)
-		assert.Equal(t, expRes, r)
-	}
-
-	expectMapping(inv, "4", false)
-	expectMapping(inv, "3", true)
-	expectMapping(inv, "3/x", true)
-	expectMapping(inv, "1", false)
-	expectMapping(inv, "1/2", true)
-	expectMapping(inv, "1/2/3", false)
-	expectMapping(inv, "1/2/a/b", true)
-
-}

cmd/control.go

@@ -1,100 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"github.com/spf13/cobra"
-	"io"
-	golog "log"
-	"net"
-	"net/http"
-	"net/url"
-	"os"
-)
-
-var controlCmd = &cobra.Command{
-	Use:   "control",
-	Short: "control zrepl daemon",
-}
-
-var pprofCmd = &cobra.Command{
-	Use:   "pprof cpu OUTFILE",
-	Short: "pprof CPU of daemon to OUTFILE (- for stdout)",
-	Run:   doControlPProf,
-}
-var pprofCmdArgs struct {
-	seconds int64
-}
-
-func init() {
-	RootCmd.AddCommand(controlCmd)
-	controlCmd.AddCommand(pprofCmd)
-	pprofCmd.Flags().Int64Var(&pprofCmdArgs.seconds, "seconds", 30, "seconds to profile")
-}
-
-func doControlPProf(cmd *cobra.Command, args []string) {
-
-	log := golog.New(os.Stderr, "", 0)
-
-	die := func() {
-		log.Printf("exiting after error")
-		os.Exit(1)
-	}
-
-	conf, err := ParseConfig(rootArgs.configFile)
-	if err != nil {
-		log.Printf("error parsing config: %s", err)
-		die()
-	}
-
-	if cmd.Flags().Arg(0) != "cpu" {
-		log.Printf("only CPU profiles are supported")
-		log.Printf("%s", cmd.UsageString())
-		die()
-	}
-
-	outfn := cmd.Flags().Arg(1)
-	if outfn == "" {
-		log.Printf("must specify output filename")
-		log.Printf("%s", cmd.UsageString())
-		die()
-	}
-	var out io.Writer
-	if outfn == "-" {
-		out = os.Stdout
-	} else {
-		out, err = os.Create(outfn)
-		if err != nil {
-			log.Printf("error creating output file: %s", err)
-			die()
-		}
-	}
-
-	log.Printf("connecting to daemon")
-	httpc := http.Client{
-		Transport: &http.Transport{
-			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
-				return net.Dial("unix", conf.Global.Control.Sockpath)
-			},
-		},
-	}
-
-	log.Printf("profiling...")
-	v := url.Values{}
-	v.Set("seconds", fmt.Sprintf("%d", pprofCmdArgs.seconds))
-	v.Encode()
-	resp, err := httpc.Get("http://unix" + ControlJobEndpointProfile + "?" + v.Encode())
-	if err != nil {
-		log.Printf("error: %s", err)
-		die()
-	}
-
-	_, err = io.Copy(out, resp.Body)
-	if err != nil {
-		log.Printf("error writing profile: %s", err)
-		die()
-	}
-
-	log.Printf("finished")
-
-}

cmd/daemon.go

@@ -1,110 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/logger"
-	"os"
-	"os/signal"
-	"syscall"
-	"time"
-)
-
-// daemonCmd represents the daemon command
-var daemonCmd = &cobra.Command{
-	Use:   "daemon",
-	Short: "start daemon",
-	Run:   doDaemon,
-}
-
-func init() {
-	RootCmd.AddCommand(daemonCmd)
-}
-
-type Job interface {
-	JobName() string
-	JobStart(ctxt context.Context)
-}
-
-func doDaemon(cmd *cobra.Command, args []string) {
-
-	conf, err := ParseConfig(rootArgs.configFile)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "error parsing config: %s", err)
-		os.Exit(1)
-	}
-
-	log := logger.NewLogger(conf.Global.logging.Outlets, 1*time.Second)
-
-	log.Debug("starting daemon")
-	ctx := context.WithValue(context.Background(), contextKeyLog, log)
-	ctx = context.WithValue(ctx, contextKeyLog, log)
-
-	d := NewDaemon(conf)
-	d.Loop(ctx)
-
-}
-
-type contextKey string
-
-const (
-	contextKeyLog contextKey = contextKey("log")
-)
-
-type Daemon struct {
-	conf *Config
-}
-
-func NewDaemon(initialConf *Config) *Daemon {
-	return &Daemon{initialConf}
-}
-
-func (d *Daemon) Loop(ctx context.Context) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-
-	ctx, cancel := context.WithCancel(ctx)
-
-	sigChan := make(chan os.Signal, 1)
-	finishs := make(chan Job)
-
-	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
-
-	log.Info("starting jobs from config")
-	i := 0
-	for _, job := range d.conf.Jobs {
-		logger := log.WithField(logJobField, job.JobName())
-		logger.Info("starting")
-		i++
-		jobCtx := context.WithValue(ctx, contextKeyLog, logger)
-		go func(j Job) {
-			j.JobStart(jobCtx)
-			finishs <- j
-		}(job)
-	}
-
-	finishCount := 0
-outer:
-	for {
-		select {
-		case <-finishs:
-			finishCount++
-			if finishCount == len(d.conf.Jobs) {
-				log.Info("all jobs finished")
-				break outer
-			}
-
-		case sig := <-sigChan:
-			log.WithField("signal", sig).Info("received signal")
-			log.Info("cancelling all jobs")
-			cancel()
-		}
-	}
-
-	signal.Stop(sigChan)
-	cancel() // make go vet happy
-
-	log.Info("exiting")
-
-}

cmd/handler.go

@@ -2,179 +2,175 @@ package cmd
 
 import (
 	"fmt"
-	"io"
-
-	"github.com/pkg/errors"
 	"github.com/zrepl/zrepl/rpc"
 	"github.com/zrepl/zrepl/zfs"
+	"io"
 )
 
 type DatasetMapping interface {
 	Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error)
 }
 
-type FilesystemRequest struct {
-	Roots []string // may be nil, indicating interest in all filesystems
-}
-
-type FilesystemVersionsRequest struct {
-	Filesystem *zfs.DatasetPath
-}
-
-type InitialTransferRequest struct {
-	Filesystem        *zfs.DatasetPath
-	FilesystemVersion zfs.FilesystemVersion
-}
-
-type IncrementalTransferRequest struct {
-	Filesystem *zfs.DatasetPath
-	From       zfs.FilesystemVersion
-	To         zfs.FilesystemVersion
-}
-
 type Handler struct {
-	logger Logger
+	Logger          Logger
-	dsf    zfs.DatasetFilter
+	PullACL         zfs.DatasetFilter
-	fsvf   zfs.FilesystemVersionFilter
+	SinkMappingFunc func(clientIdentity string) (mapping DatasetMapping, err error)
 }
 
-func NewHandler(logger Logger, dsfilter zfs.DatasetFilter, snapfilter zfs.FilesystemVersionFilter) (h Handler) {
+func (h Handler) HandleFilesystemRequest(r rpc.FilesystemRequest) (roots []*zfs.DatasetPath, err error) {
-	return Handler{logger, dsfilter, snapfilter}
-}
 
-func registerEndpoints(server rpc.RPCServer, handler Handler) (err error) {
+	h.Logger.Printf("handling fsr: %#v", r)
-	err = server.RegisterEndpoint("FilesystemRequest", handler.HandleFilesystemRequest)
-	if err != nil {
-		panic(err)
-	}
-	err = server.RegisterEndpoint("FilesystemVersionsRequest", handler.HandleFilesystemVersionsRequest)
-	if err != nil {
-		panic(err)
-	}
-	err = server.RegisterEndpoint("InitialTransferRequest", handler.HandleInitialTransferRequest)
-	if err != nil {
-		panic(err)
-	}
-	err = server.RegisterEndpoint("IncrementalTransferRequest", handler.HandleIncrementalTransferRequest)
-	if err != nil {
-		panic(err)
-	}
-	return nil
-}
 
-func (h Handler) HandleFilesystemRequest(r *FilesystemRequest, roots *[]*zfs.DatasetPath) (err error) {
+	h.Logger.Printf("using PullACL: %#v", h.PullACL)
 
-	log := h.logger.WithField("endpoint", "FilesystemRequest")
+	if roots, err = zfs.ZFSListMapping(h.PullACL); err != nil {
-
+		h.Logger.Printf("handle fsr err: %v\n", err)
-	log.WithField("request", r).Debug("request")
-	log.WithField("dataset_filter", h.dsf).Debug("dsf")
-
-	allowed, err := zfs.ZFSListMapping(h.dsf)
-	if err != nil {
-		log.WithError(err).Error("error listing filesystems")
 		return
 	}
 
-	log.WithField("response", allowed).Debug("response")
+	h.Logger.Printf("returning: %#v", roots)
-	*roots = allowed
+
 	return
 }
 
-func (h Handler) HandleFilesystemVersionsRequest(r *FilesystemVersionsRequest, versions *[]zfs.FilesystemVersion) (err error) {
+func (h Handler) HandleFilesystemVersionsRequest(r rpc.FilesystemVersionsRequest) (versions []zfs.FilesystemVersion, err error) {
 
-	log := h.logger.WithField("endpoint", "FilesystemVersionsRequest")
+	h.Logger.Printf("handling filesystem versions request: %#v", r)
-
-	log.WithField("request", r).Debug("request")
 
 	// allowed to request that?
-	if h.pullACLCheck(r.Filesystem, nil); err != nil {
+	if h.pullACLCheck(r.Filesystem); err != nil {
-		log.WithError(err).Warn("pull ACL check failed")
 		return
 	}
 
 	// find our versions
-	vs, err := zfs.ZFSListFilesystemVersions(r.Filesystem, h.fsvf)
+	if versions, err = zfs.ZFSListFilesystemVersions(r.Filesystem, nil); err != nil {
-	if err != nil {
+		h.Logger.Printf("our versions error: %#v\n", err)
-		log.WithError(err).Error("cannot list filesystem versions")
 		return
 	}
 
-	log.WithField("resposne", vs).Debug("response")
+	h.Logger.Printf("our versions: %#v\n", versions)
-
-	*versions = vs
 	return
 
 }
 
-func (h Handler) HandleInitialTransferRequest(r *InitialTransferRequest, stream *io.Reader) (err error) {
+func (h Handler) HandleInitialTransferRequest(r rpc.InitialTransferRequest) (stream io.Reader, err error) {
 
-	log := h.logger.WithField("endpoint", "InitialTransferRequest")
+	h.Logger.Printf("handling initial transfer request: %#v", r)
-
+	if err = h.pullACLCheck(r.Filesystem); err != nil {
-	log.WithField("request", r).Debug("request")
-	if err = h.pullACLCheck(r.Filesystem, &r.FilesystemVersion); err != nil {
-		log.WithError(err).Warn("pull ACL check failed")
 		return
 	}
 
-	log.Debug("invoking zfs send")
+	h.Logger.Printf("invoking zfs send")
 
-	s, err := zfs.ZFSSend(r.Filesystem, &r.FilesystemVersion, nil)
+	if stream, err = zfs.ZFSSend(r.Filesystem, &r.FilesystemVersion, nil); err != nil {
-	if err != nil {
+		h.Logger.Printf("error sending filesystem: %#v", err)
-		log.WithError(err).Error("cannot send filesystem")
 	}
-	*stream = s
 
 	return
 
 }
 
-func (h Handler) HandleIncrementalTransferRequest(r *IncrementalTransferRequest, stream *io.Reader) (err error) {
+func (h Handler) HandleIncrementalTransferRequest(r rpc.IncrementalTransferRequest) (stream io.Reader, err error) {
 
-	log := h.logger.WithField("endpoint", "IncrementalTransferRequest")
+	h.Logger.Printf("handling incremental transfer request: %#v", r)
-	log.WithField("request", r).Debug("request")
+	if err = h.pullACLCheck(r.Filesystem); err != nil {
-	if err = h.pullACLCheck(r.Filesystem, &r.From); err != nil {
-		log.WithError(err).Warn("pull ACL check failed")
-		return
-	}
-	if err = h.pullACLCheck(r.Filesystem, &r.To); err != nil {
-		log.WithError(err).Warn("pull ACL check failed")
 		return
 	}
 
-	log.Debug("invoking zfs send")
+	h.Logger.Printf("invoking zfs send")
 
-	s, err := zfs.ZFSSend(r.Filesystem, &r.From, &r.To)
+	if stream, err = zfs.ZFSSend(r.Filesystem, &r.From, &r.To); err != nil {
-	if err != nil {
+		h.Logger.Printf("error sending filesystem: %#v", err)
-		log.WithError(err).Error("cannot send filesystem")
 	}
 
-	*stream = s
 	return
 
 }
 
-func (h Handler) pullACLCheck(p *zfs.DatasetPath, v *zfs.FilesystemVersion) (err error) {
+func (h Handler) HandleResumeTransferRequest(r rpc.ResumeTransferRequest) (stream io.Reader, err error) {
-	var fsAllowed, vAllowed bool
+	// decode receive_resume_token: zfs send -nvP <token>
-	fsAllowed, err = h.dsf.Filter(p)
+	// A) use exit code to determine if could send (exit != 0 means could not send)
+	// B) check ACL if the filesystem in toname field (it is a snapshot so strip its snapshot name first)
+	//    is allowed for the given client
+	/*
+
+			# zfs send -nvt will print nvlist contents on ZoL and FreeBSD, should be good enough
+			# will print regardless of whether it can send or not -> good for us
+			# need clean way to extract that...
+				# expect 'resume token contents:\nnvlist version: 0\n'
+				# parse everything after that is tab-indented as key value pairs separated by = sign
+				# => return dict
+
+			zfs send -nv -t 1-c6491acbe-c8-789c636064000310a500c4ec50360710e72765a526973030b0419460caa7a515a79630c001489e0d493ea9b224b5182451885d7f497e7a69660a0343f79b1b9a8a2b3db65b20c97382e5f312735319188a4bf28b12d353f5931293b34b0b8af5ab8a520b72f4d3f2f31d8a53f35220660300c1091dbe
+		resume token contents:
+		nvlist version: 0
+		        object = 0x6
+		        offset = 0x0
+		        bytes = 0x7100
+		        toguid = 0xb748a92129d8ec8b
+		        toname = storage/backups/zrepl/foo@send
+		cannot resume send: 'storage/backups/zrepl/foo@send' used in the initial send no longer exists
+
+		zfs send -nvt 1-ebbbbea7e-f0-789c636064000310a501c49c50360710a715e5e7a69766a6304041f79b1b9a8a2b3db62b00d9ec48eaf293b252934b181858a0ea30e4d3d28a534b18e00024cf86249f5459925a0ca44fc861d75f920f71c59c5fdf6f7b3eea32b24092e704cbe725e6a632301497e41725a6a7ea27252667971614eb5715a516e4e8a7e5e73b14a7e6a51881cd06005a222749
+		resume token contents:
+		nvlist version: 0
+		        fromguid = 0xb748a92129d8ec8b #NOTE the fromguid field which is only in this one, so don't hardcode
+		        object = 0x4
+		        offset = 0x0
+		        bytes = 0x1ec8
+		        toguid = 0x328ae249dbf7fa9c
+		        toname = storage/backups/zrepl/foo@send2
+		send from storage/backups/zrepl/foo@send to storage/backups/zrepl/foo@send2 estimated size is 1.02M
+
+	*/
+	panic("not implemented")
+	return nil, nil
+}
+
+func (h Handler) HandlePullMeRequest(r rpc.PullMeRequest, clientIdentity string, client rpc.RPCRequester) (err error) {
+
+	// Check if we have a sink for this request
+	// Use that mapping to do what happens in doPull
+
+	h.Logger.Printf("handling PullMeRequest: %#v", r)
+
+	var sinkMapping DatasetMapping
+	sinkMapping, err = h.SinkMappingFunc(clientIdentity)
+	if err != nil {
+		h.Logger.Printf("no sink mapping for client identity '%s', denying PullMeRequest", clientIdentity)
+		err = fmt.Errorf("no sink for client identity '%s'", clientIdentity)
+		return
+	}
+
+	h.Logger.Printf("doing pull...")
+
+	err = doPull(PullContext{
+		Remote:            client,
+		Log:               h.Logger,
+		Mapping:           sinkMapping,
+		InitialReplPolicy: r.InitialReplPolicy,
+	})
+	if err != nil {
+		h.Logger.Printf("PullMeRequest failed with error: %s", err)
+		return
+	}
+
+	h.Logger.Printf("finished handling PullMeRequest: %#v", r)
+
+	return
+}
+
+func (h Handler) pullACLCheck(p *zfs.DatasetPath) (err error) {
+	var allowed bool
+	allowed, err = h.PullACL.Filter(p)
 	if err != nil {
 		err = fmt.Errorf("error evaluating ACL: %s", err)
+		h.Logger.Printf(err.Error())
 		return
 	}
-	if !fsAllowed {
+	if !allowed {
 		err = fmt.Errorf("ACL prohibits access to %s", p.ToString())
-		return
+		h.Logger.Printf(err.Error())
-	}
-	if v == nil {
-		return
-	}
-
-	vAllowed, err = h.fsvf.Filter(*v)
-	if err != nil {
-		err = errors.Wrap(err, "error evaluating version filter")
-		return
-	}
-	if !vAllowed {
-		err = fmt.Errorf("ACL prohibits access to %s", v.ToAbsPath(p))
 		return
 	}
 	return

cmd/helpers.go

@@ -1,35 +0,0 @@
-package cmd
-
-import (
-	"github.com/pkg/errors"
-	"net"
-	"os"
-	"path/filepath"
-)
-
-func ListenUnixPrivate(sockaddr *net.UnixAddr) (*net.UnixListener, error) {
-
-	sockdir := filepath.Dir(sockaddr.Name)
-	sdstat, err := os.Stat(sockdir)
-	if err != nil {
-		return nil, errors.Wrapf(err, "cannot stat(2) '%s'", sockdir)
-	}
-	if !sdstat.IsDir() {
-		return nil, errors.Errorf("not a directory: %s", sockdir)
-	}
-	p := sdstat.Mode().Perm()
-	if p&0007 != 0 {
-		return nil, errors.Errorf("socket directory not be world-accessible: %s (permissions are %#o)", sockdir, p)
-	}
-
-	// Maybe things have not been cleaned up before
-	s, err := os.Stat(sockaddr.Name)
-	if err == nil {
-		if s.Mode()&os.ModeSocket != 0 {
-			// opportunistically try to remove it, but if this fails, it is not an error
-			os.Remove(sockaddr.Name)
-		}
-	}
-
-	return net.ListenUnix("unix", sockaddr)
-}

cmd/logging_formatters.go

@@ -1,174 +0,0 @@
-package cmd
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"github.com/go-logfmt/logfmt"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/logger"
-	"strings"
-	"time"
-)
-
-type EntryFormatter interface {
-	Format(e *logger.Entry) ([]byte, error)
-}
-
-const (
-	FieldLevel   = "level"
-	FieldMessage = "msg"
-	FieldTime    = "time"
-)
-
-const (
-	logJobField     string = "job"
-	logTaskField    string = "task"
-	logFSField      string = "filesystem"
-	logMapFromField string = "map_from"
-	logMapToField   string = "map_to"
-	logIncFromField string = "inc_from"
-	logIncToField   string = "inc_to"
-)
-
-type NoFormatter struct{}
-
-func (f NoFormatter) Format(e *logger.Entry) ([]byte, error) {
-	return []byte(e.Message), nil
-}
-
-type HumanFormatter struct {
-	NoMetadata bool
-}
-
-var _ SetNoMetadataFormatter = &HumanFormatter{}
-
-func (f *HumanFormatter) SetNoMetadata(noMetadata bool) {
-	f.NoMetadata = noMetadata
-}
-
-func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
-
-	var line bytes.Buffer
-
-	if !f.NoMetadata {
-		fmt.Fprintf(&line, "[%s]", e.Level.Short())
-	}
-
-	prefixFields := []string{logJobField, logTaskField, logFSField}
-	prefixed := make(map[string]bool, len(prefixFields)+2)
-	for _, field := range prefixFields {
-		val, ok := e.Fields[field].(string)
-		if ok {
-			fmt.Fprintf(&line, "[%s]", val)
-			prefixed[field] = true
-		} else {
-			break
-		}
-	}
-	// even more prefix fields
-	mapFrom, mapFromOk := e.Fields[logMapFromField].(string)
-	mapTo, mapToOk := e.Fields[logMapToField].(string)
-	if mapFromOk && mapToOk {
-		fmt.Fprintf(&line, "[%s => %s]", mapFrom, mapTo)
-		prefixed[logMapFromField], prefixed[logMapToField] = true, true
-	}
-	incFrom, incFromOk := e.Fields[logIncFromField].(string)
-	incTo, incToOk := e.Fields[logIncToField].(string)
-	if incFromOk && incToOk {
-		fmt.Fprintf(&line, "[%s => %s]", incFrom, incTo)
-		prefixed[logIncFromField], prefixed[logIncToField] = true, true
-	}
-
-	if line.Len() > 0 {
-		fmt.Fprint(&line, ": ")
-	}
-	fmt.Fprint(&line, e.Message)
-
-	for field, value := range e.Fields {
-
-		if prefixed[field] {
-			continue
-		}
-
-		if strings.ContainsAny(field, " \t") {
-			return nil, errors.Errorf("field must not contain whitespace: '%s'", field)
-		}
-		fmt.Fprintf(&line, " %s=\"%s\"", field, value)
-	}
-
-	return line.Bytes(), nil
-}
-
-type JSONFormatter struct{}
-
-func (f *JSONFormatter) Format(e *logger.Entry) ([]byte, error) {
-	data := make(logger.Fields, len(e.Fields)+3)
-	for k, v := range e.Fields {
-		switch v := v.(type) {
-		case error:
-			// Otherwise errors are ignored by `encoding/json`
-			// https://github.com/sirupsen/logrus/issues/137
-			data[k] = v.Error()
-		default:
-			_, err := json.Marshal(v)
-			if err != nil {
-				return nil, errors.Errorf("field is not JSON encodable: %s", k)
-			}
-			data[k] = v
-		}
-	}
-
-	data[FieldMessage] = e.Message
-	data[FieldTime] = e.Time.Format(time.RFC3339)
-	data[FieldLevel] = e.Level
-
-	return json.Marshal(data)
-
-}
-
-type LogfmtFormatter struct {
-	NoMetadata bool
-}
-
-var _ SetNoMetadataFormatter = &LogfmtFormatter{}
-
-func (f *LogfmtFormatter) SetNoMetadata(noMetadata bool) {
-	f.NoMetadata = noMetadata
-}
-
-func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
-	var buf bytes.Buffer
-	enc := logfmt.NewEncoder(&buf)
-
-	if !f.NoMetadata {
-		enc.EncodeKeyval(FieldTime, e.Time)
-		enc.EncodeKeyval(FieldLevel, e.Level)
-	}
-
-	// at least try and put job and task in front
-	prefixed := make(map[string]bool, 2)
-	prefix := []string{logJobField, logTaskField}
-	for _, pf := range prefix {
-		v, ok := e.Fields[pf]
-		if !ok {
-			break
-		}
-		enc.EncodeKeyval(pf, v)
-		prefixed[pf] = true
-	}
-
-	enc.EncodeKeyval(FieldMessage, e.Message)
-
-	for k, v := range e.Fields {
-		if !prefixed[k] {
-			enc.EncodeKeyval(k, v)
-		}
-	}
-
-	if err := enc.EndRecord(); err != nil {
-		return nil, err
-	}
-
-	return buf.Bytes(), nil
-}

cmd/logging_outlets.go

@@ -1,124 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"crypto/tls"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/logger"
-	"io"
-	"log/syslog"
-	"net"
-	"time"
-)
-
-type WriterOutlet struct {
-	Formatter EntryFormatter
-	Writer    io.Writer
-}
-
-func (h WriterOutlet) WriteEntry(ctx context.Context, entry logger.Entry) error {
-	bytes, err := h.Formatter.Format(&entry)
-	if err != nil {
-		return err
-	}
-	_, err = h.Writer.Write(bytes)
-	h.Writer.Write([]byte("\n"))
-	return err
-}
-
-type TCPOutlet struct {
-	Formatter    EntryFormatter
-	Net, Address string
-	Dialer       net.Dialer
-	TLS          *tls.Config
-	// Specifies how much time must pass between a connection error and a reconnection attempt
-	// Log entries written to the outlet during this time interval are silently dropped.
-	RetryInterval time.Duration
-	// nil if there was an error sending / connecting to remote server
-	conn net.Conn
-	// Last time an error occurred when sending / connecting to remote server
-	retry time.Time
-}
-
-func (h *TCPOutlet) WriteEntry(ctx context.Context, e logger.Entry) error {
-
-	b, err := h.Formatter.Format(&e)
-	if err != nil {
-		return err
-	}
-
-	if h.conn == nil {
-		if time.Now().Sub(h.retry) < h.RetryInterval {
-			// cool-down phase, drop the log entry
-			return nil
-		}
-
-		if h.TLS != nil {
-			h.conn, err = tls.DialWithDialer(&h.Dialer, h.Net, h.Address, h.TLS)
-		} else {
-			h.conn, err = h.Dialer.DialContext(ctx, h.Net, h.Address)
-		}
-		if err != nil {
-			h.conn = nil
-			h.retry = time.Now()
-			return errors.Wrap(err, "cannot dial")
-		}
-	}
-
-	_, err = h.conn.Write(b)
-	if err == nil {
-		_, err = h.conn.Write([]byte("\n"))
-	}
-	if err != nil {
-		h.conn.Close()
-		h.conn = nil
-		h.retry = time.Now()
-		return errors.Wrap(err, "cannot write")
-	}
-
-	return nil
-}
-
-type SyslogOutlet struct {
-	Formatter          EntryFormatter
-	RetryInterval      time.Duration
-	writer             *syslog.Writer
-	lastConnectAttempt time.Time
-}
-
-func (o *SyslogOutlet) WriteEntry(ctx context.Context, entry logger.Entry) error {
-
-	bytes, err := o.Formatter.Format(&entry)
-	if err != nil {
-		return err
-	}
-
-	s := string(bytes)
-
-	if o.writer == nil {
-		now := time.Now()
-		if now.Sub(o.lastConnectAttempt) < o.RetryInterval {
-			return nil // not an error toward logger
-		}
-		o.writer, err = syslog.New(syslog.LOG_LOCAL0, "zrepl")
-		o.lastConnectAttempt = time.Now()
-		if err != nil {
-			o.writer = nil
-			return err
-		}
-	}
-
-	switch entry.Level {
-	case logger.Debug:
-		return o.writer.Debug(s)
-	case logger.Info:
-		return o.writer.Info(s)
-	case logger.Warn:
-		return o.writer.Warning(s)
-	case logger.Error:
-		return o.writer.Err(s)
-	default:
-		return o.writer.Err(s) // write as error as reaching this case is in fact an error
-	}
-
-}

cmd/main.go

@@ -11,16 +11,29 @@
 package cmd
 
 import (
+	"fmt"
 	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/jobrun"
+	"golang.org/x/sys/unix"
+	"io"
+	golog "log"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
 )
 
-//
+type Logger interface {
-//type Logger interface {
+	Printf(format string, v ...interface{})
-//	Printf(format string, v ...interface{})
+}
-//}
 
-type Logger = *logger.Logger
+// global state / facilities
+var (
+	conf     Config
+	runner   *jobrun.JobRunner
+	logFlags int = golog.LUTC | golog.Ldate | golog.Ltime
+	logOut   io.Writer
+	log      Logger
+)
 
 var RootCmd = &cobra.Command{
 	Use:   "zrepl",
@@ -35,9 +48,57 @@ var RootCmd = &cobra.Command{
 
 var rootArgs struct {
 	configFile string
+	stderrFile string
+	httpPprof  string
 }
 
 func init() {
-	//cobra.OnInitialize(initConfig)
+	cobra.OnInitialize(initConfig)
 	RootCmd.PersistentFlags().StringVar(&rootArgs.configFile, "config", "", "config file path")
+	RootCmd.PersistentFlags().StringVar(&rootArgs.stderrFile, "stderrFile", "-", "redirect stderr to given path")
+	RootCmd.PersistentFlags().StringVar(&rootArgs.httpPprof, "debug.pprof.http", "", "run pprof http server on given port")
+}
+
+func initConfig() {
+
+	// Logging & stderr redirection
+	if rootArgs.stderrFile != "-" {
+		file, err := os.OpenFile(rootArgs.stderrFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
+		if err != nil {
+			return
+		}
+
+		if err = unix.Dup2(int(file.Fd()), int(os.Stderr.Fd())); err != nil {
+			file.WriteString(fmt.Sprintf("error redirecting stderr file %s: %s\n", rootArgs.stderrFile, err))
+			return
+		}
+		logOut = file
+	} else {
+		logOut = os.Stderr
+	}
+
+	log = golog.New(logOut, "", logFlags)
+
+	// CPU profiling
+	if rootArgs.httpPprof != "" {
+		go func() {
+			http.ListenAndServe(rootArgs.httpPprof, nil)
+		}()
+	}
+
+	// Config
+	if rootArgs.configFile == "" {
+		log.Printf("config file not set")
+		os.Exit(1)
+	}
+	var err error
+	if conf, err = ParseConfig(rootArgs.configFile); err != nil {
+		log.Printf("error parsing config: %s", err)
+		os.Exit(1)
+	}
+
+	jobrunLogger := golog.New(os.Stderr, "jobrun ", logFlags)
+	runner = jobrun.NewJobRunner(jobrunLogger)
+	return
+
 }

cmd/prune.go

@@ -1,103 +1,139 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/util"
 	"github.com/zrepl/zrepl/zfs"
+	"os"
+	"sort"
 	"time"
 )
 
-type Pruner struct {
+var pruneArgs struct {
-	Now            time.Time
+	job    string
-	DryRun         bool
+	dryRun bool
-	DatasetFilter  zfs.DatasetFilter
-	SnapshotPrefix string
-	PrunePolicy    PrunePolicy
 }
 
-type PruneResult struct {
+var PruneCmd = &cobra.Command{
-	Filesystem *zfs.DatasetPath
+	Use:   "prune",
-	All        []zfs.FilesystemVersion
+	Short: "perform pruning",
-	Keep       []zfs.FilesystemVersion
+	Run:   cmdPrune,
-	Remove     []zfs.FilesystemVersion
 }
 
-func (p *Pruner) Run(ctx context.Context) (r []PruneResult, err error) {
+func init() {
+	PruneCmd.Flags().StringVar(&pruneArgs.job, "job", "", "job to run")
+	PruneCmd.Flags().BoolVarP(&pruneArgs.dryRun, "dryrun", "n", false, "dry run")
+	RootCmd.AddCommand(PruneCmd)
+}
 
-	log := ctx.Value(contextKeyLog).(Logger)
+func cmdPrune(cmd *cobra.Command, args []string) {
 
-	if p.DryRun {
+	if len(args) < 1 {
-		log.Info("doing dry run")
+		log.Printf("must specify exactly one job as positional argument")
+		os.Exit(1)
 	}
 
-	filesystems, err := zfs.ZFSListMapping(p.DatasetFilter)
+	job, ok := conf.Prunes[args[0]]
+	if !ok {
+		log.Printf("could not find prune job: %s", args[0])
+		os.Exit(1)
+	}
+
+	log.Printf("Beginning prune job:\n%s", job)
+	ctx := PruneContext{job, time.Now(), pruneArgs.dryRun}
+	err := doPrune(ctx, log)
 	if err != nil {
-		log.WithError(err).Error("error applying filesystem filter")
+		log.Printf("Prune job failed with error: %s", err)
-		return nil, err
+		os.Exit(1)
-	}
-	if len(filesystems) <= 0 {
-		log.Info("no filesystems matching filter")
-		return nil, err
 	}
 
-	r = make([]PruneResult, 0, len(filesystems))
+}
+
+type PruneContext struct {
+	Prune  *Prune
+	Now    time.Time
+	DryRun bool
+}
+
+type retentionGridAdaptor struct {
+	zfs.FilesystemVersion
+}
+
+func (a retentionGridAdaptor) Date() time.Time {
+	return a.Creation
+}
+
+func (a retentionGridAdaptor) LessThan(b util.RetentionGridEntry) bool {
+	return a.CreateTXG < b.(retentionGridAdaptor).CreateTXG
+}
+
+func doPrune(ctx PruneContext, log Logger) error {
+
+	if ctx.DryRun {
+		log.Printf("doing dry run")
+	}
+
+	prune := ctx.Prune
+
+	// ZFSListSnapsFiltered --> todo can extend fsfilter or need new? Have already something per fs
+	// Dedicated snapshot object? Adaptor object to FilesystemVersion?
+
+	filesystems, err := zfs.ZFSListMapping(prune.DatasetFilter)
+	if err != nil {
+		return fmt.Errorf("error applying filesystem filter: %s", err)
+	}
 
 	for _, fs := range filesystems {
 
-		log := log.WithField(logFSField, fs.ToString())
+		fsversions, err := zfs.ZFSListFilesystemVersions(fs, prune.SnapshotFilter)
-
-		fsversions, err := zfs.ZFSListFilesystemVersions(fs, &PrefixSnapshotFilter{p.SnapshotPrefix})
 		if err != nil {
-			log.WithError(err).Error("error listing filesytem versions")
+			return fmt.Errorf("error listing filesytem versions of %s: %s", fs, err)
-			continue
 		}
+
 		if len(fsversions) == 0 {
-			log.WithField("prefix", p.SnapshotPrefix).Info("no filesystem versions matching prefix")
 			continue
 		}
 
-		keep, remove, err := p.PrunePolicy.Prune(fs, fsversions)
+		adaptors := make([]util.RetentionGridEntry, len(fsversions))
-		if err != nil {
+		for fsv := range fsversions {
-			log.WithError(err).Error("error evaluating prune policy")
+			adaptors[fsv] = retentionGridAdaptor{fsversions[fsv]}
-			continue
 		}
 
-		log.WithField("fsversions", fsversions).
+		sort.SliceStable(adaptors, func(i, j int) bool {
-			WithField("keep", keep).
+			return adaptors[i].LessThan(adaptors[j])
-			WithField("remove", remove).
+		})
-			Debug("prune policy debug dump")
+		ctx.Now = adaptors[len(adaptors)-1].Date()
 
-		r = append(r, PruneResult{fs, fsversions, keep, remove})
+		describe := func(a retentionGridAdaptor) string {
-
+			timeSince := a.Creation.Sub(ctx.Now)
-		makeFields := func(v zfs.FilesystemVersion) (fields map[string]interface{}) {
-			fields = make(map[string]interface{})
-			fields["version"] = v.ToAbsPath(fs)
-			timeSince := v.Creation.Sub(p.Now)
-			fields["age_ns"] = timeSince
 			const day time.Duration = 24 * time.Hour
 			days := timeSince / day
 			remainder := timeSince % day
-			fields["age_str"] = fmt.Sprintf("%dd%s", days, remainder)
+			return fmt.Sprintf("%s@%dd%s from now", a.ToAbsPath(fs), days, remainder)
-			return
 		}
 
-		for _, v := range remove {
+		keep, remove := prune.RetentionPolicy.FitEntries(ctx.Now, adaptors)
-			fields := makeFields(v)
+		for _, a := range remove {
-			log.WithFields(fields).Info("destroying version")
+			r := a.(retentionGridAdaptor)
-			// echo what we'll do and exec zfs destroy if not dry run
+			log.Printf("remove %s", describe(r))
-			// TODO special handling for EBUSY (zfs hold)
+			// do echo what we'll do and exec zfs destroy if not dry run
-			// TODO error handling for clones? just echo to cli, skip over, and exit with non-zero status code (we're idempotent)
+			// special handling for EBUSY (zfs hold)
-			if !p.DryRun {
+			// error handling for clones? just echo to cli, skip over, and exit with non-zero status code (we're idempotent)
-				err := zfs.ZFSDestroyFilesystemVersion(fs, v)
+			if !ctx.DryRun {
+				err := zfs.ZFSDestroy(r.ToAbsPath(fs))
 				if err != nil {
 					// handle
-					log.WithFields(fields).WithError(err).Error("error destroying version")
+					log.Printf("error: %s", err)
 				}
 			}
 		}
-
+		for _, a := range keep {
+			r := a.(retentionGridAdaptor)
+			log.Printf("would keep %s", describe(r))
+		}
 	}
 
-	return
+	return nil
 
 }

cmd/replication.go

@@ -3,38 +3,221 @@ package cmd
 import (
 	"fmt"
 	"io"
+	"os"
+	"sync"
 	"time"
 
-	"bytes"
+	"github.com/spf13/cobra"
-	"encoding/json"
+	"github.com/zrepl/zrepl/jobrun"
 	"github.com/zrepl/zrepl/rpc"
 	"github.com/zrepl/zrepl/util"
 	"github.com/zrepl/zrepl/zfs"
 )
 
+var runArgs struct {
+	job  string
+	once bool
+}
+
+var RunCmd = &cobra.Command{
+	Use:   "run",
+	Short: "run push & pull replication",
+	Run:   cmdRun,
+}
+
+var PushCmd = &cobra.Command{
+	Use:   "push",
+	Short: "run push job (first positional argument)",
+	Run:   cmdPush,
+}
+
+var PullCmd = &cobra.Command{
+	Use:   "pull",
+	Short: "run pull job (first positional argument)",
+	Run:   cmdPull,
+}
+
+func init() {
+	RootCmd.AddCommand(RunCmd)
+	RunCmd.Flags().BoolVar(&runArgs.once, "once", false, "run jobs only once, regardless of configured repeat behavior")
+	RunCmd.Flags().StringVar(&runArgs.job, "job", "", "run only the given job")
+
+	RootCmd.AddCommand(PushCmd)
+	RootCmd.AddCommand(PullCmd)
+}
+
+func cmdPush(cmd *cobra.Command, args []string) {
+
+	if len(args) != 1 {
+		log.Printf("must specify exactly one job as positional argument")
+		os.Exit(1)
+	}
+	job, ok := conf.Pushs[args[0]]
+	if !ok {
+		log.Printf("could not find push job %s", args[0])
+		os.Exit(1)
+	}
+	if err := jobPush(job, log); err != nil {
+		log.Printf("error doing push: %s", err)
+		os.Exit(1)
+	}
+
+}
+
+func cmdPull(cmd *cobra.Command, args []string) {
+
+	if len(args) != 1 {
+		log.Printf("must specify exactly one job as positional argument")
+		os.Exit(1)
+	}
+	job, ok := conf.Pulls[args[0]]
+	if !ok {
+		log.Printf("could not find pull job %s", args[0])
+		os.Exit(1)
+	}
+
+	if err := jobPull(job, log); err != nil {
+		log.Printf("error doing pull: %s", err)
+		os.Exit(1)
+	}
+
+}
+
+func cmdRun(cmd *cobra.Command, args []string) {
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		runner.Start()
+	}()
+
+	jobs := make([]jobrun.Job, len(conf.Pulls)+len(conf.Pushs))
+	i := 0
+	for _, pull := range conf.Pulls {
+		jobs[i] = jobrun.Job{
+			Name:           fmt.Sprintf("pull.%d", i),
+			RepeatStrategy: pull.RepeatStrategy,
+			RunFunc: func(log jobrun.Logger) error {
+				log.Printf("doing pull: %v", pull)
+				return jobPull(pull, log)
+			},
+		}
+		i++
+	}
+	for _, push := range conf.Pushs {
+		jobs[i] = jobrun.Job{
+			Name:           fmt.Sprintf("push.%d", i),
+			RepeatStrategy: push.RepeatStrategy,
+			RunFunc: func(log jobrun.Logger) error {
+				log.Printf("doing push: %v", push)
+				return jobPush(push, log)
+			},
+		}
+		i++
+	}
+
+	for _, j := range jobs {
+		if runArgs.once {
+			j.RepeatStrategy = jobrun.NoRepeatStrategy{}
+		}
+		if runArgs.job != "" {
+			if runArgs.job == j.Name {
+				runner.AddJob(j)
+				break
+			}
+			continue
+		}
+		runner.AddJob(j)
+	}
+
+	for {
+		select {
+		case job := <-runner.NotificationChan():
+			log.Printf("job %s reported error: %v\n", job.Name, job.LastError)
+		}
+	}
+
+	wg.Wait()
+
+}
+
 type localPullACL struct{}
 
 func (a localPullACL) Filter(p *zfs.DatasetPath) (pass bool, err error) {
 	return true, nil
 }
 
-const LOCAL_TRANSPORT_IDENTITY string = "local"
+func jobPull(pull *Pull, log jobrun.Logger) (err error) {
 
-const DEFAULT_INITIAL_REPL_POLICY = InitialReplPolicyMostRecent
+	if lt, ok := pull.From.Transport.(LocalTransport); ok {
 
-type InitialReplPolicy string
+		lt.SetHandler(Handler{
+			Logger: log,
+			// Allow access to any dataset since we control what mapping
+			// is passed to the pull routine.
+			// All local datasets will be passed to its Map() function,
+			// but only those for which a mapping exists will actually be pulled.
+			// We can pay this small performance penalty for now.
+			PullACL: localPullACL{},
+		})
+		pull.From.Transport = lt
+		log.Printf("fixing up local transport: %#v", pull.From.Transport)
+	}
 
-const (
+	var remote rpc.RPCRequester
-	InitialReplPolicyMostRecent InitialReplPolicy = "most_recent"
-	InitialReplPolicyAll        InitialReplPolicy = "all"
-)
 
-func closeRPCWithTimeout(log Logger, remote rpc.RPCClient, timeout time.Duration, goodbye string) {
+	if remote, err = pull.From.Transport.Connect(log); err != nil {
-	log.Info("closing rpc connection")
+		return
+	}
+
+	defer closeRPCWithTimeout(log, remote, time.Second*10, "")
+
+	return doPull(PullContext{remote, log, pull.Mapping, pull.InitialReplPolicy})
+}
+
+func jobPush(push *Push, log jobrun.Logger) (err error) {
+
+	if _, ok := push.To.Transport.(LocalTransport); ok {
+		panic("no support for local pushs")
+	}
+
+	var remote rpc.RPCRequester
+	if remote, err = push.To.Transport.Connect(log); err != nil {
+		return err
+	}
+
+	defer closeRPCWithTimeout(log, remote, time.Second*10, "")
+
+	log.Printf("building handler for PullMeRequest")
+	handler := Handler{
+		Logger:          log,
+		PullACL:         push.Filter,
+		SinkMappingFunc: nil, // no need for that in the handler for PullMe
+	}
+	log.Printf("handler: %#v", handler)
+
+	r := rpc.PullMeRequest{
+		InitialReplPolicy: push.InitialReplPolicy,
+	}
+	log.Printf("doing PullMeRequest: %#v", r)
+
+	if err = remote.PullMeRequest(r, handler); err != nil {
+		log.Printf("PullMeRequest failed: %s", err)
+		return
+	}
+
+	log.Printf("push job finished")
+	return
+
+}
+
+func closeRPCWithTimeout(log Logger, remote rpc.RPCRequester, timeout time.Duration, goodbye string) {
+	log.Printf("closing rpc connection")
 
 	ch := make(chan error)
 	go func() {
-		ch <- remote.Close()
+		ch <- remote.CloseRequest(rpc.CloseRequest{goodbye})
 		close(ch)
 	}()
 
@@ -47,16 +230,20 @@ func closeRPCWithTimeout(log Logger, remote rpc.RPCClient, timeout time.Duration
 	}
 
 	if err != nil {
-		log.WithError(err).Error("error closing connection")
+		log.Printf("error closing connection: %s", err)
+		err = remote.ForceClose()
+		if err != nil {
+			log.Printf("error force-closing connection: %s", err)
+		}
 	}
 	return
 }
 
 type PullContext struct {
-	Remote            rpc.RPCClient
+	Remote            rpc.RPCRequester
 	Log               Logger
 	Mapping           DatasetMapping
-	InitialReplPolicy InitialReplPolicy
+	InitialReplPolicy rpc.InitialReplPolicy
 }
 
 func doPull(pull PullContext) (err error) {
@@ -64,14 +251,14 @@ func doPull(pull PullContext) (err error) {
 	remote := pull.Remote
 	log := pull.Log
 
-	log.Info("request remote filesystem list")
+	log.Printf("requesting remote filesystem list")
-	fsr := FilesystemRequest{}
+	fsr := rpc.FilesystemRequest{}
 	var remoteFilesystems []*zfs.DatasetPath
-	if err = remote.Call("FilesystemRequest", &fsr, &remoteFilesystems); err != nil {
+	if remoteFilesystems, err = remote.FilesystemRequest(fsr); err != nil {
 		return
 	}
 
-	log.Debug("map remote filesystems to local paths and determine order for per-filesystem sync")
+	log.Printf("map remote filesystems to local paths and determine order for per-filesystem sync")
 	type RemoteLocalMapping struct {
 		Remote *zfs.DatasetPath
 		Local  *zfs.DatasetPath
@@ -83,42 +270,39 @@ func doPull(pull PullContext) (err error) {
 		var localFs *zfs.DatasetPath
 		localFs, err = pull.Mapping.Map(remoteFilesystems[fs])
 		if err != nil {
-			err := fmt.Errorf("error mapping %s: %s", remoteFilesystems[fs], err)
+			if err != NoMatchError {
-			log.WithError(err).WithField(logMapFromField, remoteFilesystems[fs]).Error("cannot map")
+				err := fmt.Errorf("error mapping %s: %s", remoteFilesystems[fs], err)
-			return err
+				log.Printf("%s", err)
-		}
+				return err
-		if localFs == nil {
+			}
 			continue
 		}
-		log.WithField(logMapFromField, remoteFilesystems[fs].ToString()).
+		log.Printf("%s => %s", remoteFilesystems[fs].ToString(), localFs.ToString())
-			WithField(logMapToField, localFs.ToString()).Debug("mapping")
 		m := RemoteLocalMapping{remoteFilesystems[fs], localFs}
 		replMapping[m.Local.ToString()] = m
 		localTraversal.Add(m.Local)
 	}
 
-	log.Debug("build cache for already present local filesystem state")
+	log.Printf("build cache for already present local filesystem state")
 	localFilesystemState, err := zfs.ZFSListFilesystemState()
 	if err != nil {
-		log.WithError(err).Error("cannot request local filesystem state")
+		log.Printf("error requesting local filesystem state: %s", err)
 		return err
 	}
 
-	log.Info("start per-filesystem sync")
+	log.Printf("start per-filesystem sync")
 	localTraversal.WalkTopDown(func(v zfs.DatasetPathVisit) bool {
 
-		log := log.WithField(logFSField, v.Path.ToString())
-
 		if v.FilledIn {
 			if _, exists := localFilesystemState[v.Path.ToString()]; exists {
 				// No need to verify if this is a placeholder or not. It is sufficient
 				// to know we can add child filesystems to it
 				return true
 			}
-			log.Debug("create placeholder filesystem")
+			log.Printf("creating placeholder filesystem %s", v.Path.ToString())
 			err = zfs.ZFSCreatePlaceholderFilesystem(v.Path)
 			if err != nil {
-				log.Error("cannot create placeholder filesystem")
+				err = fmt.Errorf("aborting, cannot create placeholder filesystem %s: %s", v.Path, err)
 				return false
 			}
 			return true
@@ -129,40 +313,122 @@ func doPull(pull PullContext) (err error) {
 			panic("internal inconsistency: replMapping should contain mapping for any path that was not filled in by WalkTopDown()")
 		}
 
-		log = log.WithField(logMapToField, m.Remote.ToString()).
+		log := func(format string, args ...interface{}) {
-			WithField(logMapFromField, m.Local.ToString())
+			log.Printf("[%s => %s]: %s", m.Remote.ToString(), m.Local.ToString(), fmt.Sprintf(format, args...))
+		}
 
-		log.Debug("examing local filesystem state")
+		log("examing local filesystem state")
 		localState, localExists := localFilesystemState[m.Local.ToString()]
 		var versions []zfs.FilesystemVersion
 		switch {
 		case !localExists:
-			log.Info("local filesystem does not exist")
+			log("local filesystem does not exist")
 		case localState.Placeholder:
-			log.Info("local filesystem is marked as placeholder")
+			log("local filesystem is marked as placeholder")
+		case localState.ResumeToken != "":
+
+			log("local filesystem has receive_resume_token")
+			if false { // TODO Check if managed resume is disabled (explain information leakage in docs!)
+				log("managed resume tokens are disabled via zrepl config, assuming config change or external administrative action")
+				log("policy forbids aborting the partial recv automatically, skipping this filesystem")
+				log("for zrepl to resume replication for this dataset, administrators should finishing the recv manually or abort the recv via `zfs recv -A %s`", m.Local.ToString())
+				return true // TODO right choice to allow children? ... should be, since the fs exists...
+			}
+
+			log("decoding receive_resume_token")
+			// TODO, see handler comments
+			// TODO override logger with fromguid toguid
+
+			log("requesting resume of transfer")
+			r := rpc.ResumeTransferRequest{m.Remote, localState.ResumeToken}
+			stream, err := remote.ResumeTransferRequest(r)
+			if err != nil {
+
+				log("resume transfer request failed: %s", err)
+				rre, ok := err.(*rpc.ResumeTransferError)
+				if !ok {
+					log("skipping this filesystem, could be temporary issue")
+					return true // TODO right choice to allow children
+				}
+
+				// Determine if we should clear the resume token
+				clearAndUseSnaps := false
+				switch rre.Reason {
+				case rpc.ResumeTransferErrorReasonNotImplemented:
+					fallthrough
+				case rpc.ResumeTransferErrorReasonDisabled:
+					fallthrough
+				case rpc.ResumeTransferErrorReasonZFSErrorPermanent:
+					clearAndUseSnaps = true
+
+				case rpc.ResumeTransferErrorReasonZFSErrorMaybeTemporary:
+					fallthrough
+				default:
+					clearAndUseSnaps = false
+				}
+
+				if !clearAndUseSnaps {
+					log("skipping this filesystem, error identified as temporary")
+					return true // TODO right choice to allow children
+				}
+
+				log("clearing local receive_resume_token")
+				if err := zfs.ZFSRecvAbort(m.Local); err != nil {
+					log("error clearing receive_resume_token: %s", err)
+					return true // TODO right choice to allow children? the filesystem exists, so it should be ok
+				}
+				// TODO go back to top of function (put all this into separate function, then tail recursive call)
+				// TODO return this_function()
+			}
+
+			// TODO warning code duplication, see below. need to unify this
+			log("invoking zfs receive")
+			watcher := util.IOProgressWatcher{Reader: stream}
+			watcher.KickOff(1*time.Second, func(p util.IOProgress) {
+				log("progress on receive operation: %v bytes received", p.TotalRX)
+			})
+
+			recvArgs := []string{"-u"}
+			if localState.Placeholder {
+				log("receive with forced rollback to replace placeholder filesystem")
+				recvArgs = append(recvArgs, "-F")
+			}
+
+			if err = zfs.ZFSRecv(m.Local, &watcher, recvArgs...); err != nil {
+				log("error receiving stream: %s", err)
+				return false
+			}
+			log("finished receiving stream, %v bytes total", watcher.Progress().TotalRX)
+
+			// TODO further problem property handling code must be duplicated here...
+
+			// TODO tail recursive call to this function, we might not be dony syncing yet
+			// TODO return this_function()
+			return true
+
 		default:
-			log.Debug("local filesystem exists")
+			log("local filesystem exists")
-			log.Debug("requesting local filesystem versions")
+			log("requesting local filesystem versions")
 			if versions, err = zfs.ZFSListFilesystemVersions(m.Local, nil); err != nil {
-				log.WithError(err).Error("cannot get local filesystem versions")
+				log("cannot get local filesystem versions: %s", err)
 				return false
 			}
 		}
 
-		log.Info("requesting remote filesystem versions")
+		log("requesting remote filesystem versions")
-		r := FilesystemVersionsRequest{
-			Filesystem: m.Remote,
-		}
 		var theirVersions []zfs.FilesystemVersion
-		if err = remote.Call("FilesystemVersionsRequest", &r, &theirVersions); err != nil {
+		theirVersions, err = remote.FilesystemVersionsRequest(rpc.FilesystemVersionsRequest{
-			log.WithError(err).Error("cannot get remote filesystem versions")
+			Filesystem: m.Remote,
-			log.Warn("stopping replication for all filesystems mapped as children of receiving filesystem")
+		})
+		if err != nil {
+			log("error requesting remote filesystem versions: %s", err)
+			log("stopping replication for all filesystems mapped as children of %s", m.Local.ToString())
 			return false
 		}
 
-		log.Debug("computing diff between remote and local filesystem versions")
+		log("computing diff between remote and local filesystem versions")
 		diff := zfs.MakeFilesystemDiff(versions, theirVersions)
-		log.WithField("diff", diff).Debug("diff between local and remote filesystem")
+		log("%s", diff)
 
 		if localState.Placeholder && diff.Conflict != zfs.ConflictAllRight {
 			panic("internal inconsistency: local placeholder implies ConflictAllRight")
@@ -171,9 +437,9 @@ func doPull(pull PullContext) (err error) {
 		switch diff.Conflict {
 		case zfs.ConflictAllRight:
 
-			log.WithField("replication_policy", pull.InitialReplPolicy).Info("performing initial sync, following policy")
+			log("performing initial sync, following policy: '%s'", pull.InitialReplPolicy)
 
-			if pull.InitialReplPolicy != InitialReplPolicyMostRecent {
+			if pull.InitialReplPolicy != rpc.InitialReplPolicyMostRecent {
 				panic(fmt.Sprintf("policy '%s' not implemented", pull.InitialReplPolicy))
 			}
 
@@ -185,132 +451,134 @@ func doPull(pull PullContext) (err error) {
 			}
 
 			if len(snapsOnly) < 1 {
-				log.Warn("cannot perform initial sync: no remote snapshots")
+				log("cannot perform initial sync: no remote snapshots. stopping...")
 				return false
 			}
 
-			r := InitialTransferRequest{
+			r := rpc.InitialTransferRequest{
 				Filesystem:        m.Remote,
 				FilesystemVersion: snapsOnly[len(snapsOnly)-1],
 			}
 
-			log.WithField("version", r.FilesystemVersion).Debug("requesting snapshot stream")
+			log("requesting snapshot stream for %s", r.FilesystemVersion)
 
 			var stream io.Reader
-
+			if stream, err = remote.InitialTransferRequest(r); err != nil {
-			if err = remote.Call("InitialTransferRequest", &r, &stream); err != nil {
+				log("error requesting initial transfer: %s", err)
-				log.WithError(err).Error("cannot request initial transfer")
 				return false
 			}
-			log.Debug("received initial transfer request response")
+			log("received initial transfer request response")
 
-			log.Debug("invoke zfs receive")
+			log("invoking zfs receive")
 			watcher := util.IOProgressWatcher{Reader: stream}
 			watcher.KickOff(1*time.Second, func(p util.IOProgress) {
-				log.WithField("total_rx", p.TotalRX).Info("progress on receive operation")
+				log("progress on receive operation: %v bytes received", p.TotalRX)
 			})
 
 			recvArgs := []string{"-u"}
 			if localState.Placeholder {
-				log.Info("receive with forced rollback to replace placeholder filesystem")
+				log("receive with forced rollback to replace placeholder filesystem")
 				recvArgs = append(recvArgs, "-F")
 			}
 
 			if err = zfs.ZFSRecv(m.Local, &watcher, recvArgs...); err != nil {
-				log.WithError(err).Error("canot receive stream")
+				log("error receiving stream: %s", err)
 				return false
 			}
-			log.WithField("total_rx", watcher.Progress().TotalRX).
+			log("finished receiving stream, %v bytes total", watcher.Progress().TotalRX)
-				Info("finished receiving stream")
 
-			log.Debug("configuring properties of received filesystem")
+			log("configuring properties of received filesystem")
 			if err = zfs.ZFSSet(m.Local, "readonly", "on"); err != nil {
-				log.WithError(err).Error("cannot set readonly property")
+
 			}
 
-			log.Info("finished initial transfer")
+			log("finished initial transfer")
 			return true
 
 		case zfs.ConflictIncremental:
 
 			if len(diff.IncrementalPath) < 2 {
-				log.Info("remote and local are in sync")
+				log("remote and local are in sync")
 				return true
 			}
 
-			log.Info("following incremental path from diff")
+			log("following incremental path from diff")
 			var pathRx uint64
 
 			for i := 0; i < len(diff.IncrementalPath)-1; i++ {
 
 				from, to := diff.IncrementalPath[i], diff.IncrementalPath[i+1]
 
-				log, _ := log.WithField(logIncFromField, from.Name).WithField(logIncToField, to.Name), 0
+				log := func(format string, args ...interface{}) {
+					log("[%v/%v][%s => %s]: %s", i+1, len(diff.IncrementalPath)-1,
+						from.Name, to.Name, fmt.Sprintf(format, args...))
+				}
 
-				log.Debug("requesting incremental snapshot stream")
+				log("requesting incremental snapshot stream")
-				r := IncrementalTransferRequest{
+				r := rpc.IncrementalTransferRequest{
 					Filesystem: m.Remote,
 					From:       from,
 					To:         to,
 				}
 				var stream io.Reader
-				if err = remote.Call("IncrementalTransferRequest", &r, &stream); err != nil {
+				if stream, err = remote.IncrementalTransferRequest(r); err != nil {
-					log.WithError(err).Error("cannot request incremental snapshot stream")
+					log("error requesting incremental snapshot stream: %s", err)
 					return false
 				}
 
-				log.Debug("invoking zfs receive")
+				log("invoking zfs receive")
 				watcher := util.IOProgressWatcher{Reader: stream}
 				watcher.KickOff(1*time.Second, func(p util.IOProgress) {
-					log.WithField("total_rx", p.TotalRX).Info("progress on receive operation")
+					log("progress on receive operation: %v bytes received", p.TotalRX)
 				})
 
 				if err = zfs.ZFSRecv(m.Local, &watcher); err != nil {
-					log.WithError(err).Error("cannot receive stream")
+					log("error receiving stream: %s", err)
 					return false
 				}
 
 				totalRx := watcher.Progress().TotalRX
 				pathRx += totalRx
-				log.WithField("total_rx", totalRx).Info("finished incremental transfer")
+				log("finished incremental transfer, %v bytes total", totalRx)
 
 			}
 
-			log.WithField("total_rx", pathRx).Info("finished following incremental path")
+			log("finished following incremental path, %v bytes total", pathRx)
 			return true
 
 		case zfs.ConflictNoCommonAncestor:
-			fallthrough
+
+			log("remote and local filesystem have snapshots, but no common one")
+			log("perform manual replication to establish a common snapshot history")
+			log("remote versions:")
+			for _, v := range diff.MRCAPathRight {
+				log(" %s (GUID %v)", v, v.Guid)
+			}
+			log("local versions:")
+			for _, v := range diff.MRCAPathLeft {
+				log(" %s (GUID %v)", v, v.Guid)
+			}
+			return false
+
 		case zfs.ConflictDiverged:
 
-			var jsonDiff bytes.Buffer
+			log("remote and local filesystem share a history but have diverged")
-			if err := json.NewEncoder(&jsonDiff).Encode(diff); err != nil {
+			log("perform manual replication or delete snapshots on the receiving" +
-				log.WithError(err).Error("cannot JSON-encode diff")
+				"side  to establish an incremental replication parse")
-				return false
+			log("remote-only versions:")
+			for _, v := range diff.MRCAPathRight {
+				log(" %s (GUID %v)", v, v.Guid)
 			}
-
+			log("local-only versions:")
-			var problem, resolution string
+			for _, v := range diff.MRCAPathLeft {
-
+				log(" %s (GUID %v)", v, v.Guid)
-			switch diff.Conflict {
-			case zfs.ConflictNoCommonAncestor:
-				problem = "remote and local filesystem have snapshots, but no common one"
-				resolution = "perform manual establish a common snapshot history"
-			case zfs.ConflictDiverged:
-				problem = "remote and local filesystem share a history but have diverged"
-				resolution = "perform manual replication or delete snapshots on the receiving" +
-					"side  to establish an incremental replication parse"
 			}
-
-			log.WithField("diff", jsonDiff.String()).
-				WithField("problem", problem).
-				WithField("resolution", resolution).
-				Error("manual conflict resolution required")
-
 			return false
 
 		}
 
-		panic("should not be reached")
+		panic("implementation error: this should not be reached")
+		return false
 
 	})
 

cmd/sampleconf/localbackup/host1.yml

@@ -1,27 +0,0 @@
-jobs:
-- name: mirror_local
-  type: local
-
-  # snapshot the filesystems matched by the left-hand-side of the mapping
-  # every 10m with zrepl_ as prefix
-  mapping: {
-    "zroot/var/db<":    "storage/backups/local/zroot/var/db",
-    "zroot/usr/home<":  "storage/backups/local/zroot/usr/home",
-    "zroot/var/tmp": "!", #don't backup /tmp
-  }
-  snapshot_prefix: zrepl_
-  interval: 10m
-  initial_repl_policy: most_recent
-
-  # keep one hour of 10m interval snapshots of filesystems matched by
-  # the left-hand-side of the mapping
-  prune_lhs:
-    policy: grid
-    grid: 1x1h(keep=all)
-
-  # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
-  prune_rhs:
-    policy: grid
-    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-
-

cmd/sampleconf/pullbackup/backuphost.yml

@@ -1,27 +0,0 @@
-jobs:
-- name: fullbackup_prod1
-  type: pull
-  # connect to remote using ssh / stdinserver command
-  connect:
-    type: ssh+stdinserver
-    host: prod1.example.com
-    user: root
-    port: 22
-    identity_file: /root/.ssh/id_ed25519
-
-  # pull (=ask for new snapshots) every 10m, prune afterwards
-  # this will leave us at most 10m behind production
-  interval: 10m
-
-  # pull all offered filesystems to storage/backups/zrepl/pull/prod1.example.com
-  mapping: {
-    "<":"storage/backups/zrepl/pull/prod1.example.com"
-  }
-  initial_repl_policy: most_recent
-
-  # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
-  snapshot_prefix: zrepl_
-  prune:
-    policy: grid
-    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-

cmd/sampleconf/pullbackup/productionhost.yml

@@ -1,45 +0,0 @@
-global:
-  serve:
-    stdinserver:
-      # Directory where AF_UNIX sockets for stdinserver command are placed.
-      #
-      # `zrepl stdinserver CLIENT_IDENTITY`
-      #   * connects to the socket in $sockdir/CLIENT_IDENTITY
-      #   * sends its stdin / stdout file descriptors to the `zrepl daemon` process (see cmsg(3))
-      #   * does nothing more
-      #
-      # This enables a setup where `zrepl daemon` is not directly exposed to the internet
-      # but instead all traffic is tunnelled through SSH.
-      # The server with the source job has an authorized_keys file entry for the public key
-      # used by the corresponding pull job
-      #
-      #   command="/mnt/zrepl stdinserver CLIENT_IDENTITY" ssh-ed25519 AAAAC3NzaC1E... zrepl@pullingserver
-      #
-      # Below is the default value.
-      sockdir: /var/run/zrepl/stdinserver
-
-jobs:
-
-- name: fullbackup_prod1
-  # expect remote to connect via ssh+stdinserver with fullbackup_prod1 as client_identity
-  type: source
-  serve:
-    type: stdinserver # see global.serve.stdinserver for explanation
-    client_identity: fullbackup_prod1
-
-  # snapshot these filesystems every 10m with zrepl_ as prefix
-  datasets: {
-    "zroot/var/db<": "ok",
-    "zroot/usr/home<": "ok",
-    "zroot/var/tmp": "!", #don't backup /tmp
-  }
-  snapshot_prefix: zrepl_
-  interval: 10m
-
-
-  # keep a one day window 10m interval snapshots in case pull doesn't work (link down, etc)
-  # (we cannot keep more than one day because this host will run out of disk space)
-  prune:
-    policy: grid
-    grid: 1x1d(keep=all)
-

cmd/sampleconf/pushbackup/backuphost.yml

@@ -1,20 +0,0 @@
-jobs:
-- name: fullbackup_prod1
-
-  # expect remote to connect via ssh+stdinserver with fullbackup_prod1 as client_identity
-  type: push-sink
-  serve:
-    type: stdinserver
-    client_identity: fullbackup_prod1
-
-  # map all pushed datasets to storage/backups/zrepl/sink/prod1.example.com
-  mapping: {
-    "<":"storage/backups/zrepl/sink/prod1.example.com"
-  }
-
-  # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
-  prune:
-    policy: grid
-    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-
-

cmd/sampleconf/pushbackup/productionhost.yml

@@ -1,26 +0,0 @@
-jobs:
-- name: fullbackup_prod1
-
-  # connect to remote using ssh / stdinserver command
-  type: push
-  connect:
-      type: ssh+stdinserver
-      host: prod1.example.com
-      user: root
-      port: 22
-      identity_file: /root/.ssh/id_ed25519
-
-  # snapshot these datsets every 10m with zrepl_ as prefix
-  datasets: {
-    "zroot/var/db<": "ok",
-    "zroot/usr/home<": "!",
-  }
-  snapshot_prefix: zrepl_
-  interval: 10m
-
-  # keep a one day window 10m interval snapshots in case push doesn't work (link down, etc)
-  # (we cannot keep more than one day because this host will run out of disk space)
-  prune:
-    policy: grid
-    grid: 1x1d(keep=all)
-

cmd/sampleconf/random/debugging.yml

@@ -1,33 +0,0 @@
-global:
-  serve:
-    stdinserver:
-      sockdir: /var/run/zrepl/stdinserver
-
-jobs:
-
-- name: debian2_pull
-  # JOB DEBUGGING OPTIONS
-  # should be equal for all job types, but each job implements the debugging itself
-  # => consult job documentation for supported options
-  debug:
-    conn: # debug the io.ReadWriteCloser connection
-      read_dump: /tmp/connlog_read   # dump results of Read() invocations to this file
-      write_dump: /tmp/connlog_write # dump results of Write() invocations to this file
-    rpc: # debug the RPC protocol implementation
-      log: true # log output from rpc layer to the job log
-
-  # ... just to make the unit tests pass.
-  # check other examples, e.g. localbackup or pullbackup for what the sutff below means
-  type: source
-  serve:
-    type: stdinserver
-    client_identity: debian2
-  datasets: {
-    "pool1/db<": ok
-  }
-  snapshot_prefix: zrepl_
-  interval: 1s
-  prune:
-    policy: grid
-    grid: 1x10s(keep=all)
-

cmd/sampleconf/random/logging.yml

@@ -1,20 +0,0 @@
-global:
-  logging:
-    stdout:
-      level: warn
-      format: human
-    tcp:
-      level: debug
-      format: json
-      net: tcp
-      address: 127.0.0.1:8080
-      retry_interval: 1s
-      tls: # if not specified, use plain TCP
-        ca: sampleconf/random/logging/logserver.crt
-        cert: sampleconf/random/logging/client.crt
-        key: sampleconf/random/logging/client.key
-    syslog:
-      enable: true
-      format: logfmt
-
-jobs: []

cmd/sampleconf/random/logging/client.crt

@@ -1,19 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDIzCCAgsCAQEwDQYJKoZIhvcNAQELBQAwWTELMAkGA1UEBhMCQVUxEzARBgNV
-BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0
-ZDESMBAGA1UEAwwJbG9nc2VydmVyMB4XDTE3MDkyNDEyMzAzNloXDTE3MTAyNDEy
-MzAzNlowVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNV
-BAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xpZW50MIIB
-IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/xJTUlqApeJGzRD+w2J8sZS
-Bo+s+04T987L/M6gaCo8aDSTEb/ZH3XSoU5JEmO6kPpwNNapOsaEhTCjndZQdm5F
-uqiUtAg1uW0HCkBEIDkGr9bFHDKzpewGmmMgfQ2+hfiBR/4ZCrc/vd9P0W9BiWQS
-Dtc7p22XraWPVL8HlSz5K/Ih+V6i8O+kBltZkusiJh2bWPoRp/netiTZuc6du+Wp
-kpWp1OBaTU4GXIAlLj5afF14BBphRQK983Yhaz53BkA7OQ76XxowynMjmuLQVGmK
-f1R9zEJuohTX9XIr1tp/ueRHcS4Awk6LcNZUMCV6270FNSIw2f4hbOZvep+t2wID
-AQABMA0GCSqGSIb3DQEBCwUAA4IBAQACK3OeNzScpiNwz/jpg/usQzvXbZ/wDvml
-YLjtzn/A65ox8a8BhxvH1ydyoCM2YAGYX7+y7qXJnMgRO/v8565CQIVcznHhg9ST
-3828/WqZ3bXf2DV5GxKKQf7hPmBnyVUUhn/Ny91MECED27lZucWiX/bczN8ffDeh
-M3+ngezcJxsOBd4x0gLrqIJCoaFRSeepOaFEW6GHQ8loxE9GmA7FQd2phIpJHFSd
-Z7nQl7X5C1iN2OboEApJHwtmNVC45UlOpg53vo2sDTLhSfdogstiWi8x1HmvhIGM
-j3XHs0Illvo9OwVrmgUph8zQ7pvr/AFrTOIbhgzl/9uVUk5ApwFM
------END CERTIFICATE-----

cmd/sampleconf/random/logging/client.csr

@@ -1,16 +0,0 @@
------BEGIN CERTIFICATE REQUEST-----
-MIICmzCCAYMCAQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx
-ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xp
-ZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/xJTUlqApeJGzRD
-+w2J8sZSBo+s+04T987L/M6gaCo8aDSTEb/ZH3XSoU5JEmO6kPpwNNapOsaEhTCj
-ndZQdm5FuqiUtAg1uW0HCkBEIDkGr9bFHDKzpewGmmMgfQ2+hfiBR/4ZCrc/vd9P
-0W9BiWQSDtc7p22XraWPVL8HlSz5K/Ih+V6i8O+kBltZkusiJh2bWPoRp/netiTZ
-uc6du+WpkpWp1OBaTU4GXIAlLj5afF14BBphRQK983Yhaz53BkA7OQ76XxowynMj
-muLQVGmKf1R9zEJuohTX9XIr1tp/ueRHcS4Awk6LcNZUMCV6270FNSIw2f4hbOZv
-ep+t2wIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAKnlr0Qs5KYF85u2YA7DJ5pL
-HwAx+qNoNbox5CS1aynrDBpDTWLaErviUJ+4WxRlRyTMEscMOIOKajbYhqqFmtGZ
-mu3SshZnFihErw8TOQMyU1LGGG+l6r+6ve5TciwJRLla2Y75z7izr6cyvQNRWdLr
-PvxL1/Yqr8LKha12+7o28R4SLf6/GY0GcedqoebRmtuwA/jES0PuGauEUD5lH4cj
-Me8sqRrB+IMHQ5j8hlJX4DbA8UQRUBL64sHkQzeQfWu+qkWmS5I19CFfLNrcH+OV
-yhyjGfN0q0jHyHdpckBhgzS7IIdo6P66AIlm4qpHM7Scra3JaGM7oaZPamJ6f8U=
------END CERTIFICATE REQUEST-----

cmd/sampleconf/random/logging/client.key

@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC3/ElNSWoCl4kb
-NEP7DYnyxlIGj6z7ThP3zsv8zqBoKjxoNJMRv9kfddKhTkkSY7qQ+nA01qk6xoSF
-MKOd1lB2bkW6qJS0CDW5bQcKQEQgOQav1sUcMrOl7AaaYyB9Db6F+IFH/hkKtz+9
-30/Rb0GJZBIO1zunbZetpY9UvweVLPkr8iH5XqLw76QGW1mS6yImHZtY+hGn+d62
-JNm5zp275amSlanU4FpNTgZcgCUuPlp8XXgEGmFFAr3zdiFrPncGQDs5DvpfGjDK
-cyOa4tBUaYp/VH3MQm6iFNf1civW2n+55EdxLgDCTotw1lQwJXrbvQU1IjDZ/iFs
-5m96n63bAgMBAAECggEAF4om0sWe06ARwbJJNFjCGpa3LfG5/xk5Qs5pmPnS2iD1
-Q5veaTnzjKvlfA/pF3o9B4mTS59fXY7Cq8vSU0J1XwGy2DPzeqlGPmgtq2kXjkvd
-iCfhZj8ybvsoyR3/rSBSDRADcnOXPqC9fgyRSMmESBDOoql1D3HdIzF4ii46ySIU
-/XQvExS6NWifbP+Ue6DETV8NhreO5PqjeXLITQhhndtc8MDL/8eCNOyN8XjYIWKX
-smlBYtRQYOOY9BHOQgUn6yvPHrtKJNKci+qcQNvWir66mBhY1o40MH5wTIV+8yP2
-Vbm/VzoNKIYgeROsilBW7QTwGvkDn3R11zeTqfUNSQKBgQD0eFzhJAEZi4uBw6Tg
-NKmBC5Y1IHPOsb5gKPNz9Z9j4qYRDySgYl6ISk+2EdhgUCo1NmTk8EIPQjIerUVf
-S+EogFnpsj8U9LR3OM79DaGkNULxrHqhd209/g8DtVgk7yjkxL4vmVOv8qpHMp/7
-eWsylN7AOxj2RB/eXYQBPrw+jQKBgQDAqae9HasLmvpJ9ktTv30yZSKXC+LP4A0D
-RBBmx410VpPd4CvcpCJxXmjer6B7+9L1xHYP2pvsnMBid5i0knuvyK28dYy7fldl
-CzWvb+lqNA5YYPFXQED4oEdihlQczoI1Bm06SFizeAKD1Q9e2c+lgbR/51j8xuXi
-twvhMj/YBwKBgQCZw97/iQrcC2Zq7yiUEOuQjD4lGk1c83U/vGIsTJC9XcCAOFsc
-OeMlrD/oz96d7a4unBDn4qpaOJOXsfpRT0PGmrxy/jcpMiUUW/ntNpa11v5NTeQw
-DRL8DAFbnsNbL8Yz5f+Nps35fBNYBuKTZLJlNTfKByHTO9QjpAQ0WEZEvQKBgQCi
-Ovm83EuYVSKmvxcE6Tyx/8lVqTOO2Vn7wweQlD4/lVujvE0S2L8L+XSS9w5K+GzW
-eFz10p3zarbw80YJ30L5bSEmjVE43BUZR4woMzM4M6dUsiTm1HshIE2b4ALZ0uZ/
-Ye794ceXL9nmSrVLqFsaQZLNFPCwwYb4FiyRry9lZwKBgAO9VbWcN8SEeBDKo3z8
-yRbRTc6sI+AdKY44Dfx0tqOPmTjO3mE4X1GU4sbfD2Bvg3DdjwTuxxC/jHaKu0GG
-dTM0CbrZGbDAj7E87SOcN/PWEeBckSvuQq5H3DQfwIpTmlS1l5oZn9CxRGbLqC2G
-ifnel8XWUG0ROybsr1tk4mzW
------END PRIVATE KEY-----

cmd/sampleconf/random/logging/logserver.crt

@@ -1,21 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDiDCCAnCgAwIBAgIJALhp/WvTQeg/MA0GCSqGSIb3DQEBCwUAMFkxCzAJBgNV
-BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
-aWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMMCWxvZ3NlcnZlcjAeFw0xNzA5MjQxMjI3
-MDRaFw0yNzA5MjIxMjI3MDRaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21l
-LVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNV
-BAMMCWxvZ3NlcnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKs3
-TLYfXhV3hap71tOkhPQlM+m0EKRAo8Nua50Cci5UhDo4JkVpyYok1h+NFkqmjU2b
-IiIuGvsZZPOWYjbWWnSJE4+n5pBFBzcfNQ4d8xVxjANImFn6Tcehhj0WkbDIv/Ge
-364XUgywS7u3EGQj/FO7vZ8KHlUxBHNuPIOPHftwIVRyleh5K32UyBaSpSmnqGos
-rvI1byMuznavcZpOs4vlebZ+Jy6a20iKf9fj/0f0t0O+F5x3JIk07D3zSywhJ4RM
-M0mGIUmYXbh2SMh+f61KDZLDANpz/pMAPbUJe0mxEtBf0tnwK1gEqc3SLwA0EwiM
-8Hnn2iaH5Ln20UE3LOkCAwEAAaNTMFEwHQYDVR0OBBYEFDXoDcwx9SngzZcRYCeP
-BplBecfiMB8GA1UdIwQYMBaAFDXoDcwx9SngzZcRYCePBplBecfiMA8GA1UdEwEB
-/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBADyNvs4AA91x3gurQb1pcPVhK6nR
-mkYSTN1AsDKSRi/X2iCUmR7G7FlF7XW8mntTpHvVzcs+gr94WckH5wqEOA5iZnaw
-PXUWexmdXUge4hmC2q6kBQ5e2ykhSJMRVZXvOLZOZV9qitceamHESV1cKZSNMvZM
-aCSVA1RK61/nUzs04pVp5PFPv9gFxJp9ki39FYFdsgZmM5RZ5I/FqxxvTJzu4RnH
-VPjsMopzARYwJw6dV2bKdFSYOE8B/Vs3Yv0GxjrABw2ko4PkBPTjLIz22x6+Hd9r
-K9BQi4pVmQfvppF5+SORSftlHSS+N47b0DD1rW1f5R6QGi71dFuJGikOwvY=
------END CERTIFICATE-----

cmd/sampleconf/random/logging/logserver.key

@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCrN0y2H14Vd4Wq
-e9bTpIT0JTPptBCkQKPDbmudAnIuVIQ6OCZFacmKJNYfjRZKpo1NmyIiLhr7GWTz
-lmI21lp0iROPp+aQRQc3HzUOHfMVcYwDSJhZ+k3HoYY9FpGwyL/xnt+uF1IMsEu7
-txBkI/xTu72fCh5VMQRzbjyDjx37cCFUcpXoeSt9lMgWkqUpp6hqLK7yNW8jLs52
-r3GaTrOL5Xm2ficumttIin/X4/9H9LdDvhecdySJNOw980ssISeETDNJhiFJmF24
-dkjIfn+tSg2SwwDac/6TAD21CXtJsRLQX9LZ8CtYBKnN0i8ANBMIjPB559omh+S5
-9tFBNyzpAgMBAAECggEBAIY8ZwJq+WKvQLb3POjWFf8so9TY/ispGrwAeJKy9j5o
-uPrERw0o8YBDfTVjclS43BQ6Srqtly3DLSjlgL8ps+WmCxYYN2ZpGE0ZRIl65bis
-O2/fnML+wbiAZTTD2xnVatfPDeP6GLQmDFpyHoHEzPIBQZvNXRbBxZGSnhMvQ/x7
-FhqSBQG4kf3b1XDCENIbFEVOBOCg7WtMiIgjEGS7QnW3I65/Zt+Ts1LXRZbz+6na
-Gmi0PGHA/oLUh1NRzsF4zuZn6fFzja5zw4mkt+JvCWEoxg1QhRAxRp6QQwmZ6MIc
-1rw1D4Z+c5UEKyqHeIwZj4M6UNPhCfTXVm47c9eSiGECgYEA4U8pB+7eRo2fqX0C
-nWsWMcmsULJvwplQnUSFenUayPn3E8ammS/ZBHksoKhj82vwIdDbtS1hQZn8Bzsi
-atc8au0wz0YRDcVDzHX4HknXVQayHtP/FTPeSr5hwpoY8vhEbySuxBTBkXCrp4dx
-u5ErfOiYEP3Q1ZvPRywelrATu20CgYEAwonV5dgOcen/4oAirlnvufc2NfqhAQwJ
-FJ/JSVMAcXxPYu3sZMv0dGWrX8mLc+P1+XMCuV/7eBM/vU2LbDzmpeUV8sJfB2jw
-wyKqKXZwBgeq60btriA4f+0ElwRGgU2KSiniUuuTX2JmyftFQx4cVAQRCFk27NY0
-09psSsYyre0CgYBo6unabdtH029EB5iOIW3GZXk+Yrk0TxyA/4WAjsOYTv5FUT4H
-G4bdVGf5sDBLDDpYJOAKsEUXvVLlMx5FzlCuIiGWg7QxS2jU7yJJSG1jhKixPlsM
-Toj3GUyAyC1SB1Ymw1g2qsuwpFzquGG3zFQJ6G3Xi7oRnmqZY+wik3+8yQKBgB11
-SdKYOPe++2SNCrNkIw0CBk9+OEs0S1u4Jn7X9sU4kbzlUlqhF89YZe8HUfqmlmTD
-qbHwet/f6lL8HxSw1Cxi2EP+cu1oUqz53tKQgL4pAxTFlNA9SND2Ty+fEh4aY8p/
-NSphSduzxuTnC8HyGVAPnZSqDcsnVLCP7r4T7TCxAoGAbJygkkk/gZ9pT4fZoIaq
-8CMR8FTfxtkwCuZsWccSMUOWtx9nqet3gbCpKHfyoYZiKB4ke+lnUz4uFS16Y3hG
-kN0hFfvfoNa8eB2Ox7vs60cMMfWJac0H7KSaDDy+EvbhE2KtQADT0eWxMyhzGR8p
-5CbIivB0QCjeQIA8dOQpE8E=
------END PRIVATE KEY-----

cmd/sampleconf/zrepl.yml

@@ -7,7 +7,7 @@ remotes:
         port: 22
         identity_file: /etc/zrepl/identities/offsite_backups
 
-push:
+pushs:
 
   offsite:
     to: offsite_backups
@@ -17,7 +17,7 @@ push:
       "tank/usr/home<": ok,
     }
 
-pull:
+pulls:
 
   offsite:
     from: offsite_backups
@@ -35,7 +35,7 @@ pull:
       "tank/usr/home":"mirrorpool/foo/bar"
     }
 
-sink:
+sinks:
 
   db1:
     mapping: {
@@ -61,7 +61,7 @@ sink:
     }
 
 
-pull_acl:
+pull_acls:
 
   office_backup:
     filter: {
@@ -95,8 +95,6 @@ prune:
     snapshot_filter: {
       prefix: zrepl_hfbak_
     }
-    repeat:
-      interval: 10s
 
 autosnap:
 

cmd/stdinserver.go

@@ -2,18 +2,17 @@ package cmd
 
 import (
 	"fmt"
-	"os"
-
-	"github.com/ftrvxmtrx/fd"
 	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/sshbytestream"
 	"io"
-	"log"
+	golog "log"
-	"net"
+	"os"
 )
 
 var StdinserverCmd = &cobra.Command{
 	Use:   "stdinserver CLIENT_IDENTITY",
-	Short: "start in stdinserver mode (from authorized_keys file)",
+	Short: "start in stdin server mode (from authorized_keys file)",
 	Run:   cmdStdinServer,
 }
 
@@ -23,69 +22,49 @@ func init() {
 
 func cmdStdinServer(cmd *cobra.Command, args []string) {
 
-	log := log.New(os.Stderr, "", log.LUTC|log.Ldate|log.Ltime)
+	var err error
-
+	defer func() {
-	die := func() {
+		if err != nil {
-		log.Printf("stdinserver exiting after fatal error")
+			log.Printf("stdinserver exiting with error: %s", err)
-		os.Exit(1)
+			os.Exit(1)
-	}
+		}
-
+	}()
-	conf, err := ParseConfig(rootArgs.configFile)
-	if err != nil {
-		log.Printf("error parsing config: %s", err)
-		die()
-	}
 
 	if len(args) != 1 || args[0] == "" {
-		err = fmt.Errorf("must specify client_identity as positional argument")
+		err = fmt.Errorf("must specify client identity as positional argument")
-		die()
+		return
 	}
 	identity := args[0]
 
-	unixaddr, err := stdinserverListenerSocket(conf.Global.Serve.Stdinserver.SockDir, identity)
+	pullACL, ok := conf.PullACLs[identity]
-	if err != nil {
+	if !ok {
-		log.Printf("%s", err)
+		err = fmt.Errorf("could not find PullACL for identity '%s'", identity)
-		os.Exit(1)
+		return
 	}
 
-	log.Printf("opening control connection to zrepld via %s", unixaddr)
+	var sshByteStream io.ReadWriteCloser
-	conn, err := net.DialUnix("unix", nil, unixaddr)
+	if sshByteStream, err = sshbytestream.Incoming(); err != nil {
-	if err != nil {
+		return
-		log.Printf("error connecting to zrepld: %s", err)
-		die()
 	}
 
-	log.Printf("sending stdin and stdout fds to zrepld")
+	sinkMapping := func(identity string) (m DatasetMapping, err error) {
-	err = fd.Put(conn, os.Stdin, os.Stdout)
+		sink, ok := conf.Sinks[identity]
-	if err != nil {
-		log.Printf("error: %s", err)
-		die()
-	}
-
-	log.Printf("waiting for zrepld to close control connection")
-	for {
-
-		var buf [64]byte
-		n, err := conn.Read(buf[:])
-		if err == nil && n != 0 {
-			log.Printf("protocol error: read expected to timeout or EOF returned bytes")
-		}
-
-		if err == io.EOF {
-			log.Printf("zrepld closed control connection, terminating")
-			break
-		}
-
-		neterr, ok := err.(net.Error)
 		if !ok {
-			log.Printf("received unexpected error type: %T %s", err, err)
+			return nil, fmt.Errorf("could not find sink for identity '%s'", identity)
-			die()
 		}
-		if !neterr.Timeout() {
+		return sink, nil
-			log.Printf("receivd unexpected net.Error (not a timeout): %s", neterr)
+	}
-			die()
+
-		}
+	sinkLogger := golog.New(logOut, fmt.Sprintf("sink[%s] ", identity), logFlags)
-		// Read timed out, as expected
+	handler := Handler{
+		Logger:          sinkLogger,
+		SinkMappingFunc: sinkMapping,
+		PullACL:         pullACL,
+	}
+
+	if err = rpc.ListenByteStreamRPC(sshByteStream, identity, handler, sinkLogger); err != nil {
+		log.Printf("listenbytestreamerror: %#v\n", err)
+		os.Exit(1)
 	}
 
 	return

cmd/test.go

@@ -1,214 +0,0 @@
-package cmd
-
-import (
-	"os"
-
-	"bytes"
-	"context"
-	"fmt"
-	"sort"
-	"strings"
-
-	"github.com/kr/pretty"
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/zfs"
-	"time"
-)
-
-var testCmd = &cobra.Command{
-	Use:              "test",
-	Short:            "test configuration",
-	PersistentPreRun: testCmdGlobalInit,
-}
-
-var testCmdGlobal struct {
-	log  Logger
-	conf *Config
-}
-
-var testConfigSyntaxCmd = &cobra.Command{
-	Use:   "config",
-	Short: "parse config file and dump parsed datastructure",
-	Run:   doTestConfig,
-}
-
-var testDatasetMapFilter = &cobra.Command{
-	Use:     "pattern jobname test/zfs/dataset/path",
-	Short:   "test dataset mapping / filter specified in config",
-	Example: ` zrepl test pattern prune.clean_backups tank/backups/legacyscript/foo`,
-	Run:     doTestDatasetMapFilter,
-}
-
-var testPrunePolicyArgs struct {
-	side        PrunePolicySide
-	showKept    bool
-	showRemoved bool
-}
-
-var testPrunePolicyCmd = &cobra.Command{
-	Use:   "prune jobname",
-	Short: "do a dry-run of the pruning part of a job",
-	Run:   doTestPrunePolicy,
-}
-
-func init() {
-	RootCmd.AddCommand(testCmd)
-	testCmd.AddCommand(testConfigSyntaxCmd)
-	testCmd.AddCommand(testDatasetMapFilter)
-
-	testPrunePolicyCmd.Flags().VarP(&testPrunePolicyArgs.side, "side", "s", "prune_lhs (left) or prune_rhs (right)")
-	testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showKept, "kept", false, "show kept snapshots")
-	testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showRemoved, "removed", true, "show removed snapshots")
-	testCmd.AddCommand(testPrunePolicyCmd)
-}
-
-func testCmdGlobalInit(cmd *cobra.Command, args []string) {
-
-	out := logger.NewOutlets()
-	out.Add(WriterOutlet{&NoFormatter{}, os.Stdout}, logger.Info)
-	log := logger.NewLogger(out, 1*time.Second)
-	testCmdGlobal.log = log
-
-	var err error
-	if testCmdGlobal.conf, err = ParseConfig(rootArgs.configFile); err != nil {
-		testCmdGlobal.log.Printf("error parsing config file: %s", err)
-		os.Exit(1)
-	}
-
-}
-
-func doTestConfig(cmd *cobra.Command, args []string) {
-
-	log, conf := testCmdGlobal.log, testCmdGlobal.conf
-
-	log.Printf("config ok")
-	log.Printf("%# v", pretty.Formatter(conf))
-	return
-}
-
-func doTestDatasetMapFilter(cmd *cobra.Command, args []string) {
-
-	log, conf := testCmdGlobal.log, testCmdGlobal.conf
-
-	if len(args) != 2 {
-		log.Printf("specify job name as first postitional argument, test input as second")
-		log.Printf(cmd.UsageString())
-		os.Exit(1)
-	}
-	n, i := args[0], args[1]
-
-	jobi, err := conf.LookupJob(n)
-	if err != nil {
-		log.Printf("%s", err)
-		os.Exit(1)
-	}
-
-	var mf *DatasetMapFilter
-	switch j := jobi.(type) {
-	case *PullJob:
-		mf = j.Mapping
-	case *SourceJob:
-		mf = j.Datasets
-	case *LocalJob:
-		mf = j.Mapping
-	default:
-		panic("incomplete implementation")
-	}
-
-	ip, err := zfs.NewDatasetPath(i)
-	if err != nil {
-		log.Printf("cannot parse test input as ZFS dataset path: %s", err)
-		os.Exit(1)
-	}
-
-	if mf.filterMode {
-		pass, err := mf.Filter(ip)
-		if err != nil {
-			log.Printf("error evaluating filter: %s", err)
-			os.Exit(1)
-		}
-		log.Printf("filter result: %v", pass)
-	} else {
-		res, err := mf.Map(ip)
-		if err != nil {
-			log.Printf("error evaluating mapping: %s", err)
-			os.Exit(1)
-		}
-		toStr := "NO MAPPING"
-		if res != nil {
-			toStr = res.ToString()
-		}
-		log.Printf("%s => %s", ip.ToString(), toStr)
-
-	}
-
-}
-
-func doTestPrunePolicy(cmd *cobra.Command, args []string) {
-
-	log, conf := testCmdGlobal.log, testCmdGlobal.conf
-
-	if cmd.Flags().NArg() != 1 {
-		log.Printf("specify job name as first positional argument")
-		log.Printf(cmd.UsageString())
-		os.Exit(1)
-	}
-
-	jobname := cmd.Flags().Arg(0)
-	jobi, err := conf.LookupJob(jobname)
-	if err != nil {
-		log.Printf("%s", err)
-		os.Exit(1)
-	}
-
-	jobp, ok := jobi.(PruningJob)
-	if !ok {
-		log.Printf("job doesn't do any prunes")
-		os.Exit(0)
-	}
-
-	log.Printf("job dump:\n%s", pretty.Sprint(jobp))
-
-	pruner, err := jobp.Pruner(testPrunePolicyArgs.side, true)
-	if err != nil {
-		log.Printf("cannot create test pruner: %s", err)
-		os.Exit(1)
-	}
-
-	log.Printf("start pruning")
-
-	ctx := context.WithValue(context.Background(), contextKeyLog, log)
-	result, err := pruner.Run(ctx)
-	if err != nil {
-		log.Printf("error running pruner: %s", err)
-		os.Exit(1)
-	}
-
-	sort.Slice(result, func(i, j int) bool {
-		return strings.Compare(result[i].Filesystem.ToString(), result[j].Filesystem.ToString()) == -1
-	})
-
-	var b bytes.Buffer
-	for _, r := range result {
-		fmt.Fprintf(&b, "%s\n", r.Filesystem.ToString())
-
-		if testPrunePolicyArgs.showKept {
-			fmt.Fprintf(&b, "\tkept:\n")
-			for _, v := range r.Keep {
-				fmt.Fprintf(&b, "\t- %s\n", v.Name)
-			}
-		}
-
-		if testPrunePolicyArgs.showRemoved {
-			fmt.Fprintf(&b, "\tremoved:\n")
-			for _, v := range r.Remove {
-				fmt.Fprintf(&b, "\t- %s\n", v.Name)
-			}
-		}
-
-	}
-
-	log.Printf("pruning result:\n%s", b.String())
-
-}

docs/content/_index.md

@@ -4,44 +4,39 @@ title = "zrepl - ZFS replication"
 
 # zrepl - ZFS replication
 
-{{% notice info %}}
+`zrepl` is a tool for replicating ZFS filesystems.
-zrepl as well as this documentation is still under active development.
-Use & test at your own risk ;)
-{{% /notice %}}
 
-## Getting started
+{{% panel theme="danger" header="Important" %}}
-
+`zrepl` as well as this documentation is still under active
-The [5 minute tutorial setup](/tutorial/) gives you a first impression.
+development. Some of the features below are not implemented yet. Use & test at your own risk ;)
+{{% / panel  %}}
 
 ## Main Features
 
-* Filesystem Replication
+* filesystem replication
-    * [x] Local & Remote
+    * local & over network (SSH)
-    * [x] Pull mode
+    * push & pull mode
-    * [ ] Push mode
+    * snapshots & bookmarks support
-    * [x] Access control checks when pulling datasets
+    * feature-negotiation for
-    * [x] [Flexible mapping]({{< ref "configuration/map_filter_syntax.md" >}}) rules
+        * resumable `send & receive`
-    * [x] Bookmarks support
+        * compressed `send & receive`
-    * [ ] Feature-negotiation for
+        * raw encrypted `send & receive` (as soon as it is available)
-        * Resumable `send & receive`
+    * access control checks when pulling datasets
-        * Compressed `send & receive`
+    * [flexible mappings]({{< ref "configuration/overview.md#mapping-filter-syntax" >}}) for filesystems
-        * Raw encrypted `send & receive` (as soon as it is available)
+* automatic snapshot creation
-* Automatic snapshot creation
+    * periodic interval
-    * [x] Ensure fixed time interval between snapshots
+* automatic snapshot pruning
-* Automatic snapshot pruning
+    * [Retention Grid]({{< ref "configuration/snapshots.md#retention-grid" >}})
-    * [x] <i class="fa fa-arrow-right" aria-hidden="true"></i> [Retention Grid]({{< ref "configuration/prune.md#retention-grid" >}})
-* Maintainable implementation in Go
-    * [x] Cross platform
-    * [x] Type safe & testable code
    
 ## Contributing
 
-zrepl is usable but nowhere near a stable release right now -  we are happy about contributors!
+`zrepl` is usable but nowhere near a stable release right now -  we are happy
+about contributors!
 
 * Explore the codebase
     * These docs live in the `docs/` subdirectory
-* Document any non-obvious / confusing / plain broken behavior you encounter when setting up zrepl for the first time
+* Document non-obvious / confusing / plain broken things you encounter when using `zrepl` for the first time
-* Check the *Issues* and *Projects* sections for things to do
+* Check the *Issues* and *Projects* sections for things to do ;)
 
 {{% panel header="<i class='fa fa-github'></i> Getting your code merged"%}}
 [The <i class='fa fa-github'></i> GitHub repository](https://github.com/zrepl/zrepl) is where all development happens.

docs/content/configuration/_index.md

@@ -3,6 +3,4 @@ title = "Configuration"
 alwaysopen = true
 +++
 
-{{% alert theme="warning" %}}Under Construction{{% /alert %}}
-
 {{% children description="true" %}}

docs/content/configuration/example_pull_backup.md

@@ -0,0 +1,9 @@
++++
+title = "Example: Pull Backup"
+description = "Example configuration for a typical pull-backup scenario, e.g. server to server"
++++
+
+Example configuration for a typical pull-backup scenario, e.g. server to server
+
+{{% alert theme="warning"%}}TBD{{% /alert %}}
+

docs/content/configuration/example_push_backup.md

@@ -0,0 +1,8 @@
++++
+title = "Example: Push Backup"
+description = "Example configuration for a typical push-backup scenario, e.g. laptop to NAS"
++++
+
+Example configuration for a typical push-backup scenario, e.g. laptop to NAS
+
+{{% alert theme="warning"%}}TBD{{% /alert %}}

docs/content/configuration/jobs.md

@@ -1,14 +0,0 @@
-+++
-title = "Job Types"
-weight = 10
-+++
-
-{{% alert theme="warning" %}}Under Construction{{% /alert %}}
-
-See `sampleconf/*/*.yml` for examples.
-
-## Source
-
-## Pull
-
-## Local

docs/content/configuration/logging.md

@@ -1,8 +0,0 @@
-+++
-title = "Logging"
-weight = 90
-+++
-
-{{% alert theme="warning" %}}Under Construction{{% /alert %}}
-
-See `cmd/sampleconf/random/logging.yml`.

docs/content/configuration/misc.md

@@ -1,52 +0,0 @@
-+++
-title = "Miscellaneous"
-weight = 100
-+++
-
-## Runtime Directories & UNIX Sockets
-
-zrepl daemon creates various UNIX sockets to allow communicating with it:
-
-* the `stdinserver` transport connects to a socket named after `client_identity` parameter
-* the `control` subcommand connects to a defined control socket
-
-There is no further authentication on these sockets.
-Therefore we have to make sure they can only be created and accessed by `zrepl daemon`.
-
-In fact, `zrepl daemon` will not bind a socket to a path in a directory that is world-accessible.
-
-The directories can be configured in the main configuration file:
-
-```yaml
-global:
-  control:
-    sockpath: /var/run/zrepl/control
-  serve:
-    stdinserver:
-      sockdir: /var/run/zrepl/stdinserver
-```
-
-
-## Super-Verbose Job Debugging
-
-You have probably landed here because you opened an issue on GitHub and some developer told you to do this...
-So just read the annotated comments ;)
-
-```yaml
-
-job:
-- name: ...
-  ...
- # JOB DEBUGGING OPTIONS
-  # should be equal for all job types, but each job implements the debugging itself
-  debug:
-    conn: # debug the io.ReadWriteCloser connection
-      read_dump: /tmp/connlog_read   # dump results of Read() invocations to this file
-      write_dump: /tmp/connlog_write # dump results of Write() invocations to this file
-    rpc: # debug the RPC protocol implementation
-      log: true # log output from rpc layer to the job log
-```
-
-{{% notice info %}}
-Connection dumps will almost certainly contain your or other's private data. Do not share it in a bug report.
-{{% /notice %}}

docs/content/configuration/overview.md

@@ -1,10 +1,60 @@
 +++
-title = "Mapping & Filter Syntax"
+title = "Overview"
-weight = 20
+weight = 100
-description = "How to specify mappings & filters"
+description = "Configuration format, SSH authentication, etc."
 +++
 
-{{% alert theme="warning" %}}Under Construction{{% /alert %}}
+{{% panel header="Recommendation" %}}
+Keep the [sample configuration file](https://github.com/zrepl/zrepl/blob/master/cmd/sampleconf/zrepl.yml) open on the side while reading this document!
+{{% / panel %}}
+
+All configuration is managed in a single YAML file.<br />
+It is structured by sections roughly corresponding to `zrepl` subcommands:
+
+```yaml
+# REPLICATION
+# Remote zrepl instances where pull and push jobs connect to
+remotes: 
+  name_of_remote: #...
+# Push jobs (replication from local to remote)
+pushs: 
+  name_of_push_job: #...
+  name_of_other_push_job: #...
+# pull jobs (replication from remote to local & local to local)
+pulls: 
+  name_of_pull_job: #...
+# mapping incoming pushs to local datasets
+sinks: 
+  client_identity: #...
+# access control for remote pull jobs
+pull_acls: 
+  client_identity: #...
+
+# SNAPSHOT MANAGEMENT
+# Automatic snapshotting of filesystems
+autosnap:
+  name_of_autosnap_job: #... 
+# Automatic pruning of snapshots based on creation date
+prune:
+  name_of_prune_job: #... 
+```
+
+When using `zrepl(8)`, a *subcommand* is passed the *job name* as a positional argument:
+
+```yaml
+autosnap: # subcommand
+  db: # job name
+    prefix: zrepl_ 
+    interval: 10m
+    dataset_filter: {
+      "tank/db<": ok
+    }
+```
+```bash
+$ zrepl autosnap --config zrepl.yml db
+```
+
+Run `zrepl --help` for a list of subcommands and options.
 
 ## Mapping & Filter Syntax
 
@@ -90,3 +140,10 @@ pull_acls:
       "tank/usr/home<": ok,
     }
 ```
+
+## Next up
+
+* [Automating snapshot creation & pruning]({{< ref "configuration/snapshots.md" >}})
+
+* [Replicating filesystems]({{< ref "configuration/replication.md" >}})
+

docs/content/configuration/prune.md

@@ -1,9 +0,0 @@
-+++
-title = "Pruning"
-description = "Automated pruning of snapshots"
-weight = 40
-+++
-
-{{% alert theme="warning" %}}Under Construction{{% /alert %}}
-
-## Retention Grid

docs/content/configuration/replication.md

@@ -1,12 +1,27 @@
 +++
-title = "Transports"
+title = "Filesystem Replication"
-weight = 30
+description = "Replicating filesystems with existing bookmarks & snapshots"
+weight = 300
 +++
 
-{{% alert theme="warning" %}}Under Construction{{% /alert %}}
+{{% alert theme="warning"%}}Under Construction{{% /alert %}}
 
-## Stdinserver
+### Remotes
 
+The `remotes` section specifies remote `zrepl` instances from which to pull from / push backups to:
+
+```yaml
+remotes:
+  offsite_backups:
+    transport:
+      ssh:
+        host: 192.168.122.6
+        user: root
+        port: 22
+        identity_file: /etc/zrepl/identities/offsite_backups
+```
+
+#### SSH Transport
 
 The SSH transport connects to the remote server using the SSH binary in
 `$PATH` and the parameters specified in the `zrepl` config file.
@@ -22,3 +37,4 @@ Check the examples for instructions on how to set this up on your machines!
 The environment variables of the underlying SSH process are cleared. `$SSH_AUTH_SOCK` will not be available. We suggest creating a separate, unencrypted SSH key.
 {{% / panel %}}
 
+

docs/content/configuration/snapshots.md

@@ -0,0 +1,9 @@
++++
+title = "Snapshot Management"
+description = "Automated snapshot creation & pruning"
+weight = 200
++++
+
+{{% alert theme="warning"%}}TBD{{% /alert %}}
+
+## Retention Grid

docs/content/impl/_index.md

@@ -2,8 +2,6 @@
 title = "Implementation Overview"
 +++
 
-{{% alert theme="warning" %}}Under Construction{{% /alert %}}
-
 The following design aspects may convince you that `zrepl` is superior to a hacked-together shell script solution.
 
 ## Language

docs/content/install/_index.md

@@ -1,85 +0,0 @@
-+++
-title = "Installation"
-weight = 20
-+++
-
-{{% notice note %}}
-Note: check out the [tutorial]({{< relref "tutorial/_index.md" >}}) if you want a first impression of zrepl.
-{{% /notice %}}
-
-## User Privileges
-
-It is possible to run zrepl as an unprivileged user in combination with
-[ZFS delegation](https://www.freebsd.org/doc/handbook/zfs-zfs-allow.html).
-
-Also, there is the possibility to run it in a jail on FreeBSD by delegating a dataset to the jail.
-
-However, until we get around documenting those setups, you will have to run zrepl as root or experiment yourself :)
-
-## Installation
-
-zrepl is currently not packaged on any operating system. Signed & versioned releases are planned but not available yet.
-
-Check out the sources yourself, fetch dependencies using dep, compile and install to the zrepl user's `$PATH`.<br />
-**Note**: if the zrepl binary is not in `$PATH`, you will have to adjust the examples in the [tutorial]({{< relref "tutorial/_index.md" >}}).
-
-```bash
-# NOTE: you may want to checkout & build as an unprivileged user
-cd /root
-git clone https://github.com/zrepl/zrepl.git
-cd zrepl
-dep ensure
-go build -o zrepl
-cp zrepl /usr/local/bin/zrepl
-rehash
-# see if it worked
-zrepl help
-```
-
-## Configuration Files
-
-zrepl searches for its main configuration file in the following locations (in that order):
-
-* `/etc/zrepl/zrepl.yml`
-* `/usr/local/etc/zrepl/zrepl.yml`
-
-Alternatively, use CLI flags to specify a config location.
-
-Copy a config from the [tutorial](/tutorial) or the `cmd/sampleconf` directory to one of these locations and customize it to your setup.
-
-## Runtime Directories
-
-Check the the [configuration documentation]({{< relref "configuration/misc.md#runtime-directories-unix-sockets" >}}) for more information.
-For default settings, the following should to the trick.
-
-```bash
-mkdir -p /var/run/zrepl/stdinserver
-chmod -R 0700 /var/run/zrepl
-```
-
-
-## Running the Daemon
-
-All actual work zrepl does is performed by a daemon process.
-
-Logging is configurable via the config file. Please refer to the [logging documentation]({{< relref "configuration/logging.md" >}}).
-
-```bash
-zrepl daemon
-```
-
-There are no *rc(8)* or *systemd.service(5)* service definitions yet. Note the *daemon(8)* utility on FreeBSD.
-
-{{% notice info %}}
-Make sure to actually monitor the error level output of zrepl: some configuration errors will not make the daemon exit.<br />
-Example: if the daemon cannot create the [stdinserver]({{< relref "configuration/transports.md#stdinserver" >}}) sockets
-in the runtime directory, it will emit an error message but not exit because other tasks such as periodic snapshots & pruning are of equal importance.
-{{% / notice %}}
-
-### Restarting
-
-The daemon handles SIGINT and SIGTERM for graceful shutdown.
-
-Graceful shutdown means at worst that a job will not be rescheduled for the next interval.
-
-The daemon exits as soon as all jobs have reported shut down.

docs/content/tutorial/_index.md

@@ -1,173 +0,0 @@
----
-title: "Tutorial"
-weight: 1
----
-
-This tutorial shows how zrepl can be used to implement a ZFS-based pull backup.
-
-We assume the following scenario
-
-* Production server `prod1` with filesystems to back up
-    * `zroot/var/db`
-    * `zroot/usr/home` and all its child filesystems
-    * **except** `zroot/usr/home/paranoid` belonging to a user doing backups themselves
-* Backup server `backups` with
-    * Filesystem `storage/zrepl/pull/prod1` + children dedicated to backups of `prod1`
-
-Our backup solution should fulfill the following requirements:
-
-* Periodically snapshot the filesystems on `prod1` *every 10 minutes*
-* Incrementally replicate these snapshots to `storage/zrepl/pull/prod1/*` on `backups`
-* Keep only very few snapshots on `prod1` to save disk space
-* Keep a fading history (24 hourly, 30 daily, 6 monthly) of snapshots on `backups`
-
-## Analysis
-
-We can model this situation as two jobs:
-
-* A **source job** on `prod1`
-    * Creates the snapshots
-    * Keeps a few snapshots that are also on `prod1` to enable incremental replication
-* A **pull job** on `prod1`
-    * Pulls the snapshots
-    * Fades out snapshots as they age
-
-{{%expand "Side note: why doesn't `backups` take the snapshots right before replication?" %}}
-After all, a little `ssh prod1 'zfs snapshot...'` wouldn't be so bad, right?
-
-As is the case with all distributed systems, the link between `prod1` and `backups` might be down for an hour or two.
-We do not want to sacrifice our required backup resolution of 10 minute intervals for a temporary connection outage.
-
-When the link comes up again, `backups` will happily catch up the 12 snapshots taken by `prod1` in the meantime, without
-a gap in our backup history.
-{{%/expand%}}
-
-## Install zrepl
-
-Follow the [OS-specific installation instructions](/install/) and come back here.
-
-## Configure `backups`
-
-We define a **pull job** named `pull_prod1` in the [main configuration file](/install/#main-configuration-file):
-
-```yaml
-jobs:
-- name: pull_prod1
-  type: pull
-  connect:
-    type: ssh+stdinserver
-    host: prod1.example.com
-    user: root
-    port: 22
-    identity_file: /etc/zrepl/ssh/prod1
-  interval: 10m
-  mapping: {
-    "<":"storage/zrepl/pull/prod1"
-  }
-  initial_repl_policy: most_recent
-  snapshot_prefix: zrepl_pull_backup_
-  prune:
-    policy: grid
-    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-```
-
-The `connect` section instructs zrepl to use the `stdinserver` transport: instead of directly exposing zrepl on `prod1`
-to the internet, `backups` starts the `zrepl stdinserver` on `prod1` via SSH.
-(You can learn more about what happens [here]({{< relref "configuration/transports.md#stdinserver" >}}), or just continue following this tutorial.)
-
-Thus, we need to create the SSH key pair `/etc/zrepl/ssh/prod1{,.pub}` and later pass the public part to `prod1`
-which will use it to authenticate `backups`. Execute the following commands on `backups` as the root user:
-
-```bash
-cd /etc/zrepl
-mkdir -p ssh
-chmod 0700 ssh
-ssh-keygen -t ed25519 -N '' -f /etc/zrepl/ssh/prod1
-```
-You can learn more about the [**pull job** format here]({{< relref "configuration/jobs.md#pull" >}}) but for now we are good to go.
-
-## Configure `prod1`
-
-We define a corresponding **source job** named `pull_backup` in the [main configuration file](/install/#main-configuration-file)
-`zrepl.yml`:
-
-```yaml
-jobs:
-
-- name: pull_backup
-  type: source
-  serve:
-    type: stdinserver
-    client_identity: backups.example.com
-  datasets: {
-    "zroot/var/db": "ok",
-    "zroot/usr/home<": "ok",
-    "zroot/usr/home/paranoid": "!",
-  }
-  snapshot_prefix: zrepl_pull_backup_
-  interval: 10m
-  prune:
-    policy: grid
-    grid: 1x1d(keep=all)
-
-```
-
-The `serve` section corresponds to the `connect` section in the configuration of `backups`.
-
-We need to allow the SSH key on `backups` to execute `zrepl stdinserver backups.example.com` on
-`prod1`. For good measure, we will in fact enforce that only this command can be executed.
-
-Open `/root/.ssh/authorized_keys` and add either of the the following lines, replacing BACKUPS_SSH_PUBKEY at the end
-of the line with the contents of `/etc/zrepl/ssh/prod1.pub` (note the **.pub** !) from `backups`.
-
-```
-# for OpenSSH >= 7.2
-command="zrepl stdinserver backups.example.com",restrict BACKUPS_SSH_PUBKEY
-# for older OpenSSH versions
-command="zrepl stdinserver backups.example.com",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc  BACKUPS_SSH_PUBKEY
-```
-
-{{% alert theme="info" %}}The entries **must** be on a single line, including the replaced BACKUPS_SSH_PUBKEY{{% /alert %}}
-
-Again, you can learn more about the [**source job** format here]({{< ref "configuration/jobs.md#source" >}}).
-
-## Apply Configuration Changes
-
-We need to restart the zrepl daemon on **both** `prod1` and `backups`.
-
-This is [OS-specific](/install/#restarting).
-
-## Watch it Work
-
-A common setup is to `watch` the log output and `zfs list` of snapshots on both machines.
-
-If you like tmux, here is a handy script that works on FreeBSD:
-
-```bash
-pkg install gnu-watch tmux
-tmux new-window
-tmux split-window "tail -f /var/log/zrepl.log"
-tmux split-window "gnu-watch 'zfs list -t snapshot -o name,creation -s creation | grep zrepl_pull_backup_'"
-tmux select-layout tiled
-```
-
-The Linux equivalent might look like this
-
-```bash
-# make sure tmux is installed & let's assume you use systemd + journald
-tmux new-window
-tmux split-window "journalctl -f -u zrepl.service"
-tmux split-window "watch 'zfs list -t snapshot -o name,creation -s creation | grep zrepl_pull_backup_'"
-tmux select-layout tiled
-```
-
-## Summary
-
-Congratulations, you have a working pull backup. Where to go next?
-
-* Read more about [configuration format, options & job types](/configuration/)
-* Learn about [implementation details](/impl/) of zrepl.
-
-
-
-

jobrun/jobrun.go

@@ -0,0 +1,158 @@
+package jobrun
+
+import (
+	"fmt"
+	"time"
+)
+
+type Logger interface {
+	Printf(format string, v ...interface{})
+}
+
+type jobLogger struct {
+	MainLog Logger
+	JobName string
+}
+
+func (l jobLogger) Printf(format string, v ...interface{}) {
+	l.MainLog.Printf(fmt.Sprintf("job[%s]: %s", l.JobName, format), v...)
+}
+
+type Job struct {
+	Name           string
+	RunFunc        func(log Logger) (err error)
+	LastStart      time.Time
+	LastError      error
+	DueAt          time.Time
+	RepeatStrategy RepeatStrategy
+}
+
+type JobRunResult struct {
+	Start  time.Time
+	Finish time.Time
+	Error  error
+}
+
+func (r JobRunResult) RunTime() time.Duration { return r.Finish.Sub(r.Start) }
+
+type RepeatStrategy interface {
+	ShouldReschedule(lastResult JobRunResult) (nextDue time.Time, reschedule bool)
+}
+
+type JobRunner struct {
+	logger           Logger
+	notificationChan chan Job
+	newJobChan       chan Job
+	finishedJobChan  chan Job
+	scheduleTimer    <-chan time.Time
+	pending          map[string]Job
+	running          map[string]Job
+}
+
+func NewJobRunner(logger Logger) *JobRunner {
+	return &JobRunner{
+		logger:           logger,
+		notificationChan: make(chan Job),
+		newJobChan:       make(chan Job),
+		finishedJobChan:  make(chan Job),
+		pending:          make(map[string]Job),
+		running:          make(map[string]Job),
+	}
+}
+
+func (r *JobRunner) AddJobChan() chan<- Job {
+	return r.newJobChan
+}
+
+func (r *JobRunner) AddJob(j Job) {
+	r.newJobChan <- j
+}
+
+func (r *JobRunner) NotificationChan() <-chan Job {
+	return r.notificationChan
+}
+
+func (r *JobRunner) Start() {
+
+loop:
+	select {
+
+	case newJob := <-r.newJobChan:
+
+		_, jobPending := r.pending[newJob.Name]
+		_, jobRunning := r.running[newJob.Name]
+
+		if jobPending || jobRunning {
+			panic("job already in runner")
+		}
+
+		r.pending[newJob.Name] = newJob
+
+	case finishedJob := <-r.finishedJobChan:
+
+		delete(r.running, finishedJob.Name)
+
+		res := JobRunResult{
+			Start:  finishedJob.LastStart,
+			Finish: time.Now(),
+			Error:  finishedJob.LastError,
+		}
+
+		r.logger.Printf("[%s] finished after %s\n", finishedJob.Name, res.RunTime())
+
+		dueTime, resched := finishedJob.RepeatStrategy.ShouldReschedule(res)
+		if resched {
+			r.logger.Printf("[%s] rescheduling to %s", finishedJob.Name, dueTime)
+			finishedJob.DueAt = dueTime
+			r.pending[finishedJob.Name] = finishedJob
+		}
+
+	case <-r.scheduleTimer:
+	}
+
+	if len(r.pending) == 0 && len(r.running) == 0 {
+		return
+	}
+
+	// Find jobs to run
+	var now time.Time
+	var jobPending bool
+
+	now = time.Now()
+	jobPending = false
+
+	nextJobDue := now.Add(time.Minute) // max(pending.Interval)
+
+	for jobName, job := range r.pending {
+
+		if job.DueAt.After(now) {
+			if job.DueAt.Before(nextJobDue) {
+				nextJobDue = job.DueAt
+			}
+			jobPending = true
+			continue
+		}
+		// This job is due, run it
+
+		delete(r.pending, jobName)
+		r.running[jobName] = job
+		job.LastStart = now
+
+		go func(job Job) {
+			jobLog := jobLogger{r.logger, job.Name}
+			if err := job.RunFunc(jobLog); err != nil {
+				job.LastError = err
+				r.notificationChan <- job
+			}
+			r.finishedJobChan <- job
+		}(job)
+
+	}
+
+	if jobPending || len(r.running) > 0 {
+		r.logger.Printf("waiting until %v\n", nextJobDue)
+		r.scheduleTimer = time.After(nextJobDue.Sub(now))
+		goto loop
+	}
+
+}

jobrun/repeat_strategies.go

@@ -0,0 +1,25 @@
+package jobrun
+
+import (
+	"time"
+)
+
+type NoRepeatStrategy struct{}
+
+func (s NoRepeatStrategy) ShouldReschedule(lastResult JobRunResult) (time.Time, bool) {
+	return time.Time{}, false
+}
+
+type PeriodicRepeatStrategy struct {
+	Interval time.Duration
+}
+
+func (s *PeriodicRepeatStrategy) ShouldReschedule(lastResult JobRunResult) (next time.Time, shouldRun bool) {
+	// Don't care about the result
+	shouldRun = true
+	next = lastResult.Start.Add(s.Interval)
+	if next.Before(time.Now()) {
+		next = time.Now()
+	}
+	return
+}

logger/datastructures.go

@@ -1,91 +0,0 @@
-package logger
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"github.com/pkg/errors"
-	"time"
-)
-
-type Level int
-
-func (l Level) MarshalJSON() ([]byte, error) {
-	return json.Marshal(l.String())
-}
-
-const (
-	Debug Level = iota
-	Info
-	Warn
-	Error
-)
-
-func (l Level) Short() string {
-	switch l {
-	case Debug:
-		return "DEBG"
-	case Info:
-		return "INFO"
-	case Warn:
-		return "WARN"
-	case Error:
-		return "ERRO"
-	default:
-		return fmt.Sprintf("%s", l)
-	}
-}
-
-func (l Level) String() string {
-	switch l {
-	case Debug:
-		return "debug"
-	case Info:
-		return "info"
-	case Warn:
-		return "warn"
-	case Error:
-		return "error"
-	default:
-		return fmt.Sprintf("%s", string(l))
-	}
-}
-
-func ParseLevel(s string) (l Level, err error) {
-	for _, l := range AllLevels {
-		if s == l.String() {
-			return l, nil
-		}
-	}
-	return -1, errors.Errorf("unknown level '%s'", s)
-}
-
-// Levels ordered least severe to most severe
-var AllLevels []Level = []Level{Debug, Info, Warn, Error}
-
-type Fields map[string]interface{}
-
-type Entry struct {
-	Level   Level
-	Message string
-	Time    time.Time
-	Fields  Fields
-}
-
-type Outlet interface {
-	// Note: os.Stderr is also used by logger.Logger for reporting errors returned by outlets
-	//       => you probably don't want to log there
-	WriteEntry(ctx context.Context, entry Entry) error
-}
-
-type Outlets map[Level][]Outlet
-
-func NewOutlets() Outlets {
-	return make(Outlets, len(AllLevels))
-}
-
-func (os Outlets) Add(outlet Outlet, minLevel Level) {
-	for _, l := range AllLevels[minLevel:] {
-		os[l] = append(os[l], outlet)
-	}
-}

logger/logger.go

@@ -1,126 +0,0 @@
-package logger
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"runtime/debug"
-	"sync"
-	"time"
-)
-
-const (
-	// The field set by WithError function
-	FieldError = "err"
-)
-
-const DefaultUserFieldCapacity = 5
-const InternalErrorPrefix = "github.com/zrepl/zrepl/logger: "
-
-type Logger struct {
-	fields        Fields
-	outlets       Outlets
-	outletTimeout time.Duration
-
-	mtx *sync.Mutex
-}
-
-func NewLogger(outlets Outlets, outletTimeout time.Duration) *Logger {
-	return &Logger{
-		make(Fields, DefaultUserFieldCapacity),
-		outlets,
-		outletTimeout,
-		&sync.Mutex{},
-	}
-}
-
-func (l *Logger) log(level Level, msg string) {
-
-	l.mtx.Lock()
-	defer l.mtx.Unlock()
-
-	entry := Entry{level, msg, time.Now(), l.fields}
-
-	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(l.outletTimeout))
-	ech := make(chan error)
-
-	louts := l.outlets[level]
-	for i := range louts {
-		go func(ctx context.Context, outlet Outlet, entry Entry) {
-			ech <- outlet.WriteEntry(ctx, entry)
-		}(ctx, louts[i], entry)
-	}
-
-	for fin := 0; fin < len(louts); fin++ {
-		select {
-		case err := <-ech:
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "%s outlet error: %s\n", InternalErrorPrefix, err)
-			}
-		case <-ctx.Done():
-			if ctx.Err() == context.DeadlineExceeded {
-				fmt.Fprintf(os.Stderr, "%s outlets exceeded deadline, keep waiting anyways", InternalErrorPrefix)
-			}
-		}
-	}
-
-	cancel() // make go vet happy
-
-}
-
-func (l *Logger) WithField(field string, val interface{}) *Logger {
-
-	l.mtx.Lock()
-	defer l.mtx.Unlock()
-
-	if _, ok := l.fields[field]; ok {
-		fmt.Fprintf(os.Stderr, "%s caller overwrites field '%s'. Stack:\n%s\n", InternalErrorPrefix, field, string(debug.Stack()))
-	}
-
-	child := &Logger{
-		fields:        make(Fields, len(l.fields)+1),
-		outlets:       l.outlets, // cannot be changed after logger initialized
-		outletTimeout: l.outletTimeout,
-		mtx:           l.mtx,
-	}
-	for k, v := range l.fields {
-		child.fields[k] = v
-	}
-	child.fields[field] = val
-
-	return child
-
-}
-
-func (l *Logger) WithFields(fields Fields) (ret *Logger) {
-	// TODO optimize
-	ret = l
-	for field, value := range fields {
-		ret = l.WithField(field, value)
-	}
-	return ret
-}
-
-func (l *Logger) WithError(err error) *Logger {
-	return l.WithField(FieldError, err.Error())
-}
-
-func (l *Logger) Debug(msg string) {
-	l.log(Debug, msg)
-}
-
-func (l *Logger) Info(msg string) {
-	l.log(Info, msg)
-}
-
-func (l *Logger) Warn(msg string) {
-	l.log(Warn, msg)
-}
-
-func (l *Logger) Error(msg string) {
-	l.log(Error, msg)
-}
-
-func (l *Logger) Printf(format string, args ...interface{}) {
-	l.log(Error, fmt.Sprintf(format, args...))
-}

logger/logger_test.go

@@ -1,47 +0,0 @@
-package logger_test
-
-import (
-	"context"
-	"fmt"
-	"github.com/kr/pretty"
-	"github.com/zrepl/zrepl/logger"
-	"testing"
-	"time"
-)
-
-type TestOutlet struct {
-	Record []logger.Entry
-}
-
-func (o *TestOutlet) WriteEntry(ctx context.Context, entry logger.Entry) error {
-	o.Record = append(o.Record, entry)
-	return nil
-}
-
-func NewTestOutlet() *TestOutlet {
-	return &TestOutlet{make([]logger.Entry, 0)}
-}
-
-func TestLogger_Basic(t *testing.T) {
-
-	outlet_arr := []logger.Outlet{
-		NewTestOutlet(),
-		NewTestOutlet(),
-	}
-
-	outlets := logger.NewOutlets()
-	for _, o := range outlet_arr {
-		outlets.Add(o, logger.Debug)
-	}
-
-	l := logger.NewLogger(outlets, 1*time.Second)
-
-	l.Info("foobar")
-
-	l.WithField("fieldname", "fieldval").Info("log with field")
-
-	l.WithError(fmt.Errorf("fooerror")).Error("error")
-
-	t.Log(pretty.Sprint(outlet_arr))
-
-}

rpc/client.go

@@ -1,136 +0,0 @@
-package rpc
-
-import (
-	"bytes"
-	"encoding/json"
-	"io"
-	"reflect"
-
-	"github.com/pkg/errors"
-)
-
-type Client struct {
-	ml     *MessageLayer
-	logger Logger
-}
-
-func NewClient(rwc io.ReadWriteCloser) *Client {
-	return &Client{NewMessageLayer(rwc), noLogger{}}
-}
-
-func (c *Client) SetLogger(logger Logger, logMessageLayer bool) {
-	c.logger = logger
-	if logMessageLayer {
-		c.ml.logger = logger
-	} else {
-		c.ml.logger = noLogger{}
-	}
-}
-
-func (c *Client) Close() (err error) {
-
-	c.logger.Printf("sending Close request")
-	header := Header{
-		DataType: DataTypeControl,
-		Endpoint: ControlEndpointClose,
-		Accept:   DataTypeControl,
-	}
-	err = c.ml.WriteHeader(&header)
-	if err != nil {
-		return
-	}
-
-	c.logger.Printf("reading Close ACK")
-	ack, err := c.ml.ReadHeader()
-	if err != nil {
-		return err
-	}
-	c.logger.Printf("received Close ACK: %#v", ack)
-	if ack.Error != StatusOK {
-		err = errors.Errorf("error hanging up: remote error (%s) %s", ack.Error, ack.ErrorMessage)
-		return
-	}
-
-	c.logger.Printf("closing MessageLayer")
-	if err = c.ml.Close(); err != nil {
-		c.logger.Printf("error closing RWC: %+v", err)
-		return
-	}
-
-	return err
-}
-
-func (c *Client) recvResponse() (h *Header, err error) {
-	h, err = c.ml.ReadHeader()
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot read header")
-	}
-	// TODO validate
-	return
-}
-
-func (c *Client) writeRequest(h *Header) (err error) {
-	// TODO validate
-	err = c.ml.WriteHeader(h)
-	if err != nil {
-		return errors.Wrap(err, "cannot write header")
-	}
-	return
-}
-
-func (c *Client) Call(endpoint string, in, out interface{}) (err error) {
-
-	var accept DataType
-	{
-		outType := reflect.TypeOf(out)
-		if typeIsIOReaderPtr(outType) {
-			accept = DataTypeOctets
-		} else {
-			accept = DataTypeMarshaledJSON
-		}
-	}
-
-	h := Header{
-		Endpoint: endpoint,
-		DataType: DataTypeMarshaledJSON,
-		Accept:   accept,
-	}
-
-	if err = c.writeRequest(&h); err != nil {
-		return err
-	}
-
-	var buf bytes.Buffer
-	if err = json.NewEncoder(&buf).Encode(in); err != nil {
-		panic("cannot encode 'in' parameter")
-	}
-	if err = c.ml.WriteData(&buf); err != nil {
-		return err
-	}
-
-	rh, err := c.recvResponse()
-	if err != nil {
-		return err
-	}
-	if rh.Error != StatusOK {
-		return &RPCError{rh}
-	}
-
-	rd := c.ml.ReadData()
-
-	switch accept {
-	case DataTypeOctets:
-		c.logger.Printf("setting out to ML data reader")
-		outPtr := out.(*io.Reader) // we checked that above
-		*outPtr = rd
-	case DataTypeMarshaledJSON:
-		c.logger.Printf("decoding marshaled json")
-		if err = json.NewDecoder(c.ml.ReadData()).Decode(out); err != nil {
-			return errors.Wrap(err, "cannot decode marshaled reply")
-		}
-	default:
-		panic("implementation error") // accept is controlled by us
-	}
-
-	return
-}

rpc/frame_layer.go

@@ -1,301 +0,0 @@
-package rpc
-
-import (
-	"bytes"
-	"encoding/binary"
-	"encoding/json"
-	"fmt"
-	"io"
-
-	"github.com/pkg/errors"
-)
-
-type Frame struct {
-	Type          FrameType
-	NoMoreFrames  bool
-	PayloadLength uint32
-}
-
-//go:generate stringer -type=FrameType
-type FrameType uint8
-
-const (
-	FrameTypeHeader  FrameType = 0x01
-	FrameTypeData    FrameType = 0x02
-	FrameTypeTrailer FrameType = 0x03
-	FrameTypeRST     FrameType = 0xff
-)
-
-//go:generate stringer -type=Status
-type Status uint64
-
-const (
-	StatusOK Status = 1 + iota
-	StatusRequestError
-	StatusServerError
-	// Returned when an error occurred but the side at fault cannot be determined
-	StatusError
-)
-
-type Header struct {
-	// Request-only
-	Endpoint string
-	// Data type of body (request & reply)
-	DataType DataType
-	// Request-only
-	Accept DataType
-	// Reply-only
-	Error Status
-	// Reply-only
-	ErrorMessage string
-}
-
-func NewErrorHeader(status Status, format string, args ...interface{}) (h *Header) {
-	h = &Header{}
-	h.Error = status
-	h.ErrorMessage = fmt.Sprintf(format, args...)
-	return
-}
-
-type DataType uint8
-
-const (
-	DataTypeNone DataType = 1 + iota
-	DataTypeControl
-	DataTypeMarshaledJSON
-	DataTypeOctets
-)
-
-const (
-	MAX_PAYLOAD_LENGTH = 4 * 1024 * 1024
-	MAX_HEADER_LENGTH  = 4 * 1024
-)
-
-type frameBridgingReader struct {
-	l         *MessageLayer
-	frameType FrameType
-	// < 0 means no limit
-	bytesLeftToLimit int
-	f                Frame
-}
-
-func NewFrameBridgingReader(l *MessageLayer, frameType FrameType, totalLimit int) *frameBridgingReader {
-	return &frameBridgingReader{l, frameType, totalLimit, Frame{}}
-}
-
-func (r *frameBridgingReader) Read(b []byte) (n int, err error) {
-	if r.bytesLeftToLimit == 0 {
-		r.l.logger.Printf("limit reached, returning EOF")
-		return 0, io.EOF
-	}
-	log := r.l.logger
-	if r.f.PayloadLength == 0 {
-
-		if r.f.NoMoreFrames {
-			r.l.logger.Printf("no more frames flag set, returning EOF")
-			err = io.EOF
-			return
-		}
-
-		log.Printf("reading frame")
-		r.f, err = r.l.readFrame()
-		if err != nil {
-			log.Printf("error reading frame: %+v", err)
-			return 0, err
-		}
-		log.Printf("read frame: %#v", r.f)
-		if r.f.Type != r.frameType {
-			err = errors.Wrapf(err, "expected frame of type %s", r.frameType)
-			return 0, err
-		}
-	}
-	maxread := len(b)
-	if maxread > int(r.f.PayloadLength) {
-		maxread = int(r.f.PayloadLength)
-	}
-	if r.bytesLeftToLimit > 0 && maxread > r.bytesLeftToLimit {
-		maxread = r.bytesLeftToLimit
-	}
-	nb, err := r.l.rwc.Read(b[:maxread])
-	log.Printf("read  %v from rwc\n", nb)
-	if nb < 0 {
-		panic("should not return negative number of bytes")
-	}
-	r.f.PayloadLength -= uint32(nb)
-	r.bytesLeftToLimit -= nb
-	return nb, err // TODO io.EOF for maxread = r.f.PayloadLength ?
-}
-
-type frameBridgingWriter struct {
-	l         *MessageLayer
-	frameType FrameType
-	// < 0 means no limit
-	bytesLeftToLimit int
-	payloadLength    int
-	buffer           *bytes.Buffer
-}
-
-func NewFrameBridgingWriter(l *MessageLayer, frameType FrameType, totalLimit int) *frameBridgingWriter {
-	return &frameBridgingWriter{l, frameType, totalLimit, MAX_PAYLOAD_LENGTH, bytes.NewBuffer(make([]byte, 0, MAX_PAYLOAD_LENGTH))}
-}
-
-func (w *frameBridgingWriter) Write(b []byte) (n int, err error) {
-	for n = 0; n < len(b); {
-		i, err := w.writeUntilFrameFull(b[n:])
-		n += i
-		if err != nil {
-			return n, errors.WithStack(err)
-		}
-	}
-	return
-}
-
-func (w *frameBridgingWriter) writeUntilFrameFull(b []byte) (n int, err error) {
-	if len(b) <= 0 {
-		return
-	}
-	if w.bytesLeftToLimit == 0 {
-		err = errors.Errorf("message exceeds max number of allowed bytes")
-		return
-	}
-	maxwrite := len(b)
-	remainingInFrame := w.payloadLength - w.buffer.Len()
-
-	if maxwrite > remainingInFrame {
-		maxwrite = remainingInFrame
-	}
-	if w.bytesLeftToLimit > 0 && maxwrite > w.bytesLeftToLimit {
-		maxwrite = w.bytesLeftToLimit
-	}
-	w.buffer.Write(b[:maxwrite])
-	w.bytesLeftToLimit -= maxwrite
-	n = maxwrite
-	if w.bytesLeftToLimit == 0 {
-		err = w.flush(true)
-	} else if w.buffer.Len() == w.payloadLength {
-		err = w.flush(false)
-	}
-	return
-}
-
-func (w *frameBridgingWriter) flush(nomore bool) (err error) {
-
-	f := Frame{w.frameType, nomore, uint32(w.buffer.Len())}
-	err = w.l.writeFrame(f)
-	if err != nil {
-		errors.WithStack(err)
-	}
-	_, err = w.buffer.WriteTo(w.l.rwc)
-	return
-}
-
-func (w *frameBridgingWriter) Close() (err error) {
-	return w.flush(true)
-}
-
-type MessageLayer struct {
-	rwc    io.ReadWriteCloser
-	logger Logger
-}
-
-func NewMessageLayer(rwc io.ReadWriteCloser) *MessageLayer {
-	return &MessageLayer{rwc, noLogger{}}
-}
-
-func (l *MessageLayer) Close() (err error) {
-	f := Frame{
-		Type:         FrameTypeRST,
-		NoMoreFrames: true,
-	}
-	if err = l.writeFrame(f); err != nil {
-		l.logger.Printf("error sending RST frame: %s", err)
-		return errors.WithStack(err)
-	}
-	return nil
-}
-
-var RST error = fmt.Errorf("reset frame observed on connection")
-
-func (l *MessageLayer) readFrame() (f Frame, err error) {
-	err = binary.Read(l.rwc, binary.LittleEndian, &f.Type)
-	if err != nil {
-		err = errors.WithStack(err)
-		return
-	}
-	err = binary.Read(l.rwc, binary.LittleEndian, &f.NoMoreFrames)
-	if err != nil {
-		err = errors.WithStack(err)
-		return
-	}
-	err = binary.Read(l.rwc, binary.LittleEndian, &f.PayloadLength)
-	if err != nil {
-		err = errors.WithStack(err)
-		return
-	}
-	if f.Type == FrameTypeRST {
-		l.logger.Printf("read RST frame")
-		err = RST
-		return
-	}
-	if f.PayloadLength > MAX_PAYLOAD_LENGTH {
-		err = errors.Errorf("frame exceeds max payload length")
-		return
-	}
-	return
-}
-
-func (l *MessageLayer) writeFrame(f Frame) (err error) {
-	err = binary.Write(l.rwc, binary.LittleEndian, &f.Type)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-	err = binary.Write(l.rwc, binary.LittleEndian, &f.NoMoreFrames)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-	err = binary.Write(l.rwc, binary.LittleEndian, &f.PayloadLength)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-	if f.PayloadLength > MAX_PAYLOAD_LENGTH {
-		err = errors.Errorf("frame exceeds max payload length")
-		return
-	}
-	return
-}
-
-func (l *MessageLayer) ReadHeader() (h *Header, err error) {
-
-	r := NewFrameBridgingReader(l, FrameTypeHeader, MAX_HEADER_LENGTH)
-	h = &Header{}
-	if err = json.NewDecoder(r).Decode(&h); err != nil {
-		l.logger.Printf("cannot decode marshaled header: %s", err)
-		return nil, err
-	}
-	return h, nil
-}
-
-func (l *MessageLayer) WriteHeader(h *Header) (err error) {
-	w := NewFrameBridgingWriter(l, FrameTypeHeader, MAX_HEADER_LENGTH)
-	err = json.NewEncoder(w).Encode(h)
-	if err != nil {
-		return errors.Wrap(err, "cannot encode header, probably fatal")
-	}
-	w.Close()
-	return
-}
-
-func (l *MessageLayer) ReadData() (reader io.Reader) {
-	r := NewFrameBridgingReader(l, FrameTypeData, -1)
-	return r
-}
-
-func (l *MessageLayer) WriteData(source io.Reader) (err error) {
-	w := NewFrameBridgingWriter(l, FrameTypeData, -1)
-	_, err = io.Copy(w, source)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-	err = w.Close()
-	return
-}

rpc/frametype_string.go

@@ -1,27 +0,0 @@
-// Code generated by "stringer -type=FrameType"; DO NOT EDIT.
-
-package rpc
-
-import "fmt"
-
-const (
-	_FrameType_name_0 = "FrameTypeHeaderFrameTypeDataFrameTypeTrailer"
-	_FrameType_name_1 = "FrameTypeRST"
-)
-
-var (
-	_FrameType_index_0 = [...]uint8{0, 15, 28, 44}
-	_FrameType_index_1 = [...]uint8{0, 12}
-)
-
-func (i FrameType) String() string {
-	switch {
-	case 1 <= i && i <= 3:
-		i -= 1
-		return _FrameType_name_0[_FrameType_index_0[i]:_FrameType_index_0[i+1]]
-	case i == 255:
-		return _FrameType_name_1
-	default:
-		return fmt.Sprintf("FrameType(%d)", i)
-	}
-}

rpc/local.go

@@ -1,63 +0,0 @@
-package rpc
-
-import (
-	"github.com/pkg/errors"
-	"reflect"
-)
-
-type LocalRPC struct {
-	endpoints map[string]reflect.Value
-}
-
-func NewLocalRPC() *LocalRPC {
-	return &LocalRPC{make(map[string]reflect.Value, 0)}
-}
-
-func (s *LocalRPC) RegisterEndpoint(name string, handler interface{}) (err error) {
-	_, ok := s.endpoints[name]
-	if ok {
-		return errors.Errorf("already set up an endpoint for '%s'", name)
-	}
-	ep, err := makeEndpointDescr(handler)
-	if err != nil {
-		return err
-	}
-	s.endpoints[name] = ep.handler
-	return nil
-}
-
-func (s *LocalRPC) Serve() (err error) {
-	panic("local cannot serve")
-}
-
-func (c *LocalRPC) Call(endpoint string, in, out interface{}) (err error) {
-	ep, ok := c.endpoints[endpoint]
-	if !ok {
-		panic("implementation error: implementation should not call local RPC without knowing which endpoints exist")
-	}
-
-	args := []reflect.Value{reflect.ValueOf(in), reflect.ValueOf(out)}
-
-	if err = checkRPCParamTypes(args[0].Type(), args[1].Type()); err != nil {
-		return
-	}
-
-	rets := ep.Call(args)
-
-	if len(rets) != 1 {
-		panic("implementation error: endpoints must have one error ")
-	}
-	if err = checkRPCReturnType(rets[0].Type()); err != nil {
-		panic(err)
-	}
-
-	err = nil
-	if !rets[0].IsNil() {
-		err = rets[0].Interface().(error) // we checked that above
-	}
-	return
-}
-
-func (c *LocalRPC) Close() (err error) {
-	return nil
-}

rpc/resumetransfererrorreason_string.go

@@ -0,0 +1,16 @@
+// Code generated by "stringer -type ResumeTransferErrorReason"; DO NOT EDIT.
+
+package rpc
+
+import "fmt"
+
+const _ResumeTransferErrorReason_name = "ResumeTransferErrorReasonNotImplementedResumeTransferErrorReasonDisabledResumeTransferErrorReasonZFSErrorPermanentResumeTransferErrorReasonZFSErrorMaybeTemporary"
+
+var _ResumeTransferErrorReason_index = [...]uint8{0, 39, 72, 114, 161}
+
+func (i ResumeTransferErrorReason) String() string {
+	if i >= ResumeTransferErrorReason(len(_ResumeTransferErrorReason_index)-1) {
+		return fmt.Sprintf("ResumeTransferErrorReason(%d)", i)
+	}
+	return _ResumeTransferErrorReason_name[_ResumeTransferErrorReason_index[i]:_ResumeTransferErrorReason_index[i+1]]
+}

rpc/rpc.go

@@ -0,0 +1,592 @@
+package rpc
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	. "github.com/zrepl/zrepl/util"
+	"github.com/zrepl/zrepl/zfs"
+	"io"
+	"reflect"
+	"time"
+)
+
+type RPCRequester interface {
+	FilesystemRequest(r FilesystemRequest) (roots []*zfs.DatasetPath, err error)
+	FilesystemVersionsRequest(r FilesystemVersionsRequest) (versions []zfs.FilesystemVersion, err error)
+	InitialTransferRequest(r InitialTransferRequest) (io.Reader, error)
+	IncrementalTransferRequest(r IncrementalTransferRequest) (io.Reader, error)
+	ResumeTransferRequest(r ResumeTransferRequest) (io.Reader, error)
+	PullMeRequest(r PullMeRequest, handler RPCHandler) (err error)
+	CloseRequest(r CloseRequest) (err error)
+	ForceClose() (err error)
+}
+
+type RPCHandler interface {
+	HandleFilesystemRequest(r FilesystemRequest) (roots []*zfs.DatasetPath, err error)
+
+	// returned versions ordered by birthtime, oldest first
+	HandleFilesystemVersionsRequest(r FilesystemVersionsRequest) (versions []zfs.FilesystemVersion, err error)
+
+	HandleInitialTransferRequest(r InitialTransferRequest) (io.Reader, error)
+	HandleIncrementalTransferRequest(r IncrementalTransferRequest) (io.Reader, error)
+	HandleResumeTransferRequest(r ResumeTransferRequest) (io.Reader, error)
+
+	// invert roles, i.e. handler becomes server and performs the requested pull using the client connection
+	HandlePullMeRequest(r PullMeRequest, clientIdentity string, client RPCRequester) (err error)
+}
+
+type Logger interface {
+	Printf(format string, args ...interface{})
+}
+
+const ByteStreamRPCProtocolVersion = 1
+
+type ByteStream interface {
+	io.ReadWriteCloser
+}
+
+type ByteStreamRPC struct {
+	conn           ByteStream
+	log            Logger
+	clientIdentity string
+}
+
+func ConnectByteStreamRPC(conn ByteStream, log Logger) (RPCRequester, error) {
+
+	rpc := ByteStreamRPC{
+		conn: conn,
+		log:  log,
+	}
+
+	// Assert protocol versions are equal
+	err := rpc.ProtocolVersionRequest()
+	if err != nil {
+		return nil, err
+	}
+
+	return rpc, nil
+}
+
+type ByteStreamRPCDecodeJSONError struct {
+	Type       reflect.Type
+	DecoderErr error
+}
+
+func (e ByteStreamRPCDecodeJSONError) Error() string {
+	return fmt.Sprintf("cannot decode %s: %s", e.Type, e.DecoderErr)
+}
+
+func ListenByteStreamRPC(conn ByteStream, clientIdentity string, handler RPCHandler, log Logger) error {
+
+	c := ByteStreamRPC{
+		conn:           conn,
+		log:            log,
+		clientIdentity: clientIdentity,
+	}
+	return c.serverLoop(handler)
+
+}
+
+func (c ByteStreamRPC) serverLoop(handler RPCHandler) error {
+
+	// A request consists of two subsequent chunked JSON objects
+	// Object 1: RequestHeader => contains type of Request Body
+	// Object 2: RequestBody, e.g. IncrementalTransferRequest
+	// A response is always a ResponseHeader followed by bytes to be interpreted
+	// 	as indicated by the ResponseHeader.ResponseType, e.g.
+	// 	 a) a chunked response
+	//   b) or another JSON object
+
+	conn := c.conn
+	log := c.log
+	defer func() {
+		panicObj := recover()
+		// if we just exited, we don't want to close the connection (PullMeRequest depends on this)
+		log.Printf("exiting server loop, panic object %#v", panicObj)
+		if panicObj != nil {
+			conn.Close()
+		}
+	}()
+
+	send := func(r interface{}) {
+		if err := writeChunkedJSON(conn, r); err != nil {
+			panic(err)
+		}
+	}
+
+	sendError := func(id ErrorId, msg string) {
+		r := ResponseHeader{
+			ErrorId:      id,
+			ResponseType: RNONE,
+			Message:      msg,
+		}
+		log.Printf("sending error response: %#v", r)
+		if err := writeChunkedJSON(conn, r); err != nil {
+			log.Printf("error sending error response: %#v", err)
+			panic(err)
+		}
+	}
+
+	recv := func(r interface{}) (err error) {
+		return readChunkedJSON(conn, r)
+	}
+
+	for {
+
+		var header RequestHeader = RequestHeader{}
+		if err := recv(&header); err != nil {
+			sendError(EDecodeHeader, err.Error())
+			return conn.Close()
+		}
+
+		switch header.Type {
+		case RTProtocolVersionRequest:
+			var rq ByteStreamRPCProtocolVersionRequest
+			if err := recv(&rq); err != nil {
+				sendError(EDecodeRequestBody, err.Error())
+				return conn.Close()
+			}
+
+			if rq.ClientVersion != ByteStreamRPCProtocolVersion {
+				sendError(EProtocolVersionMismatch, "")
+				return conn.Close()
+			}
+
+			r := ResponseHeader{
+				RequestId:    header.Id,
+				ResponseType: ROK,
+			}
+			send(&r)
+
+		case RTCloseRequest:
+
+			var rq CloseRequest
+			if err := recv(&rq); err != nil {
+				sendError(EDecodeRequestBody, err.Error())
+				return conn.Close()
+			}
+
+			log.Printf("close request with goodbye: %s", rq.Goodbye)
+
+			send(&ResponseHeader{
+				RequestId:    header.Id,
+				ResponseType: ROK,
+			})
+
+			return conn.Close()
+
+		case RTFilesystemRequest:
+
+			var rq FilesystemRequest
+			if err := recv(&rq); err != nil {
+				sendError(EDecodeRequestBody, "")
+				return conn.Close()
+			}
+
+			roots, err := handler.HandleFilesystemRequest(rq)
+			if err != nil {
+				sendError(EHandler, err.Error())
+				return conn.Close()
+			} else {
+				r := ResponseHeader{
+					RequestId:    header.Id,
+					ResponseType: RFilesystems,
+				}
+				send(&r)
+				send(&roots)
+			}
+
+		case RTFilesystemVersionsRequest:
+
+			var rq FilesystemVersionsRequest
+			if err := recv(&rq); err != nil {
+				sendError(EDecodeRequestBody, err.Error())
+				return err
+			}
+
+			diff, err := handler.HandleFilesystemVersionsRequest(rq)
+			if err != nil {
+				sendError(EHandler, err.Error())
+				return err
+			} else {
+				r := ResponseHeader{
+					RequestId:    header.Id,
+					ResponseType: RFilesystemDiff,
+				}
+				send(&r)
+				send(&diff)
+			}
+
+			log.Printf("finished FilesystemVersionReqeust")
+
+		case RTInitialTransferRequest:
+			var rq InitialTransferRequest
+			if err := recv(&rq); err != nil {
+				sendError(EDecodeRequestBody, "")
+				return conn.Close()
+			}
+			log.Printf("initial transfer request: %#v", rq)
+
+			snapReader, err := handler.HandleInitialTransferRequest(rq)
+			if err != nil {
+				sendError(EHandler, err.Error())
+				return conn.Close()
+			} else {
+
+				r := ResponseHeader{
+					RequestId:    header.Id,
+					ResponseType: RChunkedStream,
+				}
+				send(&r)
+
+				chunker := NewChunker(snapReader)
+				watcher := IOProgressWatcher{Reader: &chunker}
+				watcher.KickOff(1*time.Second, func(p IOProgress) {
+					log.Printf("progress sending initial snapshot stream: %v bytes sent", p.TotalRX)
+				})
+				_, err := io.Copy(conn, &watcher)
+				if err != nil {
+					log.Printf("error sending initial snapshot stream: %s", err)
+					panic(err)
+				}
+				log.Printf("finished sending initial snapshot stream: total %v bytes sent", watcher.Progress().TotalRX)
+			}
+
+		case RTIncrementalTransferRequest:
+
+			var rq IncrementalTransferRequest
+			if err := recv(&rq); err != nil {
+				sendError(EDecodeRequestBody, "")
+				return conn.Close()
+			}
+
+			snapReader, err := handler.HandleIncrementalTransferRequest(rq)
+			if err != nil {
+				sendError(EHandler, err.Error())
+			} else {
+
+				r := ResponseHeader{
+					RequestId:    header.Id,
+					ResponseType: RChunkedStream,
+				}
+				send(&r)
+
+				chunker := NewChunker(snapReader)
+
+				watcher := IOProgressWatcher{Reader: &chunker}
+				watcher.KickOff(1*time.Second, func(p IOProgress) {
+					log.Printf("progress sending incremental snapshot stream: %v bytes sent", p.TotalRX)
+				})
+				_, err := io.Copy(conn, &watcher)
+				if err != nil {
+					panic(err)
+				}
+				log.Printf("finished sending incremental snapshot stream: total %v bytes sent", watcher.Progress().TotalRX)
+			}
+
+		case RTResumeTransferRequest:
+
+			var rq ResumeTransferRequest
+			if err := recv(&rq); err != nil {
+				sendError(EDecodeRequestBody, "")
+				return conn.Close()
+			}
+
+			stream, err := handler.HandleResumeTransferRequest(rq)
+			if err != nil {
+				sendError(EHandler, err.Error())
+			} else {
+				r := ResponseHeader{
+					RequestId:    header.Id,
+					ResponseType: RChunkedStream,
+				}
+				send(&r)
+
+				chunker := NewChunker(stream)
+
+				watcher := IOProgressWatcher{Reader: &chunker}
+				watcher.KickOff(1*time.Second, func(p IOProgress) {
+					log.Printf("progress sending resumed stream: %v bytes sent", p.TotalRX)
+				})
+				_, err := io.Copy(conn, &watcher)
+				if err != nil {
+					panic(err)
+				}
+				log.Printf("finished sending resumed stream: total %v bytes sent", watcher.Progress().TotalRX)
+			}
+
+		case RTPullMeRequest:
+
+			var rq PullMeRequest
+			if err := recv(&rq); err != nil {
+				sendError(EDecodeRequestBody, err.Error())
+
+				return conn.Close()
+			}
+
+			if rq.Finished {
+				// we are the client that sent a PullMeRequest with Finished = false
+				// and then entered this server loop
+				log.Printf("PullMeRequest.Finished == true, exiting server loop")
+				send(ResponseHeader{
+					RequestId:    header.Id,
+					ResponseType: ROK,
+				})
+				return nil
+			}
+
+			// We are a server receiving a PullMeRequest from a client
+
+			log.Printf("confirming PullMeRequest")
+			send(ResponseHeader{
+				RequestId:    header.Id,
+				ResponseType: ROK,
+			})
+
+			log.Printf("pulling from client '%s', expecting client is in server loop", c.clientIdentity)
+			if c.clientIdentity == "" || c.clientIdentity == LOCAL_TRANSPORT_IDENTITY {
+				err := fmt.Errorf("client has bad name: '%s'", c.clientIdentity)
+				log.Printf(err.Error())
+				panic(err)
+			}
+			pullErr := handler.HandlePullMeRequest(rq, c.clientIdentity, c)
+			if pullErr != nil {
+				log.Printf("pulling failed with error: %s", pullErr)
+				panic(pullErr)
+			}
+
+			log.Printf("finished handling PullMeRequest, sending Finished = true")
+			req := PullMeRequest{Finished: true}
+			c.sendRequestReceiveHeader(req, ROK)
+
+		default:
+			sendError(EUnknownRequestType, "")
+			return conn.Close()
+		}
+	}
+
+	return nil
+}
+
+func writeChunkedJSON(conn io.Writer, r interface{}) (err error) {
+	var buf bytes.Buffer
+	encoder := json.NewEncoder(&buf)
+	encoder.Encode(r)
+	ch := NewChunker(&buf)
+	_, err = io.Copy(conn, &ch)
+	return
+}
+
+func readChunkedJSON(conn io.ReadWriter, r interface{}) (err error) {
+	unch := NewUnchunker(conn)
+	dec := json.NewDecoder(unch)
+	err = dec.Decode(r)
+	if err != nil {
+		err = ByteStreamRPCDecodeJSONError{
+			Type:       reflect.TypeOf(r),
+			DecoderErr: err,
+		}
+	}
+	closeErr := unch.Close()
+	if err == nil && closeErr != nil {
+		err = closeErr
+	}
+	return
+}
+
+func inferRequestType(v interface{}) (RequestType, error) {
+	switch v.(type) {
+	case ByteStreamRPCProtocolVersionRequest:
+		return RTProtocolVersionRequest, nil
+	case FilesystemRequest:
+		return RTFilesystemRequest, nil
+	case FilesystemVersionsRequest:
+		return RTFilesystemVersionsRequest, nil
+	case InitialTransferRequest:
+		return RTInitialTransferRequest, nil
+	case IncrementalTransferRequest:
+		return RTIncrementalTransferRequest, nil
+	case ResumeTransferRequest:
+		return RTResumeTransferRequest, nil
+	case PullMeRequest:
+		return RTPullMeRequest, nil
+	case CloseRequest:
+		return RTCloseRequest, nil
+	default:
+		return 0, errors.New(fmt.Sprintf("cannot infer request type for type '%v'",
+			reflect.TypeOf(v)))
+	}
+}
+
+func genUUID() [16]byte {
+	return [16]byte{} // TODO
+}
+
+func (c ByteStreamRPC) sendRequest(v interface{}) (err error) {
+
+	var rt RequestType
+
+	if rt, err = inferRequestType(v); err != nil {
+		return
+	}
+
+	h := RequestHeader{
+		Type: rt,
+		Id:   genUUID(),
+	}
+
+	if err = writeChunkedJSON(c.conn, h); err != nil {
+		return
+	}
+	if err = writeChunkedJSON(c.conn, v); err != nil {
+		return
+	}
+
+	return
+}
+
+func (c ByteStreamRPC) expectResponseType(rt ResponseType) (err error) {
+
+	var h ResponseHeader
+	if err = readChunkedJSON(c.conn, &h); err != nil {
+		return
+	}
+
+	if h.ResponseType != rt {
+		return errors.New(fmt.Sprintf("unexpected response type in response header: got %#v, expected %#v. response header: %#v",
+			h.ResponseType, rt, h))
+	}
+	return
+}
+
+func (c ByteStreamRPC) sendRequestReceiveHeader(request interface{}, rt ResponseType) (err error) {
+
+	if err = c.sendRequest(request); err != nil {
+		return err
+	}
+
+	if err = c.expectResponseType(rt); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c ByteStreamRPC) ProtocolVersionRequest() (err error) {
+	b := ByteStreamRPCProtocolVersionRequest{
+		ClientVersion: ByteStreamRPCProtocolVersion,
+	}
+
+	// OK response means the remote side can cope with our protocol version
+	return c.sendRequestReceiveHeader(b, ROK)
+}
+
+func (c ByteStreamRPC) FilesystemRequest(r FilesystemRequest) (roots []*zfs.DatasetPath, err error) {
+
+	if err = c.sendRequestReceiveHeader(r, RFilesystems); err != nil {
+		return
+	}
+
+	roots = make([]*zfs.DatasetPath, 0)
+
+	if err = readChunkedJSON(c.conn, &roots); err != nil {
+		return
+	}
+
+	return
+}
+
+func (c ByteStreamRPC) FilesystemVersionsRequest(r FilesystemVersionsRequest) (versions []zfs.FilesystemVersion, err error) {
+
+	if err = c.sendRequestReceiveHeader(r, RFilesystemDiff); err != nil {
+		return
+	}
+
+	err = readChunkedJSON(c.conn, &versions)
+	return
+}
+
+func (c ByteStreamRPC) InitialTransferRequest(r InitialTransferRequest) (unchunker io.Reader, err error) {
+
+	if err = c.sendRequestReceiveHeader(r, RChunkedStream); err != nil {
+		return
+	}
+
+	unchunker = NewUnchunker(c.conn)
+	return
+}
+
+func (c ByteStreamRPC) IncrementalTransferRequest(r IncrementalTransferRequest) (unchunker io.Reader, err error) {
+	if err = c.sendRequestReceiveHeader(r, RChunkedStream); err != nil {
+		return
+	}
+	unchunker = NewUnchunker(c.conn)
+	return
+}
+
+func (c ByteStreamRPC) ResumeTransferRequest(r ResumeTransferRequest) (io.Reader, error) {
+	// TODO reconstruct ResumeTransferError
+	if err := c.sendRequestReceiveHeader(r, RChunkedStream); err != nil {
+		return nil, err
+	}
+	return NewUnchunker(c.conn), nil
+}
+
+func (c ByteStreamRPC) PullMeRequest(r PullMeRequest, handler RPCHandler) (err error) {
+	err = c.sendRequestReceiveHeader(r, ROK)
+	return c.serverLoop(handler)
+}
+
+func (c ByteStreamRPC) CloseRequest(r CloseRequest) (err error) {
+	if err = c.sendRequestReceiveHeader(r, ROK); err != nil {
+		return
+	}
+	err = c.conn.Close()
+	return
+}
+
+func (c ByteStreamRPC) ForceClose() (err error) {
+	return c.conn.Close()
+}
+
+type LocalRPC struct {
+	handler RPCHandler
+}
+
+func ConnectLocalRPC(handler RPCHandler) RPCRequester {
+	return LocalRPC{handler}
+}
+
+func (c LocalRPC) FilesystemRequest(r FilesystemRequest) (roots []*zfs.DatasetPath, err error) {
+	return c.handler.HandleFilesystemRequest(r)
+}
+
+func (c LocalRPC) FilesystemVersionsRequest(r FilesystemVersionsRequest) (versions []zfs.FilesystemVersion, err error) {
+	return c.handler.HandleFilesystemVersionsRequest(r)
+}
+
+func (c LocalRPC) InitialTransferRequest(r InitialTransferRequest) (io.Reader, error) {
+	return c.handler.HandleInitialTransferRequest(r)
+}
+
+func (c LocalRPC) IncrementalTransferRequest(r IncrementalTransferRequest) (reader io.Reader, err error) {
+	reader, err = c.handler.HandleIncrementalTransferRequest(r)
+	return
+}
+
+func (c LocalRPC) ResumeTransferRequest(r ResumeTransferRequest) (io.Reader, error) {
+	return c.handler.HandleResumeTransferRequest(r)
+}
+
+func (c LocalRPC) PullMeRequest(r PullMeRequest, handler RPCHandler) (err error) {
+	// The config syntactically only allows local Pulls, hence this is never called
+	// In theory, the following line should work:
+	// return handler.HandlePullMeRequest(r, LOCAL_TRANSPORT_IDENTITY, c)
+	panic("internal inconsistency: local pull me request unsupported")
+}
+
+func (c LocalRPC) CloseRequest(r CloseRequest) error { return nil }
+
+func (c LocalRPC) ForceClose() error { return nil }

rpc/rpc_test.go

@@ -0,0 +1,33 @@
+package rpc
+
+import (
+	"bytes"
+	"github.com/stretchr/testify/assert"
+	"github.com/zrepl/zrepl/util"
+	"io"
+	"strings"
+	"testing"
+)
+
+func TestByteStreamRPCDecodeJSONError(t *testing.T) {
+
+	r := strings.NewReader("{'a':'aber'}")
+
+	var chunked bytes.Buffer
+	ch := util.NewChunker(r)
+	io.Copy(&chunked, &ch)
+
+	type SampleType struct {
+		A uint
+	}
+	var s SampleType
+	err := readChunkedJSON(&chunked, &s)
+	assert.NotNil(t, err)
+
+	_, ok := err.(ByteStreamRPCDecodeJSONError)
+	if !ok {
+		t.Errorf("expected ByteStreamRPCDecodeJSONError, got %t\n", err)
+		t.Errorf("%s\n", err)
+	}
+
+}

rpc/server.go

@@ -1,259 +0,0 @@
-package rpc
-
-import (
-	"bytes"
-	"encoding/json"
-	"io"
-	"reflect"
-
-	"github.com/pkg/errors"
-)
-
-type Server struct {
-	ml        *MessageLayer
-	logger    Logger
-	endpoints map[string]endpointDescr
-}
-
-type typeMap struct {
-	local reflect.Type
-	proto DataType
-}
-type endpointDescr struct {
-	inType  typeMap
-	outType typeMap
-	handler reflect.Value
-}
-
-type MarshaledJSONEndpoint func(bodyJSON interface{})
-
-func NewServer(rwc io.ReadWriteCloser) *Server {
-	ml := NewMessageLayer(rwc)
-	return &Server{
-		ml, noLogger{}, make(map[string]endpointDescr),
-	}
-}
-
-func (s *Server) SetLogger(logger Logger, logMessageLayer bool) {
-	s.logger = logger
-	if logMessageLayer {
-		s.ml.logger = logger
-	} else {
-		s.ml.logger = noLogger{}
-	}
-}
-
-func (s *Server) RegisterEndpoint(name string, handler interface{}) (err error) {
-	_, ok := s.endpoints[name]
-	if ok {
-		return errors.Errorf("already set up an endpoint for '%s'", name)
-	}
-	s.endpoints[name], err = makeEndpointDescr(handler)
-	return
-}
-
-func checkResponseHeader(h *Header) (err error) {
-	var statusNotSet Status
-	if h.Error == statusNotSet {
-		return errors.Errorf("status has zero-value")
-	}
-	return nil
-}
-
-func (s *Server) writeResponse(h *Header) (err error) {
-	// TODO validate
-	return s.ml.WriteHeader(h)
-}
-
-func (s *Server) recvRequest() (h *Header, err error) {
-	h, err = s.ml.ReadHeader()
-	if err != nil {
-		s.logger.Printf("error reading header: %s", err)
-		return nil, err
-	}
-
-	s.logger.Printf("validating request")
-	err = nil // TODO validate
-	if err == nil {
-		return h, nil
-	}
-	s.logger.Printf("request validation error: %s", err)
-
-	r := NewErrorHeader(StatusRequestError, "%s", err)
-	return nil, s.writeResponse(r)
-}
-
-var doneServeNext error = errors.New("this should not cause a HangUp() in the server")
-var doneStopServing error = errors.New("this should cause the server to close the connection")
-
-var ProtocolError error = errors.New("protocol error, server should hang up")
-
-const ControlEndpointClose string = "Close"
-
-// Serve the connection until failure or the client hangs up
-func (s *Server) Serve() (err error) {
-	for {
-
-		err = s.ServeRequest()
-
-		if err == nil {
-			continue
-		}
-		if err == doneServeNext {
-			s.logger.Printf("subroutine returned pseudo-error indicating early-exit")
-			err = nil
-			continue
-		}
-
-		if err == doneStopServing {
-			s.logger.Printf("subroutine returned pseudo-error indicating close request")
-			err = nil
-			break
-		}
-
-		break
-	}
-
-	if err != nil {
-		s.logger.Printf("an error occurred that could not be handled on PRC protocol level: %+v", err)
-	}
-
-	s.logger.Printf("cloing MessageLayer")
-	if mlErr := s.ml.Close(); mlErr != nil {
-		s.logger.Printf("error closing MessageLayer: %+v", mlErr)
-	}
-
-	return err
-}
-
-// Serve a single request
-// * wait for request to come in
-// * call handler
-// * reply
-//
-// The connection is left open, the next bytes on the conn should be
-// the next request header.
-//
-// Returns an err != nil if the error is bad enough to hang up on the client.
-// Examples: 		protocol version mismatches, protocol errors in general, ...
-// Non-Examples:	a handler error
-func (s *Server) ServeRequest() (err error) {
-
-	ml := s.ml
-
-	s.logger.Printf("reading header")
-	h, err := s.recvRequest()
-	if err != nil {
-		return err
-	}
-
-	if h.DataType == DataTypeControl {
-		switch h.Endpoint {
-		case ControlEndpointClose:
-			ack := Header{Error: StatusOK, DataType: DataTypeControl}
-			err = s.writeResponse(&ack)
-			if err != nil {
-				return err
-			}
-			return doneStopServing
-		default:
-			r := NewErrorHeader(StatusRequestError, "unregistered control endpoint %s", h.Endpoint)
-			return s.writeResponse(r)
-		}
-	}
-
-	ep, ok := s.endpoints[h.Endpoint]
-	if !ok {
-		r := NewErrorHeader(StatusRequestError, "unregistered endpoint %s", h.Endpoint)
-		return s.writeResponse(r)
-	}
-
-	if ep.inType.proto != h.DataType {
-		r := NewErrorHeader(StatusRequestError, "wrong DataType for endpoint %s (has %s, you provided %s)", h.Endpoint, ep.inType.proto, h.DataType)
-		return s.writeResponse(r)
-	}
-
-	if ep.outType.proto != h.Accept {
-		r := NewErrorHeader(StatusRequestError, "wrong Accept for endpoint %s (has %s, you provided %s)", h.Endpoint, ep.outType.proto, h.Accept)
-		return s.writeResponse(r)
-	}
-
-	dr := ml.ReadData()
-
-	// Determine inval
-	var inval reflect.Value
-	switch ep.inType.proto {
-	case DataTypeMarshaledJSON:
-		// Unmarshal input
-		inval = reflect.New(ep.inType.local.Elem())
-		invalIface := inval.Interface()
-		err = json.NewDecoder(dr).Decode(invalIface)
-		if err != nil {
-			r := NewErrorHeader(StatusRequestError, "cannot decode marshaled JSON: %s", err)
-			return s.writeResponse(r)
-		}
-	case DataTypeOctets:
-		// Take data as is
-		inval = reflect.ValueOf(dr)
-	default:
-		panic("not implemented")
-	}
-
-	outval := reflect.New(ep.outType.local.Elem()) // outval is a double pointer
-
-	s.logger.Printf("before handler, inval=%v outval=%v", inval, outval)
-
-	// Call the handler
-	errs := ep.handler.Call([]reflect.Value{inval, outval})
-
-	if !errs[0].IsNil() {
-		he := errs[0].Interface().(error) // we checked that before...
-		s.logger.Printf("handler returned error: %s", err)
-		r := NewErrorHeader(StatusError, "%s", he.Error())
-		return s.writeResponse(r)
-	}
-
-	switch ep.outType.proto {
-
-	case DataTypeMarshaledJSON:
-
-		var dataBuf bytes.Buffer
-		// Marshal output
-		err = json.NewEncoder(&dataBuf).Encode(outval.Interface())
-		if err != nil {
-			r := NewErrorHeader(StatusServerError, "cannot marshal response: %s", err)
-			return s.writeResponse(r)
-		}
-
-		replyHeader := Header{
-			Error:    StatusOK,
-			DataType: ep.outType.proto,
-		}
-		if err = s.writeResponse(&replyHeader); err != nil {
-			return err
-		}
-
-		if err = ml.WriteData(&dataBuf); err != nil {
-			return
-		}
-
-	case DataTypeOctets:
-
-		h := Header{
-			Error:    StatusOK,
-			DataType: DataTypeOctets,
-		}
-		if err = s.writeResponse(&h); err != nil {
-			return
-		}
-
-		reader := outval.Interface().(*io.Reader) // we checked that when adding the endpoint
-		err = ml.WriteData(*reader)
-		if err != nil {
-			return err
-		}
-
-	}
-
-	return nil
-}

rpc/shared.go

@@ -1,111 +0,0 @@
-package rpc
-
-import (
-	"fmt"
-	"github.com/pkg/errors"
-	"io"
-	"reflect"
-)
-
-type RPCServer interface {
-	Serve() (err error)
-	RegisterEndpoint(name string, handler interface{}) (err error)
-}
-
-type RPCClient interface {
-	Call(endpoint string, in, out interface{}) (err error)
-	Close() (err error)
-}
-
-type Logger interface {
-	Printf(format string, args ...interface{})
-}
-
-type noLogger struct{}
-
-func (l noLogger) Printf(format string, args ...interface{}) {}
-func typeIsIOReader(t reflect.Type) bool {
-	return t == reflect.TypeOf((*io.Reader)(nil)).Elem()
-}
-
-func typeIsIOReaderPtr(t reflect.Type) bool {
-	return t == reflect.TypeOf((*io.Reader)(nil))
-}
-
-// An error returned by the Client if the response indicated a status code other than StatusOK
-type RPCError struct {
-	ResponseHeader *Header
-}
-
-func (e *RPCError) Error() string {
-	return fmt.Sprintf("%s: %s", e.ResponseHeader.Error, e.ResponseHeader.ErrorMessage)
-}
-
-type RPCProtoError struct {
-	Message         string
-	UnderlyingError error
-}
-
-func (e *RPCProtoError) Error() string {
-	return e.Message
-}
-
-func checkRPCParamTypes(in, out reflect.Type) (err error) {
-	if !(in.Kind() == reflect.Ptr || typeIsIOReader(in)) {
-		err = errors.Errorf("input parameter must be a pointer or an io.Reader, is of kind %s, type %s", in.Kind(), in)
-		return
-	}
-	if !(out.Kind() == reflect.Ptr) {
-		err = errors.Errorf("second input parameter (the non-error output parameter) must be a pointer or an *io.Reader")
-		return
-	}
-	return nil
-}
-
-func checkRPCReturnType(rt reflect.Type) (err error) {
-	errInterfaceType := reflect.TypeOf((*error)(nil)).Elem()
-	if !rt.Implements(errInterfaceType) {
-		err = errors.Errorf("handler must return an error")
-		return
-	}
-	return nil
-}
-
-func makeEndpointDescr(handler interface{}) (descr endpointDescr, err error) {
-
-	ht := reflect.TypeOf(handler)
-
-	if ht.Kind() != reflect.Func {
-		err = errors.Errorf("handler must be of kind reflect.Func")
-		return
-	}
-
-	if ht.NumIn() != 2 || ht.NumOut() != 1 {
-		err = errors.Errorf("handler must have exactly two input parameters and one output parameter")
-		return
-	}
-	if err = checkRPCParamTypes(ht.In(0), ht.In(1)); err != nil {
-		return
-	}
-	if err = checkRPCReturnType(ht.Out(0)); err != nil {
-		return
-	}
-
-	descr.handler = reflect.ValueOf(handler)
-	descr.inType.local = ht.In(0)
-	descr.outType.local = ht.In(1)
-
-	if typeIsIOReader(ht.In(0)) {
-		descr.inType.proto = DataTypeOctets
-	} else {
-		descr.inType.proto = DataTypeMarshaledJSON
-	}
-
-	if typeIsIOReaderPtr(ht.In(1)) {
-		descr.outType.proto = DataTypeOctets
-	} else {
-		descr.outType.proto = DataTypeMarshaledJSON
-	}
-
-	return
-}

rpc/status_string.go

@@ -1,17 +0,0 @@
-// Code generated by "stringer -type=Status"; DO NOT EDIT.
-
-package rpc
-
-import "fmt"
-
-const _Status_name = "StatusOKStatusRequestErrorStatusServerErrorStatusError"
-
-var _Status_index = [...]uint8{0, 8, 26, 43, 54}
-
-func (i Status) String() string {
-	i -= 1
-	if i >= Status(len(_Status_index)-1) {
-		return fmt.Sprintf("Status(%d)", i+1)
-	}
-	return _Status_name[_Status_index[i]:_Status_index[i+1]]
-}

rpc/structs.go

@@ -0,0 +1,144 @@
+package rpc
+
+import (
+	"encoding/json"
+	"io"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var _ json.Marshaler = &zfs.DatasetPath{}
+var _ json.Unmarshaler = &zfs.DatasetPath{}
+
+type RequestId [16]byte
+type RequestType uint8
+
+const (
+	RTProtocolVersionRequest     RequestType = 0x01
+	RTFilesystemRequest                      = 0x10
+	RTFilesystemVersionsRequest              = 0x11
+	RTInitialTransferRequest                 = 0x12
+	RTIncrementalTransferRequest             = 0x13
+	RTResumeTransferRequest                  = 0x14
+	RTPullMeRequest                          = 0x20
+	RTCloseRequest                           = 0xf0
+)
+
+type RequestHeader struct {
+	Type RequestType
+	Id   [16]byte // UUID
+}
+
+type FilesystemRequest struct {
+	Roots []string // may be nil, indicating interest in all filesystems
+}
+
+type FilesystemVersionsRequest struct {
+	Filesystem *zfs.DatasetPath
+}
+
+type InitialTransferRequest struct {
+	Filesystem        *zfs.DatasetPath
+	FilesystemVersion zfs.FilesystemVersion
+}
+
+func (r InitialTransferRequest) Respond(snapshotReader io.Reader) {
+
+}
+
+type IncrementalTransferRequest struct {
+	Filesystem *zfs.DatasetPath
+	From       zfs.FilesystemVersion
+	To         zfs.FilesystemVersion
+}
+
+type ResumeTransferRequest struct {
+	Filesystem *zfs.DatasetPath
+	Token      string
+}
+
+type ResumeTransferError struct {
+	Reason   ResumeTransferErrorReason
+	ZFSError string
+}
+
+func (e *ResumeTransferError) Error() string {
+	return e.Reason.String()
+}
+
+//go:generate stringer -type ResumeTransferErrorReason
+type ResumeTransferErrorReason uint8
+
+const (
+	ResumeTransferErrorReasonNotImplemented ResumeTransferErrorReason = iota
+	ResumeTransferErrorReasonDisabled
+	ResumeTransferErrorReasonZFSErrorPermanent
+	ResumeTransferErrorReasonZFSErrorMaybeTemporary
+)
+
+func (r IncrementalTransferRequest) Respond(snapshotReader io.Reader) {
+
+}
+
+type ByteStreamRPCProtocolVersionRequest struct {
+	ClientVersion uint8
+}
+
+const LOCAL_TRANSPORT_IDENTITY string = "local"
+
+const DEFAULT_INITIAL_REPL_POLICY = InitialReplPolicyMostRecent
+
+type InitialReplPolicy string
+
+const (
+	InitialReplPolicyMostRecent InitialReplPolicy = "most_recent"
+	InitialReplPolicyAll        InitialReplPolicy = "all"
+)
+
+type PullMeRequest struct {
+	// if true, the other fields are undefined
+	Finished          bool
+	InitialReplPolicy InitialReplPolicy
+}
+
+type CloseRequest struct {
+	Goodbye string
+}
+
+type ErrorId uint8
+
+const (
+	ENoError                 ErrorId = 0
+	EDecodeHeader                    = 1
+	EUnknownRequestType              = 2
+	EDecodeRequestBody               = 3
+	EProtocolVersionMismatch         = 4
+	EHandler                         = 5
+)
+
+type ResponseType uint8
+
+const (
+	RNONE           ResponseType = 0x0
+	ROK                          = 0x1
+	RFilesystems                 = 0x10
+	RFilesystemDiff              = 0x11
+	RChunkedStream               = 0x20
+)
+
+type ResponseHeader struct {
+	RequestId    RequestId
+	ErrorId      ErrorId
+	Message      string
+	ResponseType ResponseType
+}
+
+func NewByteStreamRPCProtocolVersionRequest() ByteStreamRPCProtocolVersionRequest {
+	return ByteStreamRPCProtocolVersionRequest{
+		ClientVersion: ByteStreamRPCProtocolVersion,
+	}
+}
+
+func newUUID() [16]byte {
+	return [16]byte{}
+}

scratchpad/chunker/.gitignore

@@ -0,0 +1 @@
+chunker

scratchpad/chunker/main.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"flag"
+	"github.com/zrepl/zrepl/sshbytestream"
+	. "github.com/zrepl/zrepl/util"
+	// "bytes"
+	_ "bufio"
+	// "strings"
+	"io"
+	"log"
+	"os"
+	_ "time"
+)
+
+func main() {
+
+	mode := flag.String("mode", "", "incoming|outgoing")
+	incomingFile := flag.String("incoming.file", "", "file to deliver to callers")
+	outgoingHost := flag.String("outgoing.sshHost", "", "ssh host")
+	outgoingUser := flag.String("outgoing.sshUser", "", "ssh user")
+	outgoingIdentity := flag.String("outgoing.sshIdentity", "", "ssh private key")
+	outgoingPort := flag.Uint("outgoing.sshPort", 22, "ssh port")
+	outgoingFile := flag.String("outgoing.File", "", "")
+	flag.Parse()
+
+	switch {
+	case (*mode == "incoming"):
+
+		conn, err := sshbytestream.Incoming()
+		if err != nil {
+			panic(err)
+		}
+
+		file, err := os.Open(*incomingFile)
+		if err != nil {
+			panic(err)
+		}
+
+		chunker := NewChunker(file)
+
+		_, err = io.Copy(conn, &chunker)
+		if err != nil && err != io.EOF {
+			panic(err)
+		}
+
+		log.Printf("Chunk Count: %d\n", chunker.ChunkCount)
+
+	case *mode == "outgoing":
+
+		conn, err := sshbytestream.Outgoing(sshbytestream.SSHTransport{
+			Host:         *outgoingHost,
+			User:         *outgoingUser,
+			IdentityFile: *outgoingIdentity,
+			Port:         uint16(*outgoingPort),
+		})
+		if err != nil {
+			panic(err)
+		}
+
+		f, err := os.OpenFile(*outgoingFile, os.O_CREATE|os.O_WRONLY, 0600)
+		if err != nil {
+			panic(err)
+		}
+
+		unchunker := NewUnchunker(conn)
+
+		_, err = io.Copy(f, unchunker)
+		if err != nil {
+			panic(err)
+		}
+
+		conn.Close()
+
+		log.Printf("Chunk Count: %d\n", unchunker.ChunkCount)
+
+		os.Exit(0)
+
+	default:
+		panic("unsupported mode!")
+
+	}
+
+}

scratchpad/repeat/main.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"log"
+	"os"
+	"os/exec"
+	"time"
+)
+
+var config struct {
+	duration time.Duration
+}
+
+func usage() {
+	log.Printf("usage: repeat repeatInterval command [args...]")
+}
+
+func main() {
+
+	args := os.Args
+
+	if len(args) < 3 {
+		usage()
+		os.Exit(1)
+	}
+	repeatInterval, err := time.ParseDuration(args[1])
+	if err != nil {
+		log.Printf("cannot parse interval: %s", err)
+		usage()
+		os.Exit(1)
+	}
+
+	var lastStart time.Time
+
+	for {
+		cmd := exec.Command(args[2], args[3:]...)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		lastStart = time.Now()
+		if err := cmd.Run(); err != nil {
+			panic(err)
+		}
+		time.Sleep(lastStart.Add(repeatInterval).Sub(time.Now()))
+	}
+
+}

scratchpad/repeat/sleepx.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+echo -n ' start '
+date
+sleep $1
+echo -n 'done '
+date 

zfs/diff.go

@@ -189,15 +189,21 @@ const ZREPL_PLACEHOLDER_PROPERTY_NAME string = "zrepl:placeholder"
 
 type FilesystemState struct {
 	Placeholder bool
-	// TODO extend with resume token when that feature is finally added
+	// If != "", the receive_resume_token found on the receiving side of a resumable send & recv
+	ResumeToken string
 }
 
 // A somewhat efficient way to determine if a filesystem exists on this host.
 // Particularly useful if exists is called more than once (will only fork exec once and cache the result)
 func ZFSListFilesystemState() (localState map[string]FilesystemState, err error) {
 
+	properties := []string{"name", ZREPL_PLACEHOLDER_PROPERTY_NAME}
+	if CLICompat.ResumableSendRecv {
+		properties = append(properties, "receive_resume_token")
+	}
+
 	var actual [][]string
-	if actual, err = ZFSList([]string{"name", ZREPL_PLACEHOLDER_PROPERTY_NAME}, "-t", "filesystem,volume"); err != nil {
+	if actual, err = ZFSList(properties, "-t", "filesystem,volume"); err != nil {
 		return
 	}
 
@@ -205,11 +211,15 @@ func ZFSListFilesystemState() (localState map[string]FilesystemState, err error)
 	for _, e := range actual {
 		dp, err := NewDatasetPath(e[0])
 		if err != nil {
-			return nil, fmt.Errorf("ZFS does not return parseable dataset path: %s", e[0])
+			fmt.Errorf("ZFS does not return parseable dataset path: %s", e[0])
 		}
 		placeholder, _ := IsPlaceholder(dp, e[1])
+		receive_resume_token := ""
+		if CLICompat.ResumableSendRecv {
+			receive_resume_token = e[2]
+		}
 		localState[e[0]] = FilesystemState{
-			placeholder,
+			placeholder, receive_resume_token,
 		}
 	}
 	return

zfs/versions.go

@@ -25,6 +25,7 @@ func (t VersionType) DelimiterChar() string {
 	default:
 		panic(fmt.Sprintf("unexpected VersionType %#v", t))
 	}
+	return ""
 }
 
 type FilesystemVersion struct {

zfs/zfs.go

@@ -13,6 +13,11 @@ import (
 	"github.com/zrepl/zrepl/util"
 )
 
+var CLICompat struct {
+	// whether the CLI supports resumable send & recv
+	ResumableSendRecv bool
+}
+
 type DatasetPath struct {
 	comps []string
 }
@@ -239,6 +244,13 @@ func ZFSRecv(fs *DatasetPath, stream io.Reader, additionalArgs ...string) (err e
 	return nil
 }
 
+func ZFSRecvAbort(fs *DatasetPath) (err error) {
+	if !CLICompat.ResumableSendRecv {
+		panic("decide if this is an error")
+	}
+	panic("not implemented")
+}
+
 func ZFSSet(fs *DatasetPath, prop, val string) (err error) {
 
 	if strings.ContainsRune(prop, '=') {