docs: fix publish.sh branch whitelisting

hotfix #161 for 0.1: limit concurrency of zfs send & recv commands

.circleci/config.yml

@@ -0,0 +1,102 @@
+version: 2.0
+workflows:
+  version: 2
+  build:
+    jobs:
+      - build-1.11
+      - build-1.12
+      - build-latest
+jobs:
+  # build-latest serves as the template
+  # we use YAML anchors & aliases to exchange the docker image (and hence Go version used for the build)
+  build-latest: &build-latest
+    description: Builds zrepl
+    parameters:
+      image:
+        description: "the docker image that the job should use"
+        type: string
+    docker:
+      - image: circleci/golang:latest
+    environment:
+      # required by lazy.sh
+      TERM: xterm
+    working_directory: /go/src/github.com/zrepl/zrepl
+    steps:
+      - run:
+          name: Setup environment variables
+          command: |
+            # used by pip (for docs)
+            echo 'export PATH="$HOME/.local/bin:$PATH"' >> $BASH_ENV
+
+      - restore_cache:
+          keys:
+            - source
+            - vendor
+            - protobuf
+
+      - checkout
+
+      - save_cache:
+          key: source
+          paths:
+            - ".git"
+      
+      # install deps
+      - run: wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
+      - run: echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807  protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
+      - run: sudo unzip -d /usr protoc-3.6.1-linux-x86_64.zip
+      - save_cache:
+          key: protobuf
+          paths:
+            - "/usr/include/google/protobuf"
+            
+      - run: sudo apt install python3 python3-pip libgirepository1.0-dev
+      - run: ./lazy.sh devsetup
+
+      - run:
+          name: Install minio client, prepare minio
+          command: |
+            curl -sSL https://dl.minio.io/client/mc/release/linux-amd64/mc -o ${GOPATH}/bin/mc
+            chmod +x ${GOPATH}/bin/mc
+            mc config host add --api s3v4 zrepl-minio https://minio.cschwarz.com ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
+
+      - run: make vendordeps
+      - save_cache:
+          key: vendor
+          paths:
+            - "./vendor"
+            
+      - run: make
+      - run: make vet
+      - run: make test
+      - run: make lint
+      - run: make release
+
+
+      - store_artifacts:
+          path: ./artifacts/release
+          when: always
+
+      - run:
+          shell: /bin/bash -euo pipefail
+          when: always
+          command: |
+            echo "$CIRCLE_BUILD_URL" > ./artifacts/release/cirlceci_build_url
+            mc cp -r artifacts/release "zrepl-minio/zrepl-ci-artifacts/${CIRCLE_SHA1}/${CIRCLE_JOB}/"
+            REPO="zrepl/zrepl"
+            COMMIT="${CIRCLE_SHA1}"
+            JOB_NAME="${CIRCLE_JOB}"
+            curl "https://api.github.com/repos/$REPO/statuses/$COMMIT?access_token=$GITHUB_COMMIT_STATUS_TOKEN" \
+              -H "Content-Type: application/json" \
+              -X POST \
+              -d '{"context":"zrepl/publish-ci-artifacts", "state": "success", "description":"CI Build Artifacts for '"$JOB_NAME"'", "target_url":"https://minio.cschwarz.com/minio/zrepl-ci-artifacts/'"$COMMIT"'/"}'
+
+
+  build-1.11:
+    <<: *build-latest
+    docker:
+      - image: circleci/golang:1.11
+  build-1.12:
+    <<: *build-latest
+    docker:
+    - image: circleci/golang:1.12

.golangci.yml

@@ -0,0 +1,10 @@
+linters:
+  enable:
+  - goimports
+
+issues:
+  exclude-rules:
+    - path: _test\.go
+      linters:
+        - errcheck
+

.travis.yml

@@ -0,0 +1,70 @@
+dist: xenial
+services:
+- docker
+
+env: # for allow_failures: https://docs.travis-ci.com/user/customizing-the-build/
+matrix:
+  include:
+
+  - language: go
+    name: "Build in Docker (docs/installation.rst)"
+    script:
+    - sudo docker build -t zrepl_build -f build.Dockerfile .
+    - |
+      sudo docker run -it --rm \
+        -v "${PWD}:/go/src/github.com/zrepl/zrepl" \
+        --user "$(id -u):$(id -g)" \
+        zrepl_build make vendordeps release
+
+  - &zrepl_build_template
+    language: go
+    go_import_path: github.com/zrepl/zrepl
+    before_install:
+        - wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
+        - echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807  protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
+        - sudo unzip -d /usr protoc-3.6.1-linux-x86_64.zip
+        - ./lazy.sh godep
+        - make vendordeps
+    script:
+      - make
+      - make vet
+      - make test
+      - make lint
+      - make artifacts/zrepl-freebsd-amd64
+      - make artifacts/zrepl-linux-amd64
+      - make artifacts/zrepl-darwin-amd64
+    go:
+    - "1.11"
+
+  - <<: *zrepl_build_template
+    go:
+    - "1.12"
+
+  - <<: *zrepl_build_template
+    go:
+    - "master"
+
+  - &zrepl_docs_template 
+    language: python
+    python:
+    - "3.4"
+    install:
+    - sudo apt-get install libgirepository1.0-dev
+    - pip install -r docs/requirements.txt
+    script:
+    - make docs
+  - <<: *zrepl_docs_template
+    python:
+    - "3.5"
+  - <<: *zrepl_docs_template
+    python:
+    - "3.6"
+  - <<: *zrepl_docs_template
+    python:
+    - "3.7"
+  
+
+  allow_failures:
+  - <<: *zrepl_build_template
+    go:
+    - "master"

Gopkg.toml

@@ -1,8 +1,13 @@
-ignored = [ "github.com/inconshreveable/mousetrap" ]
+ignored = [
+    "github.com/inconshreveable/mousetrap",
+]
 
-[[constraint]]
+required = [
-  branch = "master"
+    "golang.org/x/tools/cmd/stringer",
-  name = "github.com/ftrvxmtrx/fd"
+    "github.com/alvaroloes/enumer",
+    "github.com/golangci/golangci-lint/cmd/golangci-lint",
+    "golang.org/x/tools/cmd/goimports",
+]
 
 [[constraint]]
   branch = "master"
@@ -12,14 +17,6 @@ ignored = [ "github.com/inconshreveable/mousetrap" ]
   branch = "master"
   name = "github.com/kr/pretty"
 
-[[constraint]]
-  branch = "master"
-  name = "github.com/mitchellh/go-homedir"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/mitchellh/mapstructure"
-
 [[constraint]]
   name = "github.com/pkg/errors"
   version = "0.8.0"
@@ -28,26 +25,43 @@ ignored = [ "github.com/inconshreveable/mousetrap" ]
   branch = "master"
   name = "github.com/spf13/cobra"
 
-[[constraint]]
-  name = "github.com/spf13/viper"
-  version = "1.0.0"
-
 [[constraint]]
   name = "github.com/stretchr/testify"
   version = "1.1.4"
 
 [[constraint]]
   branch = "v2"
-  name = "github.com/go-yaml/yaml"
+  name = "github.com/zrepl/yaml-config"
 
 [[constraint]]
   name = "github.com/go-logfmt/logfmt"
   version = "*"
 
 [[constraint]]
-  name = "github.com/problame/go-rwccmd"
+  name = "github.com/problame/go-netssh"
   branch = "master"
 
 [[constraint]]
-  name = "github.com/problame/go-netssh"
+  name = "github.com/prometheus/client_golang"
   branch = "master"
+
+[[constraint]]
+  name = "github.com/golang/protobuf"
+  version = "1"
+
+[[constraint]]
+  name = "github.com/fatih/color"
+  version = "1.7.0"
+
+
+[[constraint]]
+  name = "github.com/gdamore/tcell"
+  version = "1.0.0"
+
+[[constraint]]
+  name = "google.golang.org/grpc"
+  version = "1"
+  
+[[constraint]]
+  version = "1.1.0"
+  name = "github.com/google/uuid"

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2017 Christian Schwarz
+Copyright (c) 2017-2018 Christian Schwarz
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,24 +1,25 @@
-.PHONY: generate build test vet cover release docs docs-clean clean vendordeps
+.PHONY: generate build test vet cover release docs docs-clean clean vendordeps format lint
 .DEFAULT_GOAL := build
 
-ROOT := github.com/zrepl/zrepl
-SUBPKGS := cmd logger rpc util zfs
-
-_TESTPKGS := $(ROOT) $(foreach p,$(SUBPKGS),$(ROOT)/$(p))
-
 ARTIFACTDIR := artifacts
 
-ifndef ZREPL_VERSION
+ifdef ZREPL_VERSION
-    ZREPL_VERSION := $(shell git describe --dirty 2>/dev/null || echo "ZREPL_BUILD_INVALID_VERSION" )
+    _ZREPL_VERSION := $(ZREPL_VERSION)
-    ifeq ($(ZREPL_VERSION),ZREPL_BUILD_INVALID_VERSION) # can't use .SHELLSTATUS because Debian Stretch is still on gmake 4.1
+endif
+ifndef _ZREPL_VERSION
+    _ZREPL_VERSION := $(shell git describe --always --dirty 2>/dev/null || echo "ZREPL_BUILD_INVALID_VERSION" )
+    ifeq ($(_ZREPL_VERSION),ZREPL_BUILD_INVALID_VERSION) # can't use .SHELLSTATUS because Debian Stretch is still on gmake 4.1
         $(error cannot infer variable ZREPL_VERSION using git and variable is not overriden by make invocation)
     endif
 endif
-GO_LDFLAGS := "-X github.com/zrepl/zrepl/cmd.zreplVersion=$(ZREPL_VERSION)"
+GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
 
 GO_BUILD := go build -ldflags $(GO_LDFLAGS)
 
-RELEASE_BINS := $(ARTIFACTDIR)/zrepl-freebsd-amd64 $(ARTIFACTDIR)/zrepl-linux-amd64
+RELEASE_BINS := $(ARTIFACTDIR)/zrepl-freebsd-amd64
+RELEASE_BINS += $(ARTIFACTDIR)/zrepl-linux-amd64
+RELEASE_BINS += $(ARTIFACTDIR)/zrepl-darwin-amd64
+
 RELEASE_NOARCH := $(ARTIFACTDIR)/zrepl-noarch.tar
 THIS_PLATFORM_RELEASE_BIN := $(shell bash -c 'source <(go env) && echo "zrepl-$${GOOS}-$${GOARCH}"' )
 
@@ -26,34 +27,27 @@ vendordeps:
 	dep ensure -v -vendor-only
 
 generate: #not part of the build, must do that manually
-	@for pkg in $(_TESTPKGS); do\
+	protoc -I=replication/logic/pdu --go_out=plugins=grpc:replication/logic/pdu replication/logic/pdu/pdu.proto
-		go generate "$$pkg" || exit 1; \
+	go generate -x ./...
-	done;
+
+format:
+	goimports -srcdir . -local 'github.com/zrepl/zrepl' -w $(shell find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go')
+
+lint:
+	golangci-lint run ./...
 
 build:
 	@echo "INFO: In case of missing dependencies, run 'make vendordeps'"
 	$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl"
 
 test:
-	@for pkg in $(_TESTPKGS); do \
+	go test ./...
-		echo "Testing $$pkg"; \
-		go test "$$pkg" || exit 1; \
-	done;
 
 vet:
-	@for pkg in $(_TESTPKGS); do \
+	# for each supported platform to cover conditional compilation
-		echo "Vetting $$pkg"; \
+	GOOS=linux   go vet ./...
-		go vet "$$pkg" || exit 1; \
+	GOOS=darwin  go vet ./...
-	done;
+	GOOS=freebsd go vet ./...
-
-cover: artifacts
-	@for pkg in $(_TESTPKGS); do \
-		profile="$(ARTIFACTDIR)/cover-$$(basename $$pkg).out"; \
-		go test -coverprofile "$$profile" $$pkg || exit 1; \
-		if [ -f "$$profile" ]; then \
-   			go tool cover -html="$$profile" -o "$${profile}.html" || exit 2; \
-		fi; \
-	done;
 
 $(ARTIFACTDIR):
 	mkdir -p "$@"
@@ -80,13 +74,13 @@ docs-clean:
 
 .PHONY: $(RELEASE_BINS)
 # TODO: two wildcards possible
-$(RELEASE_BINS): $(ARTIFACTDIR)/zrepl-%-amd64: generate $(ARTIFACTDIR) vet test
+$(RELEASE_BINS): $(ARTIFACTDIR)/zrepl-%-amd64: generate $(ARTIFACTDIR) vet test lint
 	@echo "INFO: In case of missing dependencies, run 'make vendordeps'"
 	GOOS=$* GOARCH=amd64   $(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$*-amd64"
 
 $(RELEASE_NOARCH): docs $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_version.txt
 	tar --mtime='1970-01-01' --sort=name \
-		--transform 's/$(ARTIFACTDIR)/zrepl-$(ZREPL_VERSION)-noarch/' \
+		--transform 's/$(ARTIFACTDIR)/zrepl-$(_ZREPL_VERSION)-noarch/' \
 		-acf $@ \
 		$(ARTIFACTDIR)/docs/html \
 		$(ARTIFACTDIR)/bash_completion \
@@ -97,10 +91,14 @@ release: $(RELEASE_BINS) $(RELEASE_NOARCH)
 	mkdir -p "$(ARTIFACTDIR)/release"
 	cp $^ "$(ARTIFACTDIR)/release"
 	cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
-	@if echo "$(ZREPL_VERSION)" | grep dirty > /dev/null; then\
+	@# note that we use ZREPL_VERSION and not _ZREPL_VERSION because we want to detect the override
-		echo '[WARN] Do not publish the artifacts, make variable ZREPL_VERSION=$(ZREPL_VERSION) indicates they are dirty!'; \
+	@if git describe --always --dirty 2>/dev/null | grep dirty >/dev/null; then \
+        echo '[INFO] either git reports checkout is dirty or git is not installed or this is not a git checkout'; \
+		if [ "$(ZREPL_VERSION)" = "" ]; then \
+			echo '[WARN] git checkout is dirty and make variable ZREPL_VERSION was not used to override'; \
 			exit 1; \
-	fi
+		fi; \
+	fi;
 
 clean: docs-clean
 	rm -rf "$(ARTIFACTDIR)"

README.md

@@ -1,5 +1,12 @@
+[![GitHub license](https://img.shields.io/github/license/zrepl/zrepl.svg)](https://github.com/zrepl/zrepl/blob/master/LICENSE)
+[![Language: Go](https://img.shields.io/badge/language-Go-6ad7e5.svg)](https://golang.org/)
+[![User Docs](https://img.shields.io/badge/docs-web-blue.svg)](https://zrepl.github.io)
+[![Donate via PayPal](https://img.shields.io/badge/donate-paypal-yellow.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
+[![Donate via Liberapay](https://img.shields.io/badge/donate-liberapay-yellow.svg)](https://liberapay.com/zrepl/donate)
+[![Twitter](https://img.shields.io/twitter/url/https/github.com/zrepl/zrepl.svg?style=social)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl)
+
 # zrepl
-zrepl is a ZFS filesystem backup & replication solution written in Go.
+zrepl is a one-stop ZFS backup & replication solution.
 
 ## User Documentation
 
@@ -26,13 +33,18 @@ Check out the *Coding Workflow* section below for details.
 * Make sure your distro is compatible with the paths in `docs/installation.rst`.
 * Ship a default config that adheres to your distro's `hier` and logging system.
 * Ship a service manager file and _please_ try to upstream it to this repository.
+  * `dist/systemd` contains a Systemd unit template.
+* Ship other material provided in `./dist`, e.g. in `/usr/share/zrepl/`.
 * Use `make release ZREPL_VERSION='mydistro-1.2.3_1'`
     * Your distro's name and any versioning supplemental to zrepl's (e.g. package revision) should be in this string
 * Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
 
 ## Developer Documentation
 
-First, use `./lazy.sh devsetup` to install build dependencies and read `docs/installation.rst -> Compiling from Source`.
+zrepl is written in [Go](https://golang.org) and uses [`dep`](https://github.com/golang/dep) to manage dependencies.
+The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
+
+To get started, run `./lazy.sh devsetup` to easily install build dependencies and read `docs/installation.rst -> Compiling from Source`.
 
 ### Overall Architecture
 
@@ -42,19 +54,53 @@ Make sure to develop an understanding how zrepl is typically used by studying th
 ### Project Structure
 
 ```
-├── cmd
+├── artifacts               # build artifcats generate by make
-│   ├── sampleconf          # example configuration
+├── cli                     # wrapper around CLI package cobra
+├── client                  # all subcommands that are not `daemon`
+├── config                  # config data types (=> package yaml-config)
+│   └── samples
+├── daemon                  # the implementation of `zrepl daemon` subcommand
+│   ├── filters
+│   ├── job                 # job implementations
+│   ├── logging             # logging outlets + formatters
+│   ├── nethelpers
+│   ├── prometheus
+│   ├── pruner              # pruner implementation
+│   ├── snapper             # snapshotter implementation
 ├── docs                    # sphinx-based documentation
+├── dist                    # supplemental material for users & package maintainers
 │   ├── **/*.rst            # documentation in reStructuredText
 │   ├── sphinxconf
 │   │   └── conf.py         # sphinx config (see commit 445a280 why its not in docs/)
 │   ├── requirements.txt    # pip3 requirements to build documentation
 │   ├── publish.sh          # shell script for automated rendering & deploy to zrepl.github.io repo
-│   ├── public_git          # checkout of zrepl.github.io managed by above shell script
+│   └── public_git          # checkout of zrepl.github.io managed by above shell script
-├── logger                  # logger package used by zrepl
+├── endpoint                # implementation of replication endpoints (=> package replication)
-├── rpc                     # rpc protocol implementation
+├── logger                  # our own logger package
+├── pruning                 # pruning rules (the logic, not the actual execution)
+│   └── retentiongrid
+├── replication
+│   ├── driver              # the driver of the replication logic (status reporting, error handling)
+│   ├── logic               # planning & executing replication steps via rpc
+|   |   └── pdu             # the generated gRPC & protobuf code used in replication (and endpoints)
+│   └── report              # the JSON-serializable report datastructures exposed to the client
+├── rpc                     # the hybrid gRPC + ./dataconn RPC client: connects to a remote replication.Endpoint
+│   ├── dataconn            # Bulk data-transfer RPC protocol
+│   ├── grpcclientidentity  # adaptor to inject package transport's 'client identity' concept into gRPC contexts
+│   ├── netadaptor          # adaptor to convert a package transport's Connecter and Listener into net.* primitives
+│   ├── transportmux        # TCP connecter and listener used to split control & data traffic
+│   └── versionhandshake    # replication protocol version handshake perfomed on newly established connections
+├── tlsconf                 # abstraction for Go TLS server + client config
+├── transport               # transports implementation
+│   ├── fromconfig
+│   ├── local
+│   ├── ssh
+│   ├── tcp
+│   └── tls
 ├── util
-└── zfs                     # ZFS wrappers, filesystemm diffing
+├── vendor                  # managed by dep
+├── version                 # abstraction for versions (filled during build by Makefile)
+└── zfs                     # zfs(8) wrappers
 ```
 
 ### Coding Workflow

build.Dockerfile

@@ -1,16 +1,31 @@
 FROM golang:latest
 
 RUN apt-get update && apt-get install -y \
-    python3-pip
+    python3-pip \
+    unzip
+
+RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
+RUN echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807  protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
+RUN unzip -d /usr protoc-3.6.1-linux-x86_64.zip
 
 ADD lazy.sh /tmp/lazy.sh
 ADD docs/requirements.txt /tmp/requirements.txt
 ENV ZREPL_LAZY_DOCS_REQPATH=/tmp/requirements.txt
-RUN /tmp/lazy.sh devsetup
+RUN /tmp/lazy.sh docdep
 
 # prepare volume mount of git checkout to /zrepl
 RUN mkdir -p /go/src/github.com/zrepl/zrepl
-RUN chmod -R 0777 /go
+RUN mkdir -p /.cache && chmod -R 0777 /.cache
 WORKDIR /go/src/github.com/zrepl/zrepl
 
+ADD Gopkg.toml Gopkg.lock  ./
+
+# godep will install the Go dependencies to vendor in order to then build and install
+# build dependencies like stringer to $GOPATH/bin.
+# However, since users volume-mount their Git checkout into /go/src/github.com/zrepl/zrepl
+# the vendor directory will be empty at build time, allowing them to experiment with
+# new checkouts, etc.
+# Thus, we only use the vendored deps for building dependencies.
+RUN /tmp/lazy.sh godep
+RUN chmod -R 0777 /go
 

build/build.go

@@ -0,0 +1,23 @@
+//+build windows
+
+// This package cannot actually be built, and since the Travis
+// tests run go test ./..., we have to avoid a build attempt.
+// Windows is not supported atm, so this works ¯\_(ツ)_/¯
+
+// This package is a pseudo-user of build-time dependencies for zrepl,
+// mostly for various code-generation tools.
+//
+// The imports are necessary to satisfy go dep
+package main
+
+import (
+	"fmt"
+
+	_ "github.com/alvaroloes/enumer"
+	_ "github.com/golang/protobuf/protoc-gen-go"
+	_ "golang.org/x/tools/cmd/stringer"
+)
+
+func main() {
+	fmt.Println("just a placeholder")
+}

cli/cli.go

@@ -0,0 +1,122 @@
+package cli
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/config"
+)
+
+var rootArgs struct {
+	configPath string
+}
+
+var rootCmd = &cobra.Command{
+	Use:   "zrepl",
+	Short: "One-stop ZFS replication solution",
+}
+
+var bashcompCmd = &cobra.Command{
+	Use:   "bashcomp path/to/out/file",
+	Short: "generate bash completions",
+	Run: func(cmd *cobra.Command, args []string) {
+		if len(args) != 1 {
+			fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
+			err := cmd.Usage()
+			if err != nil {
+				panic(err)
+			}
+			os.Exit(1)
+		}
+		if err := rootCmd.GenBashCompletionFile(args[0]); err != nil {
+			fmt.Fprintf(os.Stderr, "error generating bash completion: %s", err)
+			os.Exit(1)
+		}
+	},
+	Hidden: true,
+}
+
+func init() {
+	rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
+	rootCmd.AddCommand(bashcompCmd)
+}
+
+type Subcommand struct {
+	Use              string
+	Short            string
+	Example          string
+	NoRequireConfig  bool
+	Run              func(subcommand *Subcommand, args []string) error
+	SetupFlags       func(f *pflag.FlagSet)
+	SetupSubcommands func() []*Subcommand
+
+	config    *config.Config
+	configErr error
+}
+
+func (s *Subcommand) ConfigParsingError() error {
+	return s.configErr
+}
+
+func (s *Subcommand) Config() *config.Config {
+	if !s.NoRequireConfig && s.config == nil {
+		panic("command that requires config is running and has no config set")
+	}
+	return s.config
+}
+
+func (s *Subcommand) run(cmd *cobra.Command, args []string) {
+	s.tryParseConfig()
+	err := s.Run(s, args)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+		os.Exit(1)
+	}
+}
+
+func (s *Subcommand) tryParseConfig() {
+	config, err := config.ParseConfig(rootArgs.configPath)
+	s.configErr = err
+	if err != nil {
+		if s.NoRequireConfig {
+			// doesn't matter
+			return
+		} else {
+			fmt.Fprintf(os.Stderr, "could not parse config: %s\n", err)
+			os.Exit(1)
+		}
+	}
+	s.config = config
+}
+
+func AddSubcommand(s *Subcommand) {
+	addSubcommandToCobraCmd(rootCmd, s)
+}
+
+func addSubcommandToCobraCmd(c *cobra.Command, s *Subcommand) {
+	cmd := cobra.Command{
+		Use:     s.Use,
+		Short:   s.Short,
+		Example: s.Example,
+	}
+	if s.SetupSubcommands == nil {
+		cmd.Run = s.run
+	} else {
+		for _, sub := range s.SetupSubcommands() {
+			addSubcommandToCobraCmd(&cmd, sub)
+		}
+	}
+	if s.SetupFlags != nil {
+		s.SetupFlags(cmd.Flags())
+	}
+	c.AddCommand(&cmd)
+}
+
+func Run() {
+	if err := rootCmd.Execute(); err != nil {
+		os.Exit(1)
+	}
+}

client/configcheck.go

@@ -0,0 +1,120 @@
+package client
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/kr/pretty"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+	"github.com/zrepl/yaml-config"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/logger"
+)
+
+var configcheckArgs struct {
+	format string
+	what   string
+}
+
+var ConfigcheckCmd = &cli.Subcommand{
+	Use:   "configcheck",
+	Short: "check if config can be parsed without errors",
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]")
+		f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]")
+	},
+	Run: func(subcommand *cli.Subcommand, args []string) error {
+		formatMap := map[string]func(interface{}){
+			"": func(i interface{}) {},
+			"pretty": func(i interface{}) {
+				if _, err := pretty.Println(i); err != nil {
+					panic(err)
+				}
+			},
+			"json": func(i interface{}) {
+				if err := json.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
+					panic(err)
+				}
+			},
+			"yaml": func(i interface{}) {
+				if err := yaml.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
+					panic(err)
+				}
+			},
+		}
+
+		formatter, ok := formatMap[configcheckArgs.format]
+		if !ok {
+			return fmt.Errorf("unsupported --format %q", configcheckArgs.format)
+		}
+
+		var hadErr bool
+		// further: try to build jobs
+		confJobs, err := job.JobsFromConfig(subcommand.Config())
+		if err != nil {
+			err := errors.Wrap(err, "cannot build jobs from config")
+			if configcheckArgs.what == "jobs" {
+				return err
+			} else {
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				confJobs = nil
+				hadErr = true
+			}
+		}
+
+		// further: try to build logging outlets
+		outlets, err := logging.OutletsFromConfig(*subcommand.Config().Global.Logging)
+		if err != nil {
+			err := errors.Wrap(err, "cannot build logging from config")
+			if configcheckArgs.what == "logging" {
+				return err
+			} else {
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				outlets = nil
+				hadErr = true
+			}
+		}
+
+		whatMap := map[string]func(){
+			"all": func() {
+				o := struct {
+					config  *config.Config
+					jobs    []job.Job
+					logging *logger.Outlets
+				}{
+					subcommand.Config(),
+					confJobs,
+					outlets,
+				}
+				formatter(o)
+			},
+			"config": func() {
+				formatter(subcommand.Config())
+			},
+			"jobs": func() {
+				formatter(confJobs)
+			},
+			"logging": func() {
+				formatter(outlets)
+			},
+		}
+
+		wf, ok := whatMap[configcheckArgs.what]
+		if !ok {
+			return fmt.Errorf("unsupported --format %q", configcheckArgs.what)
+		}
+		wf()
+
+		if hadErr {
+			return fmt.Errorf("config parsing failed")
+		} else {
+			return nil
+		}
+	},
+}

client/jsonclient.go

@@ -0,0 +1,49 @@
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net"
+	"net/http"
+
+	"github.com/pkg/errors"
+)
+
+func controlHttpClient(sockpath string) (client http.Client, err error) {
+	return http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", sockpath)
+			},
+		},
+	}, nil
+}
+
+func jsonRequestResponse(c http.Client, endpoint string, req interface{}, res interface{}) error {
+	var buf bytes.Buffer
+	encodeErr := json.NewEncoder(&buf).Encode(req)
+	if encodeErr != nil {
+		return encodeErr
+	}
+
+	resp, err := c.Post("http://unix"+endpoint, "application/json", &buf)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		var msg bytes.Buffer
+		_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
+		return errors.Errorf("%s", msg.String())
+	}
+
+	decodeError := json.NewDecoder(resp.Body).Decode(&res)
+	if decodeError != nil {
+		return decodeError
+	}
+
+	return nil
+}

client/migrate.go

@@ -0,0 +1,104 @@
+package client
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/zfs"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+)
+
+var (
+	MigrateCmd = &cli.Subcommand{
+		Use:   "migrate",
+		Short: "perform migration of the on-disk / zfs properties",
+		SetupSubcommands: func() []*cli.Subcommand {
+			return migrations
+		},
+	}
+)
+
+var migrations = []*cli.Subcommand{
+	&cli.Subcommand{
+		Use: "0.0.X:0.1:placeholder",
+		Run: doMigratePlaceholder0_1,
+		SetupFlags: func(f *pflag.FlagSet) {
+			f.BoolVar(&migratePlaceholder0_1Args.dryRun, "dry-run", false, "dry run")
+		},
+	},
+}
+
+var migratePlaceholder0_1Args struct {
+	dryRun bool
+}
+
+func doMigratePlaceholder0_1(sc *cli.Subcommand, args []string) error {
+	if len(args) != 0 {
+		return fmt.Errorf("migration does not take arguments, got %v", args)
+	}
+
+	cfg := sc.Config()
+
+	ctx := context.Background()
+	allFSS, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
+	if err != nil {
+		return errors.Wrap(err, "cannot list filesystems")
+	}
+
+	type workItem struct {
+		jobName string
+		rootFS  *zfs.DatasetPath
+		fss     []*zfs.DatasetPath
+	}
+	var wis []workItem
+	for i, j := range cfg.Jobs {
+		var rfsS string
+		switch job := j.Ret.(type) {
+		case *config.SinkJob:
+			rfsS = job.RootFS
+		case *config.PullJob:
+			rfsS = job.RootFS
+		default:
+			fmt.Printf("ignoring job %q (%d/%d, type %T)\n", j.Name(), i, len(cfg.Jobs), j.Ret)
+			continue
+		}
+		rfs, err := zfs.NewDatasetPath(rfsS)
+		if err != nil {
+			return errors.Wrapf(err, "root fs for job %q is not a valid dataset path", j.Name())
+		}
+		var fss []*zfs.DatasetPath
+		for _, fs := range allFSS {
+			if fs.HasPrefix(rfs) {
+				fss = append(fss, fs)
+			}
+		}
+		wis = append(wis, workItem{j.Name(), rfs, fss})
+	}
+
+	for _, wi := range wis {
+		fmt.Printf("job %q => migrate filesystems below root_fs %q\n", wi.jobName, wi.rootFS.ToString())
+		if len(wi.fss) == 0 {
+			fmt.Printf("\tno filesystems\n")
+			continue
+		}
+		for _, fs := range wi.fss {
+			fmt.Printf("\t%q ... ", fs.ToString())
+			r, err := zfs.ZFSMigrateHashBasedPlaceholderToCurrent(fs, migratePlaceholder0_1Args.dryRun)
+			if err != nil {
+				fmt.Printf("error: %s\n", err)
+			} else if !r.NeedsModification {
+				fmt.Printf("unchanged (placeholder=%v)\n", r.OriginalState.IsPlaceholder)
+			} else {
+				fmt.Printf("migrate (placeholder=%v) (old value = %q)\n",
+					r.OriginalState.IsPlaceholder, r.OriginalState.RawLocalPropertyValue)
+			}
+		}
+	}
+
+	return nil
+}

client/migrate_test.go

@@ -0,0 +1,16 @@
+package client
+
+import "testing"
+
+func TestMigrationsUnambiguousNames(t *testing.T) {
+	names := make(map[string]bool)
+	for _, mig := range migrations {
+		if _, ok := names[mig.Use]; ok {
+			t.Errorf("duplicate migration name %q", mig.Use)
+			t.FailNow()
+			return
+		} else {
+			names[mig.Use] = true
+		}
+	}
+}

client/pprof.go

@@ -0,0 +1,67 @@
+package client
+
+import (
+	"errors"
+	"log"
+	"os"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+)
+
+var pprofArgs struct {
+	daemon.PprofServerControlMsg
+}
+
+var PprofCmd = &cli.Subcommand{
+	Use:   "pprof off | [on TCP_LISTEN_ADDRESS]",
+	Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
+	Run: func(subcommand *cli.Subcommand, args []string) error {
+		if len(args) < 1 {
+			goto enargs
+		}
+		switch args[0] {
+		case "on":
+			pprofArgs.Run = true
+			if len(args) != 2 {
+				return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
+			}
+			pprofArgs.HttpListenAddress = args[1]
+		case "off":
+			if len(args) != 1 {
+				goto enargs
+			}
+			pprofArgs.Run = false
+		}
+
+		RunPProf(subcommand.Config())
+		return nil
+	enargs:
+		return errors.New("invalid number of positional arguments")
+
+	},
+}
+
+func RunPProf(conf *config.Config) {
+	log := log.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Printf("exiting after error")
+		os.Exit(1)
+	}
+
+	log.Printf("connecting to zrepl daemon")
+
+	httpc, err := controlHttpClient(conf.Global.Control.SockPath)
+	if err != nil {
+		log.Printf("error creating http client: %s", err)
+		die()
+	}
+	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofArgs.PprofServerControlMsg, struct{}{})
+	if err != nil {
+		log.Printf("error sending control message: %s", err)
+		die()
+	}
+	log.Printf("finished")
+}

client/signal.go

@@ -0,0 +1,40 @@
+package client
+
+import (
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+)
+
+var SignalCmd = &cli.Subcommand{
+	Use:   "signal [wakeup|reset] JOB",
+	Short: "wake up a job from wait state or abort its current invocation",
+	Run: func(subcommand *cli.Subcommand, args []string) error {
+		return runSignalCmd(subcommand.Config(), args)
+	},
+}
+
+func runSignalCmd(config *config.Config, args []string) error {
+	if len(args) != 2 {
+		return errors.Errorf("Expected 2 arguments: [wakeup|reset] JOB")
+	}
+
+	httpc, err := controlHttpClient(config.Global.Control.SockPath)
+	if err != nil {
+		return err
+	}
+
+	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointSignal,
+		struct {
+			Name string
+			Op   string
+		}{
+			Name: args[1],
+			Op:   args[0],
+		},
+		struct{}{},
+	)
+	return err
+}

client/status.go

@@ -0,0 +1,632 @@
+package client
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"os"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gdamore/tcell/termbox"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+	"github.com/zrepl/yaml-config"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/daemon"
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/daemon/pruner"
+	"github.com/zrepl/zrepl/replication/report"
+)
+
+type byteProgressMeasurement struct {
+	time time.Time
+	val  int64
+}
+
+type bytesProgressHistory struct {
+	last        *byteProgressMeasurement // pointer as poor man's optional
+	changeCount int
+	lastChange  time.Time
+	bpsAvg      float64
+}
+
+func (p *bytesProgressHistory) Update(currentVal int64) (bytesPerSecondAvg int64, changeCount int) {
+
+	if p.last == nil {
+		p.last = &byteProgressMeasurement{
+			time: time.Now(),
+			val:  currentVal,
+		}
+		return 0, 0
+	}
+
+	if p.last.val != currentVal {
+		p.changeCount++
+		p.lastChange = time.Now()
+	}
+
+	if time.Since(p.lastChange) > 3*time.Second {
+		p.last = nil
+		return 0, 0
+	}
+
+	deltaV := currentVal - p.last.val
+	deltaT := time.Since(p.last.time)
+	rate := float64(deltaV) / deltaT.Seconds()
+
+	factor := 0.3
+	p.bpsAvg = (1-factor)*p.bpsAvg + factor*rate
+
+	p.last.time = time.Now()
+	p.last.val = currentVal
+
+	return int64(p.bpsAvg), p.changeCount
+}
+
+type tui struct {
+	x, y   int
+	indent int
+
+	lock   sync.Mutex //For report and error
+	report map[string]job.Status
+	err    error
+
+	replicationProgress map[string]*bytesProgressHistory // by job name
+}
+
+func newTui() tui {
+	return tui{
+		replicationProgress: make(map[string]*bytesProgressHistory),
+	}
+}
+
+const INDENT_MULTIPLIER = 4
+
+func (t *tui) moveLine(dl int, col int) {
+	t.y += dl
+	t.x = t.indent*INDENT_MULTIPLIER + col
+}
+
+func (t *tui) write(text string) {
+	for _, c := range text {
+		if c == '\n' {
+			t.newline()
+			continue
+		}
+		termbox.SetCell(t.x, t.y, c, termbox.ColorDefault, termbox.ColorDefault)
+		t.x += 1
+	}
+}
+
+func (t *tui) printf(text string, a ...interface{}) {
+	t.write(fmt.Sprintf(text, a...))
+}
+
+func wrap(s string, width int) string {
+	var b strings.Builder
+	for len(s) > 0 {
+		rem := width
+		if rem > len(s) {
+			rem = len(s)
+		}
+		if idx := strings.IndexAny(s, "\n\r"); idx != -1 && idx < rem {
+			rem = idx + 1
+		}
+		untilNewline := strings.TrimRight(s[:rem], "\n\r")
+		s = s[rem:]
+		if len(untilNewline) == 0 {
+			continue
+		}
+		b.WriteString(untilNewline)
+		b.WriteString("\n")
+	}
+	return strings.TrimRight(b.String(), "\n\r")
+}
+
+func (t *tui) printfDrawIndentedAndWrappedIfMultiline(format string, a ...interface{}) {
+	whole := fmt.Sprintf(format, a...)
+	width, _ := termbox.Size()
+	if !strings.ContainsAny(whole, "\n\r") && t.x+len(whole) <= width {
+		t.printf(format, a...)
+	} else {
+		t.addIndent(1)
+		t.newline()
+		t.write(wrap(whole, width-INDENT_MULTIPLIER*t.indent))
+		t.addIndent(-1)
+	}
+}
+
+func (t *tui) newline() {
+	t.moveLine(1, 0)
+}
+
+func (t *tui) setIndent(indent int) {
+	t.indent = indent
+	t.moveLine(0, 0)
+}
+
+func (t *tui) addIndent(indent int) {
+	t.indent += indent
+	t.moveLine(0, 0)
+}
+
+var statusFlags struct {
+	Raw bool
+}
+
+var StatusCmd = &cli.Subcommand{
+	Use:   "status",
+	Short: "show job activity or dump as JSON for monitoring",
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.BoolVar(&statusFlags.Raw, "raw", false, "dump raw status description from zrepl daemon")
+	},
+	Run: runStatus,
+}
+
+func runStatus(s *cli.Subcommand, args []string) error {
+	httpc, err := controlHttpClient(s.Config().Global.Control.SockPath)
+	if err != nil {
+		return err
+	}
+
+	if statusFlags.Raw {
+		resp, err := httpc.Get("http://unix" + daemon.ControlJobEndpointStatus)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != http.StatusOK {
+			fmt.Fprintf(os.Stderr, "Received error response:\n")
+			_, err := io.CopyN(os.Stderr, resp.Body, 4096)
+			if err != nil {
+				return err
+			}
+			return errors.Errorf("exit")
+		}
+		if _, err := io.Copy(os.Stdout, resp.Body); err != nil {
+			return err
+		}
+		return nil
+	}
+
+	t := newTui()
+	t.lock.Lock()
+	t.err = errors.New("Got no report yet")
+	t.lock.Unlock()
+
+	err = termbox.Init()
+	if err != nil {
+		return err
+	}
+	defer termbox.Close()
+
+	update := func() {
+		m := make(map[string]job.Status)
+
+		err2 := jsonRequestResponse(httpc, daemon.ControlJobEndpointStatus,
+			struct{}{},
+			&m,
+		)
+
+		t.lock.Lock()
+		t.err = err2
+		t.report = m
+		t.lock.Unlock()
+		t.draw()
+	}
+	update()
+
+	ticker := time.NewTicker(500 * time.Millisecond)
+	defer ticker.Stop()
+	go func() {
+		for range ticker.C {
+			update()
+		}
+	}()
+
+	termbox.HideCursor()
+	termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
+
+loop:
+	for {
+		switch ev := termbox.PollEvent(); ev.Type {
+		case termbox.EventKey:
+			switch ev.Key {
+			case termbox.KeyEsc:
+				break loop
+			case termbox.KeyCtrlC:
+				break loop
+			}
+		case termbox.EventResize:
+			t.draw()
+		}
+	}
+
+	return nil
+
+}
+
+func (t *tui) getReplicationProgresHistory(jobName string) *bytesProgressHistory {
+	p, ok := t.replicationProgress[jobName]
+	if !ok {
+		p = &bytesProgressHistory{}
+		t.replicationProgress[jobName] = p
+	}
+	return p
+}
+
+func (t *tui) draw() {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
+	t.x = 0
+	t.y = 0
+	t.indent = 0
+
+	if t.err != nil {
+		t.write(t.err.Error())
+	} else {
+		//Iterate over map in alphabetical order
+		keys := make([]string, len(t.report))
+		i := 0
+		for k := range t.report {
+			keys[i] = k
+			i++
+		}
+		sort.Strings(keys)
+
+		for _, k := range keys {
+			v := t.report[k]
+			if len(k) == 0 || daemon.IsInternalJobName(k) { //Internal job
+				continue
+			}
+			t.setIndent(0)
+
+			t.printf("Job: %s", k)
+			t.setIndent(1)
+			t.newline()
+			t.printf("Type: %s", v.Type)
+			t.setIndent(1)
+			t.newline()
+
+			if v.Type == job.TypePush || v.Type == job.TypePull {
+				activeStatus, ok := v.JobSpecific.(*job.ActiveSideStatus)
+				if !ok || activeStatus == nil {
+					t.printf("ActiveSideStatus is null")
+					t.newline()
+					continue
+				}
+
+				t.printf("Replication:")
+				t.newline()
+				t.addIndent(1)
+				t.renderReplicationReport(activeStatus.Replication, t.getReplicationProgresHistory(k))
+				t.addIndent(-1)
+
+				t.printf("Pruning Sender:")
+				t.newline()
+				t.addIndent(1)
+				t.renderPrunerReport(activeStatus.PruningSender)
+				t.addIndent(-1)
+
+				t.printf("Pruning Receiver:")
+				t.newline()
+				t.addIndent(1)
+				t.renderPrunerReport(activeStatus.PruningReceiver)
+				t.addIndent(-1)
+			} else if v.Type == job.TypeSnap {
+				snapStatus, ok := v.JobSpecific.(*job.SnapJobStatus)
+				if !ok || snapStatus == nil {
+					t.printf("SnapJobStatus is null")
+					t.newline()
+					continue
+				}
+				t.printf("Pruning snapshots:")
+				t.newline()
+				t.addIndent(1)
+				t.renderPrunerReport(snapStatus.Pruning)
+				t.addIndent(-1)
+			} else {
+				t.printf("No status representation for job type '%s', dumping as YAML", v.Type)
+				t.newline()
+				asYaml, err := yaml.Marshal(v.JobSpecific)
+				if err != nil {
+					t.printf("Error marshaling status to YAML: %s", err)
+					t.newline()
+					continue
+				}
+				t.write(string(asYaml))
+				t.newline()
+				continue
+			}
+		}
+	}
+	termbox.Flush()
+}
+
+func (t *tui) renderReplicationReport(rep *report.Report, history *bytesProgressHistory) {
+	if rep == nil {
+		t.printf("...\n")
+		return
+	}
+
+	if rep.WaitReconnectError != nil {
+		t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: %s", rep.WaitReconnectError)
+		t.newline()
+	}
+	if !rep.WaitReconnectSince.IsZero() {
+		delta := time.Until(rep.WaitReconnectUntil).Round(time.Second)
+		if rep.WaitReconnectUntil.IsZero() || delta > 0 {
+			var until string
+			if rep.WaitReconnectUntil.IsZero() {
+				until = "waiting indefinitely"
+			} else {
+				until = fmt.Sprintf("hard fail in %s @ %s", delta, rep.WaitReconnectUntil)
+			}
+			t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnecting with exponential backoff (since %s) (%s)",
+				rep.WaitReconnectSince, until)
+		} else {
+			t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnects reached hard-fail timeout @ %s", rep.WaitReconnectUntil)
+		}
+		t.newline()
+	}
+
+	// TODO visualize more than the latest attempt by folding all attempts into one
+	if len(rep.Attempts) == 0 {
+		t.printf("no attempts made yet")
+		return
+	} else {
+		t.printf("Attempt #%d", len(rep.Attempts))
+		if len(rep.Attempts) > 1 {
+			t.printf(". Previous attempts failed with the follwing statuses:")
+			t.newline()
+			t.addIndent(1)
+			for i, a := range rep.Attempts[:len(rep.Attempts)-1] {
+				t.printfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt))
+				t.newline()
+			}
+			t.addIndent(-1)
+		} else {
+			t.newline()
+		}
+	}
+
+	latest := rep.Attempts[len(rep.Attempts)-1]
+	sort.Slice(latest.Filesystems, func(i, j int) bool {
+		return latest.Filesystems[i].Info.Name < latest.Filesystems[j].Info.Name
+	})
+
+	t.printf("Status: %s", latest.State)
+	t.newline()
+	if latest.State == report.AttemptPlanningError {
+		t.printf("Problem: ")
+		t.printfDrawIndentedAndWrappedIfMultiline("%s", latest.PlanError)
+		t.newline()
+	} else if latest.State == report.AttemptFanOutError {
+		t.printf("Problem: one or more of the filesystems encountered errors")
+		t.newline()
+	}
+
+	if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
+		// Draw global progress bar
+		// Progress: [---------------]
+		expected, replicated := latest.BytesSum()
+		rate, changeCount := history.Update(replicated)
+		t.write("Progress: ")
+		t.drawBar(50, replicated, expected, changeCount)
+		t.write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinary(replicated), ByteCountBinary(expected), ByteCountBinary(rate)))
+		t.newline()
+
+		var maxFSLen int
+		for _, fs := range latest.Filesystems {
+			if len(fs.Info.Name) > maxFSLen {
+				maxFSLen = len(fs.Info.Name)
+			}
+		}
+		for _, fs := range latest.Filesystems {
+			t.printFilesystemStatus(fs, false, maxFSLen) // FIXME bring 'active' flag back
+		}
+
+	}
+
+}
+
+func (t *tui) renderPrunerReport(r *pruner.Report) {
+	if r == nil {
+		t.printf("...\n")
+		return
+	}
+
+	state, err := pruner.StateString(r.State)
+	if err != nil {
+		t.printf("Status: %q (parse error: %q)\n", r.State, err)
+		return
+	}
+
+	t.printf("Status: %s", state)
+	t.newline()
+
+	if r.Error != "" {
+		t.printf("Error: %s\n", r.Error)
+	}
+
+	type commonFS struct {
+		*pruner.FSReport
+		completed bool
+	}
+	all := make([]commonFS, 0, len(r.Pending)+len(r.Completed))
+	for i := range r.Pending {
+		all = append(all, commonFS{&r.Pending[i], false})
+	}
+	for i := range r.Completed {
+		all = append(all, commonFS{&r.Completed[i], true})
+	}
+
+	switch state {
+	case pruner.Plan:
+		fallthrough
+	case pruner.PlanErr:
+		return
+	}
+
+	if len(all) == 0 {
+		t.printf("nothing to do\n")
+		return
+	}
+
+	var totalDestroyCount, completedDestroyCount int
+	var maxFSname int
+	for _, fs := range all {
+		totalDestroyCount += len(fs.DestroyList)
+		if fs.completed {
+			completedDestroyCount += len(fs.DestroyList)
+		}
+		if maxFSname < len(fs.Filesystem) {
+			maxFSname = len(fs.Filesystem)
+		}
+	}
+
+	// global progress bar
+	progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
+	t.write("Progress: ")
+	t.write("[")
+	t.write(times("=", progress))
+	t.write(">")
+	t.write(times("-", 80-progress))
+	t.write("]")
+	t.printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
+	t.newline()
+
+	sort.SliceStable(all, func(i, j int) bool {
+		return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
+	})
+
+	// Draw a table-like representation of 'all'
+	for _, fs := range all {
+		t.write(rightPad(fs.Filesystem, maxFSname, " "))
+		t.write(" ")
+		if !fs.SkipReason.NotSkipped() {
+			t.printf("skipped: %s\n", fs.SkipReason)
+			continue
+		}
+		if fs.LastError != "" {
+			if strings.ContainsAny(fs.LastError, "\r\n") {
+				t.printf("ERROR:")
+				t.printfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError)
+			} else {
+				t.printfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError)
+			}
+			t.newline()
+			continue
+		}
+
+		pruneRuleActionStr := fmt.Sprintf("(destroy %d of %d snapshots)",
+			len(fs.DestroyList), len(fs.SnapshotList))
+
+		if fs.completed {
+			t.printf("Completed  %s\n", pruneRuleActionStr)
+			continue
+		}
+
+		t.write("Pending    ") // whitespace is padding 10
+		if len(fs.DestroyList) == 1 {
+			t.write(fs.DestroyList[0].Name)
+		} else {
+			t.write(pruneRuleActionStr)
+		}
+		t.newline()
+	}
+
+}
+
+func times(str string, n int) (out string) {
+	for i := 0; i < n; i++ {
+		out += str
+	}
+	return
+}
+
+func rightPad(str string, length int, pad string) string {
+	if len(str) > length {
+		return str[:length]
+	}
+	return str + times(pad, length-len(str))
+}
+
+var arrowPositions = `>\|/`
+
+// changeCount = 0 indicates stall / no progresss
+func (t *tui) drawBar(length int, bytes, totalBytes int64, changeCount int) {
+	var completedLength int
+	if totalBytes > 0 {
+		completedLength = int(int64(length) * bytes / totalBytes)
+		if completedLength > length {
+			completedLength = length
+		}
+	} else if totalBytes == bytes {
+		completedLength = length
+	}
+
+	t.write("[")
+	t.write(times("=", completedLength))
+	t.write(string(arrowPositions[changeCount%len(arrowPositions)]))
+	t.write(times("-", length-completedLength))
+	t.write("]")
+}
+
+func (t *tui) printFilesystemStatus(rep *report.FilesystemReport, active bool, maxFS int) {
+
+	expected, replicated := rep.BytesSum()
+	status := fmt.Sprintf("%s (step %d/%d, %s/%s)",
+		strings.ToUpper(string(rep.State)),
+		rep.CurrentStep, len(rep.Steps),
+		ByteCountBinary(replicated), ByteCountBinary(expected),
+	)
+
+	activeIndicator := " "
+	if active {
+		activeIndicator = "*"
+	}
+	t.printf("%s %s %s ",
+		activeIndicator,
+		rightPad(rep.Info.Name, maxFS, " "),
+		status)
+
+	next := ""
+	if err := rep.Error(); err != nil {
+		next = err.Err
+	} else if rep.State != report.FilesystemDone {
+		if nextStep := rep.NextStep(); nextStep != nil {
+			if nextStep.IsIncremental() {
+				next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
+			} else {
+				next = fmt.Sprintf("next: %s (full)", nextStep.Info.To)
+			}
+		} else {
+			next = "" // individual FSes may still be in planning state
+		}
+
+	}
+	t.printfDrawIndentedAndWrappedIfMultiline("%s", next)
+
+	t.newline()
+}
+
+func ByteCountBinary(b int64) string {
+	const unit = 1024
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := int64(unit), 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
+}

client/stdinserver.go

@@ -1,55 +1,51 @@
-package cmd
+package client
 
 import (
 	"os"
 
-	"context"
 	"github.com/problame/go-netssh"
-	"github.com/spf13/cobra"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+
+	"context"
+	"errors"
 	"log"
 	"path"
 )
 
-var StdinserverCmd = &cobra.Command{
+var StdinserverCmd = &cli.Subcommand{
 	Use:   "stdinserver CLIENT_IDENTITY",
-	Short: "start in stdinserver mode (from authorized_keys file)",
+	Short: "stdinserver transport mode (started from authorized_keys file as forced command)",
-	Run:   cmdStdinServer,
+	Run: func(subcommand *cli.Subcommand, args []string) error {
+		return runStdinserver(subcommand.Config(), args)
+	},
 }
 
-func init() {
+func runStdinserver(config *config.Config, args []string) error {
-	RootCmd.AddCommand(StdinserverCmd)
-}
-
-func cmdStdinServer(cmd *cobra.Command, args []string) {
 
 	// NOTE: the netssh proxying protocol requires exiting with non-zero status if anything goes wrong
 	defer os.Exit(1)
 
 	log := log.New(os.Stderr, "", log.LUTC|log.Ldate|log.Ltime)
 
-	conf, err := ParseConfig(rootArgs.configFile)
-	if err != nil {
-		log.Printf("error parsing config: %s", err)
-		return
-	}
-
 	if len(args) != 1 || args[0] == "" {
-		log.Print("must specify client_identity as positional argument")
+		err := errors.New("must specify client_identity as positional argument")
-		return
+		return err
 	}
 
 	identity := args[0]
-	unixaddr := path.Join(conf.Global.Serve.Stdinserver.SockDir, identity)
+	unixaddr := path.Join(config.Global.Serve.StdinServer.SockDir, identity)
 
 	log.Printf("proxying client identity '%s' to zrepl daemon '%s'", identity, unixaddr)
 
 	ctx := netssh.ContextWithLog(context.TODO(), log)
 
-	err = netssh.Proxy(ctx, unixaddr)
+	err := netssh.Proxy(ctx, unixaddr)
 	if err == nil {
 		log.Print("proxying finished successfully, exiting with status 0")
 		os.Exit(0)
 	}
 	log.Printf("error proxying: %s", err)
-
+	return nil
 }

client/testcmd.go

@@ -0,0 +1,178 @@
+package client
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var TestCmd = &cli.Subcommand{
+	Use: "test",
+	SetupSubcommands: func() []*cli.Subcommand {
+		return []*cli.Subcommand{testFilter, testPlaceholder}
+	},
+}
+
+var testFilterArgs struct {
+	job   string
+	all   bool
+	input string
+}
+
+var testFilter = &cli.Subcommand{
+	Use:   "filesystems --job JOB [--all | --input INPUT]",
+	Short: "test filesystems filter specified in push or source job",
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&testFilterArgs.job, "job", "", "the name of the push or source job")
+		f.StringVar(&testFilterArgs.input, "input", "", "a filesystem name to test against the job's filters")
+		f.BoolVar(&testFilterArgs.all, "all", false, "test all local filesystems")
+	},
+	Run: runTestFilterCmd,
+}
+
+func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
+
+	if testFilterArgs.job == "" {
+		return fmt.Errorf("must specify --job flag")
+	}
+	if !(testFilterArgs.all != (testFilterArgs.input != "")) { // xor
+		return fmt.Errorf("must set one: --all or --input")
+	}
+
+	conf := subcommand.Config()
+
+	var confFilter config.FilesystemsFilter
+	job, err := conf.Job(testFilterArgs.job)
+	if err != nil {
+		return err
+	}
+	switch j := job.Ret.(type) {
+	case *config.SourceJob:
+		confFilter = j.Filesystems
+	case *config.PushJob:
+		confFilter = j.Filesystems
+	default:
+		return fmt.Errorf("job type %T does not have filesystems filter", j)
+	}
+
+	f, err := filters.DatasetMapFilterFromConfig(confFilter)
+	if err != nil {
+		return fmt.Errorf("filter invalid: %s", err)
+	}
+
+	var fsnames []string
+	if testFilterArgs.input != "" {
+		fsnames = []string{testFilterArgs.input}
+	} else {
+		out, err := zfs.ZFSList([]string{"name"})
+		if err != nil {
+			return fmt.Errorf("could not list ZFS filesystems: %s", err)
+		}
+		for _, row := range out {
+
+			fsnames = append(fsnames, row[0])
+		}
+	}
+
+	fspaths := make([]*zfs.DatasetPath, len(fsnames))
+	for i, fsname := range fsnames {
+		path, err := zfs.NewDatasetPath(fsname)
+		if err != nil {
+			return err
+		}
+		fspaths[i] = path
+	}
+
+	hadFilterErr := false
+	for _, in := range fspaths {
+		var res string
+		var errStr string
+		pass, err := f.Filter(in)
+		if err != nil {
+			res = "ERROR"
+			errStr = err.Error()
+			hadFilterErr = true
+		} else if pass {
+			res = "ACCEPT"
+		} else {
+			res = "REJECT"
+		}
+		fmt.Printf("%s\t%s\t%s\n", res, in.ToString(), errStr)
+	}
+
+	if hadFilterErr {
+		return fmt.Errorf("filter errors occurred")
+	}
+	return nil
+}
+
+var testPlaceholderArgs struct {
+	ds  string
+	all bool
+}
+
+var testPlaceholder = &cli.Subcommand{
+	Use:   "placeholder [--all | --dataset DATASET]",
+	Short: fmt.Sprintf("list received placeholder filesystems (zfs property %q)", zfs.PlaceholderPropertyName),
+	Example: `
+	placeholder --all
+	placeholder --dataset path/to/sink/clientident/fs`,
+	NoRequireConfig: true,
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&testPlaceholderArgs.ds, "dataset", "", "dataset path (not required to exist)")
+		f.BoolVar(&testPlaceholderArgs.all, "all", false, "list tab-separated placeholder status of all filesystems")
+	},
+	Run: runTestPlaceholder,
+}
+
+func runTestPlaceholder(subcommand *cli.Subcommand, args []string) error {
+
+	var checkDPs []*zfs.DatasetPath
+
+	// all actions first
+	if testPlaceholderArgs.all {
+		out, err := zfs.ZFSList([]string{"name"})
+		if err != nil {
+			return errors.Wrap(err, "could not list ZFS filesystems")
+		}
+		for _, row := range out {
+			dp, err := zfs.NewDatasetPath(row[0])
+			if err != nil {
+				panic(err)
+			}
+			checkDPs = append(checkDPs, dp)
+		}
+	} else {
+		dp, err := zfs.NewDatasetPath(testPlaceholderArgs.ds)
+		if err != nil {
+			return err
+		}
+		if dp.Empty() {
+			return fmt.Errorf("must specify --dataset DATASET or --all")
+		}
+		checkDPs = append(checkDPs, dp)
+	}
+
+	fmt.Printf("IS_PLACEHOLDER\tDATASET\tzrepl:placeholder\n")
+	for _, dp := range checkDPs {
+		ph, err := zfs.ZFSGetFilesystemPlaceholderState(dp)
+		if err != nil {
+			return errors.Wrap(err, "cannot get placeholder state")
+		}
+		if !ph.FSExists {
+			panic("placeholder state inconsistent: filesystem " + ph.FS + " must exist in this context")
+		}
+		is := "yes"
+		if !ph.IsPlaceholder {
+			is = "no"
+		}
+		fmt.Printf("%s\t%s\t%s\n", is, dp.ToString(), ph.RawLocalPropertyValue)
+	}
+	return nil
+}

client/version.go

@@ -0,0 +1,74 @@
+package client
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+	"github.com/zrepl/zrepl/version"
+)
+
+var versionArgs struct {
+	Show      string
+	Config    *config.Config
+	ConfigErr error
+}
+
+var VersionCmd = &cli.Subcommand{
+	Use:             "version",
+	Short:           "print version of zrepl binary and running daemon",
+	NoRequireConfig: true,
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)")
+	},
+	Run: func(subcommand *cli.Subcommand, args []string) error {
+		versionArgs.Config = subcommand.Config()
+		versionArgs.ConfigErr = subcommand.ConfigParsingError()
+		return runVersionCmd()
+	},
+}
+
+func runVersionCmd() error {
+	args := versionArgs
+
+	if args.Show != "daemon" && args.Show != "client" && args.Show != "" {
+		return fmt.Errorf("show flag must be 'client' or 'server' or be left empty")
+	}
+
+	var clientVersion, daemonVersion *version.ZreplVersionInformation
+	if args.Show == "client" || args.Show == "" {
+		clientVersion = version.NewZreplVersionInformation()
+		fmt.Printf("client: %s\n", clientVersion.String())
+	}
+	if args.Show == "daemon" || args.Show == "" {
+
+		if args.ConfigErr != nil {
+			return fmt.Errorf("config parsing error: %s", args.ConfigErr)
+		}
+
+		httpc, err := controlHttpClient(args.Config.Global.Control.SockPath)
+		if err != nil {
+			return fmt.Errorf("server: error: %s\n", err)
+		}
+
+		var info version.ZreplVersionInformation
+		err = jsonRequestResponse(httpc, daemon.ControlJobEndpointVersion, "", &info)
+		if err != nil {
+			return fmt.Errorf("server: error: %s\n", err)
+		}
+		daemonVersion = &info
+		fmt.Printf("server: %s\n", daemonVersion.String())
+	}
+
+	if args.Show == "" {
+		if clientVersion.Version != daemonVersion.Version {
+			fmt.Fprintf(os.Stderr, "WARNING: client version != daemon version, restart zrepl daemon\n")
+		}
+	}
+
+	return nil
+}

cmd/autosnap.go

@@ -1,201 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"github.com/zrepl/zrepl/zfs"
-	"sort"
-	"time"
-)
-
-type IntervalAutosnap struct {
-	task             *Task
-	DatasetFilter    zfs.DatasetFilter
-	Prefix           string
-	SnapshotInterval time.Duration
-}
-
-func (a *IntervalAutosnap) filterFilesystems() (fss []*zfs.DatasetPath, stop bool) {
-	a.task.Enter("filter_filesystems")
-	defer a.task.Finish()
-	fss, err := zfs.ZFSListMapping(a.DatasetFilter)
-	stop = err != nil
-	if err != nil {
-		a.task.Log().WithError(err).Error("cannot list datasets")
-	}
-	if len(fss) == 0 {
-		a.task.Log().Warn("no filesystem matching filesystem filter")
-	}
-	return fss, stop
-}
-
-func (a *IntervalAutosnap) findSyncPoint(fss []*zfs.DatasetPath) (syncPoint time.Time, err error) {
-	a.task.Enter("find_sync_point")
-	defer a.task.Finish()
-	type snapTime struct {
-		ds   *zfs.DatasetPath
-		time time.Time
-	}
-
-	if len(fss) == 0 {
-		return time.Now(), nil
-	}
-
-	snaptimes := make([]snapTime, 0, len(fss))
-
-	now := time.Now()
-
-	a.task.Log().Debug("examine filesystem state")
-	for _, d := range fss {
-
-		l := a.task.Log().WithField(logFSField, d.ToString())
-
-		fsvs, err := zfs.ZFSListFilesystemVersions(d, NewPrefixFilter(a.Prefix))
-		if err != nil {
-			l.WithError(err).Error("cannot list filesystem versions")
-			continue
-		}
-		if len(fsvs) <= 0 {
-			l.WithField("prefix", a.Prefix).Info("no filesystem versions with prefix")
-			continue
-		}
-
-		// Sort versions by creation
-		sort.SliceStable(fsvs, func(i, j int) bool {
-			return fsvs[i].CreateTXG < fsvs[j].CreateTXG
-		})
-
-		latest := fsvs[len(fsvs)-1]
-		l.WithField("creation", latest.Creation).
-			Debug("found latest snapshot")
-
-		since := now.Sub(latest.Creation)
-		if since < 0 {
-			l.WithField("snapshot", latest.Name).
-				WithField("creation", latest.Creation).
-				Error("snapshot is from the future")
-			continue
-		}
-		next := now
-		if since < a.SnapshotInterval {
-			next = latest.Creation.Add(a.SnapshotInterval)
-		}
-		snaptimes = append(snaptimes, snapTime{d, next})
-	}
-
-	if len(snaptimes) == 0 {
-		snaptimes = append(snaptimes, snapTime{nil, now})
-	}
-
-	sort.Slice(snaptimes, func(i, j int) bool {
-		return snaptimes[i].time.Before(snaptimes[j].time)
-	})
-
-	return snaptimes[0].time, nil
-
-}
-
-func (a *IntervalAutosnap) waitForSyncPoint(ctx context.Context, syncPoint time.Time) {
-	a.task.Enter("wait_sync_point")
-	defer a.task.Finish()
-
-	const LOG_TIME_FMT string = time.ANSIC
-
-	a.task.Log().WithField("sync_point", syncPoint.Format(LOG_TIME_FMT)).
-		Info("wait for sync point")
-
-	select {
-	case <-ctx.Done():
-		a.task.Log().WithError(ctx.Err()).Info("context done")
-		return
-	case <-time.After(syncPoint.Sub(time.Now())):
-	}
-}
-
-func (a *IntervalAutosnap) syncUpRun(ctx context.Context, didSnaps chan struct{}) (stop bool) {
-	a.task.Enter("sync_up")
-	defer a.task.Finish()
-
-	fss, stop := a.filterFilesystems()
-	if stop {
-		return true
-	}
-
-	syncPoint, err := a.findSyncPoint(fss)
-	if err != nil {
-		return true
-	}
-
-	a.waitForSyncPoint(ctx, syncPoint)
-
-	a.task.Log().Debug("snapshot all filesystems to enable further snaps in lockstep")
-	a.doSnapshots(didSnaps)
-	return false
-}
-
-func (a *IntervalAutosnap) Run(ctx context.Context, didSnaps chan struct{}) {
-
-	if a.syncUpRun(ctx, didSnaps) {
-		a.task.Log().Error("stoppping autosnap after error in sync up")
-		return
-	}
-
-	// task drops back to idle here
-
-	a.task.Log().Debug("setting up ticker in SnapshotInterval")
-	ticker := time.NewTicker(a.SnapshotInterval)
-	for {
-		select {
-		case <-ctx.Done():
-			ticker.Stop()
-			a.task.Log().WithError(ctx.Err()).Info("context done")
-			return
-
-		case <-ticker.C:
-			a.doSnapshots(didSnaps)
-		}
-	}
-
-}
-
-func (a *IntervalAutosnap) doSnapshots(didSnaps chan struct{}) {
-
-	a.task.Enter("do_snapshots")
-	defer a.task.Finish()
-
-	// don't cache the result from previous run in case the user added
-	// a new dataset in the meantime
-	ds, stop := a.filterFilesystems()
-	if stop {
-		return
-	}
-
-	// TODO channel programs -> allow a little jitter?
-	for _, d := range ds {
-		suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
-		snapname := fmt.Sprintf("%s%s", a.Prefix, suffix)
-
-		l := a.task.Log().WithField(logFSField, d.ToString()).
-			WithField("snapname", snapname)
-
-		l.Info("create snapshot")
-		err := zfs.ZFSSnapshot(d, snapname, false)
-		if err != nil {
-			a.task.Log().WithError(err).Error("cannot create snapshot")
-		}
-
-		l.Info("create corresponding bookmark")
-		err = zfs.ZFSBookmark(d, snapname, snapname)
-		if err != nil {
-			a.task.Log().WithError(err).Error("cannot create bookmark")
-		}
-
-	}
-
-	select {
-	case didSnaps <- struct{}{}:
-	default:
-		a.task.Log().Error("warning: callback channel is full, discarding")
-	}
-
-}

cmd/bashcomp.go

@@ -1,29 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-
-	"github.com/spf13/cobra"
-	"os"
-)
-
-var bashcompCmd = &cobra.Command{
-	Use:   "bashcomp path/to/out/file",
-	Short: "generate bash completions",
-	Run: func(cmd *cobra.Command, args []string) {
-		if len(args) != 1 {
-			fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
-			cmd.Usage()
-			os.Exit(1)
-		}
-		if err := RootCmd.GenBashCompletionFile(args[0]); err != nil {
-			fmt.Fprintf(os.Stderr, "error generating bash completion: %s", err)
-			os.Exit(1)
-		}
-	},
-	Hidden: true,
-}
-
-func init() {
-	RootCmd.AddCommand(bashcompCmd)
-}

cmd/config.go

@@ -1,102 +0,0 @@
-package cmd
-
-import (
-	"io"
-
-	"fmt"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type Config struct {
-	Global Global
-	Jobs   map[string]Job
-}
-
-func (c *Config) LookupJob(name string) (j Job, err error) {
-	j, ok := c.Jobs[name]
-	if !ok {
-		return nil, errors.Errorf("job '%s' is not defined", name)
-	}
-	return j, nil
-}
-
-type Global struct {
-	Serve struct {
-		Stdinserver struct {
-			SockDir string
-		}
-	}
-	Control struct {
-		Sockpath string
-	}
-	logging *LoggingConfig
-}
-
-type JobDebugSettings struct {
-	Conn struct {
-		ReadDump  string `mapstructure:"read_dump"`
-		WriteDump string `mapstructure:"write_dump"`
-	}
-	RPC struct {
-		Log bool
-	}
-}
-
-type RWCConnecter interface {
-	Connect() (io.ReadWriteCloser, error)
-}
-type AuthenticatedChannelListenerFactory interface {
-	Listen() (AuthenticatedChannelListener, error)
-}
-
-type AuthenticatedChannelListener interface {
-	Accept() (ch io.ReadWriteCloser, err error)
-	Close() (err error)
-}
-
-type SSHStdinServerConnectDescr struct {
-}
-
-type PrunePolicy interface {
-	// Prune filters versions and decide which to keep and which to remove.
-	// Prune **does not** implement the actual removal of the versions.
-	Prune(fs *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error)
-}
-
-type PruningJob interface {
-	Pruner(task *Task, side PrunePolicySide, dryRun bool) (Pruner, error)
-}
-
-// A type for constants describing different prune policies of a PruningJob
-// This is mostly a special-case for LocalJob, which is the only job that has two prune policies
-// instead of one.
-// It implements github.com/spf13/pflag.Value to be used as CLI flag for the test subcommand
-type PrunePolicySide string
-
-const (
-	PrunePolicySideDefault PrunePolicySide = ""
-	PrunePolicySideLeft    PrunePolicySide = "left"
-	PrunePolicySideRight   PrunePolicySide = "right"
-)
-
-func (s *PrunePolicySide) String() string {
-	return string(*s)
-}
-
-func (s *PrunePolicySide) Set(news string) error {
-	p := PrunePolicySide(news)
-	switch p {
-	case PrunePolicySideRight:
-		fallthrough
-	case PrunePolicySideLeft:
-		*s = p
-	default:
-		return errors.Errorf("must be either %s or %s", PrunePolicySideLeft, PrunePolicySideRight)
-	}
-	return nil
-}
-
-func (s *PrunePolicySide) Type() string {
-	return fmt.Sprintf("%s | %s", PrunePolicySideLeft, PrunePolicySideRight)
-}

cmd/config_connect.go

@@ -1,48 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"io"
-
-	"context"
-	"github.com/jinzhu/copier"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/problame/go-netssh"
-)
-
-type SSHStdinserverConnecter struct {
-	Host                 string
-	User                 string
-	Port                 uint16
-	IdentityFile         string   `mapstructure:"identity_file"`
-	TransportOpenCommand []string `mapstructure:"transport_open_command"`
-	SSHCommand           string   `mapstructure:"ssh_command"`
-	Options              []string
-}
-
-func parseSSHStdinserverConnecter(i map[string]interface{}) (c *SSHStdinserverConnecter, err error) {
-
-	c = &SSHStdinserverConnecter{}
-	if err = mapstructure.Decode(i, c); err != nil {
-		err = errors.New(fmt.Sprintf("could not parse ssh transport: %s", err))
-		return nil, err
-	}
-
-	// TODO assert fields are filled
-	return
-
-}
-
-func (c *SSHStdinserverConnecter) Connect() (rwc io.ReadWriteCloser, err error) {
-
-	var endpoint netssh.Endpoint
-	if err = copier.Copy(&endpoint, c); err != nil {
-		return
-	}
-	if rwc, err = netssh.Dial(context.TODO(), endpoint); err != nil {
-		err = errors.WithStack(err)
-		return
-	}
-	return
-}

cmd/config_fsvfilter.go

@@ -1,36 +0,0 @@
-package cmd
-
-import (
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/zfs"
-	"strings"
-)
-
-type PrefixFilter struct {
-	prefix    string
-	fstype    zfs.VersionType
-	fstypeSet bool // optionals anyone?
-}
-
-func NewPrefixFilter(prefix string) *PrefixFilter {
-	return &PrefixFilter{prefix: prefix}
-}
-
-func NewTypedPrefixFilter(prefix string, versionType zfs.VersionType) *PrefixFilter {
-	return &PrefixFilter{prefix, versionType, true}
-}
-
-func parseSnapshotPrefix(i string) (p string, err error) {
-	if len(i) <= 0 {
-		err = errors.Errorf("snapshot prefix must not be empty string")
-		return
-	}
-	p = i
-	return
-}
-
-func (f *PrefixFilter) Filter(fsv zfs.FilesystemVersion) (accept bool, err error) {
-	fstypeMatches := (!f.fstypeSet || fsv.Type == f.fstype)
-	prefixMatches := strings.HasPrefix(fsv.Name, f.prefix)
-	return fstypeMatches && prefixMatches, nil
-}

cmd/config_job_control.go

@@ -1,143 +0,0 @@
-package cmd
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"github.com/pkg/errors"
-	"io"
-	"net"
-	"net/http"
-)
-
-type ControlJob struct {
-	Name     string
-	sockaddr *net.UnixAddr
-}
-
-func NewControlJob(name, sockpath string) (j *ControlJob, err error) {
-	j = &ControlJob{Name: name}
-
-	j.sockaddr, err = net.ResolveUnixAddr("unix", sockpath)
-	if err != nil {
-		err = errors.Wrap(err, "cannot resolve unix address")
-		return
-	}
-
-	return
-}
-
-func (j *ControlJob) JobName() string {
-	return j.Name
-}
-
-func (j *ControlJob) JobStatus(ctx context.Context) (*JobStatus, error) {
-	return &JobStatus{Tasks: nil}, nil
-}
-
-const (
-	ControlJobEndpointPProf   string = "/debug/pprof"
-	ControlJobEndpointVersion string = "/version"
-	ControlJobEndpointStatus  string = "/status"
-)
-
-func (j *ControlJob) JobStart(ctx context.Context) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-	defer log.Info("control job finished")
-
-	daemon := ctx.Value(contextKeyDaemon).(*Daemon)
-
-	l, err := ListenUnixPrivate(j.sockaddr)
-	if err != nil {
-		log.WithError(err).Error("error listening")
-		return
-	}
-
-	pprofServer := NewPProfServer(ctx)
-
-	mux := http.NewServeMux()
-	mux.Handle(ControlJobEndpointPProf, requestLogger{log: log, handlerFunc: func(w http.ResponseWriter, r *http.Request) {
-		var msg PprofServerControlMsg
-		err := json.NewDecoder(r.Body).Decode(&msg)
-		if err != nil {
-			log.WithError(err).Error("bad pprof request from client")
-			w.WriteHeader(http.StatusBadRequest)
-		}
-		pprofServer.Control(msg)
-		w.WriteHeader(200)
-	}})
-	mux.Handle(ControlJobEndpointVersion,
-		requestLogger{log: log, handler: jsonResponder{func() (interface{}, error) {
-			return NewZreplVersionInformation(), nil
-		}}})
-	mux.Handle(ControlJobEndpointStatus,
-		requestLogger{log: log, handler: jsonResponder{func() (interface{}, error) {
-			return daemon.Status(), nil
-		}}})
-	server := http.Server{Handler: mux}
-
-outer:
-	for {
-
-		served := make(chan error)
-		go func() {
-			served <- server.Serve(l)
-			close(served)
-		}()
-
-		select {
-		case <-ctx.Done():
-			log.WithError(ctx.Err()).Info("context done")
-			server.Shutdown(context.Background())
-			break outer
-		case err = <-served:
-			if err != nil {
-				log.WithError(err).Error("error serving")
-				break outer
-			}
-		}
-
-	}
-
-}
-
-type jsonResponder struct {
-	producer func() (interface{}, error)
-}
-
-func (j jsonResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	res, err := j.producer()
-	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		io.WriteString(w, err.Error())
-		return
-	}
-	var buf bytes.Buffer
-	err = json.NewEncoder(&buf).Encode(res)
-	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		io.WriteString(w, err.Error())
-	} else {
-		io.Copy(w, &buf)
-	}
-}
-
-type requestLogger struct {
-	log         Logger
-	handler     http.Handler
-	handlerFunc http.HandlerFunc
-}
-
-func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	log := l.log.WithField("method", r.Method).WithField("url", r.URL)
-	log.Info("start")
-	if l.handlerFunc != nil {
-		l.handlerFunc(w, r)
-	} else if l.handler != nil {
-		l.handler.ServeHTTP(w, r)
-	} else {
-		log.Error("no handler or handlerFunc configured")
-	}
-	log.Info("finish")
-}

cmd/config_job_local.go

@@ -1,214 +0,0 @@
-package cmd
-
-import (
-	"time"
-
-	"context"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/zfs"
-	"sync"
-)
-
-type LocalJob struct {
-	Name              string
-	Mapping           *DatasetMapFilter
-	SnapshotPrefix    string
-	Interval          time.Duration
-	InitialReplPolicy InitialReplPolicy
-	PruneLHS          PrunePolicy
-	PruneRHS          PrunePolicy
-	Debug             JobDebugSettings
-	snapperTask       *Task
-	mainTask          *Task
-	handlerTask       *Task
-	pruneRHSTask      *Task
-	pruneLHSTask      *Task
-}
-
-func parseLocalJob(c JobParsingContext, name string, i map[string]interface{}) (j *LocalJob, err error) {
-
-	var asMap struct {
-		Mapping           map[string]string
-		SnapshotPrefix    string `mapstructure:"snapshot_prefix"`
-		Interval          string
-		InitialReplPolicy string                 `mapstructure:"initial_repl_policy"`
-		PruneLHS          map[string]interface{} `mapstructure:"prune_lhs"`
-		PruneRHS          map[string]interface{} `mapstructure:"prune_rhs"`
-		Debug             map[string]interface{}
-	}
-
-	if err = mapstructure.Decode(i, &asMap); err != nil {
-		err = errors.Wrap(err, "mapstructure error")
-		return nil, err
-	}
-
-	j = &LocalJob{Name: name}
-
-	if j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false); err != nil {
-		return
-	}
-
-	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
-		return
-	}
-
-	if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
-		err = errors.Wrap(err, "cannot parse interval")
-		return
-	}
-
-	if j.InitialReplPolicy, err = parseInitialReplPolicy(asMap.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY); err != nil {
-		return
-	}
-
-	if j.PruneLHS, err = parsePrunePolicy(asMap.PruneLHS, true); err != nil {
-		err = errors.Wrap(err, "cannot parse 'prune_lhs'")
-		return
-	}
-	if j.PruneRHS, err = parsePrunePolicy(asMap.PruneRHS, false); err != nil {
-		err = errors.Wrap(err, "cannot parse 'prune_rhs'")
-		return
-	}
-
-	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
-		err = errors.Wrap(err, "cannot parse 'debug'")
-		return
-	}
-
-	return
-}
-
-func (j *LocalJob) JobName() string {
-	return j.Name
-}
-
-func (j *LocalJob) JobStart(ctx context.Context) {
-
-	rootLog := ctx.Value(contextKeyLog).(Logger)
-
-	j.snapperTask = NewTask("snapshot", rootLog)
-	j.mainTask = NewTask("main", rootLog)
-	j.handlerTask = NewTask("handler", rootLog)
-	j.pruneRHSTask = NewTask("prune_rhs", rootLog)
-	j.pruneLHSTask = NewTask("prune_lhs", rootLog)
-
-	local := rpc.NewLocalRPC()
-	// Allow access to any dataset since we control what mapping
-	// is passed to the pull routine.
-	// All local datasets will be passed to its Map() function,
-	// but only those for which a mapping exists will actually be pulled.
-	// We can pay this small performance penalty for now.
-	handler := NewHandler(j.handlerTask.Log(), localPullACL{}, NewPrefixFilter(j.SnapshotPrefix))
-
-	registerEndpoints(local, handler)
-
-	snapper := IntervalAutosnap{
-		task:             j.snapperTask,
-		DatasetFilter:    j.Mapping.AsFilter(),
-		Prefix:           j.SnapshotPrefix,
-		SnapshotInterval: j.Interval,
-	}
-
-	plhs, err := j.Pruner(j.pruneLHSTask, PrunePolicySideLeft, false)
-	if err != nil {
-		rootLog.WithError(err).Error("error creating lhs pruner")
-		return
-	}
-	prhs, err := j.Pruner(j.pruneRHSTask, PrunePolicySideRight, false)
-	if err != nil {
-		rootLog.WithError(err).Error("error creating rhs pruner")
-		return
-	}
-
-	didSnaps := make(chan struct{})
-	go snapper.Run(ctx, didSnaps)
-
-outer:
-	for {
-
-		select {
-		case <-ctx.Done():
-			j.mainTask.Log().WithError(ctx.Err()).Info("context")
-			break outer
-		case <-didSnaps:
-			j.mainTask.Log().Debug("finished taking snapshots")
-			j.mainTask.Log().Info("starting replication procedure")
-		}
-
-		j.mainTask.Log().Debug("replicating from lhs to rhs")
-		j.mainTask.Enter("replicate")
-		puller := Puller{j.mainTask, local, j.Mapping, j.InitialReplPolicy}
-		puller.Pull()
-		j.mainTask.Finish()
-
-		// use a ctx as soon as Pull gains ctx support
-		select {
-		case <-ctx.Done():
-			break outer
-		default:
-		}
-
-		var wg sync.WaitGroup
-
-		j.mainTask.Log().Info("pruning lhs")
-		wg.Add(1)
-		go func() {
-			plhs.Run(ctx)
-			wg.Done()
-		}()
-
-		j.mainTask.Log().Info("pruning rhs")
-		wg.Add(1)
-		go func() {
-			prhs.Run(ctx)
-			wg.Done()
-		}()
-
-		wg.Wait()
-
-	}
-
-}
-
-func (j *LocalJob) JobStatus(ctxt context.Context) (*JobStatus, error) {
-	return &JobStatus{Tasks: []*TaskStatus{
-		j.snapperTask.Status(),
-		j.pruneLHSTask.Status(),
-		j.pruneRHSTask.Status(),
-		j.mainTask.Status(),
-	}}, nil
-}
-
-func (j *LocalJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) {
-
-	var dsfilter zfs.DatasetFilter
-	var pp PrunePolicy
-	switch side {
-	case PrunePolicySideLeft:
-		pp = j.PruneLHS
-		dsfilter = j.Mapping.AsFilter()
-	case PrunePolicySideRight:
-		pp = j.PruneRHS
-		dsfilter, err = j.Mapping.InvertedFilter()
-		if err != nil {
-			err = errors.Wrap(err, "cannot invert mapping for prune_rhs")
-			return
-		}
-	default:
-		err = errors.Errorf("must be either left or right side")
-		return
-	}
-
-	p = Pruner{
-		task,
-		time.Now(),
-		dryRun,
-		dsfilter,
-		j.SnapshotPrefix,
-		pp,
-	}
-
-	return
-}

cmd/config_job_pull.go

@@ -1,195 +0,0 @@
-package cmd
-
-import (
-	"time"
-
-	"context"
-	"fmt"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/util"
-)
-
-type PullJob struct {
-	Name     string
-	Connect  RWCConnecter
-	Interval time.Duration
-	Mapping  *DatasetMapFilter
-	// constructed from mapping during parsing
-	pruneFilter       *DatasetMapFilter
-	SnapshotPrefix    string
-	InitialReplPolicy InitialReplPolicy
-	Prune             PrunePolicy
-	Debug             JobDebugSettings
-
-	task *Task
-}
-
-func parsePullJob(c JobParsingContext, name string, i map[string]interface{}) (j *PullJob, err error) {
-
-	var asMap struct {
-		Connect           map[string]interface{}
-		Interval          string
-		Mapping           map[string]string
-		InitialReplPolicy string `mapstructure:"initial_repl_policy"`
-		Prune             map[string]interface{}
-		SnapshotPrefix    string `mapstructure:"snapshot_prefix"`
-		Debug             map[string]interface{}
-	}
-
-	if err = mapstructure.Decode(i, &asMap); err != nil {
-		err = errors.Wrap(err, "mapstructure error")
-		return nil, err
-	}
-
-	j = &PullJob{Name: name}
-
-	j.Connect, err = parseSSHStdinserverConnecter(asMap.Connect)
-	if err != nil {
-		err = errors.Wrap(err, "cannot parse 'connect'")
-		return nil, err
-	}
-
-	if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
-		err = errors.Wrap(err, "cannot parse 'interval'")
-		return nil, err
-	}
-
-	j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false)
-	if err != nil {
-		err = errors.Wrap(err, "cannot parse 'mapping'")
-		return nil, err
-	}
-
-	if j.pruneFilter, err = j.Mapping.InvertedFilter(); err != nil {
-		err = errors.Wrap(err, "cannot automatically invert 'mapping' for prune job")
-		return nil, err
-	}
-
-	j.InitialReplPolicy, err = parseInitialReplPolicy(asMap.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY)
-	if err != nil {
-		err = errors.Wrap(err, "cannot parse 'initial_repl_policy'")
-		return
-	}
-
-	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
-		return
-	}
-
-	if j.Prune, err = parsePrunePolicy(asMap.Prune, false); err != nil {
-		err = errors.Wrap(err, "cannot parse prune policy")
-		return
-	}
-
-	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
-		err = errors.Wrap(err, "cannot parse 'debug'")
-		return
-	}
-
-	return
-}
-
-func (j *PullJob) JobName() string {
-	return j.Name
-}
-
-func (j *PullJob) JobStart(ctx context.Context) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-	defer log.Info("exiting")
-	j.task = NewTask("main", log)
-
-	// j.task is idle here idle here
-
-	ticker := time.NewTicker(j.Interval)
-	for {
-		j.doRun(ctx)
-		select {
-		case <-ctx.Done():
-			j.task.Log().WithError(ctx.Err()).Info("context")
-			return
-		case <-ticker.C:
-		}
-	}
-}
-
-func (j *PullJob) doRun(ctx context.Context) {
-
-	j.task.Enter("run")
-	defer j.task.Finish()
-
-	j.task.Log().Info("connecting")
-	rwc, err := j.Connect.Connect()
-	if err != nil {
-		j.task.Log().WithError(err).Error("error connecting")
-		return
-	}
-
-	rwc, err = util.NewReadWriteCloserLogger(rwc, j.Debug.Conn.ReadDump, j.Debug.Conn.WriteDump)
-	if err != nil {
-		return
-	}
-
-	client := rpc.NewClient(rwc)
-	if j.Debug.RPC.Log {
-		client.SetLogger(j.task.Log(), true)
-	}
-
-	j.task.Enter("pull")
-	puller := Puller{j.task, client, j.Mapping, j.InitialReplPolicy}
-	puller.Pull()
-	closeRPCWithTimeout(j.task, client, time.Second*1, "")
-	rwc.Close()
-	j.task.Finish()
-
-	j.task.Enter("prune")
-	pruner, err := j.Pruner(j.task, PrunePolicySideDefault, false)
-	if err != nil {
-		j.task.Log().WithError(err).Error("error creating pruner")
-	} else {
-		pruner.Run(ctx)
-	}
-	j.task.Finish()
-
-}
-
-func (j *PullJob) JobStatus(ctxt context.Context) (*JobStatus, error) {
-	return &JobStatus{Tasks: []*TaskStatus{j.task.Status()}}, nil
-}
-
-func (j *PullJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) {
-	p = Pruner{
-		task,
-		time.Now(),
-		dryRun,
-		j.pruneFilter,
-		j.SnapshotPrefix,
-		j.Prune,
-	}
-	return
-}
-
-func closeRPCWithTimeout(task *Task, remote rpc.RPCClient, timeout time.Duration, goodbye string) {
-
-	task.Log().Info("closing rpc connection")
-
-	ch := make(chan error)
-	go func() {
-		ch <- remote.Close()
-		close(ch)
-	}()
-
-	var err error
-	select {
-	case <-time.After(timeout):
-		err = fmt.Errorf("timeout exceeded (%s)", timeout)
-	case closeRequestErr := <-ch:
-		err = closeRequestErr
-	}
-
-	if err != nil {
-		task.Log().WithError(err).Error("error closing connection")
-	}
-	return
-}

cmd/config_job_source.go

@@ -1,250 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"io"
-	"time"
-
-	mapstructure "github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/util"
-)
-
-type SourceJob struct {
-	Name           string
-	Serve          AuthenticatedChannelListenerFactory
-	Filesystems    *DatasetMapFilter
-	SnapshotPrefix string
-	Interval       time.Duration
-	Prune          PrunePolicy
-	Debug          JobDebugSettings
-	serveTask      *Task
-	autosnapTask   *Task
-	pruneTask      *Task
-}
-
-func parseSourceJob(c JobParsingContext, name string, i map[string]interface{}) (j *SourceJob, err error) {
-
-	var asMap struct {
-		Serve          map[string]interface{}
-		Filesystems    map[string]string
-		SnapshotPrefix string `mapstructure:"snapshot_prefix"`
-		Interval       string
-		Prune          map[string]interface{}
-		Debug          map[string]interface{}
-	}
-
-	if err = mapstructure.Decode(i, &asMap); err != nil {
-		err = errors.Wrap(err, "mapstructure error")
-		return nil, err
-	}
-
-	j = &SourceJob{Name: name}
-
-	if j.Serve, err = parseAuthenticatedChannelListenerFactory(c, asMap.Serve); err != nil {
-		return
-	}
-
-	if j.Filesystems, err = parseDatasetMapFilter(asMap.Filesystems, true); err != nil {
-		return
-	}
-
-	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
-		return
-	}
-
-	if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
-		err = errors.Wrap(err, "cannot parse 'interval'")
-		return
-	}
-
-	if j.Prune, err = parsePrunePolicy(asMap.Prune, true); err != nil {
-		err = errors.Wrap(err, "cannot parse 'prune'")
-		return
-	}
-
-	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
-		err = errors.Wrap(err, "cannot parse 'debug'")
-		return
-	}
-
-	return
-}
-
-func (j *SourceJob) JobName() string {
-	return j.Name
-}
-
-func (j *SourceJob) JobStart(ctx context.Context) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-	defer log.Info("exiting")
-
-	j.autosnapTask = NewTask("autosnap", log)
-	j.pruneTask = NewTask("prune", log)
-	j.serveTask = NewTask("serve", log)
-
-	a := IntervalAutosnap{j.autosnapTask, j.Filesystems, j.SnapshotPrefix, j.Interval}
-	p, err := j.Pruner(j.pruneTask, PrunePolicySideDefault, false)
-
-	if err != nil {
-		log.WithError(err).Error("error creating pruner")
-		return
-	}
-
-	didSnaps := make(chan struct{})
-
-	go j.serve(ctx, j.serveTask)
-	go a.Run(ctx, didSnaps)
-
-outer:
-	for {
-		select {
-		case <-ctx.Done():
-			break outer
-		case <-didSnaps:
-			log.Info("starting pruner")
-			p.Run(ctx)
-			log.Info("pruner done")
-		}
-	}
-	log.WithError(ctx.Err()).Info("context")
-
-}
-
-func (j *SourceJob) JobStatus(ctxt context.Context) (*JobStatus, error) {
-	return &JobStatus{
-		Tasks: []*TaskStatus{
-			j.autosnapTask.Status(),
-			j.pruneTask.Status(),
-			j.serveTask.Status(),
-		}}, nil
-}
-
-func (j *SourceJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) {
-	p = Pruner{
-		task,
-		time.Now(),
-		dryRun,
-		j.Filesystems,
-		j.SnapshotPrefix,
-		j.Prune,
-	}
-	return
-}
-
-func (j *SourceJob) serve(ctx context.Context, task *Task) {
-
-	listener, err := j.Serve.Listen()
-	if err != nil {
-		task.Log().WithError(err).Error("error listening")
-		return
-	}
-
-	type rwcChanMsg struct {
-		rwc io.ReadWriteCloser
-		err error
-	}
-	rwcChan := make(chan rwcChanMsg)
-
-	// Serve connections until interrupted or error
-outer:
-	for {
-
-		go func() {
-			rwc, err := listener.Accept()
-			if err != nil {
-				rwcChan <- rwcChanMsg{rwc, err}
-				close(rwcChan)
-				return
-			}
-			rwcChan <- rwcChanMsg{rwc, err}
-		}()
-
-		select {
-
-		case rwcMsg := <-rwcChan:
-
-			if rwcMsg.err != nil {
-				task.Log().WithError(err).Error("error accepting connection")
-				break outer
-			}
-
-			j.handleConnection(rwcMsg.rwc, task)
-
-		case <-ctx.Done():
-			task.Log().WithError(ctx.Err()).Info("context")
-			break outer
-
-		}
-
-	}
-
-	task.Enter("close_listener")
-	defer task.Finish()
-	err = listener.Close()
-	if err != nil {
-		task.Log().WithError(err).Error("error closing listener")
-	}
-
-	return
-
-}
-
-func (j *SourceJob) handleConnection(rwc io.ReadWriteCloser, task *Task) {
-
-	task.Enter("handle_connection")
-	defer task.Finish()
-
-	task.Log().Info("handling client connection")
-
-	rwc, err := util.NewReadWriteCloserLogger(rwc, j.Debug.Conn.ReadDump, j.Debug.Conn.WriteDump)
-	if err != nil {
-		panic(err)
-	}
-
-	// construct connection handler
-	handler := NewHandler(task.Log(), j.Filesystems, NewPrefixFilter(j.SnapshotPrefix))
-
-	// handle connection
-	rpcServer := rpc.NewServer(rwc)
-	if j.Debug.RPC.Log {
-		rpclog := task.Log().WithField("subsystem", "rpc")
-		rpcServer.SetLogger(rpclog, true)
-	}
-	registerEndpoints(rpcServer, handler)
-	if err = rpcServer.Serve(); err != nil {
-		task.Log().WithError(err).Error("error serving connection")
-	}
-
-	// wait for client to close connection
-	// FIXME: we cannot just close it like we would to with a TCP socket because
-	// FIXME: go-nettsh's Close() may overtake the remaining data in the pipe
-	const CLIENT_HANGUP_TIMEOUT = 1 * time.Second
-	task.Log().
-		WithField("timeout", CLIENT_HANGUP_TIMEOUT).
-		Debug("waiting for client to hang up")
-
-	wchan := make(chan error)
-	go func() {
-		var pseudo [1]byte
-		_, err := io.ReadFull(rwc, pseudo[:])
-		wchan <- err
-	}()
-	var werr error
-	select {
-	case werr = <-wchan:
-		// all right
-	case <-time.After(CLIENT_HANGUP_TIMEOUT):
-		werr = errors.New("client did not close connection within timeout")
-	}
-	if werr != nil && werr != io.EOF {
-		task.Log().WithError(werr).
-			Error("error waiting for client to hang up")
-	}
-	task.Log().Info("closing client connection")
-	if err = rwc.Close(); err != nil {
-		task.Log().WithError(err).Error("error force-closing connection")
-	}
-}

cmd/config_logging.go

@@ -1,251 +0,0 @@
-package cmd
-
-import (
-	"crypto/tls"
-	"crypto/x509"
-	"github.com/mattn/go-isatty"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/logger"
-	"io/ioutil"
-	"os"
-	"time"
-)
-
-type LoggingConfig struct {
-	Outlets *logger.Outlets
-}
-
-type MetadataFlags int64
-
-const (
-	MetadataTime MetadataFlags = 1 << iota
-	MetadataLevel
-
-	MetadataNone MetadataFlags = 0
-	MetadataAll  MetadataFlags = ^0
-)
-
-func parseLogging(i interface{}) (c *LoggingConfig, err error) {
-
-	c = &LoggingConfig{}
-	c.Outlets = logger.NewOutlets()
-
-	var asList []interface{}
-	if err = mapstructure.Decode(i, &asList); err != nil {
-		return nil, errors.Wrap(err, "mapstructure error")
-	}
-	if len(asList) == 0 {
-		// Default config
-		out := WriterOutlet{&HumanFormatter{}, os.Stdout}
-		c.Outlets.Add(out, logger.Warn)
-		return
-	}
-
-	var syslogOutlets, stdoutOutlets int
-	for lei, le := range asList {
-
-		outlet, minLevel, err := parseOutlet(le)
-		if err != nil {
-			return nil, errors.Wrapf(err, "cannot parse outlet #%d", lei)
-		}
-		var _ logger.Outlet = WriterOutlet{}
-		var _ logger.Outlet = &SyslogOutlet{}
-		switch outlet.(type) {
-		case *SyslogOutlet:
-			syslogOutlets++
-		case WriterOutlet:
-			stdoutOutlets++
-		}
-
-		c.Outlets.Add(outlet, minLevel)
-
-	}
-
-	if syslogOutlets > 1 {
-		return nil, errors.Errorf("can only define one 'syslog' outlet")
-	}
-	if stdoutOutlets > 1 {
-		return nil, errors.Errorf("can only define one 'stdout' outlet")
-	}
-
-	return c, nil
-
-}
-
-func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
-	var is string
-	switch j := i.(type) {
-	case string:
-		is = j
-	default:
-		return nil, errors.Errorf("invalid log format: wrong type: %T", i)
-	}
-
-	switch is {
-	case "human":
-		return &HumanFormatter{}, nil
-	case "logfmt":
-		return &LogfmtFormatter{}, nil
-	case "json":
-		return &JSONFormatter{}, nil
-	default:
-		return nil, errors.Errorf("invalid log format: '%s'", is)
-	}
-
-}
-
-func parseOutlet(i interface{}) (o logger.Outlet, level logger.Level, err error) {
-
-	var in struct {
-		Outlet string
-		Level  string
-		Format string
-	}
-	if err = mapstructure.Decode(i, &in); err != nil {
-		err = errors.Wrap(err, "mapstructure error")
-		return
-	}
-	if in.Outlet == "" || in.Level == "" || in.Format == "" {
-		err = errors.Errorf("must specify 'outlet', 'level' and 'format' field")
-		return
-	}
-
-	minLevel, err := logger.ParseLevel(in.Level)
-	if err != nil {
-		err = errors.Wrap(err, "cannot parse 'level' field")
-		return
-	}
-	formatter, err := parseLogFormat(in.Format)
-	if err != nil {
-		err = errors.Wrap(err, "cannot parse")
-		return
-	}
-
-	switch in.Outlet {
-	case "stdout":
-		o, err = parseStdoutOutlet(i, formatter)
-	case "tcp":
-		o, err = parseTCPOutlet(i, formatter)
-	case "syslog":
-		o, err = parseSyslogOutlet(i, formatter)
-	default:
-		err = errors.Errorf("unknown outlet type '%s'", in.Outlet)
-	}
-	return o, minLevel, err
-
-}
-
-func parseStdoutOutlet(i interface{}, formatter EntryFormatter) (WriterOutlet, error) {
-
-	var in struct {
-		Time bool
-	}
-	if err := mapstructure.Decode(i, &in); err != nil {
-		return WriterOutlet{}, errors.Wrap(err, "invalid structure for stdout outlet")
-	}
-
-	flags := MetadataAll
-	writer := os.Stdout
-	if !isatty.IsTerminal(writer.Fd()) && !in.Time {
-		flags &= ^MetadataTime
-	}
-
-	formatter.SetMetadataFlags(flags)
-	return WriterOutlet{
-		formatter,
-		os.Stdout,
-	}, nil
-}
-
-func parseTCPOutlet(i interface{}, formatter EntryFormatter) (out *TCPOutlet, err error) {
-
-	var in struct {
-		Net           string
-		Address       string
-		RetryInterval string `mapstructure:"retry_interval"`
-		TLS           *struct {
-			CA   string
-			Cert string
-			Key  string
-		}
-	}
-	if err = mapstructure.Decode(i, &in); err != nil {
-		return nil, errors.Wrap(err, "mapstructure error")
-	}
-
-	retryInterval, err := time.ParseDuration(in.RetryInterval)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
-	}
-
-	if len(in.Net) == 0 {
-		return nil, errors.New("field 'net' must not be empty")
-	}
-	if len(in.Address) == 0 {
-		return nil, errors.New("field 'address' must not be empty")
-	}
-
-	var tlsConfig *tls.Config
-	if in.TLS != nil {
-
-		cert, err := tls.LoadX509KeyPair(in.TLS.Cert, in.TLS.Key)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot load client cert")
-		}
-
-		var rootCAs *x509.CertPool
-		if in.TLS.CA == "" {
-			if rootCAs, err = x509.SystemCertPool(); err != nil {
-				return nil, errors.Wrap(err, "cannot open system cert pool")
-			}
-		} else {
-			rootCAs = x509.NewCertPool()
-			rootCAPEM, err := ioutil.ReadFile(in.TLS.CA)
-			if err != nil {
-				return nil, errors.Wrap(err, "cannot load CA cert")
-			}
-			if !rootCAs.AppendCertsFromPEM(rootCAPEM) {
-				return nil, errors.New("cannot parse CA cert")
-			}
-		}
-		if err != nil && in.TLS.CA == "" {
-			return nil, errors.Wrap(err, "cannot load root ca pool")
-		}
-
-		tlsConfig = &tls.Config{
-			Certificates: []tls.Certificate{cert},
-			RootCAs:      rootCAs,
-		}
-
-		tlsConfig.BuildNameToCertificate()
-	}
-
-	formatter.SetMetadataFlags(MetadataAll)
-	return NewTCPOutlet(formatter, in.Net, in.Address, tlsConfig, retryInterval), nil
-
-}
-
-func parseSyslogOutlet(i interface{}, formatter EntryFormatter) (out *SyslogOutlet, err error) {
-
-	var in struct {
-		RetryInterval string `mapstructure:"retry_interval"`
-	}
-	if err = mapstructure.Decode(i, &in); err != nil {
-		return nil, errors.Wrap(err, "mapstructure error")
-	}
-
-	out = &SyslogOutlet{}
-	out.Formatter = formatter
-	out.Formatter.SetMetadataFlags(MetadataNone)
-
-	out.RetryInterval = 0 // default to 0 as we assume local syslog will just work
-	if in.RetryInterval != "" {
-		out.RetryInterval, err = time.ParseDuration(in.RetryInterval)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
-		}
-	}
-
-	return
-}

cmd/config_parse.go

@@ -1,294 +0,0 @@
-package cmd
-
-import (
-	"io/ioutil"
-
-	"fmt"
-	yaml "github.com/go-yaml/yaml"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"os"
-	"regexp"
-	"strconv"
-	"time"
-)
-
-var ConfigFileDefaultLocations []string = []string{
-	"/etc/zrepl/zrepl.yml",
-	"/usr/local/etc/zrepl/zrepl.yml",
-}
-
-const (
-	JobNameControl string = "control"
-)
-
-var ReservedJobNames []string = []string{
-	JobNameControl,
-}
-
-type ConfigParsingContext struct {
-	Global *Global
-}
-
-func ParseConfig(path string) (config *Config, err error) {
-
-	if path == "" {
-		// Try default locations
-		for _, l := range ConfigFileDefaultLocations {
-			stat, err := os.Stat(l)
-			if err != nil {
-				continue
-			}
-			if !stat.Mode().IsRegular() {
-				err = errors.Errorf("file at default location is not a regular file: %s", l)
-				continue
-			}
-			path = l
-			break
-		}
-	}
-
-	var i interface{}
-
-	var bytes []byte
-
-	if bytes, err = ioutil.ReadFile(path); err != nil {
-		err = errors.WithStack(err)
-		return
-	}
-
-	if err = yaml.Unmarshal(bytes, &i); err != nil {
-		err = errors.WithStack(err)
-		return
-	}
-
-	return parseConfig(i)
-}
-
-func parseConfig(i interface{}) (c *Config, err error) {
-
-	var asMap struct {
-		Global map[string]interface{}
-		Jobs   []map[string]interface{}
-	}
-	if err := mapstructure.Decode(i, &asMap); err != nil {
-		return nil, errors.Wrap(err, "config root must be a dict")
-	}
-
-	c = &Config{}
-
-	// Parse global with defaults
-	c.Global.Serve.Stdinserver.SockDir = "/var/run/zrepl/stdinserver"
-	c.Global.Control.Sockpath = "/var/run/zrepl/control"
-
-	err = mapstructure.Decode(asMap.Global, &c.Global)
-	if err != nil {
-		err = errors.Wrap(err, "mapstructure error on 'global' section: %s")
-		return
-	}
-
-	if c.Global.logging, err = parseLogging(asMap.Global["logging"]); err != nil {
-		return nil, errors.Wrap(err, "cannot parse logging section")
-	}
-
-	cpc := ConfigParsingContext{&c.Global}
-	jpc := JobParsingContext{cpc}
-
-	// Jobs
-	c.Jobs = make(map[string]Job, len(asMap.Jobs))
-	for i := range asMap.Jobs {
-		job, err := parseJob(jpc, asMap.Jobs[i])
-		if err != nil {
-			// Try to find its name
-			namei, ok := asMap.Jobs[i]["name"]
-			if !ok {
-				namei = fmt.Sprintf("<no name, entry #%d in list>", i)
-			}
-			err = errors.Wrapf(err, "cannot parse job '%v'", namei)
-			return nil, err
-		}
-		jn := job.JobName()
-		if _, ok := c.Jobs[jn]; ok {
-			err = errors.Errorf("duplicate job name: %s", jn)
-			return nil, err
-		}
-		c.Jobs[job.JobName()] = job
-	}
-
-	cj, err := NewControlJob(JobNameControl, jpc.Global.Control.Sockpath)
-	if err != nil {
-		err = errors.Wrap(err, "cannot create control job")
-		return
-	}
-	c.Jobs[JobNameControl] = cj
-
-	return c, nil
-
-}
-
-func extractStringField(i map[string]interface{}, key string, notempty bool) (field string, err error) {
-	vi, ok := i[key]
-	if !ok {
-		err = errors.Errorf("must have field '%s'", key)
-		return "", err
-	}
-	field, ok = vi.(string)
-	if !ok {
-		err = errors.Errorf("'%s' field must have type string", key)
-		return "", err
-	}
-	if notempty && len(field) <= 0 {
-		err = errors.Errorf("'%s' field must not be empty", key)
-		return "", err
-	}
-	return
-}
-
-type JobParsingContext struct {
-	ConfigParsingContext
-}
-
-func parseJob(c JobParsingContext, i map[string]interface{}) (j Job, err error) {
-
-	name, err := extractStringField(i, "name", true)
-	if err != nil {
-		return
-	}
-
-	for _, r := range ReservedJobNames {
-		if name == r {
-			err = errors.Errorf("job name '%s' is reserved", name)
-			return
-		}
-	}
-
-	jobtype, err := extractStringField(i, "type", true)
-	if err != nil {
-		return
-	}
-
-	switch jobtype {
-	case "pull":
-		return parsePullJob(c, name, i)
-	case "source":
-		return parseSourceJob(c, name, i)
-	case "local":
-		return parseLocalJob(c, name, i)
-	default:
-		return nil, errors.Errorf("unknown job type '%s'", jobtype)
-	}
-
-}
-
-func parseConnect(i map[string]interface{}) (c RWCConnecter, err error) {
-
-	t, err := extractStringField(i, "type", true)
-	if err != nil {
-		return nil, err
-	}
-
-	switch t {
-	case "ssh+stdinserver":
-		return parseSSHStdinserverConnecter(i)
-	default:
-		return nil, errors.Errorf("unknown connection type '%s'", t)
-	}
-
-}
-
-func parseInitialReplPolicy(v interface{}, defaultPolicy InitialReplPolicy) (p InitialReplPolicy, err error) {
-	s, ok := v.(string)
-	if !ok {
-		goto err
-	}
-
-	switch {
-	case s == "":
-		p = defaultPolicy
-	case s == "most_recent":
-		p = InitialReplPolicyMostRecent
-	case s == "all":
-		p = InitialReplPolicyAll
-	default:
-		goto err
-	}
-
-	return
-
-err:
-	err = errors.New(fmt.Sprintf("expected InitialReplPolicy, got %#v", v))
-	return
-}
-
-func parsePrunePolicy(v map[string]interface{}, willSeeBookmarks bool) (p PrunePolicy, err error) {
-
-	policyName, err := extractStringField(v, "policy", true)
-	if err != nil {
-		return
-	}
-
-	switch policyName {
-	case "grid":
-		return parseGridPrunePolicy(v, willSeeBookmarks)
-	case "noprune":
-		return NoPrunePolicy{}, nil
-	default:
-		err = errors.Errorf("unknown policy '%s'", policyName)
-		return
-	}
-}
-
-func parseAuthenticatedChannelListenerFactory(c JobParsingContext, v map[string]interface{}) (p AuthenticatedChannelListenerFactory, err error) {
-
-	t, err := extractStringField(v, "type", true)
-	if err != nil {
-		return nil, err
-	}
-
-	switch t {
-	case "stdinserver":
-		return parseStdinserverListenerFactory(c, v)
-	default:
-		err = errors.Errorf("unknown type '%s'", t)
-		return
-	}
-
-}
-
-var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
-
-func parsePostitiveDuration(e string) (d time.Duration, err error) {
-	comps := durationStringRegex.FindStringSubmatch(e)
-	if len(comps) != 3 {
-		err = fmt.Errorf("does not match regex: %s %#v", e, comps)
-		return
-	}
-
-	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
-	if err != nil {
-		return 0, err
-	}
-	if durationFactor <= 0 {
-		return 0, errors.New("duration must be positive integer")
-	}
-
-	var durationUnit time.Duration
-	switch comps[2] {
-	case "s":
-		durationUnit = time.Second
-	case "m":
-		durationUnit = time.Minute
-	case "h":
-		durationUnit = time.Hour
-	case "d":
-		durationUnit = 24 * time.Hour
-	case "w":
-		durationUnit = 24 * 7 * time.Hour
-	default:
-		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
-		return
-	}
-
-	d = time.Duration(durationFactor) * durationUnit
-	return
-}

cmd/config_prune_grid.go

@@ -1,245 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/util"
-	"github.com/zrepl/zrepl/zfs"
-	"math"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-	"time"
-)
-
-type GridPrunePolicy struct {
-	RetentionGrid *util.RetentionGrid
-	MaxBookmarks  int
-}
-
-const GridPrunePolicyMaxBookmarksKeepAll = -1
-
-type retentionGridAdaptor struct {
-	zfs.FilesystemVersion
-}
-
-func (a retentionGridAdaptor) Date() time.Time {
-	return a.Creation
-}
-
-func (a retentionGridAdaptor) LessThan(b util.RetentionGridEntry) bool {
-	return a.CreateTXG < b.(retentionGridAdaptor).CreateTXG
-}
-
-// Prune filters snapshots with the retention grid.
-// Bookmarks are deleted such that KeepBookmarks are kept in the end.
-// The oldest bookmarks are removed first.
-func (p *GridPrunePolicy) Prune(_ *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error) {
-	skeep, sremove := p.pruneSnapshots(versions)
-	keep, remove = p.pruneBookmarks(skeep)
-	remove = append(remove, sremove...)
-	return keep, remove, nil
-}
-
-func (p *GridPrunePolicy) pruneSnapshots(versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion) {
-
-	// Build adaptors for retention grid
-	keep = []zfs.FilesystemVersion{}
-	adaptors := make([]util.RetentionGridEntry, 0)
-	for fsv := range versions {
-		if versions[fsv].Type != zfs.Snapshot {
-			keep = append(keep, versions[fsv])
-			continue
-		}
-		adaptors = append(adaptors, retentionGridAdaptor{versions[fsv]})
-	}
-
-	sort.SliceStable(adaptors, func(i, j int) bool {
-		return adaptors[i].LessThan(adaptors[j])
-	})
-	now := adaptors[len(adaptors)-1].Date()
-
-	// Evaluate retention grid
-	keepa, removea := p.RetentionGrid.FitEntries(now, adaptors)
-
-	// Revert adaptors
-	for i := range keepa {
-		keep = append(keep, keepa[i].(retentionGridAdaptor).FilesystemVersion)
-	}
-	remove = make([]zfs.FilesystemVersion, len(removea))
-	for i := range removea {
-		remove[i] = removea[i].(retentionGridAdaptor).FilesystemVersion
-	}
-	return
-
-}
-
-func (p *GridPrunePolicy) pruneBookmarks(versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion) {
-
-	if p.MaxBookmarks == GridPrunePolicyMaxBookmarksKeepAll {
-		return versions, []zfs.FilesystemVersion{}
-	}
-
-	keep = []zfs.FilesystemVersion{}
-	bookmarks := make([]zfs.FilesystemVersion, 0)
-	for fsv := range versions {
-		if versions[fsv].Type != zfs.Bookmark {
-			keep = append(keep, versions[fsv])
-			continue
-		}
-		bookmarks = append(bookmarks, versions[fsv])
-	}
-
-	if len(bookmarks) == 0 {
-		return keep, []zfs.FilesystemVersion{}
-	}
-	if len(bookmarks) < p.MaxBookmarks {
-		keep = append(keep, bookmarks...)
-		return keep, []zfs.FilesystemVersion{}
-	}
-
-	// NOTE: sorting descending by descending by createtxg <=> sorting ascending wrt creation time
-	sort.SliceStable(bookmarks, func(i, j int) bool {
-		return (bookmarks[i].CreateTXG > bookmarks[j].CreateTXG)
-	})
-
-	keep = append(keep, bookmarks[:p.MaxBookmarks]...)
-	remove = bookmarks[p.MaxBookmarks:]
-
-	return keep, remove
-}
-
-func parseGridPrunePolicy(e map[string]interface{}, willSeeBookmarks bool) (p *GridPrunePolicy, err error) {
-
-	const KeepBookmarksAllString = "all"
-	var i struct {
-		Grid          string
-		KeepBookmarks string `mapstructure:"keep_bookmarks"`
-	}
-
-	dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{Result: &i, WeaklyTypedInput: true})
-	if err != nil {
-		err = errors.Wrap(err, "mapstructure error")
-		return
-	}
-	if err = dec.Decode(e); err != nil {
-		err = errors.Wrapf(err, "mapstructure error")
-		return
-	}
-
-	// Parse grid
-	intervals, err := parseRetentionGridIntervalsString(i.Grid)
-	if err != nil {
-		err = fmt.Errorf("cannot parse retention grid: %s", err)
-		return
-	}
-	// Assert intervals are of increasing length (not necessarily required, but indicates config mistake)
-	lastDuration := time.Duration(0)
-	for i := range intervals {
-
-		if intervals[i].Length < lastDuration {
-			// If all intervals before were keep=all, this is ok
-			allPrevKeepCountAll := true
-			for j := i - 1; allPrevKeepCountAll && j >= 0; j-- {
-				allPrevKeepCountAll = intervals[j].KeepCount == util.RetentionGridKeepCountAll
-			}
-			if allPrevKeepCountAll {
-				goto isMonotonicIncrease
-			}
-			err = errors.New("retention grid interval length must be monotonically increasing")
-			return
-		}
-	isMonotonicIncrease:
-		lastDuration = intervals[i].Length
-
-	}
-
-	// Parse KeepBookmarks
-	keepBookmarks := 0
-	if i.KeepBookmarks == KeepBookmarksAllString || (i.KeepBookmarks == "" && !willSeeBookmarks) {
-		keepBookmarks = GridPrunePolicyMaxBookmarksKeepAll
-	} else {
-		i, err := strconv.ParseInt(i.KeepBookmarks, 10, 32)
-		if err != nil || i <= 0 || i > math.MaxInt32 {
-			return nil, errors.Errorf("keep_bookmarks must be positive integer or 'all'")
-		}
-		keepBookmarks = int(i)
-	}
-	return &GridPrunePolicy{
-		util.NewRetentionGrid(intervals),
-		keepBookmarks,
-	}, nil
-}
-
-var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
-
-func parseRetentionGridIntervalString(e string) (intervals []util.RetentionInterval, err error) {
-
-	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
-	if comps == nil {
-		err = fmt.Errorf("retention string does not match expected format")
-		return
-	}
-
-	times, err := strconv.Atoi(comps[1])
-	if err != nil {
-		return nil, err
-	} else if times <= 0 {
-		return nil, fmt.Errorf("contains factor <= 0")
-	}
-
-	duration, err := parsePostitiveDuration(comps[2])
-	if err != nil {
-		return nil, err
-	}
-
-	keepCount := 1
-	if comps[3] != "" {
-		// Decompose key=value, comma separated
-		// For now, only keep_count is supported
-		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
-		res := re.FindStringSubmatch(comps[4])
-		if res == nil || len(res) != 2 {
-			err = fmt.Errorf("interval parameter contains unknown parameters")
-			return
-		}
-		if res[1] == "all" {
-			keepCount = util.RetentionGridKeepCountAll
-		} else {
-			keepCount, err = strconv.Atoi(res[1])
-			if err != nil {
-				err = fmt.Errorf("cannot parse keep_count value")
-				return
-			}
-		}
-	}
-
-	intervals = make([]util.RetentionInterval, times)
-	for i := range intervals {
-		intervals[i] = util.RetentionInterval{
-			Length:    duration,
-			KeepCount: keepCount,
-		}
-	}
-
-	return
-
-}
-
-func parseRetentionGridIntervalsString(s string) (intervals []util.RetentionInterval, err error) {
-
-	ges := strings.Split(s, "|")
-	intervals = make([]util.RetentionInterval, 0, 7*len(ges))
-
-	for intervalIdx, e := range ges {
-		parsed, err := parseRetentionGridIntervalString(e)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
-		}
-		intervals = append(intervals, parsed...)
-	}
-
-	return
-}

cmd/config_prune_none.go

@@ -1,11 +0,0 @@
-package cmd
-
-import "github.com/zrepl/zrepl/zfs"
-
-type NoPrunePolicy struct{}
-
-func (p NoPrunePolicy) Prune(fs *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error) {
-	keep = versions
-	remove = []zfs.FilesystemVersion{}
-	return
-}

cmd/config_serve_stdinserver.go

@@ -1,56 +0,0 @@
-package cmd
-
-import (
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/problame/go-netssh"
-	"io"
-	"path"
-)
-
-type StdinserverListenerFactory struct {
-	ClientIdentity string `mapstructure:"client_identity"`
-	sockpath       string
-}
-
-func parseStdinserverListenerFactory(c JobParsingContext, i map[string]interface{}) (f *StdinserverListenerFactory, err error) {
-
-	f = &StdinserverListenerFactory{}
-
-	if err = mapstructure.Decode(i, f); err != nil {
-		return nil, errors.Wrap(err, "mapstructure error")
-	}
-	if !(len(f.ClientIdentity) > 0) {
-		err = errors.Errorf("must specify 'client_identity'")
-		return
-	}
-
-	f.sockpath = path.Join(c.Global.Serve.Stdinserver.SockDir, f.ClientIdentity)
-
-	return
-}
-
-func (f *StdinserverListenerFactory) Listen() (al AuthenticatedChannelListener, err error) {
-
-	if err = PreparePrivateSockpath(f.sockpath); err != nil {
-		return nil, err
-	}
-
-	l, err := netssh.Listen(f.sockpath)
-	if err != nil {
-		return nil, err
-	}
-	return StdinserverListener{l}, nil
-}
-
-type StdinserverListener struct {
-	l *netssh.Listener
-}
-
-func (l StdinserverListener) Accept() (ch io.ReadWriteCloser, err error) {
-	return l.l.Accept()
-}
-
-func (l StdinserverListener) Close() (err error) {
-	return l.l.Close()
-}

cmd/config_test.go

@@ -1,222 +0,0 @@
-package cmd
-
-import (
-	"testing"
-	"time"
-
-	"github.com/kr/pretty"
-	"github.com/stretchr/testify/assert"
-	"github.com/zrepl/zrepl/util"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
-
-	paths := []string{
-		"./sampleconf/localbackup/host1.yml",
-		"./sampleconf/pullbackup/backuphost.yml",
-		"./sampleconf/pullbackup/productionhost.yml",
-		"./sampleconf/random/debugging.yml",
-		"./sampleconf/random/logging.yml",
-	}
-
-	for _, p := range paths {
-
-		c, err := ParseConfig(p)
-		if err != nil {
-			t.Errorf("error parsing %s:\n%+v", p, err)
-		}
-
-		t.Logf("file: %s", p)
-		t.Log(pretty.Sprint(c))
-
-	}
-
-}
-
-func TestParseRetentionGridStringParsing(t *testing.T) {
-
-	intervals, err := parseRetentionGridIntervalsString("2x10m(keep=2) | 1x1h | 3x1w")
-
-	assert.Nil(t, err)
-	assert.Len(t, intervals, 6)
-	proto := util.RetentionInterval{
-		KeepCount: 2,
-		Length:    10 * time.Minute,
-	}
-	assert.EqualValues(t, proto, intervals[0])
-	assert.EqualValues(t, proto, intervals[1])
-
-	proto.KeepCount = 1
-	proto.Length = 1 * time.Hour
-	assert.EqualValues(t, proto, intervals[2])
-
-	proto.Length = 7 * 24 * time.Hour
-	assert.EqualValues(t, proto, intervals[3])
-	assert.EqualValues(t, proto, intervals[4])
-	assert.EqualValues(t, proto, intervals[5])
-
-	intervals, err = parseRetentionGridIntervalsString("|")
-	assert.Error(t, err)
-	intervals, err = parseRetentionGridIntervalsString("2x10m")
-	assert.NoError(t, err)
-
-	intervals, err = parseRetentionGridIntervalsString("1x10m(keep=all)")
-	assert.NoError(t, err)
-	assert.Len(t, intervals, 1)
-	assert.EqualValues(t, util.RetentionGridKeepCountAll, intervals[0].KeepCount)
-
-}
-
-func TestDatasetMapFilter(t *testing.T) {
-
-	expectMapping := func(m map[string]string, from, to string) {
-		dmf, err := parseDatasetMapFilter(m, false)
-		if err != nil {
-			t.Logf("expect test map to be valid: %s", err)
-			t.FailNow()
-		}
-		fromPath, err := zfs.NewDatasetPath(from)
-		if err != nil {
-			t.Logf("expect test from path to be valid: %s", err)
-			t.FailNow()
-		}
-
-		res, err := dmf.Map(fromPath)
-		if to == "" {
-			assert.Nil(t, res)
-			assert.Nil(t, err)
-			t.Logf("%s => NOT MAPPED", fromPath.ToString())
-			return
-		}
-
-		assert.Nil(t, err)
-		toPath, err := zfs.NewDatasetPath(to)
-		if err != nil {
-			t.Logf("expect test to path to be valid: %s", err)
-			t.FailNow()
-		}
-		assert.True(t, res.Equal(toPath))
-	}
-
-	expectFilter := func(m map[string]string, path string, pass bool) {
-		dmf, err := parseDatasetMapFilter(m, true)
-		if err != nil {
-			t.Logf("expect test filter to be valid: %s", err)
-			t.FailNow()
-		}
-		p, err := zfs.NewDatasetPath(path)
-		if err != nil {
-			t.Logf("expect test path to be valid: %s", err)
-			t.FailNow()
-		}
-		res, err := dmf.Filter(p)
-		assert.Nil(t, err)
-		assert.Equal(t, pass, res)
-	}
-
-	map1 := map[string]string{
-		"a/b/c<":     "root1",
-		"a/b<":       "root2",
-		"<":          "root3/b/c",
-		"b":          "!",
-		"a/b/c/d/e<": "!",
-		"q<":         "root4/1/2",
-	}
-
-	expectMapping(map1, "a/b/c", "root1")
-	expectMapping(map1, "a/b/c/d", "root1/d")
-	expectMapping(map1, "a/b/c/d/e", "")
-	expectMapping(map1, "a/b/e", "root2/e")
-	expectMapping(map1, "a/b", "root2")
-	expectMapping(map1, "x", "root3/b/c/x")
-	expectMapping(map1, "x/y", "root3/b/c/x/y")
-	expectMapping(map1, "q", "root4/1/2")
-	expectMapping(map1, "b", "")
-	expectMapping(map1, "q/r", "root4/1/2/r")
-
-	filter1 := map[string]string{
-		"<":    "!",
-		"a<":   "ok",
-		"a/b<": "!",
-	}
-
-	expectFilter(filter1, "b", false)
-	expectFilter(filter1, "a", true)
-	expectFilter(filter1, "a/d", true)
-	expectFilter(filter1, "a/b", false)
-	expectFilter(filter1, "a/b/c", false)
-
-	filter2 := map[string]string{}
-	expectFilter(filter2, "foo", false) // default to omit
-
-}
-
-func TestDatasetMapFilter_AsFilter(t *testing.T) {
-
-	mapspec := map[string]string{
-		"a/b/c<":     "root1",
-		"a/b<":       "root2",
-		"<":          "root3/b/c",
-		"b":          "!",
-		"a/b/c/d/e<": "!",
-		"q<":         "root4/1/2",
-	}
-
-	m, err := parseDatasetMapFilter(mapspec, false)
-	assert.Nil(t, err)
-
-	f := m.AsFilter()
-
-	t.Logf("Mapping:\n%s\nFilter:\n%s", pretty.Sprint(m), pretty.Sprint(f))
-
-	tf := func(f zfs.DatasetFilter, path string, pass bool) {
-		p, err := zfs.NewDatasetPath(path)
-		assert.Nil(t, err)
-		r, err := f.Filter(p)
-		assert.Nil(t, err)
-		assert.Equal(t, pass, r)
-	}
-
-	tf(f, "a/b/c", true)
-	tf(f, "a/b", true)
-	tf(f, "b", false)
-	tf(f, "a/b/c/d/e", false)
-	tf(f, "a/b/c/d/e/f", false)
-	tf(f, "a", true)
-
-}
-
-func TestDatasetMapFilter_InvertedFilter(t *testing.T) {
-	mapspec := map[string]string{
-		"a/b":      "1/2",
-		"a/b/c<":   "3",
-		"a/b/c/d<": "1/2/a",
-		"a/b/d":    "!",
-	}
-
-	m, err := parseDatasetMapFilter(mapspec, false)
-	assert.Nil(t, err)
-
-	inv, err := m.InvertedFilter()
-	assert.Nil(t, err)
-
-	t.Log(pretty.Sprint(inv))
-
-	expectMapping := func(m *DatasetMapFilter, ps string, expRes bool) {
-		p, err := zfs.NewDatasetPath(ps)
-		assert.Nil(t, err)
-		r, err := m.Filter(p)
-		assert.Nil(t, err)
-		assert.Equal(t, expRes, r)
-	}
-
-	expectMapping(inv, "4", false)
-	expectMapping(inv, "3", true)
-	expectMapping(inv, "3/x", true)
-	expectMapping(inv, "1", false)
-	expectMapping(inv, "1/2", true)
-	expectMapping(inv, "1/2/3", false)
-	expectMapping(inv, "1/2/a/b", true)
-
-}

cmd/control.go

@@ -1,305 +0,0 @@
-package cmd
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"github.com/dustin/go-humanize"
-	"github.com/pkg/errors"
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/logger"
-	"io"
-	golog "log"
-	"net"
-	"net/http"
-	"os"
-	"sort"
-	"strings"
-	"time"
-)
-
-var controlCmd = &cobra.Command{
-	Use:   "control",
-	Short: "control zrepl daemon",
-}
-
-var pprofCmd = &cobra.Command{
-	Use:   "pprof off | [on TCP_LISTEN_ADDRESS]",
-	Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
-	Run:   doControlPProf,
-	PreRunE: func(cmd *cobra.Command, args []string) error {
-		if cmd.Flags().NArg() < 1 {
-			goto enargs
-		}
-		switch cmd.Flags().Arg(0) {
-		case "on":
-			pprofCmdArgs.msg.Run = true
-			if cmd.Flags().NArg() != 2 {
-				return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
-			}
-			pprofCmdArgs.msg.HttpListenAddress = cmd.Flags().Arg(1)
-		case "off":
-			if cmd.Flags().NArg() != 1 {
-				goto enargs
-			}
-			pprofCmdArgs.msg.Run = false
-		}
-		return nil
-	enargs:
-		return errors.New("invalid number of positional arguments")
-
-	},
-}
-var pprofCmdArgs struct {
-	msg PprofServerControlMsg
-}
-
-var controlVersionCmd = &cobra.Command{
-	Use:   "version",
-	Short: "print version of running zrepl daemon",
-	Run:   doControLVersionCmd,
-}
-
-var controlStatusCmdArgs struct {
-	format      string
-	level       logger.Level
-	onlyShowJob string
-}
-
-var controlStatusCmd = &cobra.Command{
-	Use:   "status [JOB_NAME]",
-	Short: "get current status",
-	Run:   doControlStatusCmd,
-}
-
-func init() {
-	RootCmd.AddCommand(controlCmd)
-	controlCmd.AddCommand(pprofCmd)
-	controlCmd.AddCommand(controlVersionCmd)
-	controlCmd.AddCommand(controlStatusCmd)
-	controlStatusCmd.Flags().StringVar(&controlStatusCmdArgs.format, "format", "human", "output format (human|raw)")
-	controlStatusCmdArgs.level = logger.Warn
-	controlStatusCmd.Flags().Var(&controlStatusCmdArgs.level, "level", "minimum log level to show")
-}
-
-func controlHttpClient() (client http.Client, err error) {
-
-	conf, err := ParseConfig(rootArgs.configFile)
-	if err != nil {
-		return http.Client{}, err
-	}
-
-	return http.Client{
-		Transport: &http.Transport{
-			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
-				return net.Dial("unix", conf.Global.Control.Sockpath)
-			},
-		},
-	}, nil
-}
-
-func doControlPProf(cmd *cobra.Command, args []string) {
-
-	log := golog.New(os.Stderr, "", 0)
-
-	die := func() {
-		log.Printf("exiting after error")
-		os.Exit(1)
-	}
-
-	log.Printf("connecting to zrepl daemon")
-	httpc, err := controlHttpClient()
-	if err != nil {
-		log.Printf("error parsing config: %s", err)
-		die()
-	}
-
-	var buf bytes.Buffer
-	if err := json.NewEncoder(&buf).Encode(&pprofCmdArgs.msg); err != nil {
-		log.Printf("error marshaling request: %s", err)
-		die()
-	}
-	_, err = httpc.Post("http://unix"+ControlJobEndpointPProf, "application/json", &buf)
-	if err != nil {
-		log.Printf("error: %s", err)
-		die()
-	}
-
-	log.Printf("finished")
-}
-
-func doControLVersionCmd(cmd *cobra.Command, args []string) {
-
-	log := golog.New(os.Stderr, "", 0)
-
-	die := func() {
-		log.Printf("exiting after error")
-		os.Exit(1)
-	}
-
-	httpc, err := controlHttpClient()
-	if err != nil {
-		log.Printf("could not connect to daemon: %s", err)
-		die()
-	}
-
-	resp, err := httpc.Get("http://unix" + ControlJobEndpointVersion)
-	if err != nil {
-		log.Printf("error: %s", err)
-		die()
-	} else if resp.StatusCode != http.StatusOK {
-		var msg bytes.Buffer
-		io.CopyN(&msg, resp.Body, 4096)
-		log.Printf("error: %s", msg.String())
-		die()
-	}
-
-	var info ZreplVersionInformation
-	err = json.NewDecoder(resp.Body).Decode(&info)
-	if err != nil {
-		log.Printf("error unmarshaling response: %s", err)
-		die()
-	}
-
-	fmt.Println(info.String())
-
-}
-
-func doControlStatusCmd(cmd *cobra.Command, args []string) {
-
-	log := golog.New(os.Stderr, "", 0)
-
-	die := func() {
-		log.Print("exiting after error")
-		os.Exit(1)
-	}
-
-	if len(args) == 1 {
-		controlStatusCmdArgs.onlyShowJob = args[0]
-	} else if len(args) > 1 {
-		log.Print("can only specify one job as positional argument")
-		cmd.Usage()
-		die()
-	}
-
-	httpc, err := controlHttpClient()
-	if err != nil {
-		log.Printf("could not connect to daemon: %s", err)
-		die()
-	}
-
-	resp, err := httpc.Get("http://unix" + ControlJobEndpointStatus)
-	if err != nil {
-		log.Printf("error: %s", err)
-		die()
-	} else if resp.StatusCode != http.StatusOK {
-		var msg bytes.Buffer
-		io.CopyN(&msg, resp.Body, 4096)
-		log.Printf("error: %s", msg.String())
-		die()
-	}
-
-	var status DaemonStatus
-	err = json.NewDecoder(resp.Body).Decode(&status)
-	if err != nil {
-		log.Printf("error unmarshaling response: %s", err)
-		die()
-	}
-
-	switch controlStatusCmdArgs.format {
-	case "raw":
-		enc := json.NewEncoder(os.Stdout)
-		enc.SetIndent("", "  ")
-		if err := enc.Encode(status); err != nil {
-			log.Panic(err)
-		}
-	case "human":
-
-		formatter := HumanFormatter{}
-		formatter.SetMetadataFlags(MetadataAll)
-		formatter.SetIgnoreFields([]string{
-			logJobField,
-		})
-		jobNames := make([]string, 0, len(status.Jobs))
-		for name, _ := range status.Jobs {
-			jobNames = append(jobNames, name)
-		}
-		sort.Slice(jobNames, func(i, j int) bool {
-			return strings.Compare(jobNames[i], jobNames[j]) == -1
-		})
-		now := time.Now()
-		for _, name := range jobNames {
-
-			if controlStatusCmdArgs.onlyShowJob != "" && name != controlStatusCmdArgs.onlyShowJob {
-				continue
-			}
-
-			job := status.Jobs[name]
-			jobLogEntries := make([]logger.Entry, 0)
-			informAboutError := false
-
-			fmt.Printf("Job '%s':\n", name)
-			for _, task := range job.Tasks {
-
-				var header bytes.Buffer
-				fmt.Fprintf(&header, "  Task '%s': ", task.Name)
-				if !task.Idle {
-					fmt.Fprint(&header, strings.Join(task.ActivityStack, "."))
-				} else {
-					fmt.Fprint(&header, "<idle>")
-				}
-				fmt.Fprint(&header, " ")
-				const TASK_STALLED_HOLDOFF_DURATION = 10 * time.Second
-				sinceLastUpdate := now.Sub(task.LastUpdate)
-				if !task.Idle || task.ProgressRx != 0 || task.ProgressTx != 0 {
-					fmt.Fprintf(&header, "(%s / %s , Rx/Tx",
-						humanize.Bytes(uint64(task.ProgressRx)),
-						humanize.Bytes(uint64(task.ProgressTx)))
-					if task.Idle {
-						fmt.Fprint(&header, ", values from last run")
-					}
-					fmt.Fprint(&header, ")")
-				}
-				fmt.Fprint(&header, "\n")
-				if !task.Idle && !task.LastUpdate.IsZero() && sinceLastUpdate >= TASK_STALLED_HOLDOFF_DURATION {
-					informAboutError = true
-					fmt.Fprintf(&header, "    WARNING: last update %s ago at %s)",
-						sinceLastUpdate.String(),
-						task.LastUpdate.Format(HumanFormatterDateFormat))
-					fmt.Fprint(&header, "\n")
-				}
-				io.Copy(os.Stdout, &header)
-
-				jobLogEntries = append(jobLogEntries, task.LogEntries...)
-				informAboutError = informAboutError || task.MaxLogLevel >= logger.Warn
-			}
-
-			sort.Slice(jobLogEntries, func(i, j int) bool {
-				return jobLogEntries[i].Time.Before(jobLogEntries[j].Time)
-			})
-			if informAboutError {
-				fmt.Println("    WARNING: Some tasks encountered problems since the last time they left idle state:")
-				fmt.Println("             check the logs below or your log file for more information.")
-				fmt.Println("             Use the --level flag if you need debug information.")
-				fmt.Println()
-			}
-			for _, e := range jobLogEntries {
-				if e.Level < controlStatusCmdArgs.level {
-					continue
-				}
-				formatted, err := formatter.Format(&e)
-				if err != nil {
-					panic(err)
-				}
-				fmt.Printf("      %s\n", string(formatted))
-			}
-			fmt.Println()
-
-		}
-	default:
-		log.Printf("invalid output format '%s'", controlStatusCmdArgs.format)
-		die()
-	}
-
-}

cmd/daemon.go

@@ -1,394 +0,0 @@
-package cmd
-
-import (
-	"container/list"
-	"context"
-	"fmt"
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/logger"
-	"io"
-	"os"
-	"os/signal"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-)
-
-// daemonCmd represents the daemon command
-var daemonCmd = &cobra.Command{
-	Use:   "daemon",
-	Short: "start daemon",
-	Run:   doDaemon,
-}
-
-func init() {
-	RootCmd.AddCommand(daemonCmd)
-}
-
-type Job interface {
-	JobName() string
-	JobStart(ctxt context.Context)
-	JobStatus(ctxt context.Context) (*JobStatus, error)
-}
-
-func doDaemon(cmd *cobra.Command, args []string) {
-
-	conf, err := ParseConfig(rootArgs.configFile)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "error parsing config: %s\n", err)
-		os.Exit(1)
-	}
-
-	log := logger.NewLogger(conf.Global.logging.Outlets, 1*time.Second)
-
-	log.Info(NewZreplVersionInformation().String())
-	log.Debug("starting daemon")
-	ctx := context.WithValue(context.Background(), contextKeyLog, log)
-	ctx = context.WithValue(ctx, contextKeyLog, log)
-
-	d := NewDaemon(conf)
-	d.Loop(ctx)
-
-}
-
-type contextKey string
-
-const (
-	contextKeyLog    contextKey = contextKey("log")
-	contextKeyDaemon contextKey = contextKey("daemon")
-)
-
-type Daemon struct {
-	conf      *Config
-	startedAt time.Time
-}
-
-func NewDaemon(initialConf *Config) *Daemon {
-	return &Daemon{conf: initialConf}
-}
-
-func (d *Daemon) Loop(ctx context.Context) {
-
-	d.startedAt = time.Now()
-
-	log := ctx.Value(contextKeyLog).(Logger)
-
-	ctx, cancel := context.WithCancel(ctx)
-	ctx = context.WithValue(ctx, contextKeyDaemon, d)
-
-	sigChan := make(chan os.Signal, 1)
-	finishs := make(chan Job)
-
-	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
-
-	log.Info("starting jobs from config")
-	i := 0
-	for _, job := range d.conf.Jobs {
-		logger := log.WithField(logJobField, job.JobName())
-		logger.Info("starting")
-		i++
-		jobCtx := context.WithValue(ctx, contextKeyLog, logger)
-		go func(j Job) {
-			j.JobStart(jobCtx)
-			finishs <- j
-		}(job)
-	}
-
-	finishCount := 0
-outer:
-	for {
-		select {
-		case <-finishs:
-			finishCount++
-			if finishCount == len(d.conf.Jobs) {
-				log.Info("all jobs finished")
-				break outer
-			}
-
-		case sig := <-sigChan:
-			log.WithField("signal", sig).Info("received signal")
-			log.Info("cancelling all jobs")
-			cancel()
-		}
-	}
-
-	signal.Stop(sigChan)
-	cancel() // make go vet happy
-
-	log.Info("exiting")
-
-}
-
-// Representation of a Job's status that is composed of Tasks
-type JobStatus struct {
-	// Statuses of all tasks of this job
-	Tasks []*TaskStatus
-	// Error != "" if JobStatus() returned an error
-	JobStatusError string
-}
-
-// Representation of a Daemon's status that is composed of Jobs
-type DaemonStatus struct {
-	StartedAt time.Time
-	Jobs      map[string]*JobStatus
-}
-
-func (d *Daemon) Status() (s *DaemonStatus) {
-
-	s = &DaemonStatus{}
-	s.StartedAt = d.startedAt
-
-	s.Jobs = make(map[string]*JobStatus, len(d.conf.Jobs))
-
-	for name, j := range d.conf.Jobs {
-		status, err := j.JobStatus(context.TODO())
-		if err != nil {
-			s.Jobs[name] = &JobStatus{nil, err.Error()}
-			continue
-		}
-		s.Jobs[name] = status
-	}
-
-	return
-}
-
-// Representation of a Task's status
-type TaskStatus struct {
-	Name string
-	// Whether the task is idle.
-	Idle bool
-	// The stack of activities the task is currently executing.
-	// The first element is the root activity and equal to Name.
-	ActivityStack []string
-	// Number of bytes received by the task since it last left idle state.
-	ProgressRx int64
-	// Number of bytes sent by the task since it last left idle state.
-	ProgressTx int64
-	// Log entries emitted by the task since it last left idle state.
-	// Only contains the log entries emitted through the task's logger
-	// (provided by Task.Log()).
-	LogEntries []logger.Entry
-	// The maximum log level of LogEntries.
-	// Only valid if len(LogEntries) > 0.
-	MaxLogLevel logger.Level
-	// Last time something about the Task changed
-	LastUpdate time.Time
-}
-
-// An instance of Task tracks  a single thread of activity that is part of a Job.
-type Task struct {
-	// Stack of activities the task is currently in
-	// Members are instances of taskActivity
-	activities *list.List
-	// Last time activities was changed (not the activities inside, the list)
-	activitiesLastUpdate time.Time
-	// Protects Task members from modification
-	rwl sync.RWMutex
-}
-
-// Structure that describes the progress a Task has made
-type taskProgress struct {
-	rx         int64
-	tx         int64
-	lastUpdate time.Time
-	logEntries []logger.Entry
-	mtx        sync.RWMutex
-}
-
-func newTaskProgress() (p *taskProgress) {
-	return &taskProgress{
-		logEntries: make([]logger.Entry, 0),
-	}
-}
-
-func (p *taskProgress) UpdateIO(drx, dtx int64) {
-	p.mtx.Lock()
-	defer p.mtx.Unlock()
-	p.rx += drx
-	p.tx += dtx
-	p.lastUpdate = time.Now()
-}
-
-func (p *taskProgress) UpdateLogEntry(entry logger.Entry) {
-	p.mtx.Lock()
-	defer p.mtx.Unlock()
-	// FIXME: ensure maximum size (issue #48)
-	p.logEntries = append(p.logEntries, entry)
-	p.lastUpdate = time.Now()
-}
-
-func (p *taskProgress) DeepCopy() (out taskProgress) {
-	p.mtx.RLock()
-	defer p.mtx.RUnlock()
-	out.rx, out.tx = p.rx, p.tx
-	out.lastUpdate = p.lastUpdate
-	out.logEntries = make([]logger.Entry, len(p.logEntries))
-	for i := range p.logEntries {
-		out.logEntries[i] = p.logEntries[i]
-	}
-	return
-}
-
-// returns a copy of this taskProgress, the mutex carries no semantic value
-func (p *taskProgress) Read() (out taskProgress) {
-	p.mtx.RLock()
-	defer p.mtx.RUnlock()
-	return p.DeepCopy()
-}
-
-// Element of a Task's activity stack
-type taskActivity struct {
-	name   string
-	idle   bool
-	logger *logger.Logger
-	// The progress of the task that is updated by UpdateIO() and UpdateLogEntry()
-	//
-	// Progress happens on a task-level and is thus global to the task.
-	// That's why progress is just a pointer to the current taskProgress:
-	// we reset progress when leaving the idle root activity
-	progress *taskProgress
-}
-
-func NewTask(name string, lg *logger.Logger) *Task {
-	t := &Task{
-		activities: list.New(),
-	}
-	rootLogger := lg.ReplaceField(logTaskField, name).
-		WithOutlet(t, logger.Debug)
-	rootAct := &taskActivity{name, true, rootLogger, newTaskProgress()}
-	t.activities.PushFront(rootAct)
-	return t
-}
-
-// callers must hold t.rwl
-func (t *Task) cur() *taskActivity {
-	return t.activities.Front().Value.(*taskActivity)
-}
-
-// buildActivityStack returns the stack of activity names
-// t.rwl must be held, but the slice can be returned since strings are immutable
-func (t *Task) buildActivityStack() []string {
-	comps := make([]string, 0, t.activities.Len())
-	for e := t.activities.Back(); e != nil; e = e.Prev() {
-		act := e.Value.(*taskActivity)
-		comps = append(comps, act.name)
-	}
-	return comps
-}
-
-// Start a sub-activity.
-// Must always be matched with a call to t.Finish()
-// --- consider using defer for this purpose.
-func (t *Task) Enter(activity string) {
-	t.rwl.Lock()
-	defer t.rwl.Unlock()
-
-	prev := t.cur()
-	if prev.idle {
-		// reset progress when leaving idle task
-		// we leave the old progress dangling to have the user not worry about
-		prev.progress = newTaskProgress()
-	}
-	act := &taskActivity{activity, false, nil, prev.progress}
-	t.activities.PushFront(act)
-	stack := t.buildActivityStack()
-	activityField := strings.Join(stack, ".")
-	act.logger = prev.logger.ReplaceField(logTaskField, activityField)
-
-	t.activitiesLastUpdate = time.Now()
-}
-
-func (t *Task) UpdateProgress(dtx, drx int64) {
-	t.rwl.RLock()
-	p := t.cur().progress // protected by own rwlock
-	t.rwl.RUnlock()
-	p.UpdateIO(dtx, drx)
-}
-
-// Returns a wrapper io.Reader that updates this task's _current_ progress value.
-// Progress updates after this task resets its progress value are discarded.
-func (t *Task) ProgressUpdater(r io.Reader) *IOProgressUpdater {
-	t.rwl.RLock()
-	defer t.rwl.RUnlock()
-	return &IOProgressUpdater{r, t.cur().progress}
-}
-
-func (t *Task) Status() *TaskStatus {
-	t.rwl.RLock()
-	defer t.rwl.RUnlock()
-	// NOTE
-	// do not return any state in TaskStatus that is protected by t.rwl
-
-	cur := t.cur()
-	stack := t.buildActivityStack()
-	prog := cur.progress.Read()
-
-	var maxLevel logger.Level
-	for _, entry := range prog.logEntries {
-		if maxLevel < entry.Level {
-			maxLevel = entry.Level
-		}
-	}
-
-	lastUpdate := prog.lastUpdate
-	if lastUpdate.Before(t.activitiesLastUpdate) {
-		lastUpdate = t.activitiesLastUpdate
-	}
-
-	s := &TaskStatus{
-		Name:          stack[0],
-		ActivityStack: stack,
-		Idle:          cur.idle,
-		ProgressRx:    prog.rx,
-		ProgressTx:    prog.tx,
-		LogEntries:    prog.logEntries,
-		MaxLogLevel:   maxLevel,
-		LastUpdate:    lastUpdate,
-	}
-
-	return s
-}
-
-// Finish a sub-activity.
-// Corresponds to a preceding call to t.Enter()
-func (t *Task) Finish() {
-	t.rwl.Lock()
-	defer t.rwl.Unlock()
-	top := t.activities.Front()
-	if top.Next() == nil {
-		return // cannot remove root activity
-	}
-	t.activities.Remove(top)
-	t.activitiesLastUpdate = time.Now()
-
-}
-
-// Returns a logger derived from the logger passed to the constructor function.
-// The logger's task field contains the current activity stack joined by '.'.
-func (t *Task) Log() *logger.Logger {
-	t.rwl.RLock()
-	defer t.rwl.RUnlock()
-	// FIXME should influence TaskStatus's LastUpdate field
-	return t.cur().logger
-}
-
-// implement logger.Outlet interface
-func (t *Task) WriteEntry(entry logger.Entry) error {
-	t.rwl.RLock()
-	defer t.rwl.RUnlock()
-	t.cur().progress.UpdateLogEntry(entry)
-	return nil
-}
-
-type IOProgressUpdater struct {
-	r io.Reader
-	p *taskProgress
-}
-
-func (u *IOProgressUpdater) Read(p []byte) (n int, err error) {
-	n, err = u.r.Read(p)
-	u.p.UpdateIO(int64(n), 0)
-	return
-
-}

cmd/handler.go

@@ -1,181 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"io"
-
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type DatasetMapping interface {
-	Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error)
-}
-
-type FilesystemRequest struct {
-	Roots []string // may be nil, indicating interest in all filesystems
-}
-
-type FilesystemVersionsRequest struct {
-	Filesystem *zfs.DatasetPath
-}
-
-type InitialTransferRequest struct {
-	Filesystem        *zfs.DatasetPath
-	FilesystemVersion zfs.FilesystemVersion
-}
-
-type IncrementalTransferRequest struct {
-	Filesystem *zfs.DatasetPath
-	From       zfs.FilesystemVersion
-	To         zfs.FilesystemVersion
-}
-
-type Handler struct {
-	logger Logger
-	dsf    zfs.DatasetFilter
-	fsvf   zfs.FilesystemVersionFilter
-}
-
-func NewHandler(logger Logger, dsfilter zfs.DatasetFilter, snapfilter zfs.FilesystemVersionFilter) (h Handler) {
-	return Handler{logger, dsfilter, snapfilter}
-}
-
-func registerEndpoints(server rpc.RPCServer, handler Handler) (err error) {
-	err = server.RegisterEndpoint("FilesystemRequest", handler.HandleFilesystemRequest)
-	if err != nil {
-		panic(err)
-	}
-	err = server.RegisterEndpoint("FilesystemVersionsRequest", handler.HandleFilesystemVersionsRequest)
-	if err != nil {
-		panic(err)
-	}
-	err = server.RegisterEndpoint("InitialTransferRequest", handler.HandleInitialTransferRequest)
-	if err != nil {
-		panic(err)
-	}
-	err = server.RegisterEndpoint("IncrementalTransferRequest", handler.HandleIncrementalTransferRequest)
-	if err != nil {
-		panic(err)
-	}
-	return nil
-}
-
-func (h Handler) HandleFilesystemRequest(r *FilesystemRequest, roots *[]*zfs.DatasetPath) (err error) {
-
-	log := h.logger.WithField("endpoint", "FilesystemRequest")
-
-	log.WithField("request", r).Debug("request")
-	log.WithField("dataset_filter", h.dsf).Debug("dsf")
-
-	allowed, err := zfs.ZFSListMapping(h.dsf)
-	if err != nil {
-		log.WithError(err).Error("error listing filesystems")
-		return
-	}
-
-	log.WithField("response", allowed).Debug("response")
-	*roots = allowed
-	return
-}
-
-func (h Handler) HandleFilesystemVersionsRequest(r *FilesystemVersionsRequest, versions *[]zfs.FilesystemVersion) (err error) {
-
-	log := h.logger.WithField("endpoint", "FilesystemVersionsRequest")
-
-	log.WithField("request", r).Debug("request")
-
-	// allowed to request that?
-	if h.pullACLCheck(r.Filesystem, nil); err != nil {
-		log.WithError(err).Warn("pull ACL check failed")
-		return
-	}
-
-	// find our versions
-	vs, err := zfs.ZFSListFilesystemVersions(r.Filesystem, h.fsvf)
-	if err != nil {
-		log.WithError(err).Error("cannot list filesystem versions")
-		return
-	}
-
-	log.WithField("response", vs).Debug("response")
-
-	*versions = vs
-	return
-
-}
-
-func (h Handler) HandleInitialTransferRequest(r *InitialTransferRequest, stream *io.Reader) (err error) {
-
-	log := h.logger.WithField("endpoint", "InitialTransferRequest")
-
-	log.WithField("request", r).Debug("request")
-	if err = h.pullACLCheck(r.Filesystem, &r.FilesystemVersion); err != nil {
-		log.WithError(err).Warn("pull ACL check failed")
-		return
-	}
-
-	log.Debug("invoking zfs send")
-
-	s, err := zfs.ZFSSend(r.Filesystem, &r.FilesystemVersion, nil)
-	if err != nil {
-		log.WithError(err).Error("cannot send filesystem")
-	}
-	*stream = s
-
-	return
-
-}
-
-func (h Handler) HandleIncrementalTransferRequest(r *IncrementalTransferRequest, stream *io.Reader) (err error) {
-
-	log := h.logger.WithField("endpoint", "IncrementalTransferRequest")
-	log.WithField("request", r).Debug("request")
-	if err = h.pullACLCheck(r.Filesystem, &r.From); err != nil {
-		log.WithError(err).Warn("pull ACL check failed")
-		return
-	}
-	if err = h.pullACLCheck(r.Filesystem, &r.To); err != nil {
-		log.WithError(err).Warn("pull ACL check failed")
-		return
-	}
-
-	log.Debug("invoking zfs send")
-
-	s, err := zfs.ZFSSend(r.Filesystem, &r.From, &r.To)
-	if err != nil {
-		log.WithError(err).Error("cannot send filesystem")
-	}
-
-	*stream = s
-	return
-
-}
-
-func (h Handler) pullACLCheck(p *zfs.DatasetPath, v *zfs.FilesystemVersion) (err error) {
-	var fsAllowed, vAllowed bool
-	fsAllowed, err = h.dsf.Filter(p)
-	if err != nil {
-		err = fmt.Errorf("error evaluating ACL: %s", err)
-		return
-	}
-	if !fsAllowed {
-		err = fmt.Errorf("ACL prohibits access to %s", p.ToString())
-		return
-	}
-	if v == nil {
-		return
-	}
-
-	vAllowed, err = h.fsvf.Filter(*v)
-	if err != nil {
-		err = errors.Wrap(err, "error evaluating version filter")
-		return
-	}
-	if !vAllowed {
-		err = fmt.Errorf("ACL prohibits access to %s", v.ToAbsPath(p))
-		return
-	}
-	return
-}

cmd/main.go

@@ -1,47 +0,0 @@
-// zrepl replicates ZFS filesystems & volumes between pools
-//
-// Code Organization
-//
-// The cmd package uses github.com/spf13/cobra for its CLI.
-//
-// It combines the other packages in the zrepl project to implement zrepl functionality.
-//
-// Each subcommand's code is in the corresponding *.go file.
-// All other *.go files contain code shared by the subcommands.
-package cmd
-
-import (
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/logger"
-)
-
-//
-//type Logger interface {
-//	Printf(format string, v ...interface{})
-//}
-
-var (
-	zreplVersion string // set by build infrastructure
-)
-
-type Logger = *logger.Logger
-
-var RootCmd = &cobra.Command{
-	Use:   "zrepl",
-	Short: "ZFS dataset replication",
-	Long: `Replicate ZFS filesystems & volumes between pools:
-
-  - push & pull mode
-  - automatic snapshot creation & pruning
-  - local / over the network
-  - ACLs instead of blank SSH access`,
-}
-
-var rootArgs struct {
-	configFile string
-}
-
-func init() {
-	//cobra.OnInitialize(initConfig)
-	RootCmd.PersistentFlags().StringVar(&rootArgs.configFile, "config", "", "config file path")
-}

cmd/prune.go

@@ -1,138 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"github.com/zrepl/zrepl/zfs"
-	"time"
-)
-
-type Pruner struct {
-	task           *Task
-	Now            time.Time
-	DryRun         bool
-	DatasetFilter  zfs.DatasetFilter
-	SnapshotPrefix string
-	PrunePolicy    PrunePolicy
-}
-
-type PruneResult struct {
-	Filesystem *zfs.DatasetPath
-	All        []zfs.FilesystemVersion
-	Keep       []zfs.FilesystemVersion
-	Remove     []zfs.FilesystemVersion
-}
-
-func (p *Pruner) filterFilesystems() (filesystems []*zfs.DatasetPath, stop bool) {
-	p.task.Enter("filter_fs")
-	defer p.task.Finish()
-	filesystems, err := zfs.ZFSListMapping(p.DatasetFilter)
-	if err != nil {
-		p.task.Log().WithError(err).Error("error applying filesystem filter")
-		return nil, true
-	}
-	if len(filesystems) <= 0 {
-		p.task.Log().Info("no filesystems matching filter")
-		return nil, true
-	}
-	return filesystems, false
-}
-
-func (p *Pruner) filterVersions(fs *zfs.DatasetPath) (fsversions []zfs.FilesystemVersion, stop bool) {
-	p.task.Enter("filter_versions")
-	defer p.task.Finish()
-	log := p.task.Log().WithField(logFSField, fs.ToString())
-
-	filter := NewPrefixFilter(p.SnapshotPrefix)
-	fsversions, err := zfs.ZFSListFilesystemVersions(fs, filter)
-	if err != nil {
-		log.WithError(err).Error("error listing filesytem versions")
-		return nil, true
-	}
-	if len(fsversions) == 0 {
-		log.WithField("prefix", p.SnapshotPrefix).Info("no filesystem versions matching prefix")
-		return nil, true
-	}
-	return fsversions, false
-}
-
-func (p *Pruner) pruneFilesystem(fs *zfs.DatasetPath) (r PruneResult, valid bool) {
-	p.task.Enter("prune_fs")
-	defer p.task.Finish()
-	log := p.task.Log().WithField(logFSField, fs.ToString())
-
-	fsversions, stop := p.filterVersions(fs)
-	if stop {
-		return
-	}
-
-	p.task.Enter("prune_policy")
-	keep, remove, err := p.PrunePolicy.Prune(fs, fsversions)
-	p.task.Finish()
-	if err != nil {
-		log.WithError(err).Error("error evaluating prune policy")
-		return
-	}
-
-	log.WithField("fsversions", fsversions).
-		WithField("keep", keep).
-		WithField("remove", remove).
-		Debug("prune policy debug dump")
-
-	r = PruneResult{fs, fsversions, keep, remove}
-
-	makeFields := func(v zfs.FilesystemVersion) (fields map[string]interface{}) {
-		fields = make(map[string]interface{})
-		fields["version"] = v.ToAbsPath(fs)
-		timeSince := v.Creation.Sub(p.Now)
-		fields["age_ns"] = timeSince
-		const day time.Duration = 24 * time.Hour
-		days := timeSince / day
-		remainder := timeSince % day
-		fields["age_str"] = fmt.Sprintf("%dd%s", days, remainder)
-		return
-	}
-
-	for _, v := range remove {
-		fields := makeFields(v)
-		log.WithFields(fields).Info("destroying version")
-		// echo what we'll do and exec zfs destroy if not dry run
-		// TODO special handling for EBUSY (zfs hold)
-		// TODO error handling for clones? just echo to cli, skip over, and exit with non-zero status code (we're idempotent)
-		if !p.DryRun {
-			p.task.Enter("destroy")
-			err := zfs.ZFSDestroyFilesystemVersion(fs, v)
-			p.task.Finish()
-			if err != nil {
-				log.WithFields(fields).WithError(err).Error("error destroying version")
-			}
-		}
-	}
-	return r, true
-}
-
-func (p *Pruner) Run(ctx context.Context) (r []PruneResult, err error) {
-	p.task.Enter("run")
-	defer p.task.Finish()
-
-	if p.DryRun {
-		p.task.Log().Info("doing dry run")
-	}
-
-	filesystems, stop := p.filterFilesystems()
-	if stop {
-		return
-	}
-
-	r = make([]PruneResult, 0, len(filesystems))
-
-	for _, fs := range filesystems {
-		res, ok := p.pruneFilesystem(fs)
-		if ok {
-			r = append(r, res)
-		}
-	}
-
-	return
-
-}

cmd/replication.go

@@ -1,323 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"io"
-
-	"bytes"
-	"encoding/json"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type localPullACL struct{}
-
-func (a localPullACL) Filter(p *zfs.DatasetPath) (pass bool, err error) {
-	return true, nil
-}
-
-const DEFAULT_INITIAL_REPL_POLICY = InitialReplPolicyMostRecent
-
-type InitialReplPolicy string
-
-const (
-	InitialReplPolicyMostRecent InitialReplPolicy = "most_recent"
-	InitialReplPolicyAll        InitialReplPolicy = "all"
-)
-
-type Puller struct {
-	task              *Task
-	Remote            rpc.RPCClient
-	Mapping           DatasetMapping
-	InitialReplPolicy InitialReplPolicy
-}
-
-type remoteLocalMapping struct {
-	Remote *zfs.DatasetPath
-	Local  *zfs.DatasetPath
-}
-
-func (p *Puller) getRemoteFilesystems() (rfs []*zfs.DatasetPath, ok bool) {
-	p.task.Enter("fetch_remote_fs_list")
-	defer p.task.Finish()
-
-	fsr := FilesystemRequest{}
-	if err := p.Remote.Call("FilesystemRequest", &fsr, &rfs); err != nil {
-		p.task.Log().WithError(err).Error("cannot fetch remote filesystem list")
-		return nil, false
-	}
-	return rfs, true
-}
-
-func (p *Puller) buildReplMapping(remoteFilesystems []*zfs.DatasetPath) (replMapping map[string]remoteLocalMapping, ok bool) {
-	p.task.Enter("build_repl_mapping")
-	defer p.task.Finish()
-
-	replMapping = make(map[string]remoteLocalMapping, len(remoteFilesystems))
-	for fs := range remoteFilesystems {
-		var err error
-		var localFs *zfs.DatasetPath
-		localFs, err = p.Mapping.Map(remoteFilesystems[fs])
-		if err != nil {
-			err := fmt.Errorf("error mapping %s: %s", remoteFilesystems[fs], err)
-			p.task.Log().WithError(err).WithField(logMapFromField, remoteFilesystems[fs]).Error("cannot map")
-			return nil, false
-		}
-		if localFs == nil {
-			continue
-		}
-		p.task.Log().WithField(logMapFromField, remoteFilesystems[fs].ToString()).
-			WithField(logMapToField, localFs.ToString()).Debug("mapping")
-		m := remoteLocalMapping{remoteFilesystems[fs], localFs}
-		replMapping[m.Local.ToString()] = m
-	}
-	return replMapping, true
-}
-
-// returns true if the receiving filesystem (local side) exists and can have child filesystems
-func (p *Puller) replFilesystem(m remoteLocalMapping, localFilesystemState map[string]zfs.FilesystemState) (localExists bool) {
-
-	p.task.Enter("repl_fs")
-	defer p.task.Finish()
-	var err error
-	remote := p.Remote
-
-	log := p.task.Log().
-		WithField(logMapFromField, m.Remote.ToString()).
-		WithField(logMapToField, m.Local.ToString())
-
-	log.Debug("examining local filesystem state")
-	localState, localExists := localFilesystemState[m.Local.ToString()]
-	var versions []zfs.FilesystemVersion
-	switch {
-	case !localExists:
-		log.Info("local filesystem does not exist")
-	case localState.Placeholder:
-		log.Info("local filesystem is marked as placeholder")
-	default:
-		log.Debug("local filesystem exists")
-		log.Debug("requesting local filesystem versions")
-		if versions, err = zfs.ZFSListFilesystemVersions(m.Local, nil); err != nil {
-			log.WithError(err).Error("cannot get local filesystem versions")
-			return false
-		}
-	}
-
-	log.Info("requesting remote filesystem versions")
-	r := FilesystemVersionsRequest{
-		Filesystem: m.Remote,
-	}
-	var theirVersions []zfs.FilesystemVersion
-	if err = remote.Call("FilesystemVersionsRequest", &r, &theirVersions); err != nil {
-		log.WithError(err).Error("cannot get remote filesystem versions")
-		log.Warn("stopping replication for all filesystems mapped as children of receiving filesystem")
-		return false
-	}
-
-	log.Debug("computing diff between remote and local filesystem versions")
-	diff := zfs.MakeFilesystemDiff(versions, theirVersions)
-	log.WithField("diff", diff).Debug("diff between local and remote filesystem")
-
-	if localState.Placeholder && diff.Conflict != zfs.ConflictAllRight {
-		panic("internal inconsistency: local placeholder implies ConflictAllRight")
-	}
-
-	switch diff.Conflict {
-	case zfs.ConflictAllRight:
-
-		log.WithField("replication_policy", p.InitialReplPolicy).Info("performing initial sync, following policy")
-
-		if p.InitialReplPolicy != InitialReplPolicyMostRecent {
-			panic(fmt.Sprintf("policy '%s' not implemented", p.InitialReplPolicy))
-		}
-
-		snapsOnly := make([]zfs.FilesystemVersion, 0, len(diff.MRCAPathRight))
-		for s := range diff.MRCAPathRight {
-			if diff.MRCAPathRight[s].Type == zfs.Snapshot {
-				snapsOnly = append(snapsOnly, diff.MRCAPathRight[s])
-			}
-		}
-
-		if len(snapsOnly) < 1 {
-			log.Warn("cannot perform initial sync: no remote snapshots")
-			return false
-		}
-
-		r := InitialTransferRequest{
-			Filesystem:        m.Remote,
-			FilesystemVersion: snapsOnly[len(snapsOnly)-1],
-		}
-
-		log.WithField("version", r.FilesystemVersion).Debug("requesting snapshot stream")
-
-		var stream io.Reader
-
-		if err = remote.Call("InitialTransferRequest", &r, &stream); err != nil {
-			log.WithError(err).Error("cannot request initial transfer")
-			return false
-		}
-		log.Debug("received initial transfer request response")
-
-		log.Debug("invoke zfs receive")
-		recvArgs := []string{"-u"}
-		if localState.Placeholder {
-			log.Info("receive with forced rollback to replace placeholder filesystem")
-			recvArgs = append(recvArgs, "-F")
-		}
-		progressStream := p.task.ProgressUpdater(stream)
-		if err = zfs.ZFSRecv(m.Local, progressStream, recvArgs...); err != nil {
-			log.WithError(err).Error("cannot receive stream")
-			return false
-		}
-		log.Info("finished receiving stream") // TODO rx delta
-
-		// TODO unify with recv path of ConflictIncremental
-		log.Debug("configuring properties of received filesystem")
-		props := zfs.NewZFSProperties()
-		props.Set("readonly", "on")
-		if err = zfs.ZFSSet(m.Local, props); err != nil {
-			log.WithError(err).Error("cannot set readonly property")
-		}
-
-		log.Info("finished initial transfer")
-		return true
-
-	case zfs.ConflictIncremental:
-
-		if len(diff.IncrementalPath) < 2 {
-			log.Info("remote and local are in sync")
-			return true
-		}
-
-		log.Info("following incremental path from diff")
-		for i := 0; i < len(diff.IncrementalPath)-1; i++ {
-
-			from, to := diff.IncrementalPath[i], diff.IncrementalPath[i+1]
-
-			log, _ := log.WithField(logIncFromField, from.Name).WithField(logIncToField, to.Name), 0
-
-			log.Debug("requesting incremental snapshot stream")
-			r := IncrementalTransferRequest{
-				Filesystem: m.Remote,
-				From:       from,
-				To:         to,
-			}
-			var stream io.Reader
-			if err = remote.Call("IncrementalTransferRequest", &r, &stream); err != nil {
-				log.WithError(err).Error("cannot request incremental snapshot stream")
-				return false
-			}
-
-			log.Debug("invoking zfs receive")
-			progressStream := p.task.ProgressUpdater(stream)
-			// TODO protect against malicious incremental stream
-			if err = zfs.ZFSRecv(m.Local, progressStream); err != nil {
-				log.WithError(err).Error("cannot receive stream")
-				return false
-			}
-			log.Info("finished incremental transfer") // TODO increment rx
-
-		}
-		log.Info("finished following incremental path") // TODO path rx
-		return true
-
-	case zfs.ConflictNoCommonAncestor:
-		fallthrough
-	case zfs.ConflictDiverged:
-
-		var jsonDiff bytes.Buffer
-		if err := json.NewEncoder(&jsonDiff).Encode(diff); err != nil {
-			log.WithError(err).Error("cannot JSON-encode diff")
-			return false
-		}
-
-		var problem, resolution string
-
-		switch diff.Conflict {
-		case zfs.ConflictNoCommonAncestor:
-			problem = "remote and local filesystem have snapshots, but no common one"
-			resolution = "perform manual establish a common snapshot history"
-		case zfs.ConflictDiverged:
-			problem = "remote and local filesystem share a history but have diverged"
-			resolution = "perform manual replication or delete snapshots on the receiving" +
-				"side  to establish an incremental replication parse"
-		}
-
-		log.WithField("diff", jsonDiff.String()).
-			WithField("problem", problem).
-			WithField("resolution", resolution).
-			Error("manual conflict resolution required")
-
-		return false
-
-	}
-
-	panic("should not be reached")
-}
-
-func (p *Puller) Pull() {
-	p.task.Enter("run")
-	defer p.task.Finish()
-
-	p.task.Log().Info("request remote filesystem list")
-	remoteFilesystems, ok := p.getRemoteFilesystems()
-	if !ok {
-		return
-	}
-
-	p.task.Log().Debug("map remote filesystems to local paths and determine order for per-filesystem sync")
-	replMapping, ok := p.buildReplMapping(remoteFilesystems)
-	if !ok {
-
-	}
-
-	p.task.Log().Debug("build cache for already present local filesystem state")
-	p.task.Enter("cache_local_fs_state")
-	localFilesystemState, err := zfs.ZFSListFilesystemState()
-	p.task.Finish()
-	if err != nil {
-		p.task.Log().WithError(err).Error("cannot request local filesystem state")
-		return
-	}
-
-	localTraversal := zfs.NewDatasetPathForest()
-	for _, m := range replMapping {
-		localTraversal.Add(m.Local)
-	}
-
-	p.task.Log().Info("start per-filesystem sync")
-	localTraversal.WalkTopDown(func(v zfs.DatasetPathVisit) bool {
-
-		p.task.Enter("tree_walk")
-		defer p.task.Finish()
-
-		log := p.task.Log().WithField(logFSField, v.Path.ToString())
-
-		if v.FilledIn {
-			if _, exists := localFilesystemState[v.Path.ToString()]; exists {
-				// No need to verify if this is a placeholder or not. It is sufficient
-				// to know we can add child filesystems to it
-				return true
-			}
-			log.Debug("create placeholder filesystem")
-			p.task.Enter("create_placeholder")
-			err = zfs.ZFSCreatePlaceholderFilesystem(v.Path)
-			p.task.Finish()
-			if err != nil {
-				log.Error("cannot create placeholder filesystem")
-				return false
-			}
-			return true
-		}
-
-		m, ok := replMapping[v.Path.ToString()]
-		if !ok {
-			panic("internal inconsistency: replMapping should contain mapping for any path that was not filled in by WalkTopDown()")
-		}
-
-		return p.replFilesystem(m, localFilesystemState)
-	})
-
-	return
-
-}

cmd/sampleconf/localbackup/host1.yml

@@ -1,29 +0,0 @@
-jobs:
-- name: mirror_local
-  type: local
-
-  # snapshot the filesystems matched by the left-hand-side of the mapping
-  # every 10m with zrepl_ as prefix
-  mapping: {
-    "zroot/var/db<":    "storage/backups/local/zroot/var/db",
-    "zroot/usr/home<":  "storage/backups/local/zroot/usr/home",
-    "zroot/usr/home/paranoid":  "!", #don't backup paranoid user
-    "zroot/poudriere/ports<": "!", #don't backup the ports trees
-  }
-  snapshot_prefix: zrepl_
-  interval: 10m
-  initial_repl_policy: most_recent
-
-  # keep one hour of 10m interval snapshots of filesystems matched by
-  # the left-hand-side of the mapping
-  prune_lhs:
-    policy: grid
-    grid: 1x1h(keep=all)
-    keep_bookmarks: all
-
-  # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
-  prune_rhs:
-    policy: grid
-    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-
-

cmd/sampleconf/pullbackup/backuphost.yml

@@ -1,27 +0,0 @@
-jobs:
-- name: fullbackup_prod1
-  type: pull
-  # connect to remote using ssh / stdinserver command
-  connect:
-    type: ssh+stdinserver
-    host: prod1.example.com
-    user: root
-    port: 22
-    identity_file: /root/.ssh/id_ed25519
-
-  # pull (=ask for new snapshots) every 10m, prune afterwards
-  # this will leave us at most 10m behind production
-  interval: 10m
-
-  # pull all offered filesystems to storage/backups/zrepl/pull/prod1.example.com
-  mapping: {
-    "<":"storage/backups/zrepl/pull/prod1.example.com"
-  }
-  initial_repl_policy: most_recent
-
-  # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
-  snapshot_prefix: zrepl_
-  prune:
-    policy: grid
-    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-

cmd/sampleconf/pullbackup/productionhost.yml

@@ -1,47 +0,0 @@
-global:
-  serve:
-    stdinserver:
-      # Directory where AF_UNIX sockets for stdinserver command are placed.
-      #
-      # `zrepl stdinserver CLIENT_IDENTITY`
-      #   * connects to the socket in $sockdir/CLIENT_IDENTITY
-      #   * sends its stdin / stdout file descriptors to the `zrepl daemon` process (see cmsg(3))
-      #   * does nothing more
-      #
-      # This enables a setup where `zrepl daemon` is not directly exposed to the internet
-      # but instead all traffic is tunnelled through SSH.
-      # The server with the source job has an authorized_keys file entry for the public key
-      # used by the corresponding pull job
-      #
-      #   command="/mnt/zrepl stdinserver CLIENT_IDENTITY" ssh-ed25519 AAAAC3NzaC1E... zrepl@pullingserver
-      #
-      # Below is the default value.
-      sockdir: /var/run/zrepl/stdinserver
-
-jobs:
-
-- name: fullbackup_prod1
-  # expect remote to connect via ssh+stdinserver with fullbackup_prod1 as client_identity
-  type: source
-  serve:
-    type: stdinserver # see global.serve.stdinserver for explanation
-    client_identity: fullbackup_prod1
-
-  # snapshot these filesystems every 10m with zrepl_ as prefix
-  filesystems: {
-    "zroot/var/db<": "ok",
-    "zroot/usr/home<": "ok",
-    "zroot/var/tmp": "!", #don't backup /tmp
-  }
-  snapshot_prefix: zrepl_
-  interval: 10m
-
-
-  # keep 1 hour of snapshots (6 at 10m interval)
-  # and  one day of bookmarks in case pull doesn't work (link down, etc)
-  # => keep_bookmarks = 24h / interval = 24h / 10m = 144
-  prune:
-    policy: grid
-    grid: 1x1h(keep=all)
-    keep_bookmarks: 144
-

cmd/sampleconf/pushbackup/backuphost.yml

@@ -1,20 +0,0 @@
-jobs:
-- name: fullbackup_prod1
-
-  # expect remote to connect via ssh+stdinserver with fullbackup_prod1 as client_identity
-  type: push-sink
-  serve:
-    type: stdinserver
-    client_identity: fullbackup_prod1
-
-  # map all pushed datasets to storage/backups/zrepl/sink/prod1.example.com
-  mapping: {
-    "<":"storage/backups/zrepl/sink/prod1.example.com"
-  }
-
-  # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
-  prune:
-    policy: grid
-    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-
-

cmd/sampleconf/pushbackup/productionhost.yml

@@ -1,26 +0,0 @@
-jobs:
-- name: fullbackup_prod1
-
-  # connect to remote using ssh / stdinserver command
-  type: push
-  connect:
-      type: ssh+stdinserver
-      host: prod1.example.com
-      user: root
-      port: 22
-      identity_file: /root/.ssh/id_ed25519
-
-  # snapshot these datsets every 10m with zrepl_ as prefix
-  filesystems: {
-    "zroot/var/db<": "ok",
-    "zroot/usr/home<": "!",
-  }
-  snapshot_prefix: zrepl_
-  interval: 10m
-
-  # keep a one day window 10m interval snapshots in case push doesn't work (link down, etc)
-  # (we cannot keep more than one day because this host will run out of disk space)
-  prune:
-    policy: grid
-    grid: 1x1d(keep=all)
-

cmd/sampleconf/random/debugging.yml

@@ -1,33 +0,0 @@
-global:
-  serve:
-    stdinserver:
-      sockdir: /var/run/zrepl/stdinserver
-
-jobs:
-
-- name: debian2_pull
-  # JOB DEBUGGING OPTIONS
-  # should be equal for all job types, but each job implements the debugging itself
-  # => consult job documentation for supported options
-  debug:
-    conn: # debug the io.ReadWriteCloser connection
-      read_dump: /tmp/connlog_read   # dump results of Read() invocations to this file
-      write_dump: /tmp/connlog_write # dump results of Write() invocations to this file
-    rpc: # debug the RPC protocol implementation
-      log: true # log output from rpc layer to the job log
-
-  # ... just to make the unit tests pass.
-  # check other examples, e.g. localbackup or pullbackup for what the sutff below means
-  type: source
-  serve:
-    type: stdinserver
-    client_identity: debian2
-  filesystems: {
-    "pool1/db<": ok
-  }
-  snapshot_prefix: zrepl_
-  interval: 1s
-  prune:
-    policy: grid
-    grid: 1x10s(keep=all)
-    keep_bookmarks: all

cmd/sampleconf/random/logging.yml

@@ -1,24 +0,0 @@
-global:
-  logging:
-
-    - outlet: stdout
-      level: warn
-      format: human
-
-    - outlet: tcp
-      level: debug
-      format: json
-      net: tcp
-      address: 127.0.0.1:8080
-      retry_interval: 1s
-      tls: # if not specified, use plain TCP
-        ca: sampleconf/random/logging/logserver.crt
-        cert: sampleconf/random/logging/client.crt
-        key: sampleconf/random/logging/client.key
-
-    - outlet: syslog
-      level: debug
-      format: logfmt
-
-
-jobs: []

cmd/sampleconf/random/logging/client.crt

@@ -1,19 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDIzCCAgsCAQEwDQYJKoZIhvcNAQELBQAwWTELMAkGA1UEBhMCQVUxEzARBgNV
-BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0
-ZDESMBAGA1UEAwwJbG9nc2VydmVyMB4XDTE3MDkyNDEyMzAzNloXDTE3MTAyNDEy
-MzAzNlowVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNV
-BAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xpZW50MIIB
-IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/xJTUlqApeJGzRD+w2J8sZS
-Bo+s+04T987L/M6gaCo8aDSTEb/ZH3XSoU5JEmO6kPpwNNapOsaEhTCjndZQdm5F
-uqiUtAg1uW0HCkBEIDkGr9bFHDKzpewGmmMgfQ2+hfiBR/4ZCrc/vd9P0W9BiWQS
-Dtc7p22XraWPVL8HlSz5K/Ih+V6i8O+kBltZkusiJh2bWPoRp/netiTZuc6du+Wp
-kpWp1OBaTU4GXIAlLj5afF14BBphRQK983Yhaz53BkA7OQ76XxowynMjmuLQVGmK
-f1R9zEJuohTX9XIr1tp/ueRHcS4Awk6LcNZUMCV6270FNSIw2f4hbOZvep+t2wID
-AQABMA0GCSqGSIb3DQEBCwUAA4IBAQACK3OeNzScpiNwz/jpg/usQzvXbZ/wDvml
-YLjtzn/A65ox8a8BhxvH1ydyoCM2YAGYX7+y7qXJnMgRO/v8565CQIVcznHhg9ST
-3828/WqZ3bXf2DV5GxKKQf7hPmBnyVUUhn/Ny91MECED27lZucWiX/bczN8ffDeh
-M3+ngezcJxsOBd4x0gLrqIJCoaFRSeepOaFEW6GHQ8loxE9GmA7FQd2phIpJHFSd
-Z7nQl7X5C1iN2OboEApJHwtmNVC45UlOpg53vo2sDTLhSfdogstiWi8x1HmvhIGM
-j3XHs0Illvo9OwVrmgUph8zQ7pvr/AFrTOIbhgzl/9uVUk5ApwFM
------END CERTIFICATE-----

cmd/sampleconf/random/logging/client.csr

@@ -1,16 +0,0 @@
------BEGIN CERTIFICATE REQUEST-----
-MIICmzCCAYMCAQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx
-ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xp
-ZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/xJTUlqApeJGzRD
-+w2J8sZSBo+s+04T987L/M6gaCo8aDSTEb/ZH3XSoU5JEmO6kPpwNNapOsaEhTCj
-ndZQdm5FuqiUtAg1uW0HCkBEIDkGr9bFHDKzpewGmmMgfQ2+hfiBR/4ZCrc/vd9P
-0W9BiWQSDtc7p22XraWPVL8HlSz5K/Ih+V6i8O+kBltZkusiJh2bWPoRp/netiTZ
-uc6du+WpkpWp1OBaTU4GXIAlLj5afF14BBphRQK983Yhaz53BkA7OQ76XxowynMj
-muLQVGmKf1R9zEJuohTX9XIr1tp/ueRHcS4Awk6LcNZUMCV6270FNSIw2f4hbOZv
-ep+t2wIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAKnlr0Qs5KYF85u2YA7DJ5pL
-HwAx+qNoNbox5CS1aynrDBpDTWLaErviUJ+4WxRlRyTMEscMOIOKajbYhqqFmtGZ
-mu3SshZnFihErw8TOQMyU1LGGG+l6r+6ve5TciwJRLla2Y75z7izr6cyvQNRWdLr
-PvxL1/Yqr8LKha12+7o28R4SLf6/GY0GcedqoebRmtuwA/jES0PuGauEUD5lH4cj
-Me8sqRrB+IMHQ5j8hlJX4DbA8UQRUBL64sHkQzeQfWu+qkWmS5I19CFfLNrcH+OV
-yhyjGfN0q0jHyHdpckBhgzS7IIdo6P66AIlm4qpHM7Scra3JaGM7oaZPamJ6f8U=
------END CERTIFICATE REQUEST-----

cmd/sampleconf/random/logging/client.key

@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC3/ElNSWoCl4kb
-NEP7DYnyxlIGj6z7ThP3zsv8zqBoKjxoNJMRv9kfddKhTkkSY7qQ+nA01qk6xoSF
-MKOd1lB2bkW6qJS0CDW5bQcKQEQgOQav1sUcMrOl7AaaYyB9Db6F+IFH/hkKtz+9
-30/Rb0GJZBIO1zunbZetpY9UvweVLPkr8iH5XqLw76QGW1mS6yImHZtY+hGn+d62
-JNm5zp275amSlanU4FpNTgZcgCUuPlp8XXgEGmFFAr3zdiFrPncGQDs5DvpfGjDK
-cyOa4tBUaYp/VH3MQm6iFNf1civW2n+55EdxLgDCTotw1lQwJXrbvQU1IjDZ/iFs
-5m96n63bAgMBAAECggEAF4om0sWe06ARwbJJNFjCGpa3LfG5/xk5Qs5pmPnS2iD1
-Q5veaTnzjKvlfA/pF3o9B4mTS59fXY7Cq8vSU0J1XwGy2DPzeqlGPmgtq2kXjkvd
-iCfhZj8ybvsoyR3/rSBSDRADcnOXPqC9fgyRSMmESBDOoql1D3HdIzF4ii46ySIU
-/XQvExS6NWifbP+Ue6DETV8NhreO5PqjeXLITQhhndtc8MDL/8eCNOyN8XjYIWKX
-smlBYtRQYOOY9BHOQgUn6yvPHrtKJNKci+qcQNvWir66mBhY1o40MH5wTIV+8yP2
-Vbm/VzoNKIYgeROsilBW7QTwGvkDn3R11zeTqfUNSQKBgQD0eFzhJAEZi4uBw6Tg
-NKmBC5Y1IHPOsb5gKPNz9Z9j4qYRDySgYl6ISk+2EdhgUCo1NmTk8EIPQjIerUVf
-S+EogFnpsj8U9LR3OM79DaGkNULxrHqhd209/g8DtVgk7yjkxL4vmVOv8qpHMp/7
-eWsylN7AOxj2RB/eXYQBPrw+jQKBgQDAqae9HasLmvpJ9ktTv30yZSKXC+LP4A0D
-RBBmx410VpPd4CvcpCJxXmjer6B7+9L1xHYP2pvsnMBid5i0knuvyK28dYy7fldl
-CzWvb+lqNA5YYPFXQED4oEdihlQczoI1Bm06SFizeAKD1Q9e2c+lgbR/51j8xuXi
-twvhMj/YBwKBgQCZw97/iQrcC2Zq7yiUEOuQjD4lGk1c83U/vGIsTJC9XcCAOFsc
-OeMlrD/oz96d7a4unBDn4qpaOJOXsfpRT0PGmrxy/jcpMiUUW/ntNpa11v5NTeQw
-DRL8DAFbnsNbL8Yz5f+Nps35fBNYBuKTZLJlNTfKByHTO9QjpAQ0WEZEvQKBgQCi
-Ovm83EuYVSKmvxcE6Tyx/8lVqTOO2Vn7wweQlD4/lVujvE0S2L8L+XSS9w5K+GzW
-eFz10p3zarbw80YJ30L5bSEmjVE43BUZR4woMzM4M6dUsiTm1HshIE2b4ALZ0uZ/
-Ye794ceXL9nmSrVLqFsaQZLNFPCwwYb4FiyRry9lZwKBgAO9VbWcN8SEeBDKo3z8
-yRbRTc6sI+AdKY44Dfx0tqOPmTjO3mE4X1GU4sbfD2Bvg3DdjwTuxxC/jHaKu0GG
-dTM0CbrZGbDAj7E87SOcN/PWEeBckSvuQq5H3DQfwIpTmlS1l5oZn9CxRGbLqC2G
-ifnel8XWUG0ROybsr1tk4mzW
------END PRIVATE KEY-----

cmd/sampleconf/random/logging/logserver.crt

@@ -1,21 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDiDCCAnCgAwIBAgIJALhp/WvTQeg/MA0GCSqGSIb3DQEBCwUAMFkxCzAJBgNV
-BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
-aWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMMCWxvZ3NlcnZlcjAeFw0xNzA5MjQxMjI3
-MDRaFw0yNzA5MjIxMjI3MDRaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21l
-LVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNV
-BAMMCWxvZ3NlcnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKs3
-TLYfXhV3hap71tOkhPQlM+m0EKRAo8Nua50Cci5UhDo4JkVpyYok1h+NFkqmjU2b
-IiIuGvsZZPOWYjbWWnSJE4+n5pBFBzcfNQ4d8xVxjANImFn6Tcehhj0WkbDIv/Ge
-364XUgywS7u3EGQj/FO7vZ8KHlUxBHNuPIOPHftwIVRyleh5K32UyBaSpSmnqGos
-rvI1byMuznavcZpOs4vlebZ+Jy6a20iKf9fj/0f0t0O+F5x3JIk07D3zSywhJ4RM
-M0mGIUmYXbh2SMh+f61KDZLDANpz/pMAPbUJe0mxEtBf0tnwK1gEqc3SLwA0EwiM
-8Hnn2iaH5Ln20UE3LOkCAwEAAaNTMFEwHQYDVR0OBBYEFDXoDcwx9SngzZcRYCeP
-BplBecfiMB8GA1UdIwQYMBaAFDXoDcwx9SngzZcRYCePBplBecfiMA8GA1UdEwEB
-/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBADyNvs4AA91x3gurQb1pcPVhK6nR
-mkYSTN1AsDKSRi/X2iCUmR7G7FlF7XW8mntTpHvVzcs+gr94WckH5wqEOA5iZnaw
-PXUWexmdXUge4hmC2q6kBQ5e2ykhSJMRVZXvOLZOZV9qitceamHESV1cKZSNMvZM
-aCSVA1RK61/nUzs04pVp5PFPv9gFxJp9ki39FYFdsgZmM5RZ5I/FqxxvTJzu4RnH
-VPjsMopzARYwJw6dV2bKdFSYOE8B/Vs3Yv0GxjrABw2ko4PkBPTjLIz22x6+Hd9r
-K9BQi4pVmQfvppF5+SORSftlHSS+N47b0DD1rW1f5R6QGi71dFuJGikOwvY=
------END CERTIFICATE-----

cmd/sampleconf/random/logging/logserver.key

@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCrN0y2H14Vd4Wq
-e9bTpIT0JTPptBCkQKPDbmudAnIuVIQ6OCZFacmKJNYfjRZKpo1NmyIiLhr7GWTz
-lmI21lp0iROPp+aQRQc3HzUOHfMVcYwDSJhZ+k3HoYY9FpGwyL/xnt+uF1IMsEu7
-txBkI/xTu72fCh5VMQRzbjyDjx37cCFUcpXoeSt9lMgWkqUpp6hqLK7yNW8jLs52
-r3GaTrOL5Xm2ficumttIin/X4/9H9LdDvhecdySJNOw980ssISeETDNJhiFJmF24
-dkjIfn+tSg2SwwDac/6TAD21CXtJsRLQX9LZ8CtYBKnN0i8ANBMIjPB559omh+S5
-9tFBNyzpAgMBAAECggEBAIY8ZwJq+WKvQLb3POjWFf8so9TY/ispGrwAeJKy9j5o
-uPrERw0o8YBDfTVjclS43BQ6Srqtly3DLSjlgL8ps+WmCxYYN2ZpGE0ZRIl65bis
-O2/fnML+wbiAZTTD2xnVatfPDeP6GLQmDFpyHoHEzPIBQZvNXRbBxZGSnhMvQ/x7
-FhqSBQG4kf3b1XDCENIbFEVOBOCg7WtMiIgjEGS7QnW3I65/Zt+Ts1LXRZbz+6na
-Gmi0PGHA/oLUh1NRzsF4zuZn6fFzja5zw4mkt+JvCWEoxg1QhRAxRp6QQwmZ6MIc
-1rw1D4Z+c5UEKyqHeIwZj4M6UNPhCfTXVm47c9eSiGECgYEA4U8pB+7eRo2fqX0C
-nWsWMcmsULJvwplQnUSFenUayPn3E8ammS/ZBHksoKhj82vwIdDbtS1hQZn8Bzsi
-atc8au0wz0YRDcVDzHX4HknXVQayHtP/FTPeSr5hwpoY8vhEbySuxBTBkXCrp4dx
-u5ErfOiYEP3Q1ZvPRywelrATu20CgYEAwonV5dgOcen/4oAirlnvufc2NfqhAQwJ
-FJ/JSVMAcXxPYu3sZMv0dGWrX8mLc+P1+XMCuV/7eBM/vU2LbDzmpeUV8sJfB2jw
-wyKqKXZwBgeq60btriA4f+0ElwRGgU2KSiniUuuTX2JmyftFQx4cVAQRCFk27NY0
-09psSsYyre0CgYBo6unabdtH029EB5iOIW3GZXk+Yrk0TxyA/4WAjsOYTv5FUT4H
-G4bdVGf5sDBLDDpYJOAKsEUXvVLlMx5FzlCuIiGWg7QxS2jU7yJJSG1jhKixPlsM
-Toj3GUyAyC1SB1Ymw1g2qsuwpFzquGG3zFQJ6G3Xi7oRnmqZY+wik3+8yQKBgB11
-SdKYOPe++2SNCrNkIw0CBk9+OEs0S1u4Jn7X9sU4kbzlUlqhF89YZe8HUfqmlmTD
-qbHwet/f6lL8HxSw1Cxi2EP+cu1oUqz53tKQgL4pAxTFlNA9SND2Ty+fEh4aY8p/
-NSphSduzxuTnC8HyGVAPnZSqDcsnVLCP7r4T7TCxAoGAbJygkkk/gZ9pT4fZoIaq
-8CMR8FTfxtkwCuZsWccSMUOWtx9nqet3gbCpKHfyoYZiKB4ke+lnUz4uFS16Y3hG
-kN0hFfvfoNa8eB2Ox7vs60cMMfWJac0H7KSaDDy+EvbhE2KtQADT0eWxMyhzGR8p
-5CbIivB0QCjeQIA8dOQpE8E=
------END PRIVATE KEY-----

cmd/test.go

@@ -1,215 +0,0 @@
-package cmd
-
-import (
-	"os"
-
-	"bytes"
-	"context"
-	"fmt"
-	"sort"
-	"strings"
-
-	"github.com/kr/pretty"
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/zfs"
-	"time"
-)
-
-var testCmd = &cobra.Command{
-	Use:              "test",
-	Short:            "test configuration",
-	PersistentPreRun: testCmdGlobalInit,
-}
-
-var testCmdGlobal struct {
-	log  Logger
-	conf *Config
-}
-
-var testConfigSyntaxCmd = &cobra.Command{
-	Use:   "config",
-	Short: "parse config file and dump parsed datastructure",
-	Run:   doTestConfig,
-}
-
-var testDatasetMapFilter = &cobra.Command{
-	Use:     "pattern jobname test/zfs/dataset/path",
-	Short:   "test dataset mapping / filter specified in config",
-	Example: ` zrepl test pattern my_pull_job tank/tmp`,
-	Run:     doTestDatasetMapFilter,
-}
-
-var testPrunePolicyArgs struct {
-	side        PrunePolicySide
-	showKept    bool
-	showRemoved bool
-}
-
-var testPrunePolicyCmd = &cobra.Command{
-	Use:   "prune jobname",
-	Short: "do a dry-run of the pruning part of a job",
-	Run:   doTestPrunePolicy,
-}
-
-func init() {
-	RootCmd.AddCommand(testCmd)
-	testCmd.AddCommand(testConfigSyntaxCmd)
-	testCmd.AddCommand(testDatasetMapFilter)
-
-	testPrunePolicyCmd.Flags().VarP(&testPrunePolicyArgs.side, "side", "s", "prune_lhs (left) or prune_rhs (right)")
-	testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showKept, "kept", false, "show kept snapshots")
-	testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showRemoved, "removed", true, "show removed snapshots")
-	testCmd.AddCommand(testPrunePolicyCmd)
-}
-
-func testCmdGlobalInit(cmd *cobra.Command, args []string) {
-
-	out := logger.NewOutlets()
-	out.Add(WriterOutlet{&NoFormatter{}, os.Stdout}, logger.Info)
-	log := logger.NewLogger(out, 1*time.Second)
-	testCmdGlobal.log = log
-
-	var err error
-	if testCmdGlobal.conf, err = ParseConfig(rootArgs.configFile); err != nil {
-		testCmdGlobal.log.Printf("error parsing config file: %s", err)
-		os.Exit(1)
-	}
-
-}
-
-func doTestConfig(cmd *cobra.Command, args []string) {
-
-	log, conf := testCmdGlobal.log, testCmdGlobal.conf
-
-	log.Printf("config ok")
-	log.Printf("%# v", pretty.Formatter(conf))
-	return
-}
-
-func doTestDatasetMapFilter(cmd *cobra.Command, args []string) {
-
-	log, conf := testCmdGlobal.log, testCmdGlobal.conf
-
-	if len(args) != 2 {
-		log.Printf("specify job name as first postitional argument, test input as second")
-		log.Printf(cmd.UsageString())
-		os.Exit(1)
-	}
-	n, i := args[0], args[1]
-
-	jobi, err := conf.LookupJob(n)
-	if err != nil {
-		log.Printf("%s", err)
-		os.Exit(1)
-	}
-
-	var mf *DatasetMapFilter
-	switch j := jobi.(type) {
-	case *PullJob:
-		mf = j.Mapping
-	case *SourceJob:
-		mf = j.Filesystems
-	case *LocalJob:
-		mf = j.Mapping
-	default:
-		panic("incomplete implementation")
-	}
-
-	ip, err := zfs.NewDatasetPath(i)
-	if err != nil {
-		log.Printf("cannot parse test input as ZFS dataset path: %s", err)
-		os.Exit(1)
-	}
-
-	if mf.filterMode {
-		pass, err := mf.Filter(ip)
-		if err != nil {
-			log.Printf("error evaluating filter: %s", err)
-			os.Exit(1)
-		}
-		log.Printf("filter result: %v", pass)
-	} else {
-		res, err := mf.Map(ip)
-		if err != nil {
-			log.Printf("error evaluating mapping: %s", err)
-			os.Exit(1)
-		}
-		toStr := "NO MAPPING"
-		if res != nil {
-			toStr = res.ToString()
-		}
-		log.Printf("%s => %s", ip.ToString(), toStr)
-
-	}
-
-}
-
-func doTestPrunePolicy(cmd *cobra.Command, args []string) {
-
-	log, conf := testCmdGlobal.log, testCmdGlobal.conf
-
-	if cmd.Flags().NArg() != 1 {
-		log.Printf("specify job name as first positional argument")
-		log.Printf(cmd.UsageString())
-		os.Exit(1)
-	}
-
-	jobname := cmd.Flags().Arg(0)
-	jobi, err := conf.LookupJob(jobname)
-	if err != nil {
-		log.Printf("%s", err)
-		os.Exit(1)
-	}
-
-	jobp, ok := jobi.(PruningJob)
-	if !ok {
-		log.Printf("job doesn't do any prunes")
-		os.Exit(0)
-	}
-
-	log.Printf("job dump:\n%s", pretty.Sprint(jobp))
-
-	task := NewTask("", log)
-	pruner, err := jobp.Pruner(task, testPrunePolicyArgs.side, true)
-	if err != nil {
-		log.Printf("cannot create test pruner: %s", err)
-		os.Exit(1)
-	}
-
-	log.Printf("start pruning")
-
-	ctx := context.WithValue(context.Background(), contextKeyLog, log)
-	result, err := pruner.Run(ctx)
-	if err != nil {
-		log.Printf("error running pruner: %s", err)
-		os.Exit(1)
-	}
-
-	sort.Slice(result, func(i, j int) bool {
-		return strings.Compare(result[i].Filesystem.ToString(), result[j].Filesystem.ToString()) == -1
-	})
-
-	var b bytes.Buffer
-	for _, r := range result {
-		fmt.Fprintf(&b, "%s\n", r.Filesystem.ToString())
-
-		if testPrunePolicyArgs.showKept {
-			fmt.Fprintf(&b, "\tkept:\n")
-			for _, v := range r.Keep {
-				fmt.Fprintf(&b, "\t- %s\n", v.Name)
-			}
-		}
-
-		if testPrunePolicyArgs.showRemoved {
-			fmt.Fprintf(&b, "\tremoved:\n")
-			for _, v := range r.Remove {
-				fmt.Fprintf(&b, "\t- %s\n", v.Name)
-			}
-		}
-
-	}
-
-	log.Printf("pruning result:\n%s", b.String())
-
-}

config/config.go

@@ -0,0 +1,583 @@
+package config
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log/syslog"
+	"os"
+	"reflect"
+	"regexp"
+	"strconv"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/zrepl/yaml-config"
+)
+
+type Config struct {
+	Jobs   []JobEnum `yaml:"jobs"`
+	Global *Global   `yaml:"global,optional,fromdefaults"`
+}
+
+func (c *Config) Job(name string) (*JobEnum, error) {
+	for _, j := range c.Jobs {
+		if j.Name() == name {
+			return &j, nil
+		}
+	}
+	return nil, fmt.Errorf("job %q not defined in config", name)
+}
+
+type JobEnum struct {
+	Ret interface{}
+}
+
+func (j JobEnum) Name() string {
+	var name string
+	switch v := j.Ret.(type) {
+	case *SnapJob:
+		name = v.Name
+	case *PushJob:
+		name = v.Name
+	case *SinkJob:
+		name = v.Name
+	case *PullJob:
+		name = v.Name
+	case *SourceJob:
+		name = v.Name
+	default:
+		panic(fmt.Sprintf("unknown job type %T", v))
+	}
+	return name
+}
+
+type ActiveJob struct {
+	Type    string                `yaml:"type"`
+	Name    string                `yaml:"name"`
+	Connect ConnectEnum           `yaml:"connect"`
+	Pruning PruningSenderReceiver `yaml:"pruning"`
+	Debug   JobDebugSettings      `yaml:"debug,optional"`
+}
+
+type PassiveJob struct {
+	Type  string           `yaml:"type"`
+	Name  string           `yaml:"name"`
+	Serve ServeEnum        `yaml:"serve"`
+	Debug JobDebugSettings `yaml:"debug,optional"`
+}
+
+type SnapJob struct {
+	Type         string            `yaml:"type"`
+	Name         string            `yaml:"name"`
+	Pruning      PruningLocal      `yaml:"pruning"`
+	Debug        JobDebugSettings  `yaml:"debug,optional"`
+	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
+	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+}
+
+type PushJob struct {
+	ActiveJob    `yaml:",inline"`
+	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
+	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+}
+
+type PullJob struct {
+	ActiveJob `yaml:",inline"`
+	RootFS    string                   `yaml:"root_fs"`
+	Interval  PositiveDurationOrManual `yaml:"interval"`
+}
+
+type PositiveDurationOrManual struct {
+	Interval time.Duration
+	Manual   bool
+}
+
+var _ yaml.Unmarshaler = (*PositiveDurationOrManual)(nil)
+
+func (i *PositiveDurationOrManual) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	var s string
+	if err := u(&s, true); err != nil {
+		return err
+	}
+	switch s {
+	case "manual":
+		i.Manual = true
+		i.Interval = 0
+	case "":
+		return fmt.Errorf("value must not be empty")
+	default:
+		i.Manual = false
+		i.Interval, err = time.ParseDuration(s)
+		if err != nil {
+			return err
+		}
+		if i.Interval <= 0 {
+			return fmt.Errorf("value must be a positive duration, got %q", s)
+		}
+	}
+	return nil
+}
+
+type SinkJob struct {
+	PassiveJob `yaml:",inline"`
+	RootFS     string `yaml:"root_fs"`
+}
+
+type SourceJob struct {
+	PassiveJob   `yaml:",inline"`
+	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
+	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+}
+
+type FilesystemsFilter map[string]bool
+
+type SnapshottingEnum struct {
+	Ret interface{}
+}
+
+type SnapshottingPeriodic struct {
+	Type     string        `yaml:"type"`
+	Prefix   string        `yaml:"prefix"`
+	Interval time.Duration `yaml:"interval,positive"`
+}
+
+type SnapshottingManual struct {
+	Type string `yaml:"type"`
+}
+
+type PruningSenderReceiver struct {
+	KeepSender   []PruningEnum `yaml:"keep_sender"`
+	KeepReceiver []PruningEnum `yaml:"keep_receiver"`
+}
+
+type PruningLocal struct {
+	Keep []PruningEnum `yaml:"keep"`
+}
+
+type LoggingOutletEnumList []LoggingOutletEnum
+
+func (l *LoggingOutletEnumList) SetDefault() {
+	def := `
+type: "stdout"
+time: true
+level: "warn"
+format: "human"
+`
+	s := &StdoutLoggingOutlet{}
+	err := yaml.UnmarshalStrict([]byte(def), &s)
+	if err != nil {
+		panic(err)
+	}
+	*l = []LoggingOutletEnum{LoggingOutletEnum{Ret: s}}
+}
+
+var _ yaml.Defaulter = &LoggingOutletEnumList{}
+
+type Global struct {
+	Logging    *LoggingOutletEnumList `yaml:"logging,optional,fromdefaults"`
+	Monitoring []MonitoringEnum       `yaml:"monitoring,optional"`
+	Control    *GlobalControl         `yaml:"control,optional,fromdefaults"`
+	Serve      *GlobalServe           `yaml:"serve,optional,fromdefaults"`
+}
+
+func Default(i interface{}) {
+	v := reflect.ValueOf(i)
+	if v.Kind() != reflect.Ptr {
+		panic(v)
+	}
+	y := `{}`
+	err := yaml.Unmarshal([]byte(y), v.Interface())
+	if err != nil {
+		panic(err)
+	}
+}
+
+type ConnectEnum struct {
+	Ret interface{}
+}
+
+type ConnectCommon struct {
+	Type string `yaml:"type"`
+}
+
+type TCPConnect struct {
+	ConnectCommon `yaml:",inline"`
+	Address       string        `yaml:"address"`
+	DialTimeout   time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
+}
+
+type TLSConnect struct {
+	ConnectCommon `yaml:",inline"`
+	Address       string        `yaml:"address"`
+	Ca            string        `yaml:"ca"`
+	Cert          string        `yaml:"cert"`
+	Key           string        `yaml:"key"`
+	ServerCN      string        `yaml:"server_cn"`
+	DialTimeout   time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
+}
+
+type SSHStdinserverConnect struct {
+	ConnectCommon        `yaml:",inline"`
+	Host                 string        `yaml:"host"`
+	User                 string        `yaml:"user"`
+	Port                 uint16        `yaml:"port"`
+	IdentityFile         string        `yaml:"identity_file"`
+	TransportOpenCommand []string      `yaml:"transport_open_command,optional"` //TODO unused
+	SSHCommand           string        `yaml:"ssh_command,optional"`            //TODO unused
+	Options              []string      `yaml:"options,optional"`
+	DialTimeout          time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
+}
+
+type LocalConnect struct {
+	ConnectCommon  `yaml:",inline"`
+	ListenerName   string `yaml:"listener_name"`
+	ClientIdentity string `yaml:"client_identity"`
+}
+
+type ServeEnum struct {
+	Ret interface{}
+}
+
+type ServeCommon struct {
+	Type string `yaml:"type"`
+}
+
+type TCPServe struct {
+	ServeCommon `yaml:",inline"`
+	Listen      string            `yaml:"listen"`
+	Clients     map[string]string `yaml:"clients"`
+}
+
+type TLSServe struct {
+	ServeCommon      `yaml:",inline"`
+	Listen           string        `yaml:"listen"`
+	Ca               string        `yaml:"ca"`
+	Cert             string        `yaml:"cert"`
+	Key              string        `yaml:"key"`
+	ClientCNs        []string      `yaml:"client_cns"`
+	HandshakeTimeout time.Duration `yaml:"handshake_timeout,zeropositive,default=10s"`
+}
+
+type StdinserverServer struct {
+	ServeCommon      `yaml:",inline"`
+	ClientIdentities []string `yaml:"client_identities"`
+}
+
+type LocalServe struct {
+	ServeCommon  `yaml:",inline"`
+	ListenerName string `yaml:"listener_name"`
+}
+
+type PruningEnum struct {
+	Ret interface{}
+}
+
+type PruneKeepNotReplicated struct {
+	Type                 string `yaml:"type"`
+	KeepSnapshotAtCursor bool   `yaml:"keep_snapshot_at_cursor,optional,default=true"`
+}
+
+type PruneKeepLastN struct {
+	Type  string `yaml:"type"`
+	Count int    `yaml:"count"`
+}
+
+type PruneKeepRegex struct { // FIXME rename to KeepRegex
+	Type   string `yaml:"type"`
+	Regex  string `yaml:"regex"`
+	Negate bool   `yaml:"negate,optional,default=false"`
+}
+
+type LoggingOutletEnum struct {
+	Ret interface{}
+}
+
+type LoggingOutletCommon struct {
+	Type   string `yaml:"type"`
+	Level  string `yaml:"level"`
+	Format string `yaml:"format"`
+}
+
+type StdoutLoggingOutlet struct {
+	LoggingOutletCommon `yaml:",inline"`
+	Time                bool `yaml:"time,default=true"`
+	Color               bool `yaml:"color,default=true"`
+}
+
+type SyslogLoggingOutlet struct {
+	LoggingOutletCommon `yaml:",inline"`
+	Facility            *SyslogFacility `yaml:"facility,optional,fromdefaults"`
+	RetryInterval       time.Duration   `yaml:"retry_interval,positive,default=10s"`
+}
+
+type TCPLoggingOutlet struct {
+	LoggingOutletCommon `yaml:",inline"`
+	Address             string               `yaml:"address"`
+	Net                 string               `yaml:"net,default=tcp"`
+	RetryInterval       time.Duration        `yaml:"retry_interval,positive,default=10s"`
+	TLS                 *TCPLoggingOutletTLS `yaml:"tls,optional"`
+}
+
+type TCPLoggingOutletTLS struct {
+	CA   string `yaml:"ca"`
+	Cert string `yaml:"cert"`
+	Key  string `yaml:"key"`
+}
+
+type MonitoringEnum struct {
+	Ret interface{}
+}
+
+type PrometheusMonitoring struct {
+	Type   string `yaml:"type"`
+	Listen string `yaml:"listen"`
+}
+
+type SyslogFacility syslog.Priority
+
+func (f *SyslogFacility) SetDefault() {
+	*f = SyslogFacility(syslog.LOG_LOCAL0)
+}
+
+var _ yaml.Defaulter = (*SyslogFacility)(nil)
+
+type GlobalControl struct {
+	SockPath string `yaml:"sockpath,default=/var/run/zrepl/control"`
+}
+
+type GlobalServe struct {
+	StdinServer *GlobalStdinServer `yaml:"stdinserver,optional,fromdefaults"`
+}
+
+type GlobalStdinServer struct {
+	SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
+}
+
+type JobDebugSettings struct {
+	Conn *struct {
+		ReadDump  string `yaml:"read_dump"`
+		WriteDump string `yaml:"write_dump"`
+	} `yaml:"conn,optional"`
+	RPCLog bool `yaml:"rpc_log,optional,default=false"`
+}
+
+func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) {
+	var in struct {
+		Type string
+	}
+	if err := u(&in, true); err != nil {
+		return nil, err
+	}
+	if in.Type == "" {
+		return nil, &yaml.TypeError{Errors: []string{"must specify type"}}
+	}
+
+	v, ok := types[in.Type]
+	if !ok {
+		return nil, &yaml.TypeError{Errors: []string{fmt.Sprintf("invalid type name %q", in.Type)}}
+	}
+	if err := u(v, false); err != nil {
+		return nil, err
+	}
+	return v, nil
+}
+
+func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"snap":   &SnapJob{},
+		"push":   &PushJob{},
+		"sink":   &SinkJob{},
+		"pull":   &PullJob{},
+		"source": &SourceJob{},
+	})
+	return
+}
+
+func (t *ConnectEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"tcp":             &TCPConnect{},
+		"tls":             &TLSConnect{},
+		"ssh+stdinserver": &SSHStdinserverConnect{},
+		"local":           &LocalConnect{},
+	})
+	return
+}
+
+func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"tcp":         &TCPServe{},
+		"tls":         &TLSServe{},
+		"stdinserver": &StdinserverServer{},
+		"local":       &LocalServe{},
+	})
+	return
+}
+
+func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"not_replicated": &PruneKeepNotReplicated{},
+		"last_n":         &PruneKeepLastN{},
+		"grid":           &PruneGrid{},
+		"regex":          &PruneKeepRegex{},
+	})
+	return
+}
+
+func (t *SnapshottingEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"periodic": &SnapshottingPeriodic{},
+		"manual":   &SnapshottingManual{},
+	})
+	return
+}
+
+func (t *LoggingOutletEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"stdout": &StdoutLoggingOutlet{},
+		"syslog": &SyslogLoggingOutlet{},
+		"tcp":    &TCPLoggingOutlet{},
+	})
+	return
+}
+
+func (t *MonitoringEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"prometheus": &PrometheusMonitoring{},
+	})
+	return
+}
+
+func (t *SyslogFacility) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	var s string
+	if err := u(&s, true); err != nil {
+		return err
+	}
+	var level syslog.Priority
+	switch s {
+	case "kern":
+		level = syslog.LOG_KERN
+	case "user":
+		level = syslog.LOG_USER
+	case "mail":
+		level = syslog.LOG_MAIL
+	case "daemon":
+		level = syslog.LOG_DAEMON
+	case "auth":
+		level = syslog.LOG_AUTH
+	case "syslog":
+		level = syslog.LOG_SYSLOG
+	case "lpr":
+		level = syslog.LOG_LPR
+	case "news":
+		level = syslog.LOG_NEWS
+	case "uucp":
+		level = syslog.LOG_UUCP
+	case "cron":
+		level = syslog.LOG_CRON
+	case "authpriv":
+		level = syslog.LOG_AUTHPRIV
+	case "ftp":
+		level = syslog.LOG_FTP
+	case "local0":
+		level = syslog.LOG_LOCAL0
+	case "local1":
+		level = syslog.LOG_LOCAL1
+	case "local2":
+		level = syslog.LOG_LOCAL2
+	case "local3":
+		level = syslog.LOG_LOCAL3
+	case "local4":
+		level = syslog.LOG_LOCAL4
+	case "local5":
+		level = syslog.LOG_LOCAL5
+	case "local6":
+		level = syslog.LOG_LOCAL6
+	case "local7":
+		level = syslog.LOG_LOCAL7
+	default:
+		return fmt.Errorf("invalid syslog level: %q", s)
+	}
+	*t = SyslogFacility(level)
+	return nil
+}
+
+var ConfigFileDefaultLocations = []string{
+	"/etc/zrepl/zrepl.yml",
+	"/usr/local/etc/zrepl/zrepl.yml",
+}
+
+func ParseConfig(path string) (i *Config, err error) {
+
+	if path == "" {
+		// Try default locations
+		for _, l := range ConfigFileDefaultLocations {
+			stat, statErr := os.Stat(l)
+			if statErr != nil {
+				continue
+			}
+			if !stat.Mode().IsRegular() {
+				err = errors.Errorf("file at default location is not a regular file: %s", l)
+				return
+			}
+			path = l
+			break
+		}
+	}
+
+	var bytes []byte
+
+	if bytes, err = ioutil.ReadFile(path); err != nil {
+		return
+	}
+
+	return ParseConfigBytes(bytes)
+}
+
+func ParseConfigBytes(bytes []byte) (*Config, error) {
+	var c *Config
+	if err := yaml.UnmarshalStrict(bytes, &c); err != nil {
+		return nil, err
+	}
+	if c == nil {
+		return nil, fmt.Errorf("config is empty or only consists of comments")
+	}
+	return c, nil
+}
+
+var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
+
+func parsePostitiveDuration(e string) (d time.Duration, err error) {
+	comps := durationStringRegex.FindStringSubmatch(e)
+	if len(comps) != 3 {
+		err = fmt.Errorf("does not match regex: %s %#v", e, comps)
+		return
+	}
+
+	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+	if durationFactor <= 0 {
+		return 0, errors.New("duration must be positive integer")
+	}
+
+	var durationUnit time.Duration
+	switch comps[2] {
+	case "s":
+		durationUnit = time.Second
+	case "m":
+		durationUnit = time.Minute
+	case "h":
+		durationUnit = time.Hour
+	case "d":
+		durationUnit = 24 * time.Hour
+	case "w":
+		durationUnit = 24 * 7 * time.Hour
+	default:
+		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
+		return
+	}
+
+	d = time.Duration(durationFactor) * durationUnit
+	return
+}

config/config_global_test.go

@@ -0,0 +1,115 @@
+package config
+
+import (
+	"fmt"
+	"log/syslog"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/zrepl/yaml-config"
+)
+
+func testValidGlobalSection(t *testing.T, s string) *Config {
+	jobdef := `
+jobs:
+- name: dummyjob
+  type: sink
+  serve:
+    type: tcp
+    listen: ":2342"
+    clients: {
+      "10.0.0.1":"foo"
+    }
+  root_fs: zoot/foo
+`
+	_, err := ParseConfigBytes([]byte(jobdef))
+	require.NoError(t, err)
+	return testValidConfig(t, s+jobdef)
+}
+
+func TestOutletTypes(t *testing.T) {
+	conf := testValidGlobalSection(t, `
+global:
+  logging:
+  - type: stdout
+    level: debug
+    format: human
+  - type: syslog
+    level: info
+    retry_interval: 20s
+    format: human
+  - type: tcp
+    level: debug
+    format: json
+    address: logserver.example.com:1234
+  - type: tcp
+    level: debug
+    format: json
+    address: encryptedlogserver.example.com:1234
+    retry_interval: 20s 
+    tls:
+      ca: /etc/zrepl/log/ca.crt
+      cert: /etc/zrepl/log/key.pem
+      key: /etc/zrepl/log/cert.pem
+`)
+	assert.Equal(t, 4, len(*conf.Global.Logging))
+	assert.NotNil(t, (*conf.Global.Logging)[3].Ret.(*TCPLoggingOutlet).TLS)
+}
+
+func TestDefaultLoggingOutlet(t *testing.T) {
+	conf := testValidGlobalSection(t, "")
+	assert.Equal(t, 1, len(*conf.Global.Logging))
+	o := (*conf.Global.Logging)[0].Ret.(*StdoutLoggingOutlet)
+	assert.Equal(t, "warn", o.Level)
+	assert.Equal(t, "human", o.Format)
+}
+
+func TestPrometheusMonitoring(t *testing.T) {
+	conf := testValidGlobalSection(t, `
+global:
+  monitoring:
+    - type: prometheus
+      listen: ':9091'
+`)
+	assert.Equal(t, ":9091", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
+}
+
+func TestSyslogLoggingOutletFacility(t *testing.T) {
+	type SyslogFacilityPriority struct {
+		Facility string
+		Priority syslog.Priority
+	}
+	syslogFacilitiesPriorities := []SyslogFacilityPriority{
+		{"", syslog.LOG_LOCAL0}, // default
+		{"kern", syslog.LOG_KERN}, {"daemon", syslog.LOG_DAEMON}, {"auth", syslog.LOG_AUTH},
+		{"syslog", syslog.LOG_SYSLOG}, {"lpr", syslog.LOG_LPR}, {"news", syslog.LOG_NEWS},
+		{"uucp", syslog.LOG_UUCP}, {"cron", syslog.LOG_CRON}, {"authpriv", syslog.LOG_AUTHPRIV},
+		{"ftp", syslog.LOG_FTP}, {"local0", syslog.LOG_LOCAL0}, {"local1", syslog.LOG_LOCAL1},
+		{"local2", syslog.LOG_LOCAL2}, {"local3", syslog.LOG_LOCAL3}, {"local4", syslog.LOG_LOCAL4},
+		{"local5", syslog.LOG_LOCAL5}, {"local6", syslog.LOG_LOCAL6}, {"local7", syslog.LOG_LOCAL7},
+	}
+
+	for _, sFP := range syslogFacilitiesPriorities {
+		logcfg := fmt.Sprintf(`
+global:
+  logging:
+  - type: syslog
+    level: info
+    format: human
+    facility: %s
+`, sFP.Facility)
+		conf := testValidGlobalSection(t, logcfg)
+		assert.Equal(t, 1, len(*conf.Global.Logging))
+		assert.True(t, SyslogFacility(sFP.Priority) == *(*conf.Global.Logging)[0].Ret.(*SyslogLoggingOutlet).Facility)
+	}
+}
+
+func TestLoggingOutletEnumList_SetDefaults(t *testing.T) {
+	e := &LoggingOutletEnumList{}
+	var i yaml.Defaulter = e
+	require.NotPanics(t, func() {
+		i.SetDefault()
+		assert.Equal(t, "warn", (*e)[0].Ret.(*StdoutLoggingOutlet).Level)
+	})
+}

config/config_minimal_test.go

@@ -0,0 +1,40 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfigEmptyFails(t *testing.T) {
+	conf, err := testConfig(t, "\n")
+	assert.Nil(t, conf)
+	assert.Error(t, err)
+}
+
+func TestJobsOnlyWorks(t *testing.T) {
+	testValidConfig(t, `
+jobs:
+- name: push
+  type: push
+  # snapshot the filesystems matched by the left-hand-side of the mapping
+  # every 10m with zrepl_ as prefix
+  connect:
+    type: tcp
+    address: localhost:2342
+  filesystems: {
+    "pool1/var/db<": true,
+    "pool1/usr/home<": true,
+    "pool1/usr/home/paranoid": false, #don't backup paranoid user
+    "pool1/poudriere/ports<": false #don't backup the ports trees
+  }
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    keep_receiver:
+    - type: last_n 
+      count: 1
+`)
+}

config/config_positiveintervalormanual_test.go

@@ -0,0 +1,41 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/zrepl/yaml-config"
+)
+
+func TestPositiveDurationOrManual(t *testing.T) {
+	cases := []struct {
+		Comment, Input string
+		Result         *PositiveDurationOrManual
+	}{
+		{"empty is error", "", nil},
+		{"negative is error", "-1s", nil},
+		{"zero seconds is error", "0s", nil},
+		{"zero is error", "0", nil},
+		{"non-manual is error", "something", nil},
+		{"positive seconds works", "1s", &PositiveDurationOrManual{Manual: false, Interval: 1 * time.Second}},
+		{"manual works", "manual", &PositiveDurationOrManual{Manual: true, Interval: 0}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.Comment, func(t *testing.T) {
+			var out struct {
+				FieldName PositiveDurationOrManual `yaml:"fieldname"`
+			}
+			input := fmt.Sprintf("\nfieldname: %s\n", tc.Input)
+			err := yaml.UnmarshalStrict([]byte(input), &out)
+			if tc.Result == nil {
+				assert.Error(t, err)
+				t.Logf("%#v", out)
+			} else {
+				assert.Equal(t, *tc.Result, out.FieldName)
+			}
+		})
+	}
+
+}

config/config_snapshotting_test.go

@@ -0,0 +1,58 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSnapshotting(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  %s
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+	manual := `
+  snapshotting:
+    type: manual
+`
+	periodic := `
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 10m
+`
+
+	fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
+	var c *Config
+
+	t.Run("manual", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(manual))
+		snm := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingManual)
+		assert.Equal(t, "manual", snm.Type)
+	})
+
+	t.Run("periodic", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(periodic))
+		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
+		assert.Equal(t, "periodic", snp.Type)
+		assert.Equal(t, 10*time.Minute, snp.Interval)
+		assert.Equal(t, "zrepl_", snp.Prefix)
+	})
+
+}

config/config_test.go

@@ -0,0 +1,67 @@
+package config
+
+import (
+	"bytes"
+	"path"
+	"path/filepath"
+	"testing"
+	"text/template"
+
+	"github.com/kr/pretty"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
+	paths, err := filepath.Glob("./samples/*")
+	if err != nil {
+		t.Errorf("glob failed: %+v", err)
+	}
+
+	for _, p := range paths {
+
+		if path.Ext(p) != ".yml" {
+			t.Logf("skipping file %s", p)
+			continue
+		}
+
+		t.Run(p, func(t *testing.T) {
+			c, err := ParseConfig(p)
+			if err != nil {
+				t.Errorf("error parsing %s:\n%+v", p, err)
+			}
+
+			t.Logf("file: %s", p)
+			t.Log(pretty.Sprint(c))
+		})
+
+	}
+
+}
+
+// template must be a template/text template with a single '{{ . }}' as placehodler for val
+//nolint[:deadcode,unused]
+func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
+	tmp, err := template.New("master").Parse(tmpl)
+	if err != nil {
+		panic(err)
+	}
+	var buf bytes.Buffer
+	err = tmp.Execute(&buf, val)
+	if err != nil {
+		panic(err)
+	}
+	return testValidConfig(t, buf.String())
+}
+
+func testValidConfig(t *testing.T, input string) *Config {
+	t.Helper()
+	conf, err := testConfig(t, input)
+	require.NoError(t, err)
+	require.NotNil(t, conf)
+	return conf
+}
+
+func testConfig(t *testing.T, input string) (*Config, error) {
+	t.Helper()
+	return ParseConfigBytes([]byte(input))
+}

config/retentiongrid.go

@@ -0,0 +1,119 @@
+package config
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type RetentionIntervalList []RetentionInterval
+
+type PruneGrid struct {
+	Type  string                `yaml:"type"`
+	Grid  RetentionIntervalList `yaml:"grid"`
+	Regex string                `yaml:"regex"`
+}
+
+type RetentionInterval struct {
+	length    time.Duration
+	keepCount int
+}
+
+func (i *RetentionInterval) Length() time.Duration {
+	return i.length
+}
+
+func (i *RetentionInterval) KeepCount() int {
+	return i.keepCount
+}
+
+const RetentionGridKeepCountAll int = -1
+
+func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	var in string
+	if err := u(&in, true); err != nil {
+		return err
+	}
+
+	intervals, err := parseRetentionGridIntervalsString(in)
+	if err != nil {
+		return err
+	}
+
+	*t = intervals
+
+	return nil
+}
+
+var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
+
+func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {
+
+	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
+	if comps == nil {
+		err = fmt.Errorf("retention string does not match expected format")
+		return
+	}
+
+	times, err := strconv.Atoi(comps[1])
+	if err != nil {
+		return nil, err
+	} else if times <= 0 {
+		return nil, fmt.Errorf("contains factor <= 0")
+	}
+
+	duration, err := parsePostitiveDuration(comps[2])
+	if err != nil {
+		return nil, err
+	}
+
+	keepCount := 1
+	if comps[3] != "" {
+		// Decompose key=value, comma separated
+		// For now, only keep_count is supported
+		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
+		res := re.FindStringSubmatch(comps[4])
+		if res == nil || len(res) != 2 {
+			err = fmt.Errorf("interval parameter contains unknown parameters")
+			return
+		}
+		if res[1] == "all" {
+			keepCount = RetentionGridKeepCountAll
+		} else {
+			keepCount, err = strconv.Atoi(res[1])
+			if err != nil {
+				err = fmt.Errorf("cannot parse keep_count value")
+				return
+			}
+		}
+	}
+
+	intervals = make([]RetentionInterval, times)
+	for i := range intervals {
+		intervals[i] = RetentionInterval{
+			length:    duration,
+			keepCount: keepCount,
+		}
+	}
+
+	return
+
+}
+
+func parseRetentionGridIntervalsString(s string) (intervals []RetentionInterval, err error) {
+
+	ges := strings.Split(s, "|")
+	intervals = make([]RetentionInterval, 0, 7*len(ges))
+
+	for intervalIdx, e := range ges {
+		parsed, err := parseRetentionGridIntervalString(e)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
+		}
+		intervals = append(intervals, parsed...)
+	}
+
+	return
+}

config/samples/local.yml

@@ -0,0 +1,31 @@
+
+jobs:
+  - type: sink
+    name: "local_sink"
+    root_fs: "storage/zrepl/sink"
+    serve:
+      type: local
+      listener_name: localsink
+
+  - type: push
+    name: "backup_system"
+    connect:
+      type: local
+      listener_name: localsink
+      client_identity: local_backup
+    filesystems: {
+      "system<": true,
+    }
+    snapshotting:
+      type: periodic
+      interval: 10m
+      prefix: zrepl_
+    pruning:
+      keep_sender:
+      - type: not_replicated
+      - type: last_n
+        count: 10
+      keep_receiver:
+      - type: grid
+        grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+        regex: "zrepl_.*"

config/samples/pull.yml

@@ -0,0 +1,24 @@
+jobs:
+- name: pull_servers
+  type: pull
+  connect:
+    type: tls
+    address: "server1.foo.bar:8888"
+    ca: "/certs/ca.crt"
+    cert: "/certs/cert.crt"
+    key: "/certs/key.pem"
+    server_cn: "server1"
+  root_fs: "pool2/backup_servers"
+  interval: 10m
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    - type: last_n
+      count: 10
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 14x1d
+      regex: "zrepl_.*"
+    keep_receiver:
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+      regex: "zrepl_.*"

config/samples/pull_ssh.yml

@@ -0,0 +1,28 @@
+jobs:
+
+- name: pull_servers
+  type: pull
+  connect:
+    type: ssh+stdinserver
+    host: app-srv.example.com
+    user: root
+    port: 22
+    identity_file: /etc/zrepl/ssh/identity
+    options: # optional, default [], `-o` arguments passed to ssh
+    - "Compression=on"
+  root_fs: "pool2/backup_servers"
+  interval: 10m
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    - type: last_n
+      count: 10
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 14x1d
+      regex: "^zrepl_.*"
+    keep_receiver:
+    - type: regex
+      regex: keep_
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+      regex: "^zrepl_.*"

config/samples/push.yml

@@ -0,0 +1,24 @@
+jobs:
+  - type: push
+    name: "push"
+    filesystems: {
+      "<": true,
+      "tmp": false
+    }
+    connect:
+      type: tcp
+      address: "backup-server.foo.bar:8888"
+    snapshotting:
+      type: manual
+    pruning:
+      keep_sender:
+        - type: not_replicated
+        - type: last_n
+          count: 10
+        - type: grid
+          grid: 1x1h(keep=all) | 24x1h | 14x1d
+          regex: "^zrepl_.*"
+      keep_receiver:
+        - type: grid
+          grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+          regex: "^zrepl_.*"

config/samples/sink.yml

@@ -0,0 +1,13 @@
+jobs:
+  - type: sink
+    name: "laptop_sink"
+    root_fs: "pool2/backup_laptops"
+    serve:
+      type: tls
+      listen: "192.168.122.189:8888"
+      ca: "ca.pem"
+      cert: "cert.pem"
+      key: "key.pem"
+      client_cns:
+        - "laptop1"
+        - "homeserver"

config/samples/snap.yml

@@ -0,0 +1,14 @@
+jobs:
+- name: snapjob
+  type: snap
+  filesystems: {
+    "tank/frequently_changed<": true,
+  }
+  snapshotting:
+    type: periodic
+    interval: 2m
+    prefix: zrepl_snapjob_
+  pruning:
+    keep:
+      - type: last_n
+        count: 60

config/samples/source.yml

@@ -0,0 +1,17 @@
+jobs:
+- name: pull_source
+  type: source
+  serve:
+    type: tcp
+    listen: "0.0.0.0:8888"
+    clients: {
+      "192.168.122.123" : "client1"
+    }
+  filesystems: {
+    "<": true,
+    "secret": false
+  }
+  snapshotting:
+    type: periodic
+    interval: 10m
+    prefix: zrepl_

config/samples/source_ssh.yml

@@ -0,0 +1,17 @@
+jobs:
+- name: pull_source
+  type: source
+  serve:
+    type: stdinserver
+    client_identities:
+      - "client1"
+      - "client2"
+  filesystems: {
+    "<": true,
+    "secret": false
+  }
+  snapshotting:
+    type: periodic
+    interval: 10m
+    prefix: zrepl_
+

daemon/control.go

@@ -0,0 +1,272 @@
+package daemon
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/daemon/nethelpers"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/version"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type controlJob struct {
+	sockaddr *net.UnixAddr
+	jobs     *jobs
+}
+
+func newControlJob(sockpath string, jobs *jobs) (j *controlJob, err error) {
+	j = &controlJob{jobs: jobs}
+
+	j.sockaddr, err = net.ResolveUnixAddr("unix", sockpath)
+	if err != nil {
+		err = errors.Wrap(err, "cannot resolve unix address")
+		return
+	}
+
+	return
+}
+
+func (j *controlJob) Name() string { return jobNameControl }
+
+func (j *controlJob) Status() *job.Status { return &job.Status{Type: job.TypeInternal} }
+
+func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
+
+var promControl struct {
+	requestBegin    *prometheus.CounterVec
+	requestFinished *prometheus.HistogramVec
+}
+
+func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) {
+	promControl.requestBegin = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "zrepl",
+		Subsystem: "control",
+		Name:      "request_begin",
+		Help:      "number of request we started to handle",
+	}, []string{"endpoint"})
+
+	promControl.requestFinished = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "zrepl",
+		Subsystem: "control",
+		Name:      "request_finished",
+		Help:      "time it took a request to finih",
+		Buckets:   []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3, 10e-3, 100e-3, 200e-3, 400e-3, 800e-3, 1, 10, 20},
+	}, []string{"endpoint"})
+	registerer.MustRegister(promControl.requestBegin)
+	registerer.MustRegister(promControl.requestFinished)
+}
+
+const (
+	ControlJobEndpointPProf   string = "/debug/pprof"
+	ControlJobEndpointVersion string = "/version"
+	ControlJobEndpointStatus  string = "/status"
+	ControlJobEndpointSignal  string = "/signal"
+)
+
+func (j *controlJob) Run(ctx context.Context) {
+
+	log := job.GetLogger(ctx)
+	defer log.Info("control job finished")
+
+	l, err := nethelpers.ListenUnixPrivate(j.sockaddr)
+	if err != nil {
+		log.WithError(err).Error("error listening")
+		return
+	}
+
+	pprofServer := NewPProfServer(ctx)
+
+	mux := http.NewServeMux()
+	mux.Handle(ControlJobEndpointPProf,
+		requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) {
+			var msg PprofServerControlMsg
+			err := decoder(&msg)
+			if err != nil {
+				return nil, errors.Errorf("decode failed")
+			}
+			pprofServer.Control(msg)
+			return struct{}{}, nil
+		}}})
+
+	mux.Handle(ControlJobEndpointVersion,
+		requestLogger{log: log, handler: jsonResponder{log, func() (interface{}, error) {
+			return version.NewZreplVersionInformation(), nil
+		}}})
+
+	mux.Handle(ControlJobEndpointStatus,
+		// don't log requests to status endpoint, too spammy
+		jsonResponder{log, func() (interface{}, error) {
+			s := j.jobs.status()
+			return s, nil
+		}})
+
+	mux.Handle(ControlJobEndpointSignal,
+		requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) {
+			type reqT struct {
+				Name string
+				Op   string
+			}
+			var req reqT
+			if decoder(&req) != nil {
+				return nil, errors.Errorf("decode failed")
+			}
+
+			var err error
+			switch req.Op {
+			case "wakeup":
+				err = j.jobs.wakeup(req.Name)
+			case "reset":
+				err = j.jobs.reset(req.Name)
+			default:
+				err = fmt.Errorf("operation %q is invalid", req.Op)
+			}
+
+			return struct{}{}, err
+		}}})
+	server := http.Server{
+		Handler: mux,
+		// control socket is local, 1s timeout should be more than sufficient, even on a loaded system
+		WriteTimeout: 1 * time.Second,
+		ReadTimeout:  1 * time.Second,
+	}
+
+outer:
+	for {
+
+		served := make(chan error)
+		go func() {
+			served <- server.Serve(l)
+			close(served)
+		}()
+
+		select {
+		case <-ctx.Done():
+			log.WithError(ctx.Err()).Info("context done")
+			err := server.Shutdown(context.Background())
+			if err != nil {
+				log.WithError(err).Error("cannot shutdown server")
+			}
+			break outer
+		case err = <-served:
+			if err != nil {
+				log.WithError(err).Error("error serving")
+				break outer
+			}
+		}
+
+	}
+
+}
+
+type jsonResponder struct {
+	log      Logger
+	producer func() (interface{}, error)
+}
+
+func (j jsonResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	logIoErr := func(err error) {
+		if err != nil {
+			j.log.WithError(err).Error("control handler io error")
+		}
+	}
+	res, err := j.producer()
+	if err != nil {
+		j.log.WithError(err).Error("control handler error")
+		w.WriteHeader(http.StatusInternalServerError)
+		_, err = io.WriteString(w, err.Error())
+		logIoErr(err)
+		return
+	}
+	var buf bytes.Buffer
+	err = json.NewEncoder(&buf).Encode(res)
+	if err != nil {
+		j.log.WithError(err).Error("control handler json marshal error")
+		w.WriteHeader(http.StatusInternalServerError)
+		_, err = io.WriteString(w, err.Error())
+	} else {
+		_, err = io.Copy(w, &buf)
+	}
+	logIoErr(err)
+}
+
+type jsonDecoder = func(interface{}) error
+
+type jsonRequestResponder struct {
+	log      Logger
+	producer func(decoder jsonDecoder) (interface{}, error)
+}
+
+func (j jsonRequestResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	logIoErr := func(err error) {
+		if err != nil {
+			j.log.WithError(err).Error("control handler io error")
+		}
+	}
+
+	var decodeError error
+	decoder := func(i interface{}) error {
+		err := json.NewDecoder(r.Body).Decode(&i)
+		decodeError = err
+		return err
+	}
+	res, producerErr := j.producer(decoder)
+
+	//If we had a decode error ignore output of producer and return error
+	if decodeError != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		_, err := io.WriteString(w, decodeError.Error())
+		logIoErr(err)
+		return
+	}
+	if producerErr != nil {
+		j.log.WithError(producerErr).Error("control handler error")
+		w.WriteHeader(http.StatusInternalServerError)
+		_, err := io.WriteString(w, producerErr.Error())
+		logIoErr(err)
+		return
+	}
+
+	var buf bytes.Buffer
+	encodeErr := json.NewEncoder(&buf).Encode(res)
+	if encodeErr != nil {
+		j.log.WithError(producerErr).Error("control handler json marhsal error")
+		w.WriteHeader(http.StatusInternalServerError)
+		_, err := io.WriteString(w, encodeErr.Error())
+		logIoErr(err)
+	} else {
+		_, err := io.Copy(w, &buf)
+		logIoErr(err)
+	}
+}
+
+type requestLogger struct {
+	log         logger.Logger
+	handler     http.Handler
+	handlerFunc http.HandlerFunc
+}
+
+func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	log := l.log.WithField("method", r.Method).WithField("url", r.URL)
+	log.Debug("start")
+	promControl.requestBegin.WithLabelValues(r.URL.Path).Inc()
+	defer prometheus.NewTimer(promControl.requestFinished.WithLabelValues(r.URL.Path)).ObserveDuration()
+	if l.handlerFunc != nil {
+		l.handlerFunc(w, r)
+	} else if l.handler != nil {
+		l.handler.ServeHTTP(w, r)
+	} else {
+		log.Error("no handler or handlerFunc configured")
+	}
+	log.Debug("finish")
+}

daemon/daemon.go

@@ -0,0 +1,222 @@
+package daemon
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/daemon/job/reset"
+	"github.com/zrepl/zrepl/daemon/job/wakeup"
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/version"
+)
+
+func Run(conf *config.Config) error {
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	defer cancel()
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		<-sigChan
+		cancel()
+	}()
+
+	outlets, err := logging.OutletsFromConfig(*conf.Global.Logging)
+	if err != nil {
+		return errors.Wrap(err, "cannot build logging from config")
+	}
+
+	confJobs, err := job.JobsFromConfig(conf)
+	if err != nil {
+		return errors.Wrap(err, "cannot build jobs from config")
+	}
+
+	log := logger.NewLogger(outlets, 1*time.Second)
+	log.Info(version.NewZreplVersionInformation().String())
+
+	for _, job := range confJobs {
+		if IsInternalJobName(job.Name()) {
+			panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME
+		}
+	}
+
+	ctx = job.WithLogger(ctx, log)
+
+	jobs := newJobs()
+
+	// start control socket
+	controlJob, err := newControlJob(conf.Global.Control.SockPath, jobs)
+	if err != nil {
+		panic(err) // FIXME
+	}
+	jobs.start(ctx, controlJob, true)
+
+	for i, jc := range conf.Global.Monitoring {
+		var (
+			job job.Job
+			err error
+		)
+		switch v := jc.Ret.(type) {
+		case *config.PrometheusMonitoring:
+			job, err = newPrometheusJobFromConfig(v)
+		default:
+			return errors.Errorf("unknown monitoring job #%d (type %T)", i, v)
+		}
+		if err != nil {
+			return errors.Wrapf(err, "cannot build monitorin gjob #%d", i)
+		}
+		jobs.start(ctx, job, true)
+	}
+
+	log.Info("starting daemon")
+
+	// start regular jobs
+	for _, j := range confJobs {
+		jobs.start(ctx, j, false)
+	}
+
+	select {
+	case <-jobs.wait():
+		log.Info("all jobs finished")
+	case <-ctx.Done():
+		log.WithError(ctx.Err()).Info("context finished")
+	}
+	log.Info("daemon exiting")
+	return nil
+}
+
+type jobs struct {
+	wg sync.WaitGroup
+
+	// m protects all fields below it
+	m       sync.RWMutex
+	wakeups map[string]wakeup.Func // by Job.Name
+	resets  map[string]reset.Func  // by Job.Name
+	jobs    map[string]job.Job
+}
+
+func newJobs() *jobs {
+	return &jobs{
+		wakeups: make(map[string]wakeup.Func),
+		resets:  make(map[string]reset.Func),
+		jobs:    make(map[string]job.Job),
+	}
+}
+
+const (
+	logJobField string = "job"
+)
+
+func (s *jobs) wait() <-chan struct{} {
+	ch := make(chan struct{})
+	go func() {
+		s.wg.Wait()
+	}()
+	return ch
+}
+
+func (s *jobs) status() map[string]*job.Status {
+	s.m.RLock()
+	defer s.m.RUnlock()
+
+	type res struct {
+		name   string
+		status *job.Status
+	}
+	var wg sync.WaitGroup
+	c := make(chan res, len(s.jobs))
+	for name, j := range s.jobs {
+		wg.Add(1)
+		go func(name string, j job.Job) {
+			defer wg.Done()
+			c <- res{name: name, status: j.Status()}
+		}(name, j)
+	}
+	wg.Wait()
+	close(c)
+	ret := make(map[string]*job.Status, len(s.jobs))
+	for res := range c {
+		ret[res.name] = res.status
+	}
+	return ret
+}
+
+func (s *jobs) wakeup(job string) error {
+	s.m.RLock()
+	defer s.m.RUnlock()
+
+	wu, ok := s.wakeups[job]
+	if !ok {
+		return errors.Errorf("Job %s does not exist", job)
+	}
+	return wu()
+}
+
+func (s *jobs) reset(job string) error {
+	s.m.RLock()
+	defer s.m.RUnlock()
+
+	wu, ok := s.resets[job]
+	if !ok {
+		return errors.Errorf("Job %s does not exist", job)
+	}
+	return wu()
+}
+
+const (
+	jobNamePrometheus = "_prometheus"
+	jobNameControl    = "_control"
+)
+
+func IsInternalJobName(s string) bool {
+	return strings.HasPrefix(s, "_")
+}
+
+func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
+	s.m.Lock()
+	defer s.m.Unlock()
+
+	jobLog := job.GetLogger(ctx).
+		WithField(logJobField, j.Name()).
+		WithOutlet(newPrometheusLogOutlet(j.Name()), logger.Debug)
+	jobName := j.Name()
+	if !internal && IsInternalJobName(jobName) {
+		panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName))
+	}
+	if internal && !IsInternalJobName(jobName) {
+		panic(fmt.Sprintf("internal job does not use internal job name %s", jobName))
+	}
+	if _, ok := s.jobs[jobName]; ok {
+		panic(fmt.Sprintf("duplicate job name %s", jobName))
+	}
+
+	j.RegisterMetrics(prometheus.DefaultRegisterer)
+
+	s.jobs[jobName] = j
+	ctx = job.WithLogger(ctx, jobLog)
+	ctx, wakeup := wakeup.Context(ctx)
+	ctx, resetFunc := reset.Context(ctx)
+	s.wakeups[jobName] = wakeup
+	s.resets[jobName] = resetFunc
+
+	s.wg.Add(1)
+	go func() {
+		defer s.wg.Done()
+		jobLog.Info("starting job")
+		defer jobLog.Info("job exited")
+		j.Run(ctx)
+	}()
+}

daemon/filters/fsmapfilter.go

@@ -1,11 +1,12 @@
-package cmd
+package filters
 
 import (
 	"fmt"
 	"strings"
 
-	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/zfs"
 )
 
@@ -101,6 +102,7 @@ func (m DatasetMapFilter) mostSpecificPrefixMapping(path *zfs.DatasetPath) (idx
 	return
 }
 
+// Returns target == nil if there is no mapping
 func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error) {
 
 	if m.filterMode {
@@ -114,10 +116,18 @@ func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath,
 	}
 	me := m.entries[mi]
 
+	if me.mapping == "" {
+		// Special case treatment: 'foo/bar<' => ''
+		if !me.subtreeMatch {
+			return nil, fmt.Errorf("mapping to '' must be a subtree match")
+		}
+		// ok...
+	} else {
 		if strings.HasPrefix("!", me.mapping) {
 			// reject mapping
 			return nil, nil
 		}
+	}
 
 	target, err = zfs.NewDatasetPath(me.mapping)
 	if err != nil {
@@ -167,7 +177,7 @@ func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
 	for i, e := range m.entries {
 		inv.entries[i].path, err = zfs.NewDatasetPath(e.mapping)
 		if err != nil {
-			err = errors.Wrapf(err, "mapping cannot be inverted: '%s' is not a dataset path: %s", e.mapping)
+			err = errors.Wrapf(err, "mapping cannot be inverted: '%s' is not a dataset path", e.mapping)
 			return
 		}
 		inv.entries[i].mapping = MapFilterResultOk
@@ -177,12 +187,43 @@ func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
 	return inv, nil
 }
 
+// FIXME investigate whether we can support more...
+func (m DatasetMapFilter) Invert() (endpoint.FSMap, error) {
+
+	if m.filterMode {
+		return nil, errors.Errorf("can only invert mappings")
+	}
+
+	if len(m.entries) != 1 {
+		return nil, errors.Errorf("inversion of complicated mappings is not implemented") // FIXME
+	}
+
+	e := m.entries[0]
+
+	inv := &DatasetMapFilter{
+		make([]datasetMapFilterEntry, len(m.entries)),
+		false,
+	}
+	mp, err := zfs.NewDatasetPath(e.mapping)
+	if err != nil {
+		return nil, err
+	}
+
+	inv.entries[0] = datasetMapFilterEntry{
+		path:         mp,
+		mapping:      e.path.ToString(),
+		subtreeMatch: e.subtreeMatch,
+	}
+
+	return inv, nil
+}
+
 // Creates a new DatasetMapFilter in filter mode from a mapping
 // All accepting mapping results are mapped to accepting filter results
 // All rejecting mapping results are mapped to rejecting filter results
-func (m DatasetMapFilter) AsFilter() (f *DatasetMapFilter) {
+func (m DatasetMapFilter) AsFilter() endpoint.FSFilter {
 
-	f = &DatasetMapFilter{
+	f := &DatasetMapFilter{
 		make([]datasetMapFilterEntry, len(m.entries)),
 		true,
 	}
@@ -217,16 +258,14 @@ func (m DatasetMapFilter) parseDatasetFilterResult(result string) (pass bool, er
 	return false, fmt.Errorf("'%s' is not a valid filter result", result)
 }
 
-func parseDatasetMapFilter(mi interface{}, filterMode bool) (f *DatasetMapFilter, err error) {
+func DatasetMapFilterFromConfig(in map[string]bool) (f *DatasetMapFilter, err error) {
 
-	var m map[string]string
+	f = NewDatasetMapFilter(len(in), true)
-	if err = mapstructure.Decode(mi, &m); err != nil {
+	for pathPattern, accept := range in {
-		err = fmt.Errorf("maps / filters must be specified as map[string]string: %s", err)
+		mapping := MapFilterResultOmit
-		return
+		if accept {
+			mapping = MapFilterResultOk
 		}
-
-	f = NewDatasetMapFilter(len(m), filterMode)
-	for pathPattern, mapping := range m {
 		if err = f.Add(pathPattern, mapping); err != nil {
 			err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err)
 			return

daemon/filters/fsvfilter.go

@@ -0,0 +1,41 @@
+package filters
+
+import (
+	"strings"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type AnyFSVFilter struct{}
+
+func NewAnyFSVFilter() AnyFSVFilter {
+	return AnyFSVFilter{}
+}
+
+var _ zfs.FilesystemVersionFilter = AnyFSVFilter{}
+
+func (AnyFSVFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) {
+	return true, nil
+}
+
+type PrefixFilter struct {
+	prefix    string
+	fstype    zfs.VersionType
+	fstypeSet bool // optionals anyone?
+}
+
+var _ zfs.FilesystemVersionFilter = &PrefixFilter{}
+
+func NewPrefixFilter(prefix string) *PrefixFilter {
+	return &PrefixFilter{prefix: prefix}
+}
+
+func NewTypedPrefixFilter(prefix string, versionType zfs.VersionType) *PrefixFilter {
+	return &PrefixFilter{prefix, versionType, true}
+}
+
+func (f *PrefixFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) {
+	fstypeMatches := (!f.fstypeSet || t == f.fstype)
+	prefixMatches := strings.HasPrefix(name, f.prefix)
+	return fstypeMatches && prefixMatches, nil
+}

daemon/job/active.go

@@ -0,0 +1,423 @@
+package job
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/daemon/job/reset"
+	"github.com/zrepl/zrepl/daemon/job/wakeup"
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/daemon/pruner"
+	"github.com/zrepl/zrepl/daemon/snapper"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/replication"
+	"github.com/zrepl/zrepl/replication/driver"
+	"github.com/zrepl/zrepl/replication/logic"
+	"github.com/zrepl/zrepl/replication/report"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/transport"
+	"github.com/zrepl/zrepl/transport/fromconfig"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type ActiveSide struct {
+	mode      activeMode
+	name      string
+	connecter transport.Connecter
+
+	prunerFactory *pruner.PrunerFactory
+
+	promRepStateSecs    *prometheus.HistogramVec // labels: state
+	promPruneSecs       *prometheus.HistogramVec // labels: prune_side
+	promBytesReplicated *prometheus.CounterVec   // labels: filesystem
+
+	tasksMtx sync.Mutex
+	tasks    activeSideTasks
+}
+
+//go:generate enumer -type=ActiveSideState
+type ActiveSideState int
+
+const (
+	ActiveSideReplicating ActiveSideState = 1 << iota
+	ActiveSidePruneSender
+	ActiveSidePruneReceiver
+	ActiveSideDone // also errors
+)
+
+type activeSideTasks struct {
+	state ActiveSideState
+
+	// valid for state ActiveSideReplicating, ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
+	replicationReport driver.ReportFunc
+	replicationCancel context.CancelFunc
+
+	// valid for state ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
+	prunerSender, prunerReceiver *pruner.Pruner
+
+	// valid for state ActiveSidePruneReceiver, ActiveSideDone
+	prunerSenderCancel, prunerReceiverCancel context.CancelFunc
+}
+
+func (a *ActiveSide) updateTasks(u func(*activeSideTasks)) activeSideTasks {
+	a.tasksMtx.Lock()
+	defer a.tasksMtx.Unlock()
+	copy := a.tasks
+	if u == nil {
+		return copy
+	}
+	u(&copy)
+	a.tasks = copy
+	return copy
+}
+
+type activeMode interface {
+	ConnectEndpoints(rpcLoggers rpc.Loggers, connecter transport.Connecter)
+	DisconnectEndpoints()
+	SenderReceiver() (logic.Sender, logic.Receiver)
+	Type() Type
+	RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{})
+	ResetConnectBackoff()
+}
+
+type modePush struct {
+	setupMtx sync.Mutex
+	sender   *endpoint.Sender
+	receiver *rpc.Client
+	fsfilter endpoint.FSFilter
+	snapper  *snapper.PeriodicOrManual
+}
+
+func (m *modePush) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Connecter) {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	if m.receiver != nil || m.sender != nil {
+		panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
+	}
+	m.sender = endpoint.NewSender(m.fsfilter)
+	m.receiver = rpc.NewClient(connecter, loggers)
+}
+
+func (m *modePush) DisconnectEndpoints() {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	m.receiver.Close()
+	m.sender = nil
+	m.receiver = nil
+}
+
+func (m *modePush) SenderReceiver() (logic.Sender, logic.Receiver) {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	return m.sender, m.receiver
+}
+
+func (m *modePush) Type() Type { return TypePush }
+
+func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
+	m.snapper.Run(ctx, wakeUpCommon)
+}
+
+func (m *modePush) ResetConnectBackoff() {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	if m.receiver != nil {
+		m.receiver.ResetConnectBackoff()
+	}
+}
+
+func modePushFromConfig(g *config.Global, in *config.PushJob) (*modePush, error) {
+	m := &modePush{}
+	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannnot build filesystem filter")
+	}
+	m.fsfilter = fsf
+
+	if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
+		return nil, errors.Wrap(err, "cannot build snapper")
+	}
+
+	return m, nil
+}
+
+type modePull struct {
+	setupMtx sync.Mutex
+	receiver *endpoint.Receiver
+	sender   *rpc.Client
+	rootFS   *zfs.DatasetPath
+	interval config.PositiveDurationOrManual
+}
+
+func (m *modePull) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Connecter) {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	if m.receiver != nil || m.sender != nil {
+		panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
+	}
+	m.receiver = endpoint.NewReceiver(m.rootFS, false)
+	m.sender = rpc.NewClient(connecter, loggers)
+}
+
+func (m *modePull) DisconnectEndpoints() {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	m.sender.Close()
+	m.sender = nil
+	m.receiver = nil
+}
+
+func (m *modePull) SenderReceiver() (logic.Sender, logic.Receiver) {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	return m.sender, m.receiver
+}
+
+func (*modePull) Type() Type { return TypePull }
+
+func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
+	if m.interval.Manual {
+		GetLogger(ctx).Info("manual pull configured, periodic pull disabled")
+		// "waiting for wakeups" is printed in common ActiveSide.do
+		return
+	}
+	t := time.NewTicker(m.interval.Interval)
+	defer t.Stop()
+	for {
+		select {
+		case <-t.C:
+			select {
+			case wakeUpCommon <- struct{}{}:
+			default:
+				GetLogger(ctx).
+					WithField("pull_interval", m.interval).
+					Warn("pull job took longer than pull interval")
+				wakeUpCommon <- struct{}{} // block anyways, to queue up the wakeup
+			}
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (m *modePull) ResetConnectBackoff() {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	if m.sender != nil {
+		m.sender.ResetConnectBackoff()
+	}
+}
+
+func modePullFromConfig(g *config.Global, in *config.PullJob) (m *modePull, err error) {
+	m = &modePull{}
+	m.interval = in.Interval
+
+	m.rootFS, err = zfs.NewDatasetPath(in.RootFS)
+	if err != nil {
+		return nil, errors.New("RootFS is not a valid zfs filesystem path")
+	}
+	if m.rootFS.Length() <= 0 {
+		return nil, errors.New("RootFS must not be empty") // duplicates error check of receiver
+	}
+
+	return m, nil
+}
+
+func activeSide(g *config.Global, in *config.ActiveJob, mode activeMode) (j *ActiveSide, err error) {
+
+	j = &ActiveSide{mode: mode}
+	j.name = in.Name
+	j.promRepStateSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace:   "zrepl",
+		Subsystem:   "replication",
+		Name:        "state_time",
+		Help:        "seconds spent during replication",
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
+	}, []string{"state"})
+	j.promBytesReplicated = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace:   "zrepl",
+		Subsystem:   "replication",
+		Name:        "bytes_replicated",
+		Help:        "number of bytes replicated from sender to receiver per filesystem",
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
+	}, []string{"filesystem"})
+
+	j.connecter, err = fromconfig.ConnecterFromConfig(g, in.Connect)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot build client")
+	}
+
+	j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace:   "zrepl",
+		Subsystem:   "pruning",
+		Name:        "time",
+		Help:        "seconds spent in pruner",
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
+	}, []string{"prune_side"})
+	j.prunerFactory, err = pruner.NewPrunerFactory(in.Pruning, j.promPruneSecs)
+	if err != nil {
+		return nil, err
+	}
+
+	return j, nil
+}
+
+func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) {
+	registerer.MustRegister(j.promRepStateSecs)
+	registerer.MustRegister(j.promPruneSecs)
+	registerer.MustRegister(j.promBytesReplicated)
+}
+
+func (j *ActiveSide) Name() string { return j.name }
+
+type ActiveSideStatus struct {
+	Replication                    *report.Report
+	PruningSender, PruningReceiver *pruner.Report
+}
+
+func (j *ActiveSide) Status() *Status {
+	tasks := j.updateTasks(nil)
+
+	s := &ActiveSideStatus{}
+	t := j.mode.Type()
+	if tasks.replicationReport != nil {
+		s.Replication = tasks.replicationReport()
+	}
+	if tasks.prunerSender != nil {
+		s.PruningSender = tasks.prunerSender.Report()
+	}
+	if tasks.prunerReceiver != nil {
+		s.PruningReceiver = tasks.prunerReceiver.Report()
+	}
+	return &Status{Type: t, JobSpecific: s}
+}
+
+func (j *ActiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
+	pull, ok := j.mode.(*modePull)
+	if !ok {
+		_ = j.mode.(*modePush) // make sure we didn't introduce a new job type
+		return nil, false
+	}
+	return pull.rootFS.Copy(), true
+}
+
+func (j *ActiveSide) Run(ctx context.Context) {
+	log := GetLogger(ctx)
+	ctx = logging.WithSubsystemLoggers(ctx, log)
+
+	defer log.Info("job exiting")
+
+	periodicDone := make(chan struct{})
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	go j.mode.RunPeriodic(ctx, periodicDone)
+
+	invocationCount := 0
+outer:
+	for {
+		log.Info("wait for wakeups")
+		select {
+		case <-ctx.Done():
+			log.WithError(ctx.Err()).Info("context")
+			break outer
+
+		case <-wakeup.Wait(ctx):
+			j.mode.ResetConnectBackoff()
+		case <-periodicDone:
+		}
+		invocationCount++
+		invLog := log.WithField("invocation", invocationCount)
+		j.do(WithLogger(ctx, invLog))
+	}
+}
+
+func (j *ActiveSide) do(ctx context.Context) {
+
+	log := GetLogger(ctx)
+	ctx = logging.WithSubsystemLoggers(ctx, log)
+	loggers := rpc.GetLoggersOrPanic(ctx) // filled by WithSubsystemLoggers
+	j.mode.ConnectEndpoints(loggers, j.connecter)
+	defer j.mode.DisconnectEndpoints()
+
+	// allow cancellation of an invocation (this function)
+	ctx, cancelThisRun := context.WithCancel(ctx)
+	defer cancelThisRun()
+	go func() {
+		select {
+		case <-reset.Wait(ctx):
+			log.Info("reset received, cancelling current invocation")
+			cancelThisRun()
+		case <-ctx.Done():
+		}
+	}()
+
+	sender, receiver := j.mode.SenderReceiver()
+
+	{
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+		ctx, repCancel := context.WithCancel(ctx)
+		var repWait driver.WaitFunc
+		j.updateTasks(func(tasks *activeSideTasks) {
+			// reset it
+			*tasks = activeSideTasks{}
+			tasks.replicationCancel = repCancel
+			tasks.replicationReport, repWait = replication.Do(
+				ctx, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver),
+			)
+			tasks.state = ActiveSideReplicating
+		})
+		log.Info("start replication")
+		repWait(true) // wait blocking
+		repCancel()   // always cancel to free up context resources
+	}
+
+	{
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+		ctx, senderCancel := context.WithCancel(ctx)
+		tasks := j.updateTasks(func(tasks *activeSideTasks) {
+			tasks.prunerSender = j.prunerFactory.BuildSenderPruner(ctx, sender, sender)
+			tasks.prunerSenderCancel = senderCancel
+			tasks.state = ActiveSidePruneSender
+		})
+		log.Info("start pruning sender")
+		tasks.prunerSender.Prune()
+		log.Info("finished pruning sender")
+		senderCancel()
+	}
+	{
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+		ctx, receiverCancel := context.WithCancel(ctx)
+		tasks := j.updateTasks(func(tasks *activeSideTasks) {
+			tasks.prunerReceiver = j.prunerFactory.BuildReceiverPruner(ctx, receiver, sender)
+			tasks.prunerReceiverCancel = receiverCancel
+			tasks.state = ActiveSidePruneReceiver
+		})
+		log.Info("start pruning receiver")
+		tasks.prunerReceiver.Prune()
+		log.Info("finished pruning receiver")
+		receiverCancel()
+	}
+
+	j.updateTasks(func(tasks *activeSideTasks) {
+		tasks.state = ActiveSideDone
+	})
+
+}

daemon/job/activesidestate_enumer.go

@@ -0,0 +1,66 @@
+// Code generated by "enumer -type=ActiveSideState"; DO NOT EDIT.
+
+package job
+
+import (
+	"fmt"
+)
+
+const (
+	_ActiveSideStateName_0 = "ActiveSideReplicatingActiveSidePruneSender"
+	_ActiveSideStateName_1 = "ActiveSidePruneReceiver"
+	_ActiveSideStateName_2 = "ActiveSideDone"
+)
+
+var (
+	_ActiveSideStateIndex_0 = [...]uint8{0, 21, 42}
+	_ActiveSideStateIndex_1 = [...]uint8{0, 23}
+	_ActiveSideStateIndex_2 = [...]uint8{0, 14}
+)
+
+func (i ActiveSideState) String() string {
+	switch {
+	case 1 <= i && i <= 2:
+		i -= 1
+		return _ActiveSideStateName_0[_ActiveSideStateIndex_0[i]:_ActiveSideStateIndex_0[i+1]]
+	case i == 4:
+		return _ActiveSideStateName_1
+	case i == 8:
+		return _ActiveSideStateName_2
+	default:
+		return fmt.Sprintf("ActiveSideState(%d)", i)
+	}
+}
+
+var _ActiveSideStateValues = []ActiveSideState{1, 2, 4, 8}
+
+var _ActiveSideStateNameToValueMap = map[string]ActiveSideState{
+	_ActiveSideStateName_0[0:21]:  1,
+	_ActiveSideStateName_0[21:42]: 2,
+	_ActiveSideStateName_1[0:23]:  4,
+	_ActiveSideStateName_2[0:14]:  8,
+}
+
+// ActiveSideStateString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func ActiveSideStateString(s string) (ActiveSideState, error) {
+	if val, ok := _ActiveSideStateNameToValueMap[s]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to ActiveSideState values", s)
+}
+
+// ActiveSideStateValues returns all values of the enum
+func ActiveSideStateValues() []ActiveSideState {
+	return _ActiveSideStateValues
+}
+
+// IsAActiveSideState returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i ActiveSideState) IsAActiveSideState() bool {
+	for _, v := range _ActiveSideStateValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}

daemon/job/build_jobs.go

@@ -0,0 +1,120 @@
+package job
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/config"
+)
+
+func JobsFromConfig(c *config.Config) ([]Job, error) {
+	js := make([]Job, len(c.Jobs))
+	for i := range c.Jobs {
+		j, err := buildJob(c.Global, c.Jobs[i])
+		if err != nil {
+			return nil, err
+		}
+		if j == nil || j.Name() == "" {
+			panic(fmt.Sprintf("implementation error: job builder returned nil job type %T", c.Jobs[i].Ret))
+		}
+		js[i] = j
+	}
+
+	// receiving-side root filesystems must not overlap
+	{
+		rfss := make([]string, 0, len(js))
+		for _, j := range js {
+			jrfs, ok := j.OwnedDatasetSubtreeRoot()
+			if !ok {
+				continue
+			}
+			rfss = append(rfss, jrfs.ToString())
+		}
+		if err := validateReceivingSidesDoNotOverlap(rfss); err != nil {
+			return nil, err
+		}
+	}
+
+	return js, nil
+}
+
+func buildJob(c *config.Global, in config.JobEnum) (j Job, err error) {
+	cannotBuildJob := func(e error, name string) (Job, error) {
+		return nil, errors.Wrapf(e, "cannot build job %q", name)
+	}
+	// FIXME prettify this
+	switch v := in.Ret.(type) {
+	case *config.SinkJob:
+		m, err := modeSinkFromConfig(c, v)
+		if err != nil {
+			return cannotBuildJob(err, v.Name)
+		}
+		j, err = passiveSideFromConfig(c, &v.PassiveJob, m)
+		if err != nil {
+			return cannotBuildJob(err, v.Name)
+		}
+	case *config.SourceJob:
+		m, err := modeSourceFromConfig(c, v)
+		if err != nil {
+			return cannotBuildJob(err, v.Name)
+		}
+		j, err = passiveSideFromConfig(c, &v.PassiveJob, m)
+		if err != nil {
+			return cannotBuildJob(err, v.Name)
+		}
+	case *config.SnapJob:
+		j, err = snapJobFromConfig(c, v)
+		if err != nil {
+			return cannotBuildJob(err, v.Name)
+		}
+	case *config.PushJob:
+		m, err := modePushFromConfig(c, v)
+		if err != nil {
+			return cannotBuildJob(err, v.Name)
+		}
+		j, err = activeSide(c, &v.ActiveJob, m)
+		if err != nil {
+			return cannotBuildJob(err, v.Name)
+		}
+	case *config.PullJob:
+		m, err := modePullFromConfig(c, v)
+		if err != nil {
+			return cannotBuildJob(err, v.Name)
+		}
+		j, err = activeSide(c, &v.ActiveJob, m)
+		if err != nil {
+			return cannotBuildJob(err, v.Name)
+		}
+	default:
+		panic(fmt.Sprintf("implementation error: unknown job type %T", v))
+	}
+	return j, nil
+
+}
+
+func validateReceivingSidesDoNotOverlap(receivingRootFSs []string) error {
+	if len(receivingRootFSs) == 0 {
+		return nil
+	}
+	rfss := make([]string, len(receivingRootFSs))
+	copy(rfss, receivingRootFSs)
+	sort.Slice(rfss, func(i, j int) bool {
+		return strings.Compare(rfss[i], rfss[j]) == -1
+	})
+	// idea:
+	//   no path in rfss must be prefix of another
+	//
+	// rfss is now lexicographically sorted, which means that
+	// if i is prefix of j, i < j (in lexicographical order)
+	// thus,
+	// if any i is prefix of i+n (n >= 1), there is overlap
+	for i := 0; i < len(rfss)-1; i++ {
+		if strings.HasPrefix(rfss[i+1], rfss[i]) {
+			return fmt.Errorf("receiving jobs with overlapping root filesystems are forbidden")
+		}
+	}
+	return nil
+}

daemon/job/build_jobs_test.go

@@ -0,0 +1,42 @@
+package job
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
+	type testCase struct {
+		err   bool
+		input []string
+	}
+	tcs := []testCase{
+		{false, nil},
+		{false, []string{}},
+		{false, []string{""}}, // not our job to determine valid paths
+		{false, []string{"a"}},
+		{false, []string{"some/path"}},
+		{false, []string{"zroot/sink1", "zroot/sink2", "zroot/sink3"}},
+		{true, []string{"zroot/b", "zroot/b"}},
+		{true, []string{"zroot/foo", "zroot/foo/bar", "zroot/baz"}},
+		{false, []string{"a/x", "b/x"}},
+		{false, []string{"a", "b"}},
+		{true, []string{"a", "a"}},
+		{true, []string{"a/x/y", "a/x"}},
+		{true, []string{"a/x", "a/x/y"}},
+		{true, []string{"a/x", "b/x", "a/x/y"}},
+		{true, []string{"a", "a/b", "a/c", "a/b"}},
+		{true, []string{"a/b", "a/c", "a/b", "a/d", "a/c"}},
+	}
+
+	for _, tc := range tcs {
+		t.Logf("input: %v", tc.input)
+		err := validateReceivingSidesDoNotOverlap(tc.input)
+		if tc.err {
+			assert.Error(t, err)
+		} else {
+			assert.NoError(t, err)
+		}
+	}
+}

daemon/job/job.go

@@ -0,0 +1,118 @@
+package job
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type Logger = logger.Logger
+
+type contextKey int
+
+const (
+	contextKeyLog contextKey = iota
+)
+
+func GetLogger(ctx context.Context) Logger {
+	if l, ok := ctx.Value(contextKeyLog).(Logger); ok {
+		return l
+	}
+	return logger.NewNullLogger()
+}
+
+func WithLogger(ctx context.Context, l Logger) context.Context {
+	return context.WithValue(ctx, contextKeyLog, l)
+}
+
+type Job interface {
+	Name() string
+	Run(ctx context.Context)
+	Status() *Status
+	RegisterMetrics(registerer prometheus.Registerer)
+	// Jobs that return a subtree of the dataset hierarchy
+	// must return the root of that subtree as rfs and ok = true
+	OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool)
+}
+
+type Type string
+
+const (
+	TypeInternal Type = "internal"
+	TypeSnap     Type = "snap"
+	TypePush     Type = "push"
+	TypeSink     Type = "sink"
+	TypePull     Type = "pull"
+	TypeSource   Type = "source"
+)
+
+type Status struct {
+	Type        Type
+	JobSpecific interface{}
+}
+
+func (s *Status) MarshalJSON() ([]byte, error) {
+	typeJson, err := json.Marshal(s.Type)
+	if err != nil {
+		return nil, err
+	}
+	jobJSON, err := json.Marshal(s.JobSpecific)
+	if err != nil {
+		return nil, err
+	}
+	m := map[string]json.RawMessage{
+		"type":         typeJson,
+		string(s.Type): jobJSON,
+	}
+	return json.Marshal(m)
+}
+
+func (s *Status) UnmarshalJSON(in []byte) (err error) {
+	var m map[string]json.RawMessage
+	if err := json.Unmarshal(in, &m); err != nil {
+		return err
+	}
+	tJSON, ok := m["type"]
+	if !ok {
+		return fmt.Errorf("field 'type' not found")
+	}
+	if err := json.Unmarshal(tJSON, &s.Type); err != nil {
+		return err
+	}
+	key := string(s.Type)
+	jobJSON, ok := m[key]
+	if !ok {
+		return fmt.Errorf("field '%s', not found", key)
+	}
+	switch s.Type {
+	case TypeSnap:
+		var st SnapJobStatus
+		err = json.Unmarshal(jobJSON, &st)
+		s.JobSpecific = &st
+
+	case TypePull:
+		fallthrough
+	case TypePush:
+		var st ActiveSideStatus
+		err = json.Unmarshal(jobJSON, &st)
+		s.JobSpecific = &st
+
+	case TypeSource:
+		fallthrough
+	case TypeSink:
+		var st PassiveStatus
+		err = json.Unmarshal(jobJSON, &st)
+		s.JobSpecific = &st
+
+	case TypeInternal:
+		// internal jobs do not report specifics
+	default:
+		err = fmt.Errorf("unknown job type '%s'", key)
+	}
+	return err
+}

daemon/job/passive.go

@@ -0,0 +1,147 @@
+package job
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/daemon/snapper"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/transport"
+	"github.com/zrepl/zrepl/transport/fromconfig"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type PassiveSide struct {
+	mode   passiveMode
+	name   string
+	listen transport.AuthenticatedListenerFactory
+}
+
+type passiveMode interface {
+	Handler() rpc.Handler
+	RunPeriodic(ctx context.Context)
+	Type() Type
+}
+
+type modeSink struct {
+	rootDataset *zfs.DatasetPath
+}
+
+func (m *modeSink) Type() Type { return TypeSink }
+
+func (m *modeSink) Handler() rpc.Handler {
+	return endpoint.NewReceiver(m.rootDataset, true)
+}
+
+func (m *modeSink) RunPeriodic(_ context.Context) {}
+
+func modeSinkFromConfig(g *config.Global, in *config.SinkJob) (m *modeSink, err error) {
+	m = &modeSink{}
+	m.rootDataset, err = zfs.NewDatasetPath(in.RootFS)
+	if err != nil {
+		return nil, errors.New("root dataset is not a valid zfs filesystem path")
+	}
+	if m.rootDataset.Length() <= 0 {
+		return nil, errors.New("root dataset must not be empty") // duplicates error check of receiver
+	}
+	return m, nil
+}
+
+type modeSource struct {
+	fsfilter zfs.DatasetFilter
+	snapper  *snapper.PeriodicOrManual
+}
+
+func modeSourceFromConfig(g *config.Global, in *config.SourceJob) (m *modeSource, err error) {
+	// FIXME exact dedup of modePush
+	m = &modeSource{}
+	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannnot build filesystem filter")
+	}
+	m.fsfilter = fsf
+
+	if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
+		return nil, errors.Wrap(err, "cannot build snapper")
+	}
+
+	return m, nil
+}
+
+func (m *modeSource) Type() Type { return TypeSource }
+
+func (m *modeSource) Handler() rpc.Handler {
+	return endpoint.NewSender(m.fsfilter)
+}
+
+func (m *modeSource) RunPeriodic(ctx context.Context) {
+	m.snapper.Run(ctx, nil)
+}
+
+func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, mode passiveMode) (s *PassiveSide, err error) {
+
+	s = &PassiveSide{mode: mode, name: in.Name}
+	if s.listen, err = fromconfig.ListenerFactoryFromConfig(g, in.Serve); err != nil {
+		return nil, errors.Wrap(err, "cannot build listener factory")
+	}
+
+	return s, nil
+}
+
+func (j *PassiveSide) Name() string { return j.name }
+
+type PassiveStatus struct{}
+
+func (s *PassiveSide) Status() *Status {
+	return &Status{Type: s.mode.Type()} // FIXME PassiveStatus
+}
+
+func (j *PassiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
+	sink, ok := j.mode.(*modeSink)
+	if !ok {
+		_ = j.mode.(*modeSource) // make sure we didn't introduce a new job type
+		return nil, false
+	}
+	return sink.rootDataset.Copy(), true
+}
+
+func (*PassiveSide) RegisterMetrics(registerer prometheus.Registerer) {}
+
+func (j *PassiveSide) Run(ctx context.Context) {
+
+	log := GetLogger(ctx)
+	defer log.Info("job exiting")
+	ctx = logging.WithSubsystemLoggers(ctx, log)
+	{
+		ctx, cancel := context.WithCancel(ctx) // shadowing
+		defer cancel()
+		go j.mode.RunPeriodic(ctx)
+	}
+
+	handler := j.mode.Handler()
+	if handler == nil {
+		panic(fmt.Sprintf("implementation error: j.mode.Handler() returned nil: %#v", j))
+	}
+
+	ctxInterceptor := func(handlerCtx context.Context) context.Context {
+		return logging.WithSubsystemLoggers(handlerCtx, log)
+	}
+
+	rpcLoggers := rpc.GetLoggersOrPanic(ctx) // WithSubsystemLoggers above
+	server := rpc.NewServer(handler, rpcLoggers, ctxInterceptor)
+
+	listener, err := j.listen()
+	if err != nil {
+		log.WithError(err).Error("cannot listen")
+		return
+	}
+
+	server.Serve(ctx, listener)
+}

daemon/job/reset/reset.go

@@ -0,0 +1,35 @@
+package reset
+
+import (
+	"context"
+	"errors"
+)
+
+type contextKey int
+
+const contextKeyReset contextKey = iota
+
+func Wait(ctx context.Context) <-chan struct{} {
+	wc, ok := ctx.Value(contextKeyReset).(chan struct{})
+	if !ok {
+		wc = make(chan struct{})
+	}
+	return wc
+}
+
+type Func func() error
+
+var AlreadyReset = errors.New("already reset")
+
+func Context(ctx context.Context) (context.Context, Func) {
+	wc := make(chan struct{})
+	wuf := func() error {
+		select {
+		case wc <- struct{}{}:
+			return nil
+		default:
+			return AlreadyReset
+		}
+	}
+	return context.WithValue(ctx, contextKeyReset, wc), wuf
+}

daemon/job/snapjob.go

@@ -0,0 +1,168 @@
+package job
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/daemon/job/wakeup"
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/daemon/pruner"
+	"github.com/zrepl/zrepl/daemon/snapper"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/replication/logic/pdu"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type SnapJob struct {
+	name     string
+	fsfilter zfs.DatasetFilter
+	snapper  *snapper.PeriodicOrManual
+
+	prunerFactory *pruner.LocalPrunerFactory
+
+	promPruneSecs *prometheus.HistogramVec // labels: prune_side
+
+	pruner *pruner.Pruner
+}
+
+func (j *SnapJob) Name() string { return j.name }
+
+func (j *SnapJob) Type() Type { return TypeSnap }
+
+func snapJobFromConfig(g *config.Global, in *config.SnapJob) (j *SnapJob, err error) {
+	j = &SnapJob{}
+	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannnot build filesystem filter")
+	}
+	j.fsfilter = fsf
+
+	if j.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
+		return nil, errors.Wrap(err, "cannot build snapper")
+	}
+	j.name = in.Name
+	j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace:   "zrepl",
+		Subsystem:   "pruning",
+		Name:        "time",
+		Help:        "seconds spent in pruner",
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
+	}, []string{"prune_side"})
+	j.prunerFactory, err = pruner.NewLocalPrunerFactory(in.Pruning, j.promPruneSecs)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot build snapjob pruning rules")
+	}
+	return j, nil
+}
+
+func (j *SnapJob) RegisterMetrics(registerer prometheus.Registerer) {
+	registerer.MustRegister(j.promPruneSecs)
+}
+
+type SnapJobStatus struct {
+	Pruning *pruner.Report
+}
+
+func (j *SnapJob) Status() *Status {
+	s := &SnapJobStatus{}
+	t := j.Type()
+	if j.pruner != nil {
+		s.Pruning = j.pruner.Report()
+	}
+	return &Status{Type: t, JobSpecific: s}
+}
+
+func (j *SnapJob) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
+	return nil, false
+}
+
+func (j *SnapJob) Run(ctx context.Context) {
+	log := GetLogger(ctx)
+	ctx = logging.WithSubsystemLoggers(ctx, log)
+
+	defer log.Info("job exiting")
+
+	periodicDone := make(chan struct{})
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	go j.snapper.Run(ctx, periodicDone)
+
+	invocationCount := 0
+outer:
+	for {
+		log.Info("wait for wakeups")
+		select {
+		case <-ctx.Done():
+			log.WithError(ctx.Err()).Info("context")
+			break outer
+
+		case <-wakeup.Wait(ctx):
+		case <-periodicDone:
+		}
+		invocationCount++
+		invLog := log.WithField("invocation", invocationCount)
+		j.doPrune(WithLogger(ctx, invLog))
+	}
+}
+
+// Adaptor that implements pruner.History around a pruner.Target.
+// The ReplicationCursor method is Get-op only and always returns
+// the filesystem's most recent version's GUID.
+//
+// TODO:
+// This is a work-around for the current package daemon/pruner
+// and package pruning.Snapshot limitation: they require the
+//  `Replicated` getter method be present, but obviously,
+// a local job like SnapJob can't deliver on that.
+// But the pruner.Pruner gives up on an FS if no replication
+// cursor is present, which is why this pruner returns the
+// most recent filesystem version.
+type alwaysUpToDateReplicationCursorHistory struct {
+	// the Target passed as Target to BuildLocalPruner
+	target pruner.Target
+}
+
+var _ pruner.History = (*alwaysUpToDateReplicationCursorHistory)(nil)
+
+func (h alwaysUpToDateReplicationCursorHistory) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
+	if req.GetGet() == nil {
+		return nil, fmt.Errorf("unsupported ReplicationCursor request: SnapJob only supports GETting a (faked) cursor")
+	}
+	fsvReq := &pdu.ListFilesystemVersionsReq{
+		Filesystem: req.GetFilesystem(),
+	}
+	res, err := h.target.ListFilesystemVersions(ctx, fsvReq)
+	if err != nil {
+		return nil, err
+	}
+	fsvs := res.GetVersions()
+	if len(fsvs) <= 0 {
+		return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Notexist{Notexist: true}}, nil
+	}
+	// always return must recent version
+	sort.Slice(fsvs, func(i, j int) bool {
+		return fsvs[i].CreateTXG < fsvs[j].CreateTXG
+	})
+	mostRecent := fsvs[len(fsvs)-1]
+	return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Guid{Guid: mostRecent.GetGuid()}}, nil
+}
+
+func (h alwaysUpToDateReplicationCursorHistory) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error) {
+	return h.target.ListFilesystems(ctx, req)
+}
+
+func (j *SnapJob) doPrune(ctx context.Context) {
+	log := GetLogger(ctx)
+	ctx = logging.WithSubsystemLoggers(ctx, log)
+	sender := endpoint.NewSender(j.fsfilter)
+	j.pruner = j.prunerFactory.BuildLocalPruner(ctx, sender, alwaysUpToDateReplicationCursorHistory{sender})
+	log.Info("start pruning")
+	j.pruner.Prune()
+	log.Info("finished pruning")
+}

daemon/job/wakeup/wakeup.go

@@ -0,0 +1,35 @@
+package wakeup
+
+import (
+	"context"
+	"errors"
+)
+
+type contextKey int
+
+const contextKeyWakeup contextKey = iota
+
+func Wait(ctx context.Context) <-chan struct{} {
+	wc, ok := ctx.Value(contextKeyWakeup).(chan struct{})
+	if !ok {
+		wc = make(chan struct{})
+	}
+	return wc
+}
+
+type Func func() error
+
+var AlreadyWokenUp = errors.New("already woken up")
+
+func Context(ctx context.Context) (context.Context, Func) {
+	wc := make(chan struct{})
+	wuf := func() error {
+		select {
+		case wc <- struct{}{}:
+			return nil
+		default:
+			return AlreadyWokenUp
+		}
+	}
+	return context.WithValue(ctx, contextKeyWakeup, wc), wuf
+}

daemon/logging/build_logging.go

@@ -0,0 +1,231 @@
+package logging
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"log/syslog"
+	"os"
+
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/pruner"
+	"github.com/zrepl/zrepl/daemon/snapper"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/replication/driver"
+	"github.com/zrepl/zrepl/replication/logic"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/rpc/transportmux"
+	"github.com/zrepl/zrepl/tlsconf"
+	"github.com/zrepl/zrepl/transport"
+)
+
+func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error) {
+
+	outlets := logger.NewOutlets()
+
+	if len(in) == 0 {
+		// Default config
+		out := WriterOutlet{&HumanFormatter{}, os.Stdout}
+		outlets.Add(out, logger.Warn)
+		return outlets, nil
+	}
+
+	var syslogOutlets, stdoutOutlets int
+	for lei, le := range in {
+
+		outlet, minLevel, err := parseOutlet(le)
+		if err != nil {
+			return nil, errors.Wrapf(err, "cannot parse outlet #%d", lei)
+		}
+		var _ logger.Outlet = WriterOutlet{}
+		var _ logger.Outlet = &SyslogOutlet{}
+		switch outlet.(type) {
+		case *SyslogOutlet:
+			syslogOutlets++
+		case WriterOutlet:
+			stdoutOutlets++
+		}
+
+		outlets.Add(outlet, minLevel)
+
+	}
+
+	if syslogOutlets > 1 {
+		return nil, errors.Errorf("can only define one 'syslog' outlet")
+	}
+	if stdoutOutlets > 1 {
+		return nil, errors.Errorf("can only define one 'stdout' outlet")
+	}
+
+	return outlets, nil
+
+}
+
+type Subsystem string
+
+const (
+	SubsysReplication  Subsystem = "repl"
+	SubsyEndpoint      Subsystem = "endpoint"
+	SubsysPruning      Subsystem = "pruning"
+	SubsysSnapshot     Subsystem = "snapshot"
+	SubsysTransport    Subsystem = "transport"
+	SubsysTransportMux Subsystem = "transportmux"
+	SubsysRPC          Subsystem = "rpc"
+	SubsysRPCControl   Subsystem = "rpc.ctrl"
+	SubsysRPCData      Subsystem = "rpc.data"
+)
+
+func WithSubsystemLoggers(ctx context.Context, log logger.Logger) context.Context {
+	ctx = logic.WithLogger(ctx, log.WithField(SubsysField, SubsysReplication))
+	ctx = driver.WithLogger(ctx, log.WithField(SubsysField, SubsysReplication))
+	ctx = endpoint.WithLogger(ctx, log.WithField(SubsysField, SubsyEndpoint))
+	ctx = pruner.WithLogger(ctx, log.WithField(SubsysField, SubsysPruning))
+	ctx = snapper.WithLogger(ctx, log.WithField(SubsysField, SubsysSnapshot))
+	ctx = transport.WithLogger(ctx, log.WithField(SubsysField, SubsysTransport))
+	ctx = transportmux.WithLogger(ctx, log.WithField(SubsysField, SubsysTransportMux))
+	ctx = rpc.WithLoggers(ctx,
+		rpc.Loggers{
+			General: log.WithField(SubsysField, SubsysRPC),
+			Control: log.WithField(SubsysField, SubsysRPCControl),
+			Data:    log.WithField(SubsysField, SubsysRPCData),
+		},
+	)
+	return ctx
+}
+
+func LogSubsystem(log logger.Logger, subsys Subsystem) logger.Logger {
+	return log.ReplaceField(SubsysField, subsys)
+}
+
+func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
+	var is string
+	switch j := i.(type) {
+	case string:
+		is = j
+	default:
+		return nil, errors.Errorf("invalid log format: wrong type: %T", i)
+	}
+
+	switch is {
+	case "human":
+		return &HumanFormatter{}, nil
+	case "logfmt":
+		return &LogfmtFormatter{}, nil
+	case "json":
+		return &JSONFormatter{}, nil
+	default:
+		return nil, errors.Errorf("invalid log format: '%s'", is)
+	}
+
+}
+
+func parseOutlet(in config.LoggingOutletEnum) (o logger.Outlet, level logger.Level, err error) {
+
+	parseCommon := func(common config.LoggingOutletCommon) (logger.Level, EntryFormatter, error) {
+		if common.Level == "" || common.Format == "" {
+			return 0, nil, errors.Errorf("must specify 'level' and 'format' field")
+		}
+
+		minLevel, err := logger.ParseLevel(common.Level)
+		if err != nil {
+			return 0, nil, errors.Wrap(err, "cannot parse 'level' field")
+		}
+		formatter, err := parseLogFormat(common.Format)
+		if err != nil {
+			return 0, nil, errors.Wrap(err, "cannot parse 'formatter' field")
+		}
+		return minLevel, formatter, nil
+	}
+
+	var f EntryFormatter
+
+	switch v := in.Ret.(type) {
+	case *config.StdoutLoggingOutlet:
+		level, f, err = parseCommon(v.LoggingOutletCommon)
+		if err != nil {
+			break
+		}
+		o, err = parseStdoutOutlet(v, f)
+	case *config.TCPLoggingOutlet:
+		level, f, err = parseCommon(v.LoggingOutletCommon)
+		if err != nil {
+			break
+		}
+		o, err = parseTCPOutlet(v, f)
+	case *config.SyslogLoggingOutlet:
+		level, f, err = parseCommon(v.LoggingOutletCommon)
+		if err != nil {
+			break
+		}
+		o, err = parseSyslogOutlet(v, f)
+	default:
+		panic(v)
+	}
+	return o, level, err
+}
+
+func parseStdoutOutlet(in *config.StdoutLoggingOutlet, formatter EntryFormatter) (WriterOutlet, error) {
+	flags := MetadataAll
+	writer := os.Stdout
+	if !isatty.IsTerminal(writer.Fd()) && !in.Time {
+		flags &= ^MetadataTime
+	}
+	if isatty.IsTerminal(writer.Fd()) && !in.Color {
+		flags &= ^MetadataColor
+	}
+
+	formatter.SetMetadataFlags(flags)
+	return WriterOutlet{
+		formatter,
+		os.Stdout,
+	}, nil
+}
+
+func parseTCPOutlet(in *config.TCPLoggingOutlet, formatter EntryFormatter) (out *TCPOutlet, err error) {
+	var tlsConfig *tls.Config
+	if in.TLS != nil {
+		tlsConfig, err = func(m *config.TCPLoggingOutletTLS, host string) (*tls.Config, error) {
+			clientCert, err := tls.LoadX509KeyPair(m.Cert, m.Key)
+			if err != nil {
+				return nil, errors.Wrap(err, "cannot load client cert")
+			}
+
+			var rootCAs *x509.CertPool
+			if m.CA == "" {
+				if rootCAs, err = x509.SystemCertPool(); err != nil {
+					return nil, errors.Wrap(err, "cannot open system cert pool")
+				}
+			} else {
+				rootCAs, err = tlsconf.ParseCAFile(m.CA)
+				if err != nil {
+					return nil, errors.Wrap(err, "cannot parse CA cert")
+				}
+			}
+			if rootCAs == nil {
+				panic("invariant violated")
+			}
+
+			return tlsconf.ClientAuthClient(host, rootCAs, clientCert)
+		}(in.TLS, in.Address)
+		if err != nil {
+			return nil, errors.New("cannot not parse TLS config in field 'tls'")
+		}
+	}
+
+	formatter.SetMetadataFlags(MetadataAll)
+	return NewTCPOutlet(formatter, in.Net, in.Address, tlsConfig, in.RetryInterval), nil
+
+}
+
+func parseSyslogOutlet(in *config.SyslogLoggingOutlet, formatter EntryFormatter) (out *SyslogOutlet, err error) {
+	out = &SyslogOutlet{}
+	out.Formatter = formatter
+	out.Formatter.SetMetadataFlags(MetadataNone)
+	out.Facility = syslog.Priority(*in.Facility)
+	out.RetryInterval = in.RetryInterval
+	return out, nil
+}

daemon/logging/logging_formatters.go

@@ -1,19 +1,17 @@
-package cmd
+package logging
 
 import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"time"
+
+	"github.com/fatih/color"
 	"github.com/go-logfmt/logfmt"
 	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/logger"
-	"time"
-)
 
-type EntryFormatter interface {
+	"github.com/zrepl/zrepl/logger"
-	SetMetadataFlags(flags MetadataFlags)
+)
-	Format(e *logger.Entry) ([]byte, error)
-}
 
 const (
 	FieldLevel   = "level"
@@ -22,13 +20,19 @@ const (
 )
 
 const (
-	logJobField     string = "job"
+	JobField    string = "job"
-	logTaskField    string = "task"
+	SubsysField string = "subsystem"
-	logFSField      string = "filesystem"
+)
-	logMapFromField string = "map_from"
+
-	logMapToField   string = "map_to"
+type MetadataFlags int64
-	logIncFromField string = "inc_from"
+
-	logIncToField   string = "inc_to"
+const (
+	MetadataTime MetadataFlags = 1 << iota
+	MetadataLevel
+	MetadataColor
+
+	MetadataNone MetadataFlags = 0
+	MetadataAll  MetadataFlags = ^0
 )
 
 type NoFormatter struct{}
@@ -69,39 +73,29 @@ func (f *HumanFormatter) ignored(field string) bool {
 func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
 
 	var line bytes.Buffer
+	col := color.New()
+	if f.metadataFlags&MetadataColor != 0 {
+		col = e.Color()
+	}
 
 	if f.metadataFlags&MetadataTime != 0 {
 		fmt.Fprintf(&line, "%s ", e.Time.Format(HumanFormatterDateFormat))
 	}
 	if f.metadataFlags&MetadataLevel != 0 {
-		fmt.Fprintf(&line, "[%s]", e.Level.Short())
+		fmt.Fprintf(&line, "[%s]", col.Sprint(e.Level.Short()))
 	}
 
-	prefixFields := []string{logJobField, logTaskField, logFSField}
+	prefixFields := []string{JobField, SubsysField}
 	prefixed := make(map[string]bool, len(prefixFields)+2)
 	for _, field := range prefixFields {
 		val, ok := e.Fields[field].(string)
-		if ok {
+		if !ok {
+			continue
+		}
 		if !f.ignored(field) {
-				fmt.Fprintf(&line, "[%s]", val)
+			fmt.Fprintf(&line, "[%s]", col.Sprint(val))
 			prefixed[field] = true
 		}
-		} else {
-			break
-		}
-	}
-	// even more prefix fields
-	mapFrom, mapFromOk := e.Fields[logMapFromField].(string)
-	mapTo, mapToOk := e.Fields[logMapToField].(string)
-	if mapFromOk && mapToOk && !f.ignored(logMapFromField) && !f.ignored(logMapToField) {
-		fmt.Fprintf(&line, "[%s => %s]", mapFrom, mapTo)
-		prefixed[logMapFromField], prefixed[logMapToField] = true, true
-	}
-	incFrom, incFromOk := e.Fields[logIncFromField].(string)
-	incTo, incToOk := e.Fields[logIncToField].(string)
-	if incFromOk && incToOk && !f.ignored(logIncFromField) && !f.ignored(logMapToField) {
-		fmt.Fprintf(&line, "[%s => %s]", incFrom, incTo)
-		prefixed[logIncFromField], prefixed[logIncToField] = true, true
 	}
 
 	if line.Len() > 0 {
@@ -110,15 +104,11 @@ func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
 	fmt.Fprint(&line, e.Message)
 
 	if len(e.Fields)-len(prefixed) > 0 {
-		fmt.Fprint(&line, " ")
-		enc := logfmt.NewEncoder(&line)
 		for field, value := range e.Fields {
 			if prefixed[field] || f.ignored(field) {
 				continue
 			}
-			if err := logfmtTryEncodeKeyval(enc, field, value); err != nil {
+			fmt.Fprintf(&line, " %s=%q", col.Sprint(field), fmt.Sprint(value))
-				return nil, err
-			}
 		}
 	}
 
@@ -171,15 +161,21 @@ func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
 	enc := logfmt.NewEncoder(&buf)
 
 	if f.metadataFlags&MetadataTime != 0 {
-		enc.EncodeKeyval(FieldTime, e.Time)
+		err := enc.EncodeKeyval(FieldTime, e.Time)
+		if err != nil {
+			return nil, errors.Wrap(err, "logfmt: encode time")
+		}
 	}
 	if f.metadataFlags&MetadataLevel != 0 {
-		enc.EncodeKeyval(FieldLevel, e.Level)
+		err := enc.EncodeKeyval(FieldLevel, e.Level)
+		if err != nil {
+			return nil, errors.Wrap(err, "logfmt: encode level")
+		}
 	}
 
 	// at least try and put job and task in front
 	prefixed := make(map[string]bool, 2)
-	prefix := []string{logJobField, logTaskField}
+	prefix := []string{JobField, SubsysField}
 	for _, pf := range prefix {
 		v, ok := e.Fields[pf]
 		if !ok {
@@ -191,8 +187,10 @@ func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
 		prefixed[pf] = true
 	}
 
-	enc.EncodeKeyval(FieldMessage, e.Message)
+	err := enc.EncodeKeyval(FieldMessage, e.Message)
-
+	if err != nil {
+		return nil, errors.Wrap(err, "logfmt: encode message")
+	}
 	for k, v := range e.Fields {
 		if !prefixed[k] {
 			if err := logfmtTryEncodeKeyval(enc, k, v); err != nil {
@@ -211,7 +209,10 @@ func logfmtTryEncodeKeyval(enc *logfmt.Encoder, field, value interface{}) error
 	case nil: // ok
 		return nil
 	case logfmt.ErrUnsupportedValueType:
-		enc.EncodeKeyval(field, fmt.Sprintf("<%T>", value))
+		err := enc.EncodeKeyval(field, fmt.Sprintf("<%T>", value))
+		if err != nil {
+			return errors.Wrap(err, "cannot encode unsuuported value type Go type")
+		}
 		return nil
 	}
 	return errors.Wrapf(err, "cannot encode field '%s'", field)

daemon/logging/logging_outlets.go

@@ -1,29 +1,39 @@
-package cmd
+package logging
 
 import (
 	"bytes"
 	"context"
 	"crypto/tls"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/logger"
 	"io"
 	"log/syslog"
 	"net"
 	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/logger"
 )
 
+type EntryFormatter interface {
+	SetMetadataFlags(flags MetadataFlags)
+	Format(e *logger.Entry) ([]byte, error)
+}
+
 type WriterOutlet struct {
-	Formatter EntryFormatter
+	formatter EntryFormatter
-	Writer    io.Writer
+	writer    io.Writer
 }
 
 func (h WriterOutlet) WriteEntry(entry logger.Entry) error {
-	bytes, err := h.Formatter.Format(&entry)
+	bytes, err := h.formatter.Format(&entry)
 	if err != nil {
 		return err
 	}
-	_, err = h.Writer.Write(bytes)
+	_, err = h.writer.Write(bytes)
-	h.Writer.Write([]byte("\n"))
+	if err != nil {
+		return err
+	}
+	_, err = h.writer.Write([]byte("\n"))
 	return err
 }
 
@@ -87,8 +97,10 @@ func (h *TCPOutlet) outLoop(retryInterval time.Duration) {
 				conn = nil
 			}
 		}
-		conn.SetWriteDeadline(time.Now().Add(retryInterval))
+		err = conn.SetWriteDeadline(time.Now().Add(retryInterval))
+		if err == nil {
 			_, err = io.Copy(conn, msg)
+		}
 		if err != nil {
 			retry = time.Now().Add(retryInterval)
 			conn.Close()
@@ -119,6 +131,7 @@ func (h *TCPOutlet) WriteEntry(e logger.Entry) error {
 type SyslogOutlet struct {
 	Formatter          EntryFormatter
 	RetryInterval      time.Duration
+	Facility           syslog.Priority
 	writer             *syslog.Writer
 	lastConnectAttempt time.Time
 }
@@ -137,7 +150,7 @@ func (o *SyslogOutlet) WriteEntry(entry logger.Entry) error {
 		if now.Sub(o.lastConnectAttempt) < o.RetryInterval {
 			return nil // not an error toward logger
 		}
-		o.writer, err = syslog.New(syslog.LOG_LOCAL0, "zrepl")
+		o.writer, err = syslog.New(o.Facility, "zrepl")
 		o.lastConnectAttempt = time.Now()
 		if err != nil {
 			o.writer = nil

daemon/main.go

@@ -0,0 +1,16 @@
+package daemon
+
+import (
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/logger"
+)
+
+type Logger = logger.Logger
+
+var DaemonCmd = &cli.Subcommand{
+	Use:   "daemon",
+	Short: "run the zrepl daemon",
+	Run: func(subcommand *cli.Subcommand, args []string) error {
+		return Run(subcommand.Config())
+	},
+}

daemon/nethelpers/helpers.go

@@ -1,10 +1,11 @@
-package cmd
+package nethelpers
 
 import (
-	"github.com/pkg/errors"
 	"net"
 	"os"
 	"path/filepath"
+
+	"github.com/pkg/errors"
 )
 
 func PreparePrivateSockpath(sockpath string) error {

daemon/pprof.go

@@ -1,4 +1,4 @@
-package cmd
+package daemon
 
 import (
 	"net/http"
@@ -7,11 +7,12 @@ import (
 	"context"
 	"net"
 	"net/http/pprof"
+
+	"github.com/zrepl/zrepl/daemon/job"
 )
 
-type PProfServer struct {
+type pprofServer struct {
 	cc       chan PprofServerControlMsg
-	state    PprofServerControlMsg
 	listener net.Listener
 }
 
@@ -22,9 +23,9 @@ type PprofServerControlMsg struct {
 	HttpListenAddress string
 }
 
-func NewPProfServer(ctx context.Context) *PProfServer {
+func NewPProfServer(ctx context.Context) *pprofServer {
 
-	s := &PProfServer{
+	s := &pprofServer{
 		cc: make(chan PprofServerControlMsg),
 	}
 
@@ -32,7 +33,7 @@ func NewPProfServer(ctx context.Context) *PProfServer {
 	return s
 }
 
-func (s *PProfServer) controlLoop(ctx context.Context) {
+func (s *pprofServer) controlLoop(ctx context.Context) {
 outer:
 	for {
 
@@ -63,7 +64,14 @@ outer:
 			mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
 			mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
 			mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
-			go http.Serve(s.listener, mux)
+			go func() {
+				err := http.Serve(s.listener, mux)
+				if ctx.Err() != nil {
+					return
+				} else if err != nil {
+					job.GetLogger(ctx).WithError(err).Error("pprof server serve error")
+				}
+			}()
 			continue
 		}
 
@@ -75,6 +83,6 @@ outer:
 	}
 }
 
-func (s *PProfServer) Control(msg PprofServerControlMsg) {
+func (s *pprofServer) Control(msg PprofServerControlMsg) {
 	s.cc <- msg
 }

daemon/prometheus.go

@@ -0,0 +1,95 @@
+package daemon
+
+import (
+	"context"
+	"net"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/rpc/dataconn/frameconn"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type prometheusJob struct {
+	listen string
+}
+
+func newPrometheusJobFromConfig(in *config.PrometheusMonitoring) (*prometheusJob, error) {
+	if _, _, err := net.SplitHostPort(in.Listen); err != nil {
+		return nil, err
+	}
+	return &prometheusJob{in.Listen}, nil
+}
+
+var prom struct {
+	taskLogEntries *prometheus.CounterVec
+}
+
+func init() {
+	prom.taskLogEntries = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "zrepl",
+		Subsystem: "daemon",
+		Name:      "log_entries",
+		Help:      "number of log entries per job task and level",
+	}, []string{"zrepl_job", "level"})
+	prometheus.MustRegister(prom.taskLogEntries)
+}
+
+func (j *prometheusJob) Name() string { return jobNamePrometheus }
+
+func (j *prometheusJob) Status() *job.Status { return &job.Status{Type: job.TypeInternal} }
+
+func (j *prometheusJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
+
+func (j *prometheusJob) RegisterMetrics(registerer prometheus.Registerer) {}
+
+func (j *prometheusJob) Run(ctx context.Context) {
+
+	if err := zfs.PrometheusRegister(prometheus.DefaultRegisterer); err != nil {
+		panic(err)
+	}
+
+	if err := frameconn.PrometheusRegister(prometheus.DefaultRegisterer); err != nil {
+		panic(err)
+	}
+
+	log := job.GetLogger(ctx)
+
+	l, err := net.Listen("tcp", j.listen)
+	if err != nil {
+		log.WithError(err).Error("cannot listen")
+	}
+	go func() {
+		<-ctx.Done()
+		l.Close()
+	}()
+
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.Handler())
+
+	err = http.Serve(l, mux)
+	if err != nil && ctx.Err() == nil {
+		log.WithError(err).Error("error while serving")
+	}
+
+}
+
+type prometheusJobOutlet struct {
+	jobName string
+}
+
+var _ logger.Outlet = prometheusJobOutlet{}
+
+func newPrometheusLogOutlet(jobName string) prometheusJobOutlet {
+	return prometheusJobOutlet{jobName}
+}
+
+func (o prometheusJobOutlet) WriteEntry(entry logger.Entry) error {
+	prom.taskLogEntries.WithLabelValues(o.jobName, entry.Level.String()).Inc()
+	return nil
+}

daemon/pruner/pruner.go

@@ -0,0 +1,582 @@
+package pruner
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/pruning"
+	"github.com/zrepl/zrepl/replication/logic/pdu"
+	"github.com/zrepl/zrepl/util/envconst"
+)
+
+// Try to keep it compatible with gitub.com/zrepl/zrepl/endpoint.Endpoint
+type History interface {
+	ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error)
+	ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
+}
+
+// Try to keep it compatible with gitub.com/zrepl/zrepl/endpoint.Endpoint
+type Target interface {
+	ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
+	ListFilesystemVersions(ctx context.Context, req *pdu.ListFilesystemVersionsReq) (*pdu.ListFilesystemVersionsRes, error)
+	DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error)
+}
+
+type Logger = logger.Logger
+
+type contextKey int
+
+const contextKeyLogger contextKey = 0
+
+func WithLogger(ctx context.Context, log Logger) context.Context {
+	return context.WithValue(ctx, contextKeyLogger, log)
+}
+
+func GetLogger(ctx context.Context) Logger {
+	if l, ok := ctx.Value(contextKeyLogger).(Logger); ok {
+		return l
+	}
+	return logger.NewNullLogger()
+}
+
+type args struct {
+	ctx                            context.Context
+	target                         Target
+	receiver                       History
+	rules                          []pruning.KeepRule
+	retryWait                      time.Duration
+	considerSnapAtCursorReplicated bool
+	promPruneSecs                  prometheus.Observer
+}
+
+type Pruner struct {
+	args args
+
+	mtx sync.RWMutex
+
+	state State
+
+	// State PlanErr
+	err error
+
+	// State Exec
+	execQueue *execQueue
+}
+
+type PrunerFactory struct {
+	senderRules                    []pruning.KeepRule
+	receiverRules                  []pruning.KeepRule
+	retryWait                      time.Duration
+	considerSnapAtCursorReplicated bool
+	promPruneSecs                  *prometheus.HistogramVec
+}
+
+type LocalPrunerFactory struct {
+	keepRules     []pruning.KeepRule
+	retryWait     time.Duration
+	promPruneSecs *prometheus.HistogramVec
+}
+
+func NewLocalPrunerFactory(in config.PruningLocal, promPruneSecs *prometheus.HistogramVec) (*LocalPrunerFactory, error) {
+	rules, err := pruning.RulesFromConfig(in.Keep)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot build pruning rules")
+	}
+	for _, r := range in.Keep {
+		if _, ok := r.Ret.(*config.PruneKeepNotReplicated); ok {
+			// rule NotReplicated  for a local pruner doesn't make sense
+			// because no replication happens with that job type
+			return nil, fmt.Errorf("single-site pruner cannot support `not_replicated` keep rule")
+		}
+	}
+	f := &LocalPrunerFactory{
+		keepRules:     rules,
+		retryWait:     envconst.Duration("ZREPL_PRUNER_RETRY_INTERVAL", 10*time.Second),
+		promPruneSecs: promPruneSecs,
+	}
+	return f, nil
+}
+
+func NewPrunerFactory(in config.PruningSenderReceiver, promPruneSecs *prometheus.HistogramVec) (*PrunerFactory, error) {
+	keepRulesReceiver, err := pruning.RulesFromConfig(in.KeepReceiver)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot build receiver pruning rules")
+	}
+
+	keepRulesSender, err := pruning.RulesFromConfig(in.KeepSender)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot build sender pruning rules")
+	}
+
+	considerSnapAtCursorReplicated := false
+	for _, r := range in.KeepSender {
+		knr, ok := r.Ret.(*config.PruneKeepNotReplicated)
+		if !ok {
+			continue
+		}
+		considerSnapAtCursorReplicated = considerSnapAtCursorReplicated || !knr.KeepSnapshotAtCursor
+	}
+	f := &PrunerFactory{
+		senderRules:                    keepRulesSender,
+		receiverRules:                  keepRulesReceiver,
+		retryWait:                      envconst.Duration("ZREPL_PRUNER_RETRY_INTERVAL", 10*time.Second),
+		considerSnapAtCursorReplicated: considerSnapAtCursorReplicated,
+		promPruneSecs:                  promPruneSecs,
+	}
+	return f, nil
+}
+
+func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, receiver History) *Pruner {
+	p := &Pruner{
+		args: args{
+			WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "sender")),
+			target,
+			receiver,
+			f.senderRules,
+			f.retryWait,
+			f.considerSnapAtCursorReplicated,
+			f.promPruneSecs.WithLabelValues("sender"),
+		},
+		state: Plan,
+	}
+	return p
+}
+
+func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target, receiver History) *Pruner {
+	p := &Pruner{
+		args: args{
+			WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "receiver")),
+			target,
+			receiver,
+			f.receiverRules,
+			f.retryWait,
+			false, // senseless here anyways
+			f.promPruneSecs.WithLabelValues("receiver"),
+		},
+		state: Plan,
+	}
+	return p
+}
+
+func (f *LocalPrunerFactory) BuildLocalPruner(ctx context.Context, target Target, receiver History) *Pruner {
+	p := &Pruner{
+		args: args{
+			ctx,
+			target,
+			receiver,
+			f.keepRules,
+			f.retryWait,
+			false, // considerSnapAtCursorReplicated is not relevant for local pruning
+			f.promPruneSecs.WithLabelValues("local"),
+		},
+		state: Plan,
+	}
+	return p
+}
+
+//go:generate enumer -type=State
+type State int
+
+const (
+	Plan State = 1 << iota
+	PlanErr
+	Exec
+	ExecErr
+	Done
+)
+
+type updater func(func(*Pruner))
+
+func (p *Pruner) Prune() {
+	p.prune(p.args)
+}
+
+func (p *Pruner) prune(args args) {
+	u := func(f func(*Pruner)) {
+		p.mtx.Lock()
+		defer p.mtx.Unlock()
+		f(p)
+	}
+	// TODO support automatic retries
+	// It is advisable to merge this code with package replication/driver before
+	// That will likely require re-modelling struct fs like replication/driver.attempt,
+	// including figuring out how to resume a plan after being interrupted by network errors
+	// The non-retrying code in this package should move straight to replication/logic.
+	doOneAttempt(&args, u)
+}
+
+type Report struct {
+	State              string
+	Error              string
+	Pending, Completed []FSReport
+}
+
+type FSReport struct {
+	Filesystem                string
+	SnapshotList, DestroyList []SnapshotReport
+	SkipReason                FSSkipReason
+	LastError                 string
+}
+
+type SnapshotReport struct {
+	Name       string
+	Replicated bool
+	Date       time.Time
+}
+
+func (p *Pruner) Report() *Report {
+	p.mtx.Lock()
+	defer p.mtx.Unlock()
+
+	r := Report{State: p.state.String()}
+
+	if p.err != nil {
+		r.Error = p.err.Error()
+	}
+
+	if p.execQueue != nil {
+		r.Pending, r.Completed = p.execQueue.Report()
+	}
+
+	return &r
+}
+
+func (p *Pruner) State() State {
+	p.mtx.Lock()
+	defer p.mtx.Unlock()
+	return p.state
+}
+
+type fs struct {
+	path string
+
+	// permanent error during planning
+	planErr        error
+	planErrContext string
+
+	// if != "", the fs was skipped for planning and the field
+	// contains the reason
+	skipReason FSSkipReason
+
+	// snapshots presented by target
+	// (type snapshot)
+	snaps []pruning.Snapshot
+	// destroy list returned by pruning.PruneSnapshots(snaps)
+	// (type snapshot)
+	destroyList []pruning.Snapshot
+
+	mtx sync.RWMutex
+
+	// only during Exec state, also used by execQueue
+	execErrLast error
+}
+
+type FSSkipReason string
+
+const (
+	NotSkipped                   = ""
+	SkipPlaceholder              = "filesystem is placeholder"
+	SkipNoCorrespondenceOnSender = "filesystem has no correspondence on sender"
+)
+
+func (r FSSkipReason) NotSkipped() bool {
+	return r == NotSkipped
+}
+
+func (f *fs) Report() FSReport {
+	f.mtx.Lock()
+	defer f.mtx.Unlock()
+
+	r := FSReport{}
+	r.Filesystem = f.path
+	r.SkipReason = f.skipReason
+	if !r.SkipReason.NotSkipped() {
+		return r
+	}
+
+	if f.planErr != nil {
+		r.LastError = f.planErr.Error()
+	} else if f.execErrLast != nil {
+		r.LastError = f.execErrLast.Error()
+	}
+
+	r.SnapshotList = make([]SnapshotReport, len(f.snaps))
+	for i, snap := range f.snaps {
+		r.SnapshotList[i] = snap.(snapshot).Report()
+	}
+
+	r.DestroyList = make([]SnapshotReport, len(f.destroyList))
+	for i, snap := range f.destroyList {
+		r.DestroyList[i] = snap.(snapshot).Report()
+	}
+
+	return r
+}
+
+type snapshot struct {
+	replicated bool
+	date       time.Time
+	fsv        *pdu.FilesystemVersion
+}
+
+func (s snapshot) Report() SnapshotReport {
+	return SnapshotReport{
+		Name:       s.Name(),
+		Replicated: s.Replicated(),
+		Date:       s.Date(),
+	}
+}
+
+var _ pruning.Snapshot = snapshot{}
+
+func (s snapshot) Name() string { return s.fsv.Name }
+
+func (s snapshot) Replicated() bool { return s.replicated }
+
+func (s snapshot) Date() time.Time { return s.date }
+
+func doOneAttempt(a *args, u updater) {
+
+	ctx, target, receiver := a.ctx, a.target, a.receiver
+
+	sfssres, err := receiver.ListFilesystems(ctx, &pdu.ListFilesystemReq{})
+	if err != nil {
+		u(func(p *Pruner) {
+			p.state = PlanErr
+			p.err = err
+		})
+		return
+	}
+	sfss := make(map[string]*pdu.Filesystem)
+	for _, sfs := range sfssres.GetFilesystems() {
+		sfss[sfs.GetPath()] = sfs
+	}
+
+	tfssres, err := target.ListFilesystems(ctx, &pdu.ListFilesystemReq{})
+	if err != nil {
+		u(func(p *Pruner) {
+			p.state = PlanErr
+			p.err = err
+		})
+		return
+	}
+	tfss := tfssres.GetFilesystems()
+
+	pfss := make([]*fs, len(tfss))
+tfss_loop:
+	for i, tfs := range tfss {
+
+		l := GetLogger(ctx).WithField("fs", tfs.Path)
+		l.Debug("plan filesystem")
+
+		pfs := &fs{
+			path: tfs.Path,
+		}
+		pfss[i] = pfs
+
+		if tfs.GetIsPlaceholder() {
+			pfs.skipReason = SkipPlaceholder
+			l.WithField("skip_reason", pfs.skipReason).Debug("skipping filesystem")
+			continue
+		} else if sfs := sfss[tfs.GetPath()]; sfs == nil {
+			pfs.skipReason = SkipNoCorrespondenceOnSender
+			l.WithField("skip_reason", pfs.skipReason).WithField("sfs", sfs.GetPath()).Debug("skipping filesystem")
+			continue
+		}
+
+		pfsPlanErrAndLog := func(err error, message string) {
+			t := fmt.Sprintf("%T", err)
+			pfs.planErr = err
+			pfs.planErrContext = message
+			l.WithField("orig_err_type", t).WithError(err).Error(fmt.Sprintf("%s: plan error, skipping filesystem", message))
+		}
+
+		tfsvsres, err := target.ListFilesystemVersions(ctx, &pdu.ListFilesystemVersionsReq{Filesystem: tfs.Path})
+		if err != nil {
+			pfsPlanErrAndLog(err, "cannot list filesystem versions")
+			continue tfss_loop
+		}
+		tfsvs := tfsvsres.GetVersions()
+		// no progress here since we could run in a live-lock (must have used target AND receiver before progress)
+
+		pfs.snaps = make([]pruning.Snapshot, 0, len(tfsvs))
+
+		rcReq := &pdu.ReplicationCursorReq{
+			Filesystem: tfs.Path,
+			Op: &pdu.ReplicationCursorReq_Get{
+				Get: &pdu.ReplicationCursorReq_GetOp{},
+			},
+		}
+		rc, err := receiver.ReplicationCursor(ctx, rcReq)
+		if err != nil {
+			pfsPlanErrAndLog(err, "cannot get replication cursor bookmark")
+			continue tfss_loop
+		}
+		if rc.GetNotexist() {
+			err := errors.New("replication cursor bookmark does not exist (one successful replication is required before pruning works)")
+			pfsPlanErrAndLog(err, "")
+			continue tfss_loop
+		}
+
+		// scan from older to newer, all snapshots older than cursor are interpreted as replicated
+		sort.Slice(tfsvs, func(i, j int) bool {
+			return tfsvs[i].CreateTXG < tfsvs[j].CreateTXG
+		})
+
+		haveCursorSnapshot := false
+		for _, tfsv := range tfsvs {
+			if tfsv.Type != pdu.FilesystemVersion_Snapshot {
+				continue
+			}
+			if tfsv.Guid == rc.GetGuid() {
+				haveCursorSnapshot = true
+			}
+		}
+		preCursor := haveCursorSnapshot
+		for _, tfsv := range tfsvs {
+			if tfsv.Type != pdu.FilesystemVersion_Snapshot {
+				continue
+			}
+			creation, err := tfsv.CreationAsTime()
+			if err != nil {
+				err := fmt.Errorf("%s: %s", tfsv.RelName(), err)
+				pfsPlanErrAndLog(err, "fs version with invalid creation date")
+				continue tfss_loop
+			}
+			// note that we cannot use CreateTXG because target and receiver could be on different pools
+			atCursor := tfsv.Guid == rc.GetGuid()
+			preCursor = preCursor && !atCursor
+			pfs.snaps = append(pfs.snaps, snapshot{
+				replicated: preCursor || (a.considerSnapAtCursorReplicated && atCursor),
+				date:       creation,
+				fsv:        tfsv,
+			})
+		}
+		if preCursor {
+			pfsPlanErrAndLog(fmt.Errorf("replication cursor not found in prune target filesystem versions"), "")
+			continue tfss_loop
+		}
+
+		// Apply prune rules
+		pfs.destroyList = pruning.PruneSnapshots(pfs.snaps, a.rules)
+	}
+
+	u(func(pruner *Pruner) {
+		pruner.execQueue = newExecQueue(len(pfss))
+		for _, pfs := range pfss {
+			pruner.execQueue.Put(pfs, nil, false)
+		}
+		pruner.state = Exec
+	})
+
+	for {
+		var pfs *fs
+		u(func(pruner *Pruner) {
+			pfs = pruner.execQueue.Pop()
+		})
+		if pfs == nil {
+			break
+		}
+		doOneAttemptExec(a, u, pfs)
+	}
+
+	var rep *Report
+	{
+		// must not hold lock for report
+		var pruner *Pruner
+		u(func(p *Pruner) {
+			pruner = p
+		})
+		rep = pruner.Report()
+	}
+	u(func(p *Pruner) {
+		if len(rep.Pending) > 0 {
+			panic("queue should not have pending items at this point")
+		}
+		hadErr := false
+		for _, fsr := range rep.Completed {
+			hadErr = hadErr || fsr.SkipReason.NotSkipped() && fsr.LastError != ""
+		}
+		if hadErr {
+			p.state = ExecErr
+		} else {
+			p.state = Done
+		}
+	})
+
+}
+
+// attempts to exec pfs, puts it back into the queue with the result
+func doOneAttemptExec(a *args, u updater, pfs *fs) {
+
+	destroyList := make([]*pdu.FilesystemVersion, len(pfs.destroyList))
+	for i := range destroyList {
+		destroyList[i] = pfs.destroyList[i].(snapshot).fsv
+		GetLogger(a.ctx).
+			WithField("fs", pfs.path).
+			WithField("destroy_snap", destroyList[i].Name).
+			Debug("policy destroys snapshot")
+	}
+	req := pdu.DestroySnapshotsReq{
+		Filesystem: pfs.path,
+		Snapshots:  destroyList,
+	}
+	GetLogger(a.ctx).WithField("fs", pfs.path).Debug("destroying snapshots")
+	res, err := a.target.DestroySnapshots(a.ctx, &req)
+	if err != nil {
+		u(func(pruner *Pruner) {
+			pruner.execQueue.Put(pfs, err, false)
+		})
+		return
+	}
+	// check if all snapshots were destroyed
+	destroyResults := make(map[string]*pdu.DestroySnapshotRes)
+	for _, fsres := range res.Results {
+		destroyResults[fsres.Snapshot.Name] = fsres
+	}
+	err = nil
+	destroyFails := make([]*pdu.DestroySnapshotRes, 0)
+	for _, reqDestroy := range destroyList {
+		res, ok := destroyResults[reqDestroy.Name]
+		if !ok {
+			err = fmt.Errorf("missing destroy-result for %s", reqDestroy.RelName())
+			break
+		} else if res.Error != "" {
+			destroyFails = append(destroyFails, res)
+		}
+	}
+	if err == nil && len(destroyFails) > 0 {
+		names := make([]string, len(destroyFails))
+		pairs := make([]string, len(destroyFails))
+		allSame := true
+		lastMsg := destroyFails[0].Error
+		for i := 0; i < len(destroyFails); i++ {
+			allSame = allSame && destroyFails[i].Error == lastMsg
+			relname := destroyFails[i].Snapshot.RelName()
+			names[i] = relname
+			pairs[i] = fmt.Sprintf("(%s: %s)", relname, destroyFails[i].Error)
+		}
+		if allSame {
+			err = fmt.Errorf("destroys failed %s: %s",
+				strings.Join(names, ", "), lastMsg)
+		} else {
+			err = fmt.Errorf("destroys failed: %s", strings.Join(pairs, ", "))
+		}
+	}
+	u(func(pruner *Pruner) {
+		pruner.execQueue.Put(pfs, err, err == nil)
+	})
+	if err != nil {
+		GetLogger(a.ctx).WithError(err).Error("target could not destroy snapshots")
+		return
+	}
+}

daemon/pruner/pruner_queue.go

@@ -0,0 +1,82 @@
+package pruner
+
+import (
+	"sort"
+	"strings"
+	"sync"
+)
+
+type execQueue struct {
+	mtx                sync.Mutex
+	pending, completed []*fs
+}
+
+func newExecQueue(cap int) *execQueue {
+	q := execQueue{
+		pending:   make([]*fs, 0, cap),
+		completed: make([]*fs, 0, cap),
+	}
+	return &q
+}
+
+func (q *execQueue) Report() (pending, completed []FSReport) {
+	q.mtx.Lock()
+	defer q.mtx.Unlock()
+
+	pending = make([]FSReport, len(q.pending))
+	for i, fs := range q.pending {
+		pending[i] = fs.Report()
+	}
+	completed = make([]FSReport, len(q.completed))
+	for i, fs := range q.completed {
+		completed[i] = fs.Report()
+	}
+
+	return pending, completed
+}
+
+func (q *execQueue) HasCompletedFSWithErrors() bool {
+	q.mtx.Lock()
+	defer q.mtx.Unlock()
+	for _, fs := range q.completed {
+		if fs.execErrLast != nil {
+			return true
+		}
+	}
+	return false
+}
+
+func (q *execQueue) Pop() *fs {
+	if len(q.pending) == 0 {
+		return nil
+	}
+	fs := q.pending[0]
+	q.pending = q.pending[1:]
+	return fs
+}
+
+func (q *execQueue) Put(fs *fs, err error, done bool) {
+	fs.mtx.Lock()
+	fs.execErrLast = err
+	if done || err != nil {
+		fs.mtx.Unlock()
+		q.mtx.Lock()
+		q.completed = append(q.completed, fs)
+		q.mtx.Unlock()
+		return
+	}
+	fs.mtx.Unlock()
+
+	q.mtx.Lock()
+	// inefficient priority q
+	q.pending = append(q.pending, fs)
+	sort.SliceStable(q.pending, func(i, j int) bool {
+		q.pending[i].mtx.Lock()
+		defer q.pending[i].mtx.Unlock()
+		q.pending[j].mtx.Lock()
+		defer q.pending[j].mtx.Unlock()
+		return strings.Compare(q.pending[i].path, q.pending[j].path) == -1
+	})
+	q.mtx.Unlock()
+
+}

daemon/pruner/state_enumer.go

@@ -0,0 +1,71 @@
+// Code generated by "enumer -type=State"; DO NOT EDIT.
+
+package pruner
+
+import (
+	"fmt"
+)
+
+const (
+	_StateName_0 = "PlanPlanErr"
+	_StateName_1 = "Exec"
+	_StateName_2 = "ExecErr"
+	_StateName_3 = "Done"
+)
+
+var (
+	_StateIndex_0 = [...]uint8{0, 4, 11}
+	_StateIndex_1 = [...]uint8{0, 4}
+	_StateIndex_2 = [...]uint8{0, 7}
+	_StateIndex_3 = [...]uint8{0, 4}
+)
+
+func (i State) String() string {
+	switch {
+	case 1 <= i && i <= 2:
+		i -= 1
+		return _StateName_0[_StateIndex_0[i]:_StateIndex_0[i+1]]
+	case i == 4:
+		return _StateName_1
+	case i == 8:
+		return _StateName_2
+	case i == 16:
+		return _StateName_3
+	default:
+		return fmt.Sprintf("State(%d)", i)
+	}
+}
+
+var _StateValues = []State{1, 2, 4, 8, 16}
+
+var _StateNameToValueMap = map[string]State{
+	_StateName_0[0:4]:  1,
+	_StateName_0[4:11]: 2,
+	_StateName_1[0:4]:  4,
+	_StateName_2[0:7]:  8,
+	_StateName_3[0:4]:  16,
+}
+
+// StateString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func StateString(s string) (State, error) {
+	if val, ok := _StateNameToValueMap[s]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to State values", s)
+}
+
+// StateValues returns all values of the enum
+func StateValues() []State {
+	return _StateValues
+}
+
+// IsAState returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i State) IsAState() bool {
+	for _, v := range _StateValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}

daemon/snapper/snapper.go

@@ -0,0 +1,389 @@
+package snapper
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+//go:generate stringer -type=SnapState
+type SnapState uint
+
+const (
+	SnapPending SnapState = 1 << iota
+	SnapStarted
+	SnapDone
+	SnapError
+)
+
+type snapProgress struct {
+	state SnapState
+
+	// SnapStarted, SnapDone, SnapError
+	name    string
+	startAt time.Time
+
+	// SnapDone
+	doneAt time.Time
+
+	// SnapErr
+	err error
+}
+
+type args struct {
+	ctx            context.Context
+	log            Logger
+	prefix         string
+	interval       time.Duration
+	fsf            *filters.DatasetMapFilter
+	snapshotsTaken chan<- struct{}
+}
+
+type Snapper struct {
+	args args
+
+	mtx   sync.Mutex
+	state State
+
+	// set in state Plan, used in Waiting
+	lastInvocation time.Time
+
+	// valid for state Snapshotting
+	plan map[*zfs.DatasetPath]snapProgress
+
+	// valid for state SyncUp and Waiting
+	sleepUntil time.Time
+
+	// valid for state Err
+	err error
+}
+
+//go:generate stringer -type=State
+type State uint
+
+const (
+	SyncUp State = 1 << iota
+	SyncUpErrWait
+	Planning
+	Snapshotting
+	Waiting
+	ErrorWait
+	Stopped
+)
+
+func (s State) sf() state {
+	m := map[State]state{
+		SyncUp:        syncUp,
+		SyncUpErrWait: wait,
+		Planning:      plan,
+		Snapshotting:  snapshot,
+		Waiting:       wait,
+		ErrorWait:     wait,
+		Stopped:       nil,
+	}
+	return m[s]
+}
+
+type updater func(u func(*Snapper)) State
+type state func(a args, u updater) state
+
+type contextKey int
+
+const (
+	contextKeyLog contextKey = 0
+)
+
+type Logger = logger.Logger
+
+func WithLogger(ctx context.Context, log Logger) context.Context {
+	return context.WithValue(ctx, contextKeyLog, log)
+}
+
+func getLogger(ctx context.Context) Logger {
+	if log, ok := ctx.Value(contextKeyLog).(Logger); ok {
+		return log
+	}
+	return logger.NewNullLogger()
+}
+
+func PeriodicFromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in *config.SnapshottingPeriodic) (*Snapper, error) {
+	if in.Prefix == "" {
+		return nil, errors.New("prefix must not be empty")
+	}
+	if in.Interval <= 0 {
+		return nil, errors.New("interval must be positive")
+	}
+
+	args := args{
+		prefix:   in.Prefix,
+		interval: in.Interval,
+		fsf:      fsf,
+		// ctx and log is set in Run()
+	}
+
+	return &Snapper{state: SyncUp, args: args}, nil
+}
+
+func (s *Snapper) Run(ctx context.Context, snapshotsTaken chan<- struct{}) {
+
+	getLogger(ctx).Debug("start")
+	defer getLogger(ctx).Debug("stop")
+
+	s.args.snapshotsTaken = snapshotsTaken
+	s.args.ctx = ctx
+	s.args.log = getLogger(ctx)
+
+	u := func(u func(*Snapper)) State {
+		s.mtx.Lock()
+		defer s.mtx.Unlock()
+		if u != nil {
+			u(s)
+		}
+		return s.state
+	}
+
+	var st state = syncUp
+
+	for st != nil {
+		pre := u(nil)
+		st = st(s.args, u)
+		post := u(nil)
+		getLogger(ctx).
+			WithField("transition", fmt.Sprintf("%s=>%s", pre, post)).
+			Debug("state transition")
+
+	}
+
+}
+
+func onErr(err error, u updater) state {
+	return u(func(s *Snapper) {
+		s.err = err
+		preState := s.state
+		switch s.state {
+		case SyncUp:
+			s.state = SyncUpErrWait
+		case Planning:
+			fallthrough
+		case Snapshotting:
+			s.state = ErrorWait
+		}
+		s.args.log.WithError(err).WithField("pre_state", preState).WithField("post_state", s.state).Error("snapshotting error")
+	}).sf()
+}
+
+func onMainCtxDone(ctx context.Context, u updater) state {
+	return u(func(s *Snapper) {
+		s.err = ctx.Err()
+		s.state = Stopped
+	}).sf()
+}
+
+func syncUp(a args, u updater) state {
+	u(func(snapper *Snapper) {
+		snapper.lastInvocation = time.Now()
+	})
+	fss, err := listFSes(a.ctx, a.fsf)
+	if err != nil {
+		return onErr(err, u)
+	}
+	syncPoint, err := findSyncPoint(a.log, fss, a.prefix, a.interval)
+	if err != nil {
+		return onErr(err, u)
+	}
+	u(func(s *Snapper) {
+		s.sleepUntil = syncPoint
+	})
+	t := time.NewTimer(time.Until(syncPoint))
+	defer t.Stop()
+	select {
+	case <-t.C:
+		return u(func(s *Snapper) {
+			s.state = Planning
+		}).sf()
+	case <-a.ctx.Done():
+		return onMainCtxDone(a.ctx, u)
+	}
+}
+
+func plan(a args, u updater) state {
+	u(func(snapper *Snapper) {
+		snapper.lastInvocation = time.Now()
+	})
+	fss, err := listFSes(a.ctx, a.fsf)
+	if err != nil {
+		return onErr(err, u)
+	}
+
+	plan := make(map[*zfs.DatasetPath]snapProgress, len(fss))
+	for _, fs := range fss {
+		plan[fs] = snapProgress{state: SnapPending}
+	}
+	return u(func(s *Snapper) {
+		s.state = Snapshotting
+		s.plan = plan
+	}).sf()
+}
+
+func snapshot(a args, u updater) state {
+
+	var plan map[*zfs.DatasetPath]snapProgress
+	u(func(snapper *Snapper) {
+		plan = snapper.plan
+	})
+
+	hadErr := false
+	// TODO channel programs -> allow a little jitter?
+	for fs, progress := range plan {
+		suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
+		snapname := fmt.Sprintf("%s%s", a.prefix, suffix)
+
+		l := a.log.
+			WithField("fs", fs.ToString()).
+			WithField("snap", snapname)
+
+		u(func(snapper *Snapper) {
+			progress.name = snapname
+			progress.startAt = time.Now()
+			progress.state = SnapStarted
+		})
+
+		l.Debug("create snapshot")
+		err := zfs.ZFSSnapshot(fs, snapname, false)
+		if err != nil {
+			hadErr = true
+			l.WithError(err).Error("cannot create snapshot")
+		}
+		doneAt := time.Now()
+
+		u(func(snapper *Snapper) {
+			progress.doneAt = doneAt
+			progress.state = SnapDone
+			if err != nil {
+				progress.state = SnapError
+				progress.err = err
+			}
+		})
+	}
+
+	select {
+	case a.snapshotsTaken <- struct{}{}:
+	default:
+		if a.snapshotsTaken != nil {
+			a.log.Warn("callback channel is full, discarding snapshot update event")
+		}
+	}
+
+	return u(func(snapper *Snapper) {
+		if hadErr {
+			snapper.state = ErrorWait
+			snapper.err = errors.New("one or more snapshots could not be created, check logs for details")
+		} else {
+			snapper.state = Waiting
+		}
+	}).sf()
+}
+
+func wait(a args, u updater) state {
+	var sleepUntil time.Time
+	u(func(snapper *Snapper) {
+		lastTick := snapper.lastInvocation
+		snapper.sleepUntil = lastTick.Add(a.interval)
+		sleepUntil = snapper.sleepUntil
+		log := a.log.WithField("sleep_until", sleepUntil).WithField("duration", a.interval)
+		logFunc := log.Debug
+		if snapper.state == ErrorWait || snapper.state == SyncUpErrWait {
+			logFunc = log.Error
+		}
+		logFunc("enter wait-state after error")
+	})
+
+	t := time.NewTimer(time.Until(sleepUntil))
+	defer t.Stop()
+
+	select {
+	case <-t.C:
+		return u(func(snapper *Snapper) {
+			snapper.state = Planning
+		}).sf()
+	case <-a.ctx.Done():
+		return onMainCtxDone(a.ctx, u)
+	}
+}
+
+func listFSes(ctx context.Context, mf *filters.DatasetMapFilter) (fss []*zfs.DatasetPath, err error) {
+	return zfs.ZFSListMapping(ctx, mf)
+}
+
+func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval time.Duration) (syncPoint time.Time, err error) {
+	type snapTime struct {
+		ds   *zfs.DatasetPath
+		time time.Time
+	}
+
+	if len(fss) == 0 {
+		return time.Now(), nil
+	}
+
+	snaptimes := make([]snapTime, 0, len(fss))
+
+	now := time.Now()
+
+	log.Debug("examine filesystem state")
+	for _, d := range fss {
+
+		l := log.WithField("fs", d.ToString())
+
+		fsvs, err := zfs.ZFSListFilesystemVersions(d, filters.NewTypedPrefixFilter(prefix, zfs.Snapshot))
+		if err != nil {
+			l.WithError(err).Error("cannot list filesystem versions")
+			continue
+		}
+		if len(fsvs) <= 0 {
+			l.WithField("prefix", prefix).Debug("no filesystem versions with prefix")
+			continue
+		}
+
+		// Sort versions by creation
+		sort.SliceStable(fsvs, func(i, j int) bool {
+			return fsvs[i].CreateTXG < fsvs[j].CreateTXG
+		})
+
+		latest := fsvs[len(fsvs)-1]
+		l.WithField("creation", latest.Creation).
+			Debug("found latest snapshot")
+
+		since := now.Sub(latest.Creation)
+		if since < 0 {
+			l.WithField("snapshot", latest.Name).
+				WithField("creation", latest.Creation).
+				Error("snapshot is from the future")
+			continue
+		}
+		next := now
+		if since < interval {
+			next = latest.Creation.Add(interval)
+		}
+		snaptimes = append(snaptimes, snapTime{d, next})
+	}
+
+	if len(snaptimes) == 0 {
+		snaptimes = append(snaptimes, snapTime{nil, now})
+	}
+
+	sort.Slice(snaptimes, func(i, j int) bool {
+		return snaptimes[i].time.Before(snaptimes[j].time)
+	})
+
+	return snaptimes[0].time, nil
+
+}

daemon/snapper/snapper_all.go

@@ -0,0 +1,40 @@
+package snapper
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/filters"
+)
+
+// FIXME: properly abstract snapshotting:
+//   - split up things that trigger snapshotting from the mechanism
+//     - timer-based trigger (periodic)
+//     - call from control socket (manual)
+//     - mixed modes?
+//   - support a `zrepl snapshot JOBNAME` subcommand for config.SnapshottingManual
+type PeriodicOrManual struct {
+	s *Snapper
+}
+
+func (s *PeriodicOrManual) Run(ctx context.Context, wakeUpCommon chan<- struct{}) {
+	if s.s != nil {
+		s.s.Run(ctx, wakeUpCommon)
+	}
+}
+
+func FromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in config.SnapshottingEnum) (*PeriodicOrManual, error) {
+	switch v := in.Ret.(type) {
+	case *config.SnapshottingPeriodic:
+		snapper, err := PeriodicFromConfig(g, fsf, v)
+		if err != nil {
+			return nil, err
+		}
+		return &PeriodicOrManual{snapper}, nil
+	case *config.SnapshottingManual:
+		return &PeriodicOrManual{}, nil
+	default:
+		return nil, fmt.Errorf("unknown snapshotting type %T", v)
+	}
+}

daemon/snapper/snapstate_string.go

@@ -0,0 +1,29 @@
+// Code generated by "stringer -type=SnapState"; DO NOT EDIT.
+
+package snapper
+
+import "strconv"
+
+const (
+	_SnapState_name_0 = "SnapPendingSnapStarted"
+	_SnapState_name_1 = "SnapDone"
+	_SnapState_name_2 = "SnapError"
+)
+
+var (
+	_SnapState_index_0 = [...]uint8{0, 11, 22}
+)
+
+func (i SnapState) String() string {
+	switch {
+	case 1 <= i && i <= 2:
+		i -= 1
+		return _SnapState_name_0[_SnapState_index_0[i]:_SnapState_index_0[i+1]]
+	case i == 4:
+		return _SnapState_name_1
+	case i == 8:
+		return _SnapState_name_2
+	default:
+		return "SnapState(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+}

daemon/snapper/state_string.go

@@ -0,0 +1,38 @@
+// Code generated by "stringer -type=State"; DO NOT EDIT.
+
+package snapper
+
+import "strconv"
+
+const (
+	_State_name_0 = "SyncUpSyncUpErrWait"
+	_State_name_1 = "Planning"
+	_State_name_2 = "Snapshotting"
+	_State_name_3 = "Waiting"
+	_State_name_4 = "ErrorWait"
+	_State_name_5 = "Stopped"
+)
+
+var (
+	_State_index_0 = [...]uint8{0, 6, 19}
+)
+
+func (i State) String() string {
+	switch {
+	case 1 <= i && i <= 2:
+		i -= 1
+		return _State_name_0[_State_index_0[i]:_State_index_0[i+1]]
+	case i == 4:
+		return _State_name_1
+	case i == 8:
+		return _State_name_2
+	case i == 16:
+		return _State_name_3
+	case i == 32:
+		return _State_name_4
+	case i == 64:
+		return _State_name_5
+	default:
+		return "State(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+}

dist/grafana/grafana-prometheus-zrepl-0.1.json

@@ -0,0 +1,595 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "$$hashKey": "object:3351",
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 1,
+  "id": 7,
+  "iteration": 1552746418826,
+  "links": [],
+  "panels": [
+    {
+      "content": "# zrepl Prometheus Metrics\n\nzrepl exposes Prometheus metrics and ships with this Grafana dashboard.\nThe exported metrics are suitable for health checks:\n\n* The log should generally be warning & error-free\n  * The `Log Messages that require attention` graph visualizes log message counts indicating problems.\n* The number of goroutines should not grow unboundedly over time.\n  * During replication, the number of goroutines can be way higher than during idle time.\n  * If the goroutine count grows with each replication, there is clearly a goroutine leak. Please open a bug report.\n* The sys memory consumption should not grow unboundedly over time.\n  * Note that the Go runtime pre-allocates some of its heap from the OS.\n  * zrepl actually uses much less memory than allocated from the OS.\n  * Since Go 1.11, Go pre-allocates more aggressively.\n* Monitor that some data is replicated, although that metric does not guarantee that replication was successful.\n\n**In general, note that the exported metrics are not stable unless declared otherwise.**",
+      "gridPos": {
+        "h": 9,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 35,
+      "mode": "markdown",
+      "title": "Panel Title",
+      "type": "text"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 1,
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 9
+      },
+      "id": 15,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "targets": [
+        {
+          "$$hashKey": "object:3436",
+          "expr": "up{job='$prom_job_name'}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "zrepl Instances Up",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": "5",
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ]
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 1,
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 9
+      },
+      "id": 22,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "increase(zrepl_daemon_log_entries{job='$prom_job_name',level=~'warn|error'}[$__interval])",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Log Messages that require attention",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ]
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 1,
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 18
+      },
+      "id": 33,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [
+        {
+          "alias": "/replicated bytes in last.*/",
+          "yaxis": 2
+        }
+      ],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(zrepl_replication_bytes_replicated{job='$prom_job_name'}[$__interval])) by (zrepl_job)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "replication data rate zrepl_job={{zrepl_job}}",
+          "refId": "A"
+        },
+        {
+          "expr": "sum(increase(zrepl_replication_bytes_replicated{job='$prom_job_name'}[10m])) by (zrepl_job)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "replicated bytes in last 10min zrepl_job={{zrepl_job}}",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Replication Data Rate and Volume(integrates last 10min)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "Bps",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "bytes",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ]
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 1,
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 18
+      },
+      "id": 23,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(increase(zrepl_daemon_log_entries{job='$prom_job_name',zrepl_job=~\"^[^_].*\"}[$__interval])) by (instance,zrepl_job)",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Log Activity (without internal jobs)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ]
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 1,
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 27
+      },
+      "id": 17,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "$$hashKey": "object:3535",
+          "expr": "go_memstats_sys_bytes{job='$prom_job_name'}",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 1,
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Memory Allocated by the Go runtime from the OS (should not grow unboundedly)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ]
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 1,
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 27
+      },
+      "id": 19,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "go_goroutines{job='$prom_job_name'}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "number of goroutines (should not grow unboundedly)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ]
+    }
+  ],
+  "refresh": "1m",
+  "schemaVersion": 16,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {
+          "text": "zrepl",
+          "value": "zrepl"
+        },
+        "datasource": "prometheus",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Prometheus Job Name",
+        "multi": false,
+        "name": "prom_job_name",
+        "options": [],
+        "query": "label_values(up, job)",
+        "refresh": 1,
+        "regex": "",
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-2d",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "",
+  "title": "zrepl 0.1",
+  "uid": "xTljn4qmk",
+  "version": 6
+}

dist/systemd/zrepl.service

@@ -0,0 +1,41 @@
+[Unit]
+Description=zrepl daemon
+Documentation=https://zrepl.github.io
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/zrepl --config /etc/zrepl/zrepl.yml daemon
+RuntimeDirectory=zrepl
+RuntimeDirectoryMode=0700
+
+ProtectSystem=strict
+#PrivateDevices=yes # TODO ZFS needs access to /dev/zfs, could we limit this?
+ProtectKernelTunables=yes
+ProtectControlGroups=yes
+PrivateTmp=yes
+#PrivateUsers=yes # TODO Does not work, why?
+ProtectKernelModules=true
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
+RestrictNamespaces=true
+RestrictRealtime=yes
+SystemCallArchitectures=native
+
+# BEGIN ProtectHome
+ProtectHome=read-only   # DEBIAN STRETCH
+# ProtectHome=tmpfs     # FEDORA 28 / 29
+# END ProtectHome
+
+# BEGIN SystemCallFilter
+## BEGIN DEBIAN STRETCH
+SystemCallFilter=~ @mount @cpu-emulation @keyring @module @obsolete @privileged @raw-io @debug @clock @resources
+## END DEBIAN STRETCH
+## BEGIN FEDORA 28/29 
+## Syscall blacklist (should be fairly stable)
+#SystemCallFilter=~ @mount @aio @cpu-emulation @keyring @memlock @module @obsolete @privileged @raw-io @reboot @setuid @swap @sync @timer @debug @clock @chown @resources
+## Syscall whitelist (not sure how stable)
+#SystemCallFilter=@default @file-system @process @basic-io @ipc @network-io @signal @io-event brk mprotect sched_getaffinity ioctl getrandom
+## END END FEDORA 28/29
+# END SystemCallFilter
+
+[Install]
+WantedBy=multi-user.target

docs/changelog.rst

@@ -1,68 +1,135 @@
-.. |break_config| replace:: **[BREAK]**
+.. |break_config| replace:: **[CONFIG]**
 .. |break| replace:: **[BREAK]**
 .. |bugfix| replace:: [BUG]
+.. |docs| replace:: [DOCS]
 .. |feature| replace:: [FEATURE]
+.. |mig| replace:: **[MIGRATION]**
+
+.. _changelog:
 
 Changelog
 =========
 
-The changelog summarized bugfixes that are deemed relevant for users.
+The changelog summarizes bugfixes that are deemed relevant for users and package maintainers.
 Developers should consult the git commit log or GitHub issue tracker.
 
-0.0.3
+We use the following annotations for classifying changes:
------
 
-* |break_config| |feature| :issue:`34`: automatic bookmarking of snapshots
+* |break_config| Change that breaks the config.
+  As a package maintainer, make sure to warn your users about config breakage somehow.
+* |break| Change that breaks interoperability or persistent state representation with previous releases.
+  As a package maintainer, make sure to warn your users about config breakage somehow.
+  Note that even updating the package on both sides might not be sufficient, e.g. if persistent state needs to be migrated to a new format.
+* |mig| Migration that must be run by the user.
+* |feature| Change that introduces new functionality.
+* |bugfix| Change that fixes a bug, no regressions or incompatibilities expected.
+* |docs| Change to the documentation.
 
-  * Snapshots are automatically bookmarked and pruning of bookmarks **must** be configured.
+0.1
-  * This breaks existing configuration: ``grid`` :ref:`prune policy <prune-retention-grid>`  specifications require the new ``keep_bookmarks`` parameter.
+---
-  * Make sure to understand the meaning bookmarks have for :ref:`maximum replication downtime <replication-downtime>`.
-  * Example: :sampleconf:`pullbackup/productionhost.yml`
 
-* |break| :commit:`ccd062e`: both sides of a replication setup must be updated and restarted. Otherwise the connecting side will hang and not time out.
+This release is a milestone for zrepl and required significant refactoring if not rewrites of substantial parts of the application.
-* |break_config| :commit:`2bfcfa5`: first outlet in ``global.logging`` is now used for logging meta-errors, for example problems encountered when writing to other outlets.
+It breaks both configuration and transport format, and thus requires manual intervention and updates on both sides of a replication setup.
-* |feature| :issue:`10`: ``zrepl control status`` subcommand
 
-  * Allows inspection of job activity per task and their log output at runtime.
+.. DANGER::
-  * Supports ``--format raw`` option for JSON output, usable for monitoring from scripts.
+   The changes in the pruning system for this release require you to explicitly define **keep rules**:
-
+   for any snapshot that you want to keep, at least one rule must match.
-* |feature| :commit:`d7f3fb9`: subcommand bash completions
+   This is different from previous releases where pruning only affected snapshots with the configured snapshotting prefix.
-
+   Make sure that snapshots to be kept or ignored by zrepl are covered, e.g. by using the ``regex`` keep rule.
-  * Package maintainers should install this as appropriate.
+   :ref:`Learn more in the config docs... <prune>`
-
-* |bugfix| :issue:`61`: fix excessive memory usage
-* |bugfix| :issue:`8` and :issue:`56`: ``ssh+stdinserver`` transport properly reaps SSH child processes
-* |bugfix| :commit:`cef63ac`: ``human`` format now prints non-string values correctly
-* |bugfix| :issue:`26`: slow TCP outlets no longer block the daemon
-
-0.0.2
------
-
-* |break_config| :commit:`b95260f`: ``global.logging`` is no longer a dictionary but a list
-
-* |break_config| :commit:`3e647c1`: ``source`` job field ``datasets`` renamed to ``filesystems``
-
-  * **NOTE**: zrepl will parse missing ``filesystems`` field as an empty filter,
-    i.e. no filesystems are presented to the other side.
-
-* |bugfix| :commit:`72d2885` fix aliasing bug with root `<` subtree wildcard
-
-  * Filesystems paths with final match at blank `s` subtree wildcard are now appended to the target path
-  * Non-root subtree wildcards, e.g. `zroot/foo/bar<` still map directrly onto the target path
-
-* Support days (``d``) and weeks (``w``) in durations
-
-* Docs
-
-  * Ditch Hugo, move to Python Sphinx
-  * Improve & simplify tutorial (single SSH key per installation)
-  * Document pruning policies
-  * Document job types
-  * Document logging
-  * Start updating implementation overview
 
 
-0.0.1
+Notes to Package Maintainers
------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-* Initial release
+* Notify users about config changes and migrations (see changes attributed with |break| and |mig| below)
+* If the daemon crashes, the stack trace produced by the Go runtime and possibly diagnostic output of zrepl will be written to stderr.
+  This behavior is independent from the ``stdout`` outlet type.
+  Please make sure the stderr output of the daemon is captured somewhere.
+  To conserve precious stack traces, make sure that multiple service restarts do not directly discard previous stderr output.
+* Make it obvious for users how to set the ``GOTRACEBACK`` environment variable to ``GOTRACEBACK=crash``.
+  This functionality will cause SIGABRT on panics and can be used to capture a coredump of the panicking process.
+  To that extend, make sure that your package build system, your OS's coredump collection and the Go delve debugger work together.
+  Use your build system to package the Go program in `this tutorial on Go coredumps and the delve debugger <https://rakyll.org/coredumps/>`_ , and make sure the symbol resolution etc. work on coredumps captured from the binary produced by your build system. (Special focus on symbol stripping, etc.)
+* Consider using the ``zrepl configcheck`` subcommand in startup scripts to abort a restart that would fail due to an invalid config.
+
+Changes
+~~~~~~~
+
+* |break| |mig| Placeholder property representation changed
+
+  * The :ref:`placeholder property <replication-placeholder-property>` now uses ``on|off`` as values
+    instead of hashes of the dataset path. This permits renames of the sink filesystem without
+    updating all placeholder properties.
+  * Relevant for 0.0.X-0.1-rc* to 0.1 migrations
+  * Make sure your config is valid with ``zrepl configcheck``
+  * Run ``zrepl migrate 0.0.X:0.1:placeholder``
+
+* |feature| :issue:`55` : Push replication (see :ref:`push job <job-push>` and :ref:`sink job <job-sink>`)
+* |feature| :ref:`TCP Transport <transport-tcp>`
+* |feature| :ref:`TCP + TLS client authentication transport <transport-tcp+tlsclientauth>`
+* |feature| :issue:`111`: RPC protocol rewrite
+
+  * |break| Protocol breakage; Update and restart of all zrepl daemons is required.
+  * Use `gRPC <https://grpc.io/>`_ for control RPCs and a custom protocol for bulk data transfer.
+  * Automatic retries for network-temporary errors
+
+    * Limited to errors during replication for this release.
+      Addresses the common problem of ISP-forced reconnection at night, but will become
+      way more useful with resumable send & recv support.
+      Pruning errors are handled per FS, i.e., a prune RPC is attempted at least once per FS.
+
+* |feature| Proper timeout handling for the :ref:`SSH transport <transport-ssh+stdinserver>`
+
+  * |break| Requires Go 1.11 or later.
+  
+* |break| |break_config|: mappings are no longer supported
+
+  * Receiving sides (``pull`` and ``sink`` job) specify a single ``root_fs``.
+    Received filesystems are then stored *per client* in ``${root_fs}/${client_identity}``.
+    See :ref:`job-overview` for details.
+
+* |feature| |break| |break_config| Manual snapshotting + triggering of replication
+
+  * |feature| :issue:`69`: include manually created snapshots in replication
+  * |break_config| ``manual`` and ``periodic`` :ref:`snapshotting types <job-snapshotting-spec>`
+  * |feature| ``zrepl signal wakeup JOB`` subcommand to trigger replication + pruning
+  * |feature| ``zrepl signal reset JOB`` subcommand to abort current replication + pruning
+
+* |feature| |break| |break_config| New pruning system
+
+  * The active side of a replication (pull or push) decides what to prune for both sender and receiver.
+    The RPC protocol is used to execute the destroy operations on the remote side.
+  * New pruning policies (see :ref:`configuration documentation <prune>` )
+
+    * The decision what snapshots shall be pruned is now made based on *keep rules*
+    * |feature| :issue:`68`: keep rule ``not_replicated`` prevents divergence of sender and receiver
+
+  * |feature| |break| Bookmark pruning is no longer necessary
+
+    * Per filesystem, zrepl creates a single bookmark (``#zrepl_replication_cursor``) and moves it forward with the most recent successfully replicated snapshot on the receiving side.
+    * Old bookmarks created by prior versions of zrepl (named like their corresponding snapshot) must be deleted manually.
+    * |break_config| ``keep_bookmarks`` parameter of the ``grid`` keep rule has been removed
+
+* |feature| ``zrepl status`` for live-updating replication progress (it's really cool!)
+* |feature| :ref:`Snapshot- & pruning-only job type <job-snap>` (for local snapshot management)
+* |feature| :issue:`67`: Expose `Prometheus <https://prometheus.io>`_ metrics via HTTP (:ref:`config docs <monitoring-prometheus>`)
+
+  * Compatible Grafana dashboard shipping in ``dist/grafana``
+
+* |break_config| Logging outlet types must be specified using the ``type`` instead of ``outlet`` key
+* |break| :issue:`53`: CLI: ``zrepl control *`` subcommands have been made direct subcommands of ``zrepl *``
+* |bugfix| Goroutine leak on ssh transport connection timeouts
+* |bugfix| :issue:`81` :issue:`77` : handle failed accepts correctly (``source`` job)
+* |bugfix| :issue:`100`: fix incompatibility with ZoL 0.8
+* |feature| :issue:`115`: logging: configurable syslog facility
+* |feature| Systemd unit file in ``dist/systemd``
+
+.. |lastrelease| replace:: 0.0.3
+
+Previous Releases
+-----------------
+
+.. NOTE::
+    Due to limitations in our documentation system, we only show the changelog since the last release and the time this documentation is built.
+    For the changelog of previous releases, use the version selection in the hosted version of these docs at `zrepl.github.io <https://zrepl.github.io>`_.