Pruner: parallel pruning

refs #62

.circleci/config.yml

@@ -1,102 +0,0 @@
-version: 2.0
-workflows:
-  version: 2
-  build:
-    jobs:
-      - build-1.11
-      - build-1.12
-      - build-latest
-jobs:
-  # build-latest serves as the template
-  # we use YAML anchors & aliases to exchange the docker image (and hence Go version used for the build)
-  build-latest: &build-latest
-    description: Builds zrepl
-    parameters:
-      image:
-        description: "the docker image that the job should use"
-        type: string
-    docker:
-      - image: circleci/golang:latest
-    environment:
-      # required by lazy.sh
-      TERM: xterm
-    working_directory: /go/src/github.com/zrepl/zrepl
-    steps:
-      - run:
-          name: Setup environment variables
-          command: |
-            # used by pip (for docs)
-            echo 'export PATH="$HOME/.local/bin:$PATH"' >> $BASH_ENV
-
-      - restore_cache:
-          keys:
-            - source
-            - vendor
-            - protobuf
-
-      - checkout
-
-      - save_cache:
-          key: source
-          paths:
-            - ".git"
-      
-      # install deps
-      - run: wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
-      - run: echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807  protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
-      - run: sudo unzip -d /usr protoc-3.6.1-linux-x86_64.zip
-      - save_cache:
-          key: protobuf
-          paths:
-            - "/usr/include/google/protobuf"
-            
-      - run: sudo apt install python3 python3-pip libgirepository1.0-dev
-      - run: ./lazy.sh devsetup
-
-      - run:
-          name: Install minio client, prepare minio
-          command: |
-            curl -sSL https://dl.minio.io/client/mc/release/linux-amd64/mc -o ${GOPATH}/bin/mc
-            chmod +x ${GOPATH}/bin/mc
-            mc config host add --api s3v4 zrepl-minio https://minio.cschwarz.com ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
-
-      - run: make vendordeps
-      - save_cache:
-          key: vendor
-          paths:
-            - "./vendor"
-            
-      - run: make
-      - run: make vet
-      - run: make test
-      - run: make lint
-      - run: make release
-
-
-      - store_artifacts:
-          path: ./artifacts/release
-          when: always
-
-      - run:
-          shell: /bin/bash -euo pipefail
-          when: always
-          command: |
-            echo "$CIRCLE_BUILD_URL" > ./artifacts/release/cirlceci_build_url
-            mc cp -r artifacts/release "zrepl-minio/zrepl-ci-artifacts/${CIRCLE_SHA1}/${CIRCLE_JOB}/"
-            REPO="zrepl/zrepl"
-            COMMIT="${CIRCLE_SHA1}"
-            JOB_NAME="${CIRCLE_JOB}"
-            curl "https://api.github.com/repos/$REPO/statuses/$COMMIT?access_token=$GITHUB_COMMIT_STATUS_TOKEN" \
-              -H "Content-Type: application/json" \
-              -X POST \
-              -d '{"context":"zrepl/publish-ci-artifacts", "state": "success", "description":"CI Build Artifacts for '"$JOB_NAME"'", "target_url":"https://minio.cschwarz.com/minio/zrepl-ci-artifacts/'"$COMMIT"'/"}'
-
-
-  build-1.11:
-    <<: *build-latest
-    docker:
-      - image: circleci/golang:1.11
-  build-1.12:
-    <<: *build-latest
-    docker:
-    - image: circleci/golang:1.12

.golangci.yml

@@ -1,10 +0,0 @@
-linters:
-  enable:
-  - goimports
-
-issues:
-  exclude-rules:
-    - path: _test\.go
-      linters:
-        - errcheck
-

.travis.yml

@@ -1,70 +0,0 @@
-dist: xenial
-services:
-- docker
-
-env: # for allow_failures: https://docs.travis-ci.com/user/customizing-the-build/
-matrix:
-  include:
-
-  - language: go
-    name: "Build in Docker (docs/installation.rst)"
-    script:
-    - sudo docker build -t zrepl_build -f build.Dockerfile .
-    - |
-      sudo docker run -it --rm \
-        -v "${PWD}:/go/src/github.com/zrepl/zrepl" \
-        --user "$(id -u):$(id -g)" \
-        zrepl_build make vendordeps release
-
-  - &zrepl_build_template
-    language: go
-    go_import_path: github.com/zrepl/zrepl
-    before_install:
-        - wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
-        - echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807  protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
-        - sudo unzip -d /usr protoc-3.6.1-linux-x86_64.zip
-        - ./lazy.sh godep
-        - make vendordeps
-    script:
-      - make
-      - make vet
-      - make test
-      - make lint
-      - make artifacts/zrepl-freebsd-amd64
-      - make artifacts/zrepl-linux-amd64
-      - make artifacts/zrepl-darwin-amd64
-    go:
-    - "1.11"
-
-  - <<: *zrepl_build_template
-    go:
-    - "1.12"
-
-  - <<: *zrepl_build_template
-    go:
-    - "master"
-
-  - &zrepl_docs_template 
-    language: python
-    python:
-    - "3.4"
-    install:
-    - sudo apt-get install libgirepository1.0-dev
-    - pip install -r docs/requirements.txt
-    script:
-    - make docs
-  - <<: *zrepl_docs_template
-    python:
-    - "3.5"
-  - <<: *zrepl_docs_template
-    python:
-    - "3.6"
-  - <<: *zrepl_docs_template
-    python:
-    - "3.7"
-  
-
-  allow_failures:
-  - <<: *zrepl_build_template
-    go:
-    - "master"

Gopkg.toml

@@ -1,13 +1,8 @@
-ignored = [
+ignored = [ "github.com/inconshreveable/mousetrap" ]
-    "github.com/inconshreveable/mousetrap",
-]
 
-required = [
+[[constraint]]
-    "golang.org/x/tools/cmd/stringer",
+  branch = "master"
-    "github.com/alvaroloes/enumer",
+  name = "github.com/ftrvxmtrx/fd"
-    "github.com/golangci/golangci-lint/cmd/golangci-lint",
-    "golang.org/x/tools/cmd/goimports",
-]
 
 [[constraint]]
   branch = "master"
@@ -17,6 +12,14 @@ required = [
   branch = "master"
   name = "github.com/kr/pretty"
 
+[[constraint]]
+  branch = "master"
+  name = "github.com/mitchellh/go-homedir"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/mitchellh/mapstructure"
+
 [[constraint]]
   name = "github.com/pkg/errors"
   version = "0.8.0"
@@ -25,43 +28,26 @@ required = [
   branch = "master"
   name = "github.com/spf13/cobra"
 
+[[constraint]]
+  name = "github.com/spf13/viper"
+  version = "1.0.0"
+
 [[constraint]]
   name = "github.com/stretchr/testify"
   version = "1.1.4"
 
 [[constraint]]
   branch = "v2"
-  name = "github.com/zrepl/yaml-config"
+  name = "github.com/go-yaml/yaml"
 
 [[constraint]]
   name = "github.com/go-logfmt/logfmt"
   version = "*"
 
+[[constraint]]
+  name = "github.com/problame/go-rwccmd"
+  branch = "master"
+
 [[constraint]]
   name = "github.com/problame/go-netssh"
   branch = "master"
-
-[[constraint]]
-  name = "github.com/prometheus/client_golang"
-  branch = "master"
-
-[[constraint]]
-  name = "github.com/golang/protobuf"
-  version = "1"
-
-[[constraint]]
-  name = "github.com/fatih/color"
-  version = "1.7.0"
-
-
-[[constraint]]
-  name = "github.com/gdamore/tcell"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "google.golang.org/grpc"
-  version = "1"
-  
-[[constraint]]
-  version = "1.1.0"
-  name = "github.com/google/uuid"

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2017-2018 Christian Schwarz
+Copyright (c) 2017 Christian Schwarz
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,25 +1,24 @@
-.PHONY: generate build test vet cover release docs docs-clean clean vendordeps format lint
+.PHONY: generate build test vet cover release docs docs-clean clean vendordeps
 .DEFAULT_GOAL := build
 
+ROOT := github.com/zrepl/zrepl
+SUBPKGS := cmd logger rpc util zfs
+
+_TESTPKGS := $(ROOT) $(foreach p,$(SUBPKGS),$(ROOT)/$(p))
+
 ARTIFACTDIR := artifacts
 
-ifdef ZREPL_VERSION
+ifndef ZREPL_VERSION
-    _ZREPL_VERSION := $(ZREPL_VERSION)
+    ZREPL_VERSION := $(shell git describe --dirty 2>/dev/null || echo "ZREPL_BUILD_INVALID_VERSION" )
-endif
+    ifeq ($(ZREPL_VERSION),ZREPL_BUILD_INVALID_VERSION) # can't use .SHELLSTATUS because Debian Stretch is still on gmake 4.1
-ifndef _ZREPL_VERSION
-    _ZREPL_VERSION := $(shell git describe --always --dirty 2>/dev/null || echo "ZREPL_BUILD_INVALID_VERSION" )
-    ifeq ($(_ZREPL_VERSION),ZREPL_BUILD_INVALID_VERSION) # can't use .SHELLSTATUS because Debian Stretch is still on gmake 4.1
         $(error cannot infer variable ZREPL_VERSION using git and variable is not overriden by make invocation)
     endif
 endif
-GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
+GO_LDFLAGS := "-X github.com/zrepl/zrepl/cmd.zreplVersion=$(ZREPL_VERSION)"
 
 GO_BUILD := go build -ldflags $(GO_LDFLAGS)
 
-RELEASE_BINS := $(ARTIFACTDIR)/zrepl-freebsd-amd64
+RELEASE_BINS := $(ARTIFACTDIR)/zrepl-freebsd-amd64 $(ARTIFACTDIR)/zrepl-linux-amd64
-RELEASE_BINS += $(ARTIFACTDIR)/zrepl-linux-amd64
-RELEASE_BINS += $(ARTIFACTDIR)/zrepl-darwin-amd64
-
 RELEASE_NOARCH := $(ARTIFACTDIR)/zrepl-noarch.tar
 THIS_PLATFORM_RELEASE_BIN := $(shell bash -c 'source <(go env) && echo "zrepl-$${GOOS}-$${GOARCH}"' )
 
@@ -27,27 +26,34 @@ vendordeps:
 	dep ensure -v -vendor-only
 
 generate: #not part of the build, must do that manually
-	protoc -I=replication/logic/pdu --go_out=plugins=grpc:replication/logic/pdu replication/logic/pdu/pdu.proto
+	@for pkg in $(_TESTPKGS); do\
-	go generate -x ./...
+		go generate "$$pkg" || exit 1; \
-
+	done;
-format:
-	goimports -srcdir . -local 'github.com/zrepl/zrepl' -w $(shell find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go')
-
-lint:
-	golangci-lint run ./...
 
 build:
 	@echo "INFO: In case of missing dependencies, run 'make vendordeps'"
 	$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl"
 
 test:
-	go test ./...
+	@for pkg in $(_TESTPKGS); do \
+		echo "Testing $$pkg"; \
+		go test "$$pkg" || exit 1; \
+	done;
 
 vet:
-	# for each supported platform to cover conditional compilation
+	@for pkg in $(_TESTPKGS); do \
-	GOOS=linux   go vet ./...
+		echo "Vetting $$pkg"; \
-	GOOS=darwin  go vet ./...
+		go vet "$$pkg" || exit 1; \
-	GOOS=freebsd go vet ./...
+	done;
+
+cover: artifacts
+	@for pkg in $(_TESTPKGS); do \
+		profile="$(ARTIFACTDIR)/cover-$$(basename $$pkg).out"; \
+		go test -coverprofile "$$profile" $$pkg || exit 1; \
+		if [ -f "$$profile" ]; then \
+   			go tool cover -html="$$profile" -o "$${profile}.html" || exit 2; \
+		fi; \
+	done;
 
 $(ARTIFACTDIR):
 	mkdir -p "$@"
@@ -74,13 +80,13 @@ docs-clean:
 
 .PHONY: $(RELEASE_BINS)
 # TODO: two wildcards possible
-$(RELEASE_BINS): $(ARTIFACTDIR)/zrepl-%-amd64: generate $(ARTIFACTDIR) vet test lint
+$(RELEASE_BINS): $(ARTIFACTDIR)/zrepl-%-amd64: generate $(ARTIFACTDIR) vet test
 	@echo "INFO: In case of missing dependencies, run 'make vendordeps'"
 	GOOS=$* GOARCH=amd64   $(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$*-amd64"
 
 $(RELEASE_NOARCH): docs $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_version.txt
 	tar --mtime='1970-01-01' --sort=name \
-		--transform 's/$(ARTIFACTDIR)/zrepl-$(_ZREPL_VERSION)-noarch/' \
+		--transform 's/$(ARTIFACTDIR)/zrepl-$(ZREPL_VERSION)-noarch/' \
 		-acf $@ \
 		$(ARTIFACTDIR)/docs/html \
 		$(ARTIFACTDIR)/bash_completion \
@@ -91,14 +97,10 @@ release: $(RELEASE_BINS) $(RELEASE_NOARCH)
 	mkdir -p "$(ARTIFACTDIR)/release"
 	cp $^ "$(ARTIFACTDIR)/release"
 	cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
-	@# note that we use ZREPL_VERSION and not _ZREPL_VERSION because we want to detect the override
+	@if echo "$(ZREPL_VERSION)" | grep dirty > /dev/null; then\
-	@if git describe --always --dirty 2>/dev/null | grep dirty >/dev/null; then \
+		echo '[WARN] Do not publish the artifacts, make variable ZREPL_VERSION=$(ZREPL_VERSION) indicates they are dirty!'; \
-        echo '[INFO] either git reports checkout is dirty or git is not installed or this is not a git checkout'; \
+		exit 1; \
-		if [ "$(ZREPL_VERSION)" = "" ]; then \
+	fi
-			echo '[WARN] git checkout is dirty and make variable ZREPL_VERSION was not used to override'; \
-			exit 1; \
-		fi; \
-	fi;
 
 clean: docs-clean
 	rm -rf "$(ARTIFACTDIR)"

README.md

@@ -1,12 +1,5 @@
-[![GitHub license](https://img.shields.io/github/license/zrepl/zrepl.svg)](https://github.com/zrepl/zrepl/blob/master/LICENSE)
-[![Language: Go](https://img.shields.io/badge/language-Go-6ad7e5.svg)](https://golang.org/)
-[![User Docs](https://img.shields.io/badge/docs-web-blue.svg)](https://zrepl.github.io)
-[![Donate via PayPal](https://img.shields.io/badge/donate-paypal-yellow.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
-[![Donate via Liberapay](https://img.shields.io/badge/donate-liberapay-yellow.svg)](https://liberapay.com/zrepl/donate)
-[![Twitter](https://img.shields.io/twitter/url/https/github.com/zrepl/zrepl.svg?style=social)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl)
-
 # zrepl
-zrepl is a one-stop ZFS backup & replication solution.
+zrepl is a ZFS filesystem backup & replication solution written in Go.
 
 ## User Documentation
 
@@ -33,18 +26,13 @@ Check out the *Coding Workflow* section below for details.
 * Make sure your distro is compatible with the paths in `docs/installation.rst`.
 * Ship a default config that adheres to your distro's `hier` and logging system.
 * Ship a service manager file and _please_ try to upstream it to this repository.
-  * `dist/systemd` contains a Systemd unit template.
-* Ship other material provided in `./dist`, e.g. in `/usr/share/zrepl/`.
 * Use `make release ZREPL_VERSION='mydistro-1.2.3_1'`
     * Your distro's name and any versioning supplemental to zrepl's (e.g. package revision) should be in this string
 * Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
 
 ## Developer Documentation
 
-zrepl is written in [Go](https://golang.org) and uses [`dep`](https://github.com/golang/dep) to manage dependencies.
+First, use `./lazy.sh devsetup` to install build dependencies and read `docs/installation.rst -> Compiling from Source`.
-The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
-
-To get started, run `./lazy.sh devsetup` to easily install build dependencies and read `docs/installation.rst -> Compiling from Source`.
 
 ### Overall Architecture
 
@@ -54,53 +42,19 @@ Make sure to develop an understanding how zrepl is typically used by studying th
 ### Project Structure
 
 ```
-├── artifacts               # build artifcats generate by make
+├── cmd
-├── cli                     # wrapper around CLI package cobra
+│   ├── sampleconf          # example configuration
-├── client                  # all subcommands that are not `daemon`
-├── config                  # config data types (=> package yaml-config)
-│   └── samples
-├── daemon                  # the implementation of `zrepl daemon` subcommand
-│   ├── filters
-│   ├── job                 # job implementations
-│   ├── logging             # logging outlets + formatters
-│   ├── nethelpers
-│   ├── prometheus
-│   ├── pruner              # pruner implementation
-│   ├── snapper             # snapshotter implementation
 ├── docs                    # sphinx-based documentation
-├── dist                    # supplemental material for users & package maintainers
 │   ├── **/*.rst            # documentation in reStructuredText
 │   ├── sphinxconf
 │   │   └── conf.py         # sphinx config (see commit 445a280 why its not in docs/)
 │   ├── requirements.txt    # pip3 requirements to build documentation
 │   ├── publish.sh          # shell script for automated rendering & deploy to zrepl.github.io repo
-│   └── public_git          # checkout of zrepl.github.io managed by above shell script
+│   ├── public_git          # checkout of zrepl.github.io managed by above shell script
-├── endpoint                # implementation of replication endpoints (=> package replication)
+├── logger                  # logger package used by zrepl
-├── logger                  # our own logger package
+├── rpc                     # rpc protocol implementation
-├── pruning                 # pruning rules (the logic, not the actual execution)
-│   └── retentiongrid
-├── replication
-│   ├── driver              # the driver of the replication logic (status reporting, error handling)
-│   ├── logic               # planning & executing replication steps via rpc
-|   |   └── pdu             # the generated gRPC & protobuf code used in replication (and endpoints)
-│   └── report              # the JSON-serializable report datastructures exposed to the client
-├── rpc                     # the hybrid gRPC + ./dataconn RPC client: connects to a remote replication.Endpoint
-│   ├── dataconn            # Bulk data-transfer RPC protocol
-│   ├── grpcclientidentity  # adaptor to inject package transport's 'client identity' concept into gRPC contexts
-│   ├── netadaptor          # adaptor to convert a package transport's Connecter and Listener into net.* primitives
-│   ├── transportmux        # TCP connecter and listener used to split control & data traffic
-│   └── versionhandshake    # replication protocol version handshake perfomed on newly established connections
-├── tlsconf                 # abstraction for Go TLS server + client config
-├── transport               # transports implementation
-│   ├── fromconfig
-│   ├── local
-│   ├── ssh
-│   ├── tcp
-│   └── tls
 ├── util
-├── vendor                  # managed by dep
+└── zfs                     # ZFS wrappers, filesystemm diffing
-├── version                 # abstraction for versions (filled during build by Makefile)
-└── zfs                     # zfs(8) wrappers
 ```
 
 ### Coding Workflow

build.Dockerfile

@@ -1,31 +1,16 @@
 FROM golang:latest
 
 RUN apt-get update && apt-get install -y \
-    python3-pip \
+    python3-pip
-    unzip
-
-RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
-RUN echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807  protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
-RUN unzip -d /usr protoc-3.6.1-linux-x86_64.zip
 
 ADD lazy.sh /tmp/lazy.sh
 ADD docs/requirements.txt /tmp/requirements.txt
 ENV ZREPL_LAZY_DOCS_REQPATH=/tmp/requirements.txt
-RUN /tmp/lazy.sh docdep
+RUN /tmp/lazy.sh devsetup
 
 # prepare volume mount of git checkout to /zrepl
 RUN mkdir -p /go/src/github.com/zrepl/zrepl
-RUN mkdir -p /.cache && chmod -R 0777 /.cache
+RUN chmod -R 0777 /go
 WORKDIR /go/src/github.com/zrepl/zrepl
 
-ADD Gopkg.toml Gopkg.lock  ./
-
-# godep will install the Go dependencies to vendor in order to then build and install
-# build dependencies like stringer to $GOPATH/bin.
-# However, since users volume-mount their Git checkout into /go/src/github.com/zrepl/zrepl
-# the vendor directory will be empty at build time, allowing them to experiment with
-# new checkouts, etc.
-# Thus, we only use the vendored deps for building dependencies.
-RUN /tmp/lazy.sh godep
-RUN chmod -R 0777 /go
 

build/build.go

@@ -1,23 +0,0 @@
-//+build windows
-
-// This package cannot actually be built, and since the Travis
-// tests run go test ./..., we have to avoid a build attempt.
-// Windows is not supported atm, so this works ¯\_(ツ)_/¯
-
-// This package is a pseudo-user of build-time dependencies for zrepl,
-// mostly for various code-generation tools.
-//
-// The imports are necessary to satisfy go dep
-package main
-
-import (
-	"fmt"
-
-	_ "github.com/alvaroloes/enumer"
-	_ "github.com/golang/protobuf/protoc-gen-go"
-	_ "golang.org/x/tools/cmd/stringer"
-)
-
-func main() {
-	fmt.Println("just a placeholder")
-}

cli/cli.go

@@ -1,122 +0,0 @@
-package cli
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
-
-	"github.com/zrepl/zrepl/config"
-)
-
-var rootArgs struct {
-	configPath string
-}
-
-var rootCmd = &cobra.Command{
-	Use:   "zrepl",
-	Short: "One-stop ZFS replication solution",
-}
-
-var bashcompCmd = &cobra.Command{
-	Use:   "bashcomp path/to/out/file",
-	Short: "generate bash completions",
-	Run: func(cmd *cobra.Command, args []string) {
-		if len(args) != 1 {
-			fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
-			err := cmd.Usage()
-			if err != nil {
-				panic(err)
-			}
-			os.Exit(1)
-		}
-		if err := rootCmd.GenBashCompletionFile(args[0]); err != nil {
-			fmt.Fprintf(os.Stderr, "error generating bash completion: %s", err)
-			os.Exit(1)
-		}
-	},
-	Hidden: true,
-}
-
-func init() {
-	rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
-	rootCmd.AddCommand(bashcompCmd)
-}
-
-type Subcommand struct {
-	Use              string
-	Short            string
-	Example          string
-	NoRequireConfig  bool
-	Run              func(subcommand *Subcommand, args []string) error
-	SetupFlags       func(f *pflag.FlagSet)
-	SetupSubcommands func() []*Subcommand
-
-	config    *config.Config
-	configErr error
-}
-
-func (s *Subcommand) ConfigParsingError() error {
-	return s.configErr
-}
-
-func (s *Subcommand) Config() *config.Config {
-	if !s.NoRequireConfig && s.config == nil {
-		panic("command that requires config is running and has no config set")
-	}
-	return s.config
-}
-
-func (s *Subcommand) run(cmd *cobra.Command, args []string) {
-	s.tryParseConfig()
-	err := s.Run(s, args)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s\n", err)
-		os.Exit(1)
-	}
-}
-
-func (s *Subcommand) tryParseConfig() {
-	config, err := config.ParseConfig(rootArgs.configPath)
-	s.configErr = err
-	if err != nil {
-		if s.NoRequireConfig {
-			// doesn't matter
-			return
-		} else {
-			fmt.Fprintf(os.Stderr, "could not parse config: %s\n", err)
-			os.Exit(1)
-		}
-	}
-	s.config = config
-}
-
-func AddSubcommand(s *Subcommand) {
-	addSubcommandToCobraCmd(rootCmd, s)
-}
-
-func addSubcommandToCobraCmd(c *cobra.Command, s *Subcommand) {
-	cmd := cobra.Command{
-		Use:     s.Use,
-		Short:   s.Short,
-		Example: s.Example,
-	}
-	if s.SetupSubcommands == nil {
-		cmd.Run = s.run
-	} else {
-		for _, sub := range s.SetupSubcommands() {
-			addSubcommandToCobraCmd(&cmd, sub)
-		}
-	}
-	if s.SetupFlags != nil {
-		s.SetupFlags(cmd.Flags())
-	}
-	c.AddCommand(&cmd)
-}
-
-func Run() {
-	if err := rootCmd.Execute(); err != nil {
-		os.Exit(1)
-	}
-}

client/configcheck.go

@@ -1,120 +0,0 @@
-package client
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-
-	"github.com/kr/pretty"
-	"github.com/pkg/errors"
-	"github.com/spf13/pflag"
-	"github.com/zrepl/yaml-config"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/daemon/logging"
-	"github.com/zrepl/zrepl/logger"
-)
-
-var configcheckArgs struct {
-	format string
-	what   string
-}
-
-var ConfigcheckCmd = &cli.Subcommand{
-	Use:   "configcheck",
-	Short: "check if config can be parsed without errors",
-	SetupFlags: func(f *pflag.FlagSet) {
-		f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]")
-		f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]")
-	},
-	Run: func(subcommand *cli.Subcommand, args []string) error {
-		formatMap := map[string]func(interface{}){
-			"": func(i interface{}) {},
-			"pretty": func(i interface{}) {
-				if _, err := pretty.Println(i); err != nil {
-					panic(err)
-				}
-			},
-			"json": func(i interface{}) {
-				if err := json.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
-					panic(err)
-				}
-			},
-			"yaml": func(i interface{}) {
-				if err := yaml.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
-					panic(err)
-				}
-			},
-		}
-
-		formatter, ok := formatMap[configcheckArgs.format]
-		if !ok {
-			return fmt.Errorf("unsupported --format %q", configcheckArgs.format)
-		}
-
-		var hadErr bool
-		// further: try to build jobs
-		confJobs, err := job.JobsFromConfig(subcommand.Config())
-		if err != nil {
-			err := errors.Wrap(err, "cannot build jobs from config")
-			if configcheckArgs.what == "jobs" {
-				return err
-			} else {
-				fmt.Fprintf(os.Stderr, "%s\n", err)
-				confJobs = nil
-				hadErr = true
-			}
-		}
-
-		// further: try to build logging outlets
-		outlets, err := logging.OutletsFromConfig(*subcommand.Config().Global.Logging)
-		if err != nil {
-			err := errors.Wrap(err, "cannot build logging from config")
-			if configcheckArgs.what == "logging" {
-				return err
-			} else {
-				fmt.Fprintf(os.Stderr, "%s\n", err)
-				outlets = nil
-				hadErr = true
-			}
-		}
-
-		whatMap := map[string]func(){
-			"all": func() {
-				o := struct {
-					config  *config.Config
-					jobs    []job.Job
-					logging *logger.Outlets
-				}{
-					subcommand.Config(),
-					confJobs,
-					outlets,
-				}
-				formatter(o)
-			},
-			"config": func() {
-				formatter(subcommand.Config())
-			},
-			"jobs": func() {
-				formatter(confJobs)
-			},
-			"logging": func() {
-				formatter(outlets)
-			},
-		}
-
-		wf, ok := whatMap[configcheckArgs.what]
-		if !ok {
-			return fmt.Errorf("unsupported --format %q", configcheckArgs.what)
-		}
-		wf()
-
-		if hadErr {
-			return fmt.Errorf("config parsing failed")
-		} else {
-			return nil
-		}
-	},
-}

client/jsonclient.go

@@ -1,49 +0,0 @@
-package client
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"io"
-	"net"
-	"net/http"
-
-	"github.com/pkg/errors"
-)
-
-func controlHttpClient(sockpath string) (client http.Client, err error) {
-	return http.Client{
-		Transport: &http.Transport{
-			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
-				return net.Dial("unix", sockpath)
-			},
-		},
-	}, nil
-}
-
-func jsonRequestResponse(c http.Client, endpoint string, req interface{}, res interface{}) error {
-	var buf bytes.Buffer
-	encodeErr := json.NewEncoder(&buf).Encode(req)
-	if encodeErr != nil {
-		return encodeErr
-	}
-
-	resp, err := c.Post("http://unix"+endpoint, "application/json", &buf)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		var msg bytes.Buffer
-		_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
-		return errors.Errorf("%s", msg.String())
-	}
-
-	decodeError := json.NewDecoder(resp.Body).Decode(&res)
-	if decodeError != nil {
-		return decodeError
-	}
-
-	return nil
-}

client/migrate.go

@@ -1,104 +0,0 @@
-package client
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/pkg/errors"
-	"github.com/spf13/pflag"
-
-	"github.com/zrepl/zrepl/zfs"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-)
-
-var (
-	MigrateCmd = &cli.Subcommand{
-		Use:   "migrate",
-		Short: "perform migration of the on-disk / zfs properties",
-		SetupSubcommands: func() []*cli.Subcommand {
-			return migrations
-		},
-	}
-)
-
-var migrations = []*cli.Subcommand{
-	&cli.Subcommand{
-		Use: "0.0.X:0.1:placeholder",
-		Run: doMigratePlaceholder0_1,
-		SetupFlags: func(f *pflag.FlagSet) {
-			f.BoolVar(&migratePlaceholder0_1Args.dryRun, "dry-run", false, "dry run")
-		},
-	},
-}
-
-var migratePlaceholder0_1Args struct {
-	dryRun bool
-}
-
-func doMigratePlaceholder0_1(sc *cli.Subcommand, args []string) error {
-	if len(args) != 0 {
-		return fmt.Errorf("migration does not take arguments, got %v", args)
-	}
-
-	cfg := sc.Config()
-
-	ctx := context.Background()
-	allFSS, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
-	if err != nil {
-		return errors.Wrap(err, "cannot list filesystems")
-	}
-
-	type workItem struct {
-		jobName string
-		rootFS  *zfs.DatasetPath
-		fss     []*zfs.DatasetPath
-	}
-	var wis []workItem
-	for i, j := range cfg.Jobs {
-		var rfsS string
-		switch job := j.Ret.(type) {
-		case *config.SinkJob:
-			rfsS = job.RootFS
-		case *config.PullJob:
-			rfsS = job.RootFS
-		default:
-			fmt.Printf("ignoring job %q (%d/%d, type %T)\n", j.Name(), i, len(cfg.Jobs), j.Ret)
-			continue
-		}
-		rfs, err := zfs.NewDatasetPath(rfsS)
-		if err != nil {
-			return errors.Wrapf(err, "root fs for job %q is not a valid dataset path", j.Name())
-		}
-		var fss []*zfs.DatasetPath
-		for _, fs := range allFSS {
-			if fs.HasPrefix(rfs) {
-				fss = append(fss, fs)
-			}
-		}
-		wis = append(wis, workItem{j.Name(), rfs, fss})
-	}
-
-	for _, wi := range wis {
-		fmt.Printf("job %q => migrate filesystems below root_fs %q\n", wi.jobName, wi.rootFS.ToString())
-		if len(wi.fss) == 0 {
-			fmt.Printf("\tno filesystems\n")
-			continue
-		}
-		for _, fs := range wi.fss {
-			fmt.Printf("\t%q ... ", fs.ToString())
-			r, err := zfs.ZFSMigrateHashBasedPlaceholderToCurrent(fs, migratePlaceholder0_1Args.dryRun)
-			if err != nil {
-				fmt.Printf("error: %s\n", err)
-			} else if !r.NeedsModification {
-				fmt.Printf("unchanged (placeholder=%v)\n", r.OriginalState.IsPlaceholder)
-			} else {
-				fmt.Printf("migrate (placeholder=%v) (old value = %q)\n",
-					r.OriginalState.IsPlaceholder, r.OriginalState.RawLocalPropertyValue)
-			}
-		}
-	}
-
-	return nil
-}

client/migrate_test.go

@@ -1,16 +0,0 @@
-package client
-
-import "testing"
-
-func TestMigrationsUnambiguousNames(t *testing.T) {
-	names := make(map[string]bool)
-	for _, mig := range migrations {
-		if _, ok := names[mig.Use]; ok {
-			t.Errorf("duplicate migration name %q", mig.Use)
-			t.FailNow()
-			return
-		} else {
-			names[mig.Use] = true
-		}
-	}
-}

client/pprof.go

@@ -1,67 +0,0 @@
-package client
-
-import (
-	"errors"
-	"log"
-	"os"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon"
-)
-
-var pprofArgs struct {
-	daemon.PprofServerControlMsg
-}
-
-var PprofCmd = &cli.Subcommand{
-	Use:   "pprof off | [on TCP_LISTEN_ADDRESS]",
-	Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
-	Run: func(subcommand *cli.Subcommand, args []string) error {
-		if len(args) < 1 {
-			goto enargs
-		}
-		switch args[0] {
-		case "on":
-			pprofArgs.Run = true
-			if len(args) != 2 {
-				return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
-			}
-			pprofArgs.HttpListenAddress = args[1]
-		case "off":
-			if len(args) != 1 {
-				goto enargs
-			}
-			pprofArgs.Run = false
-		}
-
-		RunPProf(subcommand.Config())
-		return nil
-	enargs:
-		return errors.New("invalid number of positional arguments")
-
-	},
-}
-
-func RunPProf(conf *config.Config) {
-	log := log.New(os.Stderr, "", 0)
-
-	die := func() {
-		log.Printf("exiting after error")
-		os.Exit(1)
-	}
-
-	log.Printf("connecting to zrepl daemon")
-
-	httpc, err := controlHttpClient(conf.Global.Control.SockPath)
-	if err != nil {
-		log.Printf("error creating http client: %s", err)
-		die()
-	}
-	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofArgs.PprofServerControlMsg, struct{}{})
-	if err != nil {
-		log.Printf("error sending control message: %s", err)
-		die()
-	}
-	log.Printf("finished")
-}

client/signal.go

@@ -1,40 +0,0 @@
-package client
-
-import (
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon"
-)
-
-var SignalCmd = &cli.Subcommand{
-	Use:   "signal [wakeup|reset] JOB",
-	Short: "wake up a job from wait state or abort its current invocation",
-	Run: func(subcommand *cli.Subcommand, args []string) error {
-		return runSignalCmd(subcommand.Config(), args)
-	},
-}
-
-func runSignalCmd(config *config.Config, args []string) error {
-	if len(args) != 2 {
-		return errors.Errorf("Expected 2 arguments: [wakeup|reset] JOB")
-	}
-
-	httpc, err := controlHttpClient(config.Global.Control.SockPath)
-	if err != nil {
-		return err
-	}
-
-	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointSignal,
-		struct {
-			Name string
-			Op   string
-		}{
-			Name: args[1],
-			Op:   args[0],
-		},
-		struct{}{},
-	)
-	return err
-}

client/status.go

@@ -1,632 +0,0 @@
-package client
-
-import (
-	"fmt"
-	"io"
-	"math"
-	"net/http"
-	"os"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/gdamore/tcell/termbox"
-	"github.com/pkg/errors"
-	"github.com/spf13/pflag"
-	"github.com/zrepl/yaml-config"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/daemon"
-	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/daemon/pruner"
-	"github.com/zrepl/zrepl/replication/report"
-)
-
-type byteProgressMeasurement struct {
-	time time.Time
-	val  int64
-}
-
-type bytesProgressHistory struct {
-	last        *byteProgressMeasurement // pointer as poor man's optional
-	changeCount int
-	lastChange  time.Time
-	bpsAvg      float64
-}
-
-func (p *bytesProgressHistory) Update(currentVal int64) (bytesPerSecondAvg int64, changeCount int) {
-
-	if p.last == nil {
-		p.last = &byteProgressMeasurement{
-			time: time.Now(),
-			val:  currentVal,
-		}
-		return 0, 0
-	}
-
-	if p.last.val != currentVal {
-		p.changeCount++
-		p.lastChange = time.Now()
-	}
-
-	if time.Since(p.lastChange) > 3*time.Second {
-		p.last = nil
-		return 0, 0
-	}
-
-	deltaV := currentVal - p.last.val
-	deltaT := time.Since(p.last.time)
-	rate := float64(deltaV) / deltaT.Seconds()
-
-	factor := 0.3
-	p.bpsAvg = (1-factor)*p.bpsAvg + factor*rate
-
-	p.last.time = time.Now()
-	p.last.val = currentVal
-
-	return int64(p.bpsAvg), p.changeCount
-}
-
-type tui struct {
-	x, y   int
-	indent int
-
-	lock   sync.Mutex //For report and error
-	report map[string]job.Status
-	err    error
-
-	replicationProgress map[string]*bytesProgressHistory // by job name
-}
-
-func newTui() tui {
-	return tui{
-		replicationProgress: make(map[string]*bytesProgressHistory),
-	}
-}
-
-const INDENT_MULTIPLIER = 4
-
-func (t *tui) moveLine(dl int, col int) {
-	t.y += dl
-	t.x = t.indent*INDENT_MULTIPLIER + col
-}
-
-func (t *tui) write(text string) {
-	for _, c := range text {
-		if c == '\n' {
-			t.newline()
-			continue
-		}
-		termbox.SetCell(t.x, t.y, c, termbox.ColorDefault, termbox.ColorDefault)
-		t.x += 1
-	}
-}
-
-func (t *tui) printf(text string, a ...interface{}) {
-	t.write(fmt.Sprintf(text, a...))
-}
-
-func wrap(s string, width int) string {
-	var b strings.Builder
-	for len(s) > 0 {
-		rem := width
-		if rem > len(s) {
-			rem = len(s)
-		}
-		if idx := strings.IndexAny(s, "\n\r"); idx != -1 && idx < rem {
-			rem = idx + 1
-		}
-		untilNewline := strings.TrimRight(s[:rem], "\n\r")
-		s = s[rem:]
-		if len(untilNewline) == 0 {
-			continue
-		}
-		b.WriteString(untilNewline)
-		b.WriteString("\n")
-	}
-	return strings.TrimRight(b.String(), "\n\r")
-}
-
-func (t *tui) printfDrawIndentedAndWrappedIfMultiline(format string, a ...interface{}) {
-	whole := fmt.Sprintf(format, a...)
-	width, _ := termbox.Size()
-	if !strings.ContainsAny(whole, "\n\r") && t.x+len(whole) <= width {
-		t.printf(format, a...)
-	} else {
-		t.addIndent(1)
-		t.newline()
-		t.write(wrap(whole, width-INDENT_MULTIPLIER*t.indent))
-		t.addIndent(-1)
-	}
-}
-
-func (t *tui) newline() {
-	t.moveLine(1, 0)
-}
-
-func (t *tui) setIndent(indent int) {
-	t.indent = indent
-	t.moveLine(0, 0)
-}
-
-func (t *tui) addIndent(indent int) {
-	t.indent += indent
-	t.moveLine(0, 0)
-}
-
-var statusFlags struct {
-	Raw bool
-}
-
-var StatusCmd = &cli.Subcommand{
-	Use:   "status",
-	Short: "show job activity or dump as JSON for monitoring",
-	SetupFlags: func(f *pflag.FlagSet) {
-		f.BoolVar(&statusFlags.Raw, "raw", false, "dump raw status description from zrepl daemon")
-	},
-	Run: runStatus,
-}
-
-func runStatus(s *cli.Subcommand, args []string) error {
-	httpc, err := controlHttpClient(s.Config().Global.Control.SockPath)
-	if err != nil {
-		return err
-	}
-
-	if statusFlags.Raw {
-		resp, err := httpc.Get("http://unix" + daemon.ControlJobEndpointStatus)
-		if err != nil {
-			return err
-		}
-		defer resp.Body.Close()
-		if resp.StatusCode != http.StatusOK {
-			fmt.Fprintf(os.Stderr, "Received error response:\n")
-			_, err := io.CopyN(os.Stderr, resp.Body, 4096)
-			if err != nil {
-				return err
-			}
-			return errors.Errorf("exit")
-		}
-		if _, err := io.Copy(os.Stdout, resp.Body); err != nil {
-			return err
-		}
-		return nil
-	}
-
-	t := newTui()
-	t.lock.Lock()
-	t.err = errors.New("Got no report yet")
-	t.lock.Unlock()
-
-	err = termbox.Init()
-	if err != nil {
-		return err
-	}
-	defer termbox.Close()
-
-	update := func() {
-		m := make(map[string]job.Status)
-
-		err2 := jsonRequestResponse(httpc, daemon.ControlJobEndpointStatus,
-			struct{}{},
-			&m,
-		)
-
-		t.lock.Lock()
-		t.err = err2
-		t.report = m
-		t.lock.Unlock()
-		t.draw()
-	}
-	update()
-
-	ticker := time.NewTicker(500 * time.Millisecond)
-	defer ticker.Stop()
-	go func() {
-		for range ticker.C {
-			update()
-		}
-	}()
-
-	termbox.HideCursor()
-	termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
-
-loop:
-	for {
-		switch ev := termbox.PollEvent(); ev.Type {
-		case termbox.EventKey:
-			switch ev.Key {
-			case termbox.KeyEsc:
-				break loop
-			case termbox.KeyCtrlC:
-				break loop
-			}
-		case termbox.EventResize:
-			t.draw()
-		}
-	}
-
-	return nil
-
-}
-
-func (t *tui) getReplicationProgresHistory(jobName string) *bytesProgressHistory {
-	p, ok := t.replicationProgress[jobName]
-	if !ok {
-		p = &bytesProgressHistory{}
-		t.replicationProgress[jobName] = p
-	}
-	return p
-}
-
-func (t *tui) draw() {
-	t.lock.Lock()
-	defer t.lock.Unlock()
-
-	termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
-	t.x = 0
-	t.y = 0
-	t.indent = 0
-
-	if t.err != nil {
-		t.write(t.err.Error())
-	} else {
-		//Iterate over map in alphabetical order
-		keys := make([]string, len(t.report))
-		i := 0
-		for k := range t.report {
-			keys[i] = k
-			i++
-		}
-		sort.Strings(keys)
-
-		for _, k := range keys {
-			v := t.report[k]
-			if len(k) == 0 || daemon.IsInternalJobName(k) { //Internal job
-				continue
-			}
-			t.setIndent(0)
-
-			t.printf("Job: %s", k)
-			t.setIndent(1)
-			t.newline()
-			t.printf("Type: %s", v.Type)
-			t.setIndent(1)
-			t.newline()
-
-			if v.Type == job.TypePush || v.Type == job.TypePull {
-				activeStatus, ok := v.JobSpecific.(*job.ActiveSideStatus)
-				if !ok || activeStatus == nil {
-					t.printf("ActiveSideStatus is null")
-					t.newline()
-					continue
-				}
-
-				t.printf("Replication:")
-				t.newline()
-				t.addIndent(1)
-				t.renderReplicationReport(activeStatus.Replication, t.getReplicationProgresHistory(k))
-				t.addIndent(-1)
-
-				t.printf("Pruning Sender:")
-				t.newline()
-				t.addIndent(1)
-				t.renderPrunerReport(activeStatus.PruningSender)
-				t.addIndent(-1)
-
-				t.printf("Pruning Receiver:")
-				t.newline()
-				t.addIndent(1)
-				t.renderPrunerReport(activeStatus.PruningReceiver)
-				t.addIndent(-1)
-			} else if v.Type == job.TypeSnap {
-				snapStatus, ok := v.JobSpecific.(*job.SnapJobStatus)
-				if !ok || snapStatus == nil {
-					t.printf("SnapJobStatus is null")
-					t.newline()
-					continue
-				}
-				t.printf("Pruning snapshots:")
-				t.newline()
-				t.addIndent(1)
-				t.renderPrunerReport(snapStatus.Pruning)
-				t.addIndent(-1)
-			} else {
-				t.printf("No status representation for job type '%s', dumping as YAML", v.Type)
-				t.newline()
-				asYaml, err := yaml.Marshal(v.JobSpecific)
-				if err != nil {
-					t.printf("Error marshaling status to YAML: %s", err)
-					t.newline()
-					continue
-				}
-				t.write(string(asYaml))
-				t.newline()
-				continue
-			}
-		}
-	}
-	termbox.Flush()
-}
-
-func (t *tui) renderReplicationReport(rep *report.Report, history *bytesProgressHistory) {
-	if rep == nil {
-		t.printf("...\n")
-		return
-	}
-
-	if rep.WaitReconnectError != nil {
-		t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: %s", rep.WaitReconnectError)
-		t.newline()
-	}
-	if !rep.WaitReconnectSince.IsZero() {
-		delta := time.Until(rep.WaitReconnectUntil).Round(time.Second)
-		if rep.WaitReconnectUntil.IsZero() || delta > 0 {
-			var until string
-			if rep.WaitReconnectUntil.IsZero() {
-				until = "waiting indefinitely"
-			} else {
-				until = fmt.Sprintf("hard fail in %s @ %s", delta, rep.WaitReconnectUntil)
-			}
-			t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnecting with exponential backoff (since %s) (%s)",
-				rep.WaitReconnectSince, until)
-		} else {
-			t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnects reached hard-fail timeout @ %s", rep.WaitReconnectUntil)
-		}
-		t.newline()
-	}
-
-	// TODO visualize more than the latest attempt by folding all attempts into one
-	if len(rep.Attempts) == 0 {
-		t.printf("no attempts made yet")
-		return
-	} else {
-		t.printf("Attempt #%d", len(rep.Attempts))
-		if len(rep.Attempts) > 1 {
-			t.printf(". Previous attempts failed with the follwing statuses:")
-			t.newline()
-			t.addIndent(1)
-			for i, a := range rep.Attempts[:len(rep.Attempts)-1] {
-				t.printfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt))
-				t.newline()
-			}
-			t.addIndent(-1)
-		} else {
-			t.newline()
-		}
-	}
-
-	latest := rep.Attempts[len(rep.Attempts)-1]
-	sort.Slice(latest.Filesystems, func(i, j int) bool {
-		return latest.Filesystems[i].Info.Name < latest.Filesystems[j].Info.Name
-	})
-
-	t.printf("Status: %s", latest.State)
-	t.newline()
-	if latest.State == report.AttemptPlanningError {
-		t.printf("Problem: ")
-		t.printfDrawIndentedAndWrappedIfMultiline("%s", latest.PlanError)
-		t.newline()
-	} else if latest.State == report.AttemptFanOutError {
-		t.printf("Problem: one or more of the filesystems encountered errors")
-		t.newline()
-	}
-
-	if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
-		// Draw global progress bar
-		// Progress: [---------------]
-		expected, replicated := latest.BytesSum()
-		rate, changeCount := history.Update(replicated)
-		t.write("Progress: ")
-		t.drawBar(50, replicated, expected, changeCount)
-		t.write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinary(replicated), ByteCountBinary(expected), ByteCountBinary(rate)))
-		t.newline()
-
-		var maxFSLen int
-		for _, fs := range latest.Filesystems {
-			if len(fs.Info.Name) > maxFSLen {
-				maxFSLen = len(fs.Info.Name)
-			}
-		}
-		for _, fs := range latest.Filesystems {
-			t.printFilesystemStatus(fs, false, maxFSLen) // FIXME bring 'active' flag back
-		}
-
-	}
-
-}
-
-func (t *tui) renderPrunerReport(r *pruner.Report) {
-	if r == nil {
-		t.printf("...\n")
-		return
-	}
-
-	state, err := pruner.StateString(r.State)
-	if err != nil {
-		t.printf("Status: %q (parse error: %q)\n", r.State, err)
-		return
-	}
-
-	t.printf("Status: %s", state)
-	t.newline()
-
-	if r.Error != "" {
-		t.printf("Error: %s\n", r.Error)
-	}
-
-	type commonFS struct {
-		*pruner.FSReport
-		completed bool
-	}
-	all := make([]commonFS, 0, len(r.Pending)+len(r.Completed))
-	for i := range r.Pending {
-		all = append(all, commonFS{&r.Pending[i], false})
-	}
-	for i := range r.Completed {
-		all = append(all, commonFS{&r.Completed[i], true})
-	}
-
-	switch state {
-	case pruner.Plan:
-		fallthrough
-	case pruner.PlanErr:
-		return
-	}
-
-	if len(all) == 0 {
-		t.printf("nothing to do\n")
-		return
-	}
-
-	var totalDestroyCount, completedDestroyCount int
-	var maxFSname int
-	for _, fs := range all {
-		totalDestroyCount += len(fs.DestroyList)
-		if fs.completed {
-			completedDestroyCount += len(fs.DestroyList)
-		}
-		if maxFSname < len(fs.Filesystem) {
-			maxFSname = len(fs.Filesystem)
-		}
-	}
-
-	// global progress bar
-	progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
-	t.write("Progress: ")
-	t.write("[")
-	t.write(times("=", progress))
-	t.write(">")
-	t.write(times("-", 80-progress))
-	t.write("]")
-	t.printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
-	t.newline()
-
-	sort.SliceStable(all, func(i, j int) bool {
-		return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
-	})
-
-	// Draw a table-like representation of 'all'
-	for _, fs := range all {
-		t.write(rightPad(fs.Filesystem, maxFSname, " "))
-		t.write(" ")
-		if !fs.SkipReason.NotSkipped() {
-			t.printf("skipped: %s\n", fs.SkipReason)
-			continue
-		}
-		if fs.LastError != "" {
-			if strings.ContainsAny(fs.LastError, "\r\n") {
-				t.printf("ERROR:")
-				t.printfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError)
-			} else {
-				t.printfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError)
-			}
-			t.newline()
-			continue
-		}
-
-		pruneRuleActionStr := fmt.Sprintf("(destroy %d of %d snapshots)",
-			len(fs.DestroyList), len(fs.SnapshotList))
-
-		if fs.completed {
-			t.printf("Completed  %s\n", pruneRuleActionStr)
-			continue
-		}
-
-		t.write("Pending    ") // whitespace is padding 10
-		if len(fs.DestroyList) == 1 {
-			t.write(fs.DestroyList[0].Name)
-		} else {
-			t.write(pruneRuleActionStr)
-		}
-		t.newline()
-	}
-
-}
-
-func times(str string, n int) (out string) {
-	for i := 0; i < n; i++ {
-		out += str
-	}
-	return
-}
-
-func rightPad(str string, length int, pad string) string {
-	if len(str) > length {
-		return str[:length]
-	}
-	return str + times(pad, length-len(str))
-}
-
-var arrowPositions = `>\|/`
-
-// changeCount = 0 indicates stall / no progresss
-func (t *tui) drawBar(length int, bytes, totalBytes int64, changeCount int) {
-	var completedLength int
-	if totalBytes > 0 {
-		completedLength = int(int64(length) * bytes / totalBytes)
-		if completedLength > length {
-			completedLength = length
-		}
-	} else if totalBytes == bytes {
-		completedLength = length
-	}
-
-	t.write("[")
-	t.write(times("=", completedLength))
-	t.write(string(arrowPositions[changeCount%len(arrowPositions)]))
-	t.write(times("-", length-completedLength))
-	t.write("]")
-}
-
-func (t *tui) printFilesystemStatus(rep *report.FilesystemReport, active bool, maxFS int) {
-
-	expected, replicated := rep.BytesSum()
-	status := fmt.Sprintf("%s (step %d/%d, %s/%s)",
-		strings.ToUpper(string(rep.State)),
-		rep.CurrentStep, len(rep.Steps),
-		ByteCountBinary(replicated), ByteCountBinary(expected),
-	)
-
-	activeIndicator := " "
-	if active {
-		activeIndicator = "*"
-	}
-	t.printf("%s %s %s ",
-		activeIndicator,
-		rightPad(rep.Info.Name, maxFS, " "),
-		status)
-
-	next := ""
-	if err := rep.Error(); err != nil {
-		next = err.Err
-	} else if rep.State != report.FilesystemDone {
-		if nextStep := rep.NextStep(); nextStep != nil {
-			if nextStep.IsIncremental() {
-				next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
-			} else {
-				next = fmt.Sprintf("next: %s (full)", nextStep.Info.To)
-			}
-		} else {
-			next = "" // individual FSes may still be in planning state
-		}
-
-	}
-	t.printfDrawIndentedAndWrappedIfMultiline("%s", next)
-
-	t.newline()
-}
-
-func ByteCountBinary(b int64) string {
-	const unit = 1024
-	if b < unit {
-		return fmt.Sprintf("%d B", b)
-	}
-	div, exp := int64(unit), 0
-	for n := b / unit; n >= unit; n /= unit {
-		div *= unit
-		exp++
-	}
-	return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
-}

client/testcmd.go

@@ -1,178 +0,0 @@
-package client
-
-import (
-	"fmt"
-
-	"github.com/pkg/errors"
-	"github.com/spf13/pflag"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-var TestCmd = &cli.Subcommand{
-	Use: "test",
-	SetupSubcommands: func() []*cli.Subcommand {
-		return []*cli.Subcommand{testFilter, testPlaceholder}
-	},
-}
-
-var testFilterArgs struct {
-	job   string
-	all   bool
-	input string
-}
-
-var testFilter = &cli.Subcommand{
-	Use:   "filesystems --job JOB [--all | --input INPUT]",
-	Short: "test filesystems filter specified in push or source job",
-	SetupFlags: func(f *pflag.FlagSet) {
-		f.StringVar(&testFilterArgs.job, "job", "", "the name of the push or source job")
-		f.StringVar(&testFilterArgs.input, "input", "", "a filesystem name to test against the job's filters")
-		f.BoolVar(&testFilterArgs.all, "all", false, "test all local filesystems")
-	},
-	Run: runTestFilterCmd,
-}
-
-func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
-
-	if testFilterArgs.job == "" {
-		return fmt.Errorf("must specify --job flag")
-	}
-	if !(testFilterArgs.all != (testFilterArgs.input != "")) { // xor
-		return fmt.Errorf("must set one: --all or --input")
-	}
-
-	conf := subcommand.Config()
-
-	var confFilter config.FilesystemsFilter
-	job, err := conf.Job(testFilterArgs.job)
-	if err != nil {
-		return err
-	}
-	switch j := job.Ret.(type) {
-	case *config.SourceJob:
-		confFilter = j.Filesystems
-	case *config.PushJob:
-		confFilter = j.Filesystems
-	default:
-		return fmt.Errorf("job type %T does not have filesystems filter", j)
-	}
-
-	f, err := filters.DatasetMapFilterFromConfig(confFilter)
-	if err != nil {
-		return fmt.Errorf("filter invalid: %s", err)
-	}
-
-	var fsnames []string
-	if testFilterArgs.input != "" {
-		fsnames = []string{testFilterArgs.input}
-	} else {
-		out, err := zfs.ZFSList([]string{"name"})
-		if err != nil {
-			return fmt.Errorf("could not list ZFS filesystems: %s", err)
-		}
-		for _, row := range out {
-
-			fsnames = append(fsnames, row[0])
-		}
-	}
-
-	fspaths := make([]*zfs.DatasetPath, len(fsnames))
-	for i, fsname := range fsnames {
-		path, err := zfs.NewDatasetPath(fsname)
-		if err != nil {
-			return err
-		}
-		fspaths[i] = path
-	}
-
-	hadFilterErr := false
-	for _, in := range fspaths {
-		var res string
-		var errStr string
-		pass, err := f.Filter(in)
-		if err != nil {
-			res = "ERROR"
-			errStr = err.Error()
-			hadFilterErr = true
-		} else if pass {
-			res = "ACCEPT"
-		} else {
-			res = "REJECT"
-		}
-		fmt.Printf("%s\t%s\t%s\n", res, in.ToString(), errStr)
-	}
-
-	if hadFilterErr {
-		return fmt.Errorf("filter errors occurred")
-	}
-	return nil
-}
-
-var testPlaceholderArgs struct {
-	ds  string
-	all bool
-}
-
-var testPlaceholder = &cli.Subcommand{
-	Use:   "placeholder [--all | --dataset DATASET]",
-	Short: fmt.Sprintf("list received placeholder filesystems (zfs property %q)", zfs.PlaceholderPropertyName),
-	Example: `
-	placeholder --all
-	placeholder --dataset path/to/sink/clientident/fs`,
-	NoRequireConfig: true,
-	SetupFlags: func(f *pflag.FlagSet) {
-		f.StringVar(&testPlaceholderArgs.ds, "dataset", "", "dataset path (not required to exist)")
-		f.BoolVar(&testPlaceholderArgs.all, "all", false, "list tab-separated placeholder status of all filesystems")
-	},
-	Run: runTestPlaceholder,
-}
-
-func runTestPlaceholder(subcommand *cli.Subcommand, args []string) error {
-
-	var checkDPs []*zfs.DatasetPath
-
-	// all actions first
-	if testPlaceholderArgs.all {
-		out, err := zfs.ZFSList([]string{"name"})
-		if err != nil {
-			return errors.Wrap(err, "could not list ZFS filesystems")
-		}
-		for _, row := range out {
-			dp, err := zfs.NewDatasetPath(row[0])
-			if err != nil {
-				panic(err)
-			}
-			checkDPs = append(checkDPs, dp)
-		}
-	} else {
-		dp, err := zfs.NewDatasetPath(testPlaceholderArgs.ds)
-		if err != nil {
-			return err
-		}
-		if dp.Empty() {
-			return fmt.Errorf("must specify --dataset DATASET or --all")
-		}
-		checkDPs = append(checkDPs, dp)
-	}
-
-	fmt.Printf("IS_PLACEHOLDER\tDATASET\tzrepl:placeholder\n")
-	for _, dp := range checkDPs {
-		ph, err := zfs.ZFSGetFilesystemPlaceholderState(dp)
-		if err != nil {
-			return errors.Wrap(err, "cannot get placeholder state")
-		}
-		if !ph.FSExists {
-			panic("placeholder state inconsistent: filesystem " + ph.FS + " must exist in this context")
-		}
-		is := "yes"
-		if !ph.IsPlaceholder {
-			is = "no"
-		}
-		fmt.Printf("%s\t%s\t%s\n", is, dp.ToString(), ph.RawLocalPropertyValue)
-	}
-	return nil
-}

client/version.go

@@ -1,74 +0,0 @@
-package client
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/spf13/pflag"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon"
-	"github.com/zrepl/zrepl/version"
-)
-
-var versionArgs struct {
-	Show      string
-	Config    *config.Config
-	ConfigErr error
-}
-
-var VersionCmd = &cli.Subcommand{
-	Use:             "version",
-	Short:           "print version of zrepl binary and running daemon",
-	NoRequireConfig: true,
-	SetupFlags: func(f *pflag.FlagSet) {
-		f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)")
-	},
-	Run: func(subcommand *cli.Subcommand, args []string) error {
-		versionArgs.Config = subcommand.Config()
-		versionArgs.ConfigErr = subcommand.ConfigParsingError()
-		return runVersionCmd()
-	},
-}
-
-func runVersionCmd() error {
-	args := versionArgs
-
-	if args.Show != "daemon" && args.Show != "client" && args.Show != "" {
-		return fmt.Errorf("show flag must be 'client' or 'server' or be left empty")
-	}
-
-	var clientVersion, daemonVersion *version.ZreplVersionInformation
-	if args.Show == "client" || args.Show == "" {
-		clientVersion = version.NewZreplVersionInformation()
-		fmt.Printf("client: %s\n", clientVersion.String())
-	}
-	if args.Show == "daemon" || args.Show == "" {
-
-		if args.ConfigErr != nil {
-			return fmt.Errorf("config parsing error: %s", args.ConfigErr)
-		}
-
-		httpc, err := controlHttpClient(args.Config.Global.Control.SockPath)
-		if err != nil {
-			return fmt.Errorf("server: error: %s\n", err)
-		}
-
-		var info version.ZreplVersionInformation
-		err = jsonRequestResponse(httpc, daemon.ControlJobEndpointVersion, "", &info)
-		if err != nil {
-			return fmt.Errorf("server: error: %s\n", err)
-		}
-		daemonVersion = &info
-		fmt.Printf("server: %s\n", daemonVersion.String())
-	}
-
-	if args.Show == "" {
-		if clientVersion.Version != daemonVersion.Version {
-			fmt.Fprintf(os.Stderr, "WARNING: client version != daemon version, restart zrepl daemon\n")
-		}
-	}
-
-	return nil
-}

cmd/autosnap.go

@@ -0,0 +1,215 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"github.com/zrepl/zrepl/zfs"
+	"sort"
+	"sync"
+	"time"
+)
+
+type IntervalAutosnap struct {
+	task             *Task
+	DatasetFilter    zfs.DatasetFilter
+	Prefix           string
+	SnapshotInterval time.Duration
+}
+
+func (a *IntervalAutosnap) filterFilesystems() (fss []*zfs.DatasetPath, stop bool) {
+	a.task.Enter("filter_filesystems")
+	defer a.task.Finish()
+	fss, err := zfs.ZFSListMapping(a.DatasetFilter)
+	stop = err != nil
+	if err != nil {
+		a.task.Log().WithError(err).Error("cannot list datasets")
+	}
+	if len(fss) == 0 {
+		a.task.Log().Warn("no filesystem matching filesystem filter")
+	}
+	return fss, stop
+}
+
+func (a *IntervalAutosnap) findSyncPoint(fss []*zfs.DatasetPath) (syncPoint time.Time, err error) {
+	a.task.Enter("find_sync_point")
+	defer a.task.Finish()
+	type snapTime struct {
+		ds   *zfs.DatasetPath
+		time time.Time
+	}
+
+	if len(fss) == 0 {
+		return time.Now(), nil
+	}
+
+	snaptimes := make([]snapTime, 0, len(fss))
+
+	now := time.Now()
+
+	a.task.Log().Debug("examine filesystem state")
+	for _, d := range fss {
+
+		l := a.task.Log().WithField(logFSField, d.ToString())
+
+		fsvs, err := zfs.ZFSListFilesystemVersions(d, NewPrefixFilter(a.Prefix))
+		if err != nil {
+			l.WithError(err).Error("cannot list filesystem versions")
+			continue
+		}
+		if len(fsvs) <= 0 {
+			l.WithField("prefix", a.Prefix).Info("no filesystem versions with prefix")
+			continue
+		}
+
+		// Sort versions by creation
+		sort.SliceStable(fsvs, func(i, j int) bool {
+			return fsvs[i].CreateTXG < fsvs[j].CreateTXG
+		})
+
+		latest := fsvs[len(fsvs)-1]
+		l.WithField("creation", latest.Creation).
+			Debug("found latest snapshot")
+
+		since := now.Sub(latest.Creation)
+		if since < 0 {
+			l.WithField("snapshot", latest.Name).
+				WithField("creation", latest.Creation).
+				Error("snapshot is from the future")
+			continue
+		}
+		next := now
+		if since < a.SnapshotInterval {
+			next = latest.Creation.Add(a.SnapshotInterval)
+		}
+		snaptimes = append(snaptimes, snapTime{d, next})
+	}
+
+	if len(snaptimes) == 0 {
+		snaptimes = append(snaptimes, snapTime{nil, now})
+	}
+
+	sort.Slice(snaptimes, func(i, j int) bool {
+		return snaptimes[i].time.Before(snaptimes[j].time)
+	})
+
+	return snaptimes[0].time, nil
+
+}
+
+func (a *IntervalAutosnap) waitForSyncPoint(ctx context.Context, syncPoint time.Time) {
+	a.task.Enter("wait_sync_point")
+	defer a.task.Finish()
+
+	const LOG_TIME_FMT string = time.ANSIC
+
+	a.task.Log().WithField("sync_point", syncPoint.Format(LOG_TIME_FMT)).
+		Info("wait for sync point")
+
+	select {
+	case <-ctx.Done():
+		a.task.Log().WithError(ctx.Err()).Info("context done")
+		return
+	case <-time.After(syncPoint.Sub(time.Now())):
+	}
+}
+
+func (a *IntervalAutosnap) syncUpRun(ctx context.Context, didSnaps chan struct{}) (stop bool) {
+	a.task.Enter("sync_up")
+	defer a.task.Finish()
+
+	fss, stop := a.filterFilesystems()
+	if stop {
+		return true
+	}
+
+	syncPoint, err := a.findSyncPoint(fss)
+	if err != nil {
+		return true
+	}
+
+	a.waitForSyncPoint(ctx, syncPoint)
+
+	a.task.Log().Debug("snapshot all filesystems to enable further snaps in lockstep")
+	a.doSnapshots(didSnaps)
+	return false
+}
+
+func (a *IntervalAutosnap) Run(ctx context.Context, didSnaps chan struct{}) {
+
+	if a.syncUpRun(ctx, didSnaps) {
+		a.task.Log().Error("stoppping autosnap after error in sync up")
+		return
+	}
+
+	// task drops back to idle here
+
+	a.task.Log().Debug("setting up ticker in SnapshotInterval")
+	ticker := time.NewTicker(a.SnapshotInterval)
+	for {
+		select {
+		case <-ctx.Done():
+			ticker.Stop()
+			a.task.Log().WithError(ctx.Err()).Info("context done")
+			return
+
+		case <-ticker.C:
+			a.doSnapshots(didSnaps)
+		}
+	}
+
+}
+
+func (a *IntervalAutosnap) doSnapshots(didSnaps chan struct{}) {
+
+	a.task.Enter("do_snapshots")
+	defer a.task.Finish()
+
+	// don't cache the result from previous run in case the user added
+	// a new dataset in the meantime
+	fss, stop := a.filterFilesystems()
+	if stop {
+		return
+	}
+
+	a.task.Log().Info("beginning parallel snapshots")
+
+	// TODO channel programs -> allow a little jitter?
+	var wg sync.WaitGroup
+	for fsi := range fss {
+		wg.Add(1)
+		go func(fs *zfs.DatasetPath) {
+			defer wg.Done()
+
+			suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
+			snapname := fmt.Sprintf("%s%s", a.Prefix, suffix)
+
+			l := a.task.Log().WithField(logFSField, fs.ToString()).
+				WithField("snapname", snapname)
+
+			l.Info("create snapshot")
+			err := zfs.ZFSSnapshot(fs, snapname, false)
+			if err != nil {
+				l.WithError(err).Error("cannot create snapshot")
+				return
+			}
+
+			l.Info("create corresponding bookmark")
+			err = zfs.ZFSBookmark(fs, snapname, snapname)
+			if err != nil {
+				l.WithError(err).Error("cannot create bookmark")
+			}
+
+		}(fss[fsi])
+	}
+
+	a.task.Log().Info("waiting for parallel snapshots to finish")
+	wg.Wait()
+	a.task.Log().Info("snapshots finished")
+
+	select {
+	case didSnaps <- struct{}{}:
+	default:
+		a.task.Log().Error("warning: callback channel is full, discarding")
+	}
+
+}

cmd/bashcomp.go

@@ -0,0 +1,29 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"os"
+)
+
+var bashcompCmd = &cobra.Command{
+	Use:   "bashcomp path/to/out/file",
+	Short: "generate bash completions",
+	Run: func(cmd *cobra.Command, args []string) {
+		if len(args) != 1 {
+			fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
+			cmd.Usage()
+			os.Exit(1)
+		}
+		if err := RootCmd.GenBashCompletionFile(args[0]); err != nil {
+			fmt.Fprintf(os.Stderr, "error generating bash completion: %s", err)
+			os.Exit(1)
+		}
+	},
+	Hidden: true,
+}
+
+func init() {
+	RootCmd.AddCommand(bashcompCmd)
+}

cmd/config.go

@@ -0,0 +1,102 @@
+package cmd
+
+import (
+	"io"
+
+	"fmt"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type Config struct {
+	Global Global
+	Jobs   map[string]Job
+}
+
+func (c *Config) LookupJob(name string) (j Job, err error) {
+	j, ok := c.Jobs[name]
+	if !ok {
+		return nil, errors.Errorf("job '%s' is not defined", name)
+	}
+	return j, nil
+}
+
+type Global struct {
+	Serve struct {
+		Stdinserver struct {
+			SockDir string
+		}
+	}
+	Control struct {
+		Sockpath string
+	}
+	logging *LoggingConfig
+}
+
+type JobDebugSettings struct {
+	Conn struct {
+		ReadDump  string `mapstructure:"read_dump"`
+		WriteDump string `mapstructure:"write_dump"`
+	}
+	RPC struct {
+		Log bool
+	}
+}
+
+type RWCConnecter interface {
+	Connect() (io.ReadWriteCloser, error)
+}
+type AuthenticatedChannelListenerFactory interface {
+	Listen() (AuthenticatedChannelListener, error)
+}
+
+type AuthenticatedChannelListener interface {
+	Accept() (ch io.ReadWriteCloser, err error)
+	Close() (err error)
+}
+
+type SSHStdinServerConnectDescr struct {
+}
+
+type PrunePolicy interface {
+	// Prune filters versions and decide which to keep and which to remove.
+	// Prune **does not** implement the actual removal of the versions.
+	Prune(fs *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error)
+}
+
+type PruningJob interface {
+	Pruner(task *Task, side PrunePolicySide, dryRun bool) (Pruner, error)
+}
+
+// A type for constants describing different prune policies of a PruningJob
+// This is mostly a special-case for LocalJob, which is the only job that has two prune policies
+// instead of one.
+// It implements github.com/spf13/pflag.Value to be used as CLI flag for the test subcommand
+type PrunePolicySide string
+
+const (
+	PrunePolicySideDefault PrunePolicySide = ""
+	PrunePolicySideLeft    PrunePolicySide = "left"
+	PrunePolicySideRight   PrunePolicySide = "right"
+)
+
+func (s *PrunePolicySide) String() string {
+	return string(*s)
+}
+
+func (s *PrunePolicySide) Set(news string) error {
+	p := PrunePolicySide(news)
+	switch p {
+	case PrunePolicySideRight:
+		fallthrough
+	case PrunePolicySideLeft:
+		*s = p
+	default:
+		return errors.Errorf("must be either %s or %s", PrunePolicySideLeft, PrunePolicySideRight)
+	}
+	return nil
+}
+
+func (s *PrunePolicySide) Type() string {
+	return fmt.Sprintf("%s | %s", PrunePolicySideLeft, PrunePolicySideRight)
+}

cmd/config_connect.go

@@ -0,0 +1,48 @@
+package cmd
+
+import (
+	"fmt"
+	"io"
+
+	"context"
+	"github.com/jinzhu/copier"
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"github.com/problame/go-netssh"
+)
+
+type SSHStdinserverConnecter struct {
+	Host                 string
+	User                 string
+	Port                 uint16
+	IdentityFile         string   `mapstructure:"identity_file"`
+	TransportOpenCommand []string `mapstructure:"transport_open_command"`
+	SSHCommand           string   `mapstructure:"ssh_command"`
+	Options              []string
+}
+
+func parseSSHStdinserverConnecter(i map[string]interface{}) (c *SSHStdinserverConnecter, err error) {
+
+	c = &SSHStdinserverConnecter{}
+	if err = mapstructure.Decode(i, c); err != nil {
+		err = errors.New(fmt.Sprintf("could not parse ssh transport: %s", err))
+		return nil, err
+	}
+
+	// TODO assert fields are filled
+	return
+
+}
+
+func (c *SSHStdinserverConnecter) Connect() (rwc io.ReadWriteCloser, err error) {
+
+	var endpoint netssh.Endpoint
+	if err = copier.Copy(&endpoint, c); err != nil {
+		return
+	}
+	if rwc, err = netssh.Dial(context.TODO(), endpoint); err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	return
+}

cmd/config_fsvfilter.go

@@ -0,0 +1,36 @@
+package cmd
+
+import (
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/zfs"
+	"strings"
+)
+
+type PrefixFilter struct {
+	prefix    string
+	fstype    zfs.VersionType
+	fstypeSet bool // optionals anyone?
+}
+
+func NewPrefixFilter(prefix string) *PrefixFilter {
+	return &PrefixFilter{prefix: prefix}
+}
+
+func NewTypedPrefixFilter(prefix string, versionType zfs.VersionType) *PrefixFilter {
+	return &PrefixFilter{prefix, versionType, true}
+}
+
+func parseSnapshotPrefix(i string) (p string, err error) {
+	if len(i) <= 0 {
+		err = errors.Errorf("snapshot prefix must not be empty string")
+		return
+	}
+	p = i
+	return
+}
+
+func (f *PrefixFilter) Filter(fsv zfs.FilesystemVersion) (accept bool, err error) {
+	fstypeMatches := (!f.fstypeSet || fsv.Type == f.fstype)
+	prefixMatches := strings.HasPrefix(fsv.Name, f.prefix)
+	return fstypeMatches && prefixMatches, nil
+}

cmd/config_job_control.go

@@ -0,0 +1,143 @@
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"github.com/pkg/errors"
+	"io"
+	"net"
+	"net/http"
+)
+
+type ControlJob struct {
+	Name     string
+	sockaddr *net.UnixAddr
+}
+
+func NewControlJob(name, sockpath string) (j *ControlJob, err error) {
+	j = &ControlJob{Name: name}
+
+	j.sockaddr, err = net.ResolveUnixAddr("unix", sockpath)
+	if err != nil {
+		err = errors.Wrap(err, "cannot resolve unix address")
+		return
+	}
+
+	return
+}
+
+func (j *ControlJob) JobName() string {
+	return j.Name
+}
+
+func (j *ControlJob) JobStatus(ctx context.Context) (*JobStatus, error) {
+	return &JobStatus{Tasks: nil}, nil
+}
+
+const (
+	ControlJobEndpointPProf   string = "/debug/pprof"
+	ControlJobEndpointVersion string = "/version"
+	ControlJobEndpointStatus  string = "/status"
+)
+
+func (j *ControlJob) JobStart(ctx context.Context) {
+
+	log := ctx.Value(contextKeyLog).(Logger)
+	defer log.Info("control job finished")
+
+	daemon := ctx.Value(contextKeyDaemon).(*Daemon)
+
+	l, err := ListenUnixPrivate(j.sockaddr)
+	if err != nil {
+		log.WithError(err).Error("error listening")
+		return
+	}
+
+	pprofServer := NewPProfServer(ctx)
+
+	mux := http.NewServeMux()
+	mux.Handle(ControlJobEndpointPProf, requestLogger{log: log, handlerFunc: func(w http.ResponseWriter, r *http.Request) {
+		var msg PprofServerControlMsg
+		err := json.NewDecoder(r.Body).Decode(&msg)
+		if err != nil {
+			log.WithError(err).Error("bad pprof request from client")
+			w.WriteHeader(http.StatusBadRequest)
+		}
+		pprofServer.Control(msg)
+		w.WriteHeader(200)
+	}})
+	mux.Handle(ControlJobEndpointVersion,
+		requestLogger{log: log, handler: jsonResponder{func() (interface{}, error) {
+			return NewZreplVersionInformation(), nil
+		}}})
+	mux.Handle(ControlJobEndpointStatus,
+		requestLogger{log: log, handler: jsonResponder{func() (interface{}, error) {
+			return daemon.Status(), nil
+		}}})
+	server := http.Server{Handler: mux}
+
+outer:
+	for {
+
+		served := make(chan error)
+		go func() {
+			served <- server.Serve(l)
+			close(served)
+		}()
+
+		select {
+		case <-ctx.Done():
+			log.WithError(ctx.Err()).Info("context done")
+			server.Shutdown(context.Background())
+			break outer
+		case err = <-served:
+			if err != nil {
+				log.WithError(err).Error("error serving")
+				break outer
+			}
+		}
+
+	}
+
+}
+
+type jsonResponder struct {
+	producer func() (interface{}, error)
+}
+
+func (j jsonResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	res, err := j.producer()
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		io.WriteString(w, err.Error())
+		return
+	}
+	var buf bytes.Buffer
+	err = json.NewEncoder(&buf).Encode(res)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		io.WriteString(w, err.Error())
+	} else {
+		io.Copy(w, &buf)
+	}
+}
+
+type requestLogger struct {
+	log         Logger
+	handler     http.Handler
+	handlerFunc http.HandlerFunc
+}
+
+func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	log := l.log.WithField("method", r.Method).WithField("url", r.URL)
+	log.Info("start")
+	if l.handlerFunc != nil {
+		l.handlerFunc(w, r)
+	} else if l.handler != nil {
+		l.handler.ServeHTTP(w, r)
+	} else {
+		log.Error("no handler or handlerFunc configured")
+	}
+	log.Info("finish")
+}

cmd/config_job_local.go

@@ -0,0 +1,214 @@
+package cmd
+
+import (
+	"time"
+
+	"context"
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/zfs"
+	"sync"
+)
+
+type LocalJob struct {
+	Name              string
+	Mapping           *DatasetMapFilter
+	SnapshotPrefix    string
+	Interval          time.Duration
+	InitialReplPolicy InitialReplPolicy
+	PruneLHS          PrunePolicy
+	PruneRHS          PrunePolicy
+	Debug             JobDebugSettings
+	snapperTask       *Task
+	mainTask          *Task
+	handlerTask       *Task
+	pruneRHSTask      *Task
+	pruneLHSTask      *Task
+}
+
+func parseLocalJob(c JobParsingContext, name string, i map[string]interface{}) (j *LocalJob, err error) {
+
+	var asMap struct {
+		Mapping           map[string]string
+		SnapshotPrefix    string `mapstructure:"snapshot_prefix"`
+		Interval          string
+		InitialReplPolicy string                 `mapstructure:"initial_repl_policy"`
+		PruneLHS          map[string]interface{} `mapstructure:"prune_lhs"`
+		PruneRHS          map[string]interface{} `mapstructure:"prune_rhs"`
+		Debug             map[string]interface{}
+	}
+
+	if err = mapstructure.Decode(i, &asMap); err != nil {
+		err = errors.Wrap(err, "mapstructure error")
+		return nil, err
+	}
+
+	j = &LocalJob{Name: name}
+
+	if j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false); err != nil {
+		return
+	}
+
+	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
+		return
+	}
+
+	if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
+		err = errors.Wrap(err, "cannot parse interval")
+		return
+	}
+
+	if j.InitialReplPolicy, err = parseInitialReplPolicy(asMap.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY); err != nil {
+		return
+	}
+
+	if j.PruneLHS, err = parsePrunePolicy(asMap.PruneLHS, true); err != nil {
+		err = errors.Wrap(err, "cannot parse 'prune_lhs'")
+		return
+	}
+	if j.PruneRHS, err = parsePrunePolicy(asMap.PruneRHS, false); err != nil {
+		err = errors.Wrap(err, "cannot parse 'prune_rhs'")
+		return
+	}
+
+	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
+		err = errors.Wrap(err, "cannot parse 'debug'")
+		return
+	}
+
+	return
+}
+
+func (j *LocalJob) JobName() string {
+	return j.Name
+}
+
+func (j *LocalJob) JobStart(ctx context.Context) {
+
+	rootLog := ctx.Value(contextKeyLog).(Logger)
+
+	j.snapperTask = NewTask("snapshot", rootLog)
+	j.mainTask = NewTask("main", rootLog)
+	j.handlerTask = NewTask("handler", rootLog)
+	j.pruneRHSTask = NewTask("prune_rhs", rootLog)
+	j.pruneLHSTask = NewTask("prune_lhs", rootLog)
+
+	local := rpc.NewLocalRPC()
+	// Allow access to any dataset since we control what mapping
+	// is passed to the pull routine.
+	// All local datasets will be passed to its Map() function,
+	// but only those for which a mapping exists will actually be pulled.
+	// We can pay this small performance penalty for now.
+	handler := NewHandler(j.handlerTask.Log(), localPullACL{}, NewPrefixFilter(j.SnapshotPrefix))
+
+	registerEndpoints(local, handler)
+
+	snapper := IntervalAutosnap{
+		task:             j.snapperTask,
+		DatasetFilter:    j.Mapping.AsFilter(),
+		Prefix:           j.SnapshotPrefix,
+		SnapshotInterval: j.Interval,
+	}
+
+	plhs, err := j.Pruner(j.pruneLHSTask, PrunePolicySideLeft, false)
+	if err != nil {
+		rootLog.WithError(err).Error("error creating lhs pruner")
+		return
+	}
+	prhs, err := j.Pruner(j.pruneRHSTask, PrunePolicySideRight, false)
+	if err != nil {
+		rootLog.WithError(err).Error("error creating rhs pruner")
+		return
+	}
+
+	didSnaps := make(chan struct{})
+	go snapper.Run(ctx, didSnaps)
+
+outer:
+	for {
+
+		select {
+		case <-ctx.Done():
+			j.mainTask.Log().WithError(ctx.Err()).Info("context")
+			break outer
+		case <-didSnaps:
+			j.mainTask.Log().Debug("finished taking snapshots")
+			j.mainTask.Log().Info("starting replication procedure")
+		}
+
+		j.mainTask.Log().Debug("replicating from lhs to rhs")
+		j.mainTask.Enter("replicate")
+		puller := Puller{j.mainTask, local, j.Mapping, j.InitialReplPolicy}
+		puller.Pull()
+		j.mainTask.Finish()
+
+		// use a ctx as soon as Pull gains ctx support
+		select {
+		case <-ctx.Done():
+			break outer
+		default:
+		}
+
+		var wg sync.WaitGroup
+
+		j.mainTask.Log().Info("pruning lhs")
+		wg.Add(1)
+		go func() {
+			plhs.Run(ctx)
+			wg.Done()
+		}()
+
+		j.mainTask.Log().Info("pruning rhs")
+		wg.Add(1)
+		go func() {
+			prhs.Run(ctx)
+			wg.Done()
+		}()
+
+		wg.Wait()
+
+	}
+
+}
+
+func (j *LocalJob) JobStatus(ctxt context.Context) (*JobStatus, error) {
+	return &JobStatus{Tasks: []*TaskStatus{
+		j.snapperTask.Status(),
+		j.pruneLHSTask.Status(),
+		j.pruneRHSTask.Status(),
+		j.mainTask.Status(),
+	}}, nil
+}
+
+func (j *LocalJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) {
+
+	var dsfilter zfs.DatasetFilter
+	var pp PrunePolicy
+	switch side {
+	case PrunePolicySideLeft:
+		pp = j.PruneLHS
+		dsfilter = j.Mapping.AsFilter()
+	case PrunePolicySideRight:
+		pp = j.PruneRHS
+		dsfilter, err = j.Mapping.InvertedFilter()
+		if err != nil {
+			err = errors.Wrap(err, "cannot invert mapping for prune_rhs")
+			return
+		}
+	default:
+		err = errors.Errorf("must be either left or right side")
+		return
+	}
+
+	p = Pruner{
+		task,
+		time.Now(),
+		dryRun,
+		dsfilter,
+		j.SnapshotPrefix,
+		pp,
+	}
+
+	return
+}

cmd/config_job_pull.go

@@ -0,0 +1,195 @@
+package cmd
+
+import (
+	"time"
+
+	"context"
+	"fmt"
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/util"
+)
+
+type PullJob struct {
+	Name     string
+	Connect  RWCConnecter
+	Interval time.Duration
+	Mapping  *DatasetMapFilter
+	// constructed from mapping during parsing
+	pruneFilter       *DatasetMapFilter
+	SnapshotPrefix    string
+	InitialReplPolicy InitialReplPolicy
+	Prune             PrunePolicy
+	Debug             JobDebugSettings
+
+	task *Task
+}
+
+func parsePullJob(c JobParsingContext, name string, i map[string]interface{}) (j *PullJob, err error) {
+
+	var asMap struct {
+		Connect           map[string]interface{}
+		Interval          string
+		Mapping           map[string]string
+		InitialReplPolicy string `mapstructure:"initial_repl_policy"`
+		Prune             map[string]interface{}
+		SnapshotPrefix    string `mapstructure:"snapshot_prefix"`
+		Debug             map[string]interface{}
+	}
+
+	if err = mapstructure.Decode(i, &asMap); err != nil {
+		err = errors.Wrap(err, "mapstructure error")
+		return nil, err
+	}
+
+	j = &PullJob{Name: name}
+
+	j.Connect, err = parseSSHStdinserverConnecter(asMap.Connect)
+	if err != nil {
+		err = errors.Wrap(err, "cannot parse 'connect'")
+		return nil, err
+	}
+
+	if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
+		err = errors.Wrap(err, "cannot parse 'interval'")
+		return nil, err
+	}
+
+	j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false)
+	if err != nil {
+		err = errors.Wrap(err, "cannot parse 'mapping'")
+		return nil, err
+	}
+
+	if j.pruneFilter, err = j.Mapping.InvertedFilter(); err != nil {
+		err = errors.Wrap(err, "cannot automatically invert 'mapping' for prune job")
+		return nil, err
+	}
+
+	j.InitialReplPolicy, err = parseInitialReplPolicy(asMap.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY)
+	if err != nil {
+		err = errors.Wrap(err, "cannot parse 'initial_repl_policy'")
+		return
+	}
+
+	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
+		return
+	}
+
+	if j.Prune, err = parsePrunePolicy(asMap.Prune, false); err != nil {
+		err = errors.Wrap(err, "cannot parse prune policy")
+		return
+	}
+
+	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
+		err = errors.Wrap(err, "cannot parse 'debug'")
+		return
+	}
+
+	return
+}
+
+func (j *PullJob) JobName() string {
+	return j.Name
+}
+
+func (j *PullJob) JobStart(ctx context.Context) {
+
+	log := ctx.Value(contextKeyLog).(Logger)
+	defer log.Info("exiting")
+	j.task = NewTask("main", log)
+
+	// j.task is idle here idle here
+
+	ticker := time.NewTicker(j.Interval)
+	for {
+		j.doRun(ctx)
+		select {
+		case <-ctx.Done():
+			j.task.Log().WithError(ctx.Err()).Info("context")
+			return
+		case <-ticker.C:
+		}
+	}
+}
+
+func (j *PullJob) doRun(ctx context.Context) {
+
+	j.task.Enter("run")
+	defer j.task.Finish()
+
+	j.task.Log().Info("connecting")
+	rwc, err := j.Connect.Connect()
+	if err != nil {
+		j.task.Log().WithError(err).Error("error connecting")
+		return
+	}
+
+	rwc, err = util.NewReadWriteCloserLogger(rwc, j.Debug.Conn.ReadDump, j.Debug.Conn.WriteDump)
+	if err != nil {
+		return
+	}
+
+	client := rpc.NewClient(rwc)
+	if j.Debug.RPC.Log {
+		client.SetLogger(j.task.Log(), true)
+	}
+
+	j.task.Enter("pull")
+	puller := Puller{j.task, client, j.Mapping, j.InitialReplPolicy}
+	puller.Pull()
+	closeRPCWithTimeout(j.task, client, time.Second*1, "")
+	rwc.Close()
+	j.task.Finish()
+
+	j.task.Enter("prune")
+	pruner, err := j.Pruner(j.task, PrunePolicySideDefault, false)
+	if err != nil {
+		j.task.Log().WithError(err).Error("error creating pruner")
+	} else {
+		pruner.Run(ctx)
+	}
+	j.task.Finish()
+
+}
+
+func (j *PullJob) JobStatus(ctxt context.Context) (*JobStatus, error) {
+	return &JobStatus{Tasks: []*TaskStatus{j.task.Status()}}, nil
+}
+
+func (j *PullJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) {
+	p = Pruner{
+		task,
+		time.Now(),
+		dryRun,
+		j.pruneFilter,
+		j.SnapshotPrefix,
+		j.Prune,
+	}
+	return
+}
+
+func closeRPCWithTimeout(task *Task, remote rpc.RPCClient, timeout time.Duration, goodbye string) {
+
+	task.Log().Info("closing rpc connection")
+
+	ch := make(chan error)
+	go func() {
+		ch <- remote.Close()
+		close(ch)
+	}()
+
+	var err error
+	select {
+	case <-time.After(timeout):
+		err = fmt.Errorf("timeout exceeded (%s)", timeout)
+	case closeRequestErr := <-ch:
+		err = closeRequestErr
+	}
+
+	if err != nil {
+		task.Log().WithError(err).Error("error closing connection")
+	}
+	return
+}

cmd/config_job_source.go

@@ -0,0 +1,250 @@
+package cmd
+
+import (
+	"context"
+	"io"
+	"time"
+
+	mapstructure "github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/util"
+)
+
+type SourceJob struct {
+	Name           string
+	Serve          AuthenticatedChannelListenerFactory
+	Filesystems    *DatasetMapFilter
+	SnapshotPrefix string
+	Interval       time.Duration
+	Prune          PrunePolicy
+	Debug          JobDebugSettings
+	serveTask      *Task
+	autosnapTask   *Task
+	pruneTask      *Task
+}
+
+func parseSourceJob(c JobParsingContext, name string, i map[string]interface{}) (j *SourceJob, err error) {
+
+	var asMap struct {
+		Serve          map[string]interface{}
+		Filesystems    map[string]string
+		SnapshotPrefix string `mapstructure:"snapshot_prefix"`
+		Interval       string
+		Prune          map[string]interface{}
+		Debug          map[string]interface{}
+	}
+
+	if err = mapstructure.Decode(i, &asMap); err != nil {
+		err = errors.Wrap(err, "mapstructure error")
+		return nil, err
+	}
+
+	j = &SourceJob{Name: name}
+
+	if j.Serve, err = parseAuthenticatedChannelListenerFactory(c, asMap.Serve); err != nil {
+		return
+	}
+
+	if j.Filesystems, err = parseDatasetMapFilter(asMap.Filesystems, true); err != nil {
+		return
+	}
+
+	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
+		return
+	}
+
+	if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
+		err = errors.Wrap(err, "cannot parse 'interval'")
+		return
+	}
+
+	if j.Prune, err = parsePrunePolicy(asMap.Prune, true); err != nil {
+		err = errors.Wrap(err, "cannot parse 'prune'")
+		return
+	}
+
+	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
+		err = errors.Wrap(err, "cannot parse 'debug'")
+		return
+	}
+
+	return
+}
+
+func (j *SourceJob) JobName() string {
+	return j.Name
+}
+
+func (j *SourceJob) JobStart(ctx context.Context) {
+
+	log := ctx.Value(contextKeyLog).(Logger)
+	defer log.Info("exiting")
+
+	j.autosnapTask = NewTask("autosnap", log)
+	j.pruneTask = NewTask("prune", log)
+	j.serveTask = NewTask("serve", log)
+
+	a := IntervalAutosnap{j.autosnapTask, j.Filesystems, j.SnapshotPrefix, j.Interval}
+	p, err := j.Pruner(j.pruneTask, PrunePolicySideDefault, false)
+
+	if err != nil {
+		log.WithError(err).Error("error creating pruner")
+		return
+	}
+
+	didSnaps := make(chan struct{})
+
+	go j.serve(ctx, j.serveTask)
+	go a.Run(ctx, didSnaps)
+
+outer:
+	for {
+		select {
+		case <-ctx.Done():
+			break outer
+		case <-didSnaps:
+			log.Info("starting pruner")
+			p.Run(ctx)
+			log.Info("pruner done")
+		}
+	}
+	log.WithError(ctx.Err()).Info("context")
+
+}
+
+func (j *SourceJob) JobStatus(ctxt context.Context) (*JobStatus, error) {
+	return &JobStatus{
+		Tasks: []*TaskStatus{
+			j.autosnapTask.Status(),
+			j.pruneTask.Status(),
+			j.serveTask.Status(),
+		}}, nil
+}
+
+func (j *SourceJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) {
+	p = Pruner{
+		task,
+		time.Now(),
+		dryRun,
+		j.Filesystems,
+		j.SnapshotPrefix,
+		j.Prune,
+	}
+	return
+}
+
+func (j *SourceJob) serve(ctx context.Context, task *Task) {
+
+	listener, err := j.Serve.Listen()
+	if err != nil {
+		task.Log().WithError(err).Error("error listening")
+		return
+	}
+
+	type rwcChanMsg struct {
+		rwc io.ReadWriteCloser
+		err error
+	}
+	rwcChan := make(chan rwcChanMsg)
+
+	// Serve connections until interrupted or error
+outer:
+	for {
+
+		go func() {
+			rwc, err := listener.Accept()
+			if err != nil {
+				rwcChan <- rwcChanMsg{rwc, err}
+				close(rwcChan)
+				return
+			}
+			rwcChan <- rwcChanMsg{rwc, err}
+		}()
+
+		select {
+
+		case rwcMsg := <-rwcChan:
+
+			if rwcMsg.err != nil {
+				task.Log().WithError(err).Error("error accepting connection")
+				break outer
+			}
+
+			j.handleConnection(rwcMsg.rwc, task)
+
+		case <-ctx.Done():
+			task.Log().WithError(ctx.Err()).Info("context")
+			break outer
+
+		}
+
+	}
+
+	task.Enter("close_listener")
+	defer task.Finish()
+	err = listener.Close()
+	if err != nil {
+		task.Log().WithError(err).Error("error closing listener")
+	}
+
+	return
+
+}
+
+func (j *SourceJob) handleConnection(rwc io.ReadWriteCloser, task *Task) {
+
+	task.Enter("handle_connection")
+	defer task.Finish()
+
+	task.Log().Info("handling client connection")
+
+	rwc, err := util.NewReadWriteCloserLogger(rwc, j.Debug.Conn.ReadDump, j.Debug.Conn.WriteDump)
+	if err != nil {
+		panic(err)
+	}
+
+	// construct connection handler
+	handler := NewHandler(task.Log(), j.Filesystems, NewPrefixFilter(j.SnapshotPrefix))
+
+	// handle connection
+	rpcServer := rpc.NewServer(rwc)
+	if j.Debug.RPC.Log {
+		rpclog := task.Log().WithField("subsystem", "rpc")
+		rpcServer.SetLogger(rpclog, true)
+	}
+	registerEndpoints(rpcServer, handler)
+	if err = rpcServer.Serve(); err != nil {
+		task.Log().WithError(err).Error("error serving connection")
+	}
+
+	// wait for client to close connection
+	// FIXME: we cannot just close it like we would to with a TCP socket because
+	// FIXME: go-nettsh's Close() may overtake the remaining data in the pipe
+	const CLIENT_HANGUP_TIMEOUT = 1 * time.Second
+	task.Log().
+		WithField("timeout", CLIENT_HANGUP_TIMEOUT).
+		Debug("waiting for client to hang up")
+
+	wchan := make(chan error)
+	go func() {
+		var pseudo [1]byte
+		_, err := io.ReadFull(rwc, pseudo[:])
+		wchan <- err
+	}()
+	var werr error
+	select {
+	case werr = <-wchan:
+		// all right
+	case <-time.After(CLIENT_HANGUP_TIMEOUT):
+		werr = errors.New("client did not close connection within timeout")
+	}
+	if werr != nil && werr != io.EOF {
+		task.Log().WithError(werr).
+			Error("error waiting for client to hang up")
+	}
+	task.Log().Info("closing client connection")
+	if err = rwc.Close(); err != nil {
+		task.Log().WithError(err).Error("error force-closing connection")
+	}
+}

cmd/config_logging.go

@@ -0,0 +1,251 @@
+package cmd
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"github.com/mattn/go-isatty"
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/logger"
+	"io/ioutil"
+	"os"
+	"time"
+)
+
+type LoggingConfig struct {
+	Outlets *logger.Outlets
+}
+
+type MetadataFlags int64
+
+const (
+	MetadataTime MetadataFlags = 1 << iota
+	MetadataLevel
+
+	MetadataNone MetadataFlags = 0
+	MetadataAll  MetadataFlags = ^0
+)
+
+func parseLogging(i interface{}) (c *LoggingConfig, err error) {
+
+	c = &LoggingConfig{}
+	c.Outlets = logger.NewOutlets()
+
+	var asList []interface{}
+	if err = mapstructure.Decode(i, &asList); err != nil {
+		return nil, errors.Wrap(err, "mapstructure error")
+	}
+	if len(asList) == 0 {
+		// Default config
+		out := WriterOutlet{&HumanFormatter{}, os.Stdout}
+		c.Outlets.Add(out, logger.Warn)
+		return
+	}
+
+	var syslogOutlets, stdoutOutlets int
+	for lei, le := range asList {
+
+		outlet, minLevel, err := parseOutlet(le)
+		if err != nil {
+			return nil, errors.Wrapf(err, "cannot parse outlet #%d", lei)
+		}
+		var _ logger.Outlet = WriterOutlet{}
+		var _ logger.Outlet = &SyslogOutlet{}
+		switch outlet.(type) {
+		case *SyslogOutlet:
+			syslogOutlets++
+		case WriterOutlet:
+			stdoutOutlets++
+		}
+
+		c.Outlets.Add(outlet, minLevel)
+
+	}
+
+	if syslogOutlets > 1 {
+		return nil, errors.Errorf("can only define one 'syslog' outlet")
+	}
+	if stdoutOutlets > 1 {
+		return nil, errors.Errorf("can only define one 'stdout' outlet")
+	}
+
+	return c, nil
+
+}
+
+func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
+	var is string
+	switch j := i.(type) {
+	case string:
+		is = j
+	default:
+		return nil, errors.Errorf("invalid log format: wrong type: %T", i)
+	}
+
+	switch is {
+	case "human":
+		return &HumanFormatter{}, nil
+	case "logfmt":
+		return &LogfmtFormatter{}, nil
+	case "json":
+		return &JSONFormatter{}, nil
+	default:
+		return nil, errors.Errorf("invalid log format: '%s'", is)
+	}
+
+}
+
+func parseOutlet(i interface{}) (o logger.Outlet, level logger.Level, err error) {
+
+	var in struct {
+		Outlet string
+		Level  string
+		Format string
+	}
+	if err = mapstructure.Decode(i, &in); err != nil {
+		err = errors.Wrap(err, "mapstructure error")
+		return
+	}
+	if in.Outlet == "" || in.Level == "" || in.Format == "" {
+		err = errors.Errorf("must specify 'outlet', 'level' and 'format' field")
+		return
+	}
+
+	minLevel, err := logger.ParseLevel(in.Level)
+	if err != nil {
+		err = errors.Wrap(err, "cannot parse 'level' field")
+		return
+	}
+	formatter, err := parseLogFormat(in.Format)
+	if err != nil {
+		err = errors.Wrap(err, "cannot parse")
+		return
+	}
+
+	switch in.Outlet {
+	case "stdout":
+		o, err = parseStdoutOutlet(i, formatter)
+	case "tcp":
+		o, err = parseTCPOutlet(i, formatter)
+	case "syslog":
+		o, err = parseSyslogOutlet(i, formatter)
+	default:
+		err = errors.Errorf("unknown outlet type '%s'", in.Outlet)
+	}
+	return o, minLevel, err
+
+}
+
+func parseStdoutOutlet(i interface{}, formatter EntryFormatter) (WriterOutlet, error) {
+
+	var in struct {
+		Time bool
+	}
+	if err := mapstructure.Decode(i, &in); err != nil {
+		return WriterOutlet{}, errors.Wrap(err, "invalid structure for stdout outlet")
+	}
+
+	flags := MetadataAll
+	writer := os.Stdout
+	if !isatty.IsTerminal(writer.Fd()) && !in.Time {
+		flags &= ^MetadataTime
+	}
+
+	formatter.SetMetadataFlags(flags)
+	return WriterOutlet{
+		formatter,
+		os.Stdout,
+	}, nil
+}
+
+func parseTCPOutlet(i interface{}, formatter EntryFormatter) (out *TCPOutlet, err error) {
+
+	var in struct {
+		Net           string
+		Address       string
+		RetryInterval string `mapstructure:"retry_interval"`
+		TLS           *struct {
+			CA   string
+			Cert string
+			Key  string
+		}
+	}
+	if err = mapstructure.Decode(i, &in); err != nil {
+		return nil, errors.Wrap(err, "mapstructure error")
+	}
+
+	retryInterval, err := time.ParseDuration(in.RetryInterval)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
+	}
+
+	if len(in.Net) == 0 {
+		return nil, errors.New("field 'net' must not be empty")
+	}
+	if len(in.Address) == 0 {
+		return nil, errors.New("field 'address' must not be empty")
+	}
+
+	var tlsConfig *tls.Config
+	if in.TLS != nil {
+
+		cert, err := tls.LoadX509KeyPair(in.TLS.Cert, in.TLS.Key)
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot load client cert")
+		}
+
+		var rootCAs *x509.CertPool
+		if in.TLS.CA == "" {
+			if rootCAs, err = x509.SystemCertPool(); err != nil {
+				return nil, errors.Wrap(err, "cannot open system cert pool")
+			}
+		} else {
+			rootCAs = x509.NewCertPool()
+			rootCAPEM, err := ioutil.ReadFile(in.TLS.CA)
+			if err != nil {
+				return nil, errors.Wrap(err, "cannot load CA cert")
+			}
+			if !rootCAs.AppendCertsFromPEM(rootCAPEM) {
+				return nil, errors.New("cannot parse CA cert")
+			}
+		}
+		if err != nil && in.TLS.CA == "" {
+			return nil, errors.Wrap(err, "cannot load root ca pool")
+		}
+
+		tlsConfig = &tls.Config{
+			Certificates: []tls.Certificate{cert},
+			RootCAs:      rootCAs,
+		}
+
+		tlsConfig.BuildNameToCertificate()
+	}
+
+	formatter.SetMetadataFlags(MetadataAll)
+	return NewTCPOutlet(formatter, in.Net, in.Address, tlsConfig, retryInterval), nil
+
+}
+
+func parseSyslogOutlet(i interface{}, formatter EntryFormatter) (out *SyslogOutlet, err error) {
+
+	var in struct {
+		RetryInterval string `mapstructure:"retry_interval"`
+	}
+	if err = mapstructure.Decode(i, &in); err != nil {
+		return nil, errors.Wrap(err, "mapstructure error")
+	}
+
+	out = &SyslogOutlet{}
+	out.Formatter = formatter
+	out.Formatter.SetMetadataFlags(MetadataNone)
+
+	out.RetryInterval = 0 // default to 0 as we assume local syslog will just work
+	if in.RetryInterval != "" {
+		out.RetryInterval, err = time.ParseDuration(in.RetryInterval)
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
+		}
+	}
+
+	return
+}

cmd/config_mapfilter.go

@@ -1,12 +1,11 @@
-package filters
+package cmd
 
 import (
 	"fmt"
 	"strings"
 
+	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/zfs"
 )
 
@@ -102,7 +101,6 @@ func (m DatasetMapFilter) mostSpecificPrefixMapping(path *zfs.DatasetPath) (idx
 	return
 }
 
-// Returns target == nil if there is no mapping
 func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error) {
 
 	if m.filterMode {
@@ -116,17 +114,9 @@ func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath,
 	}
 	me := m.entries[mi]
 
-	if me.mapping == "" {
+	if strings.HasPrefix("!", me.mapping) {
-		// Special case treatment: 'foo/bar<' => ''
+		// reject mapping
-		if !me.subtreeMatch {
+		return nil, nil
-			return nil, fmt.Errorf("mapping to '' must be a subtree match")
-		}
-		// ok...
-	} else {
-		if strings.HasPrefix("!", me.mapping) {
-			// reject mapping
-			return nil, nil
-		}
 	}
 
 	target, err = zfs.NewDatasetPath(me.mapping)
@@ -177,7 +167,7 @@ func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
 	for i, e := range m.entries {
 		inv.entries[i].path, err = zfs.NewDatasetPath(e.mapping)
 		if err != nil {
-			err = errors.Wrapf(err, "mapping cannot be inverted: '%s' is not a dataset path", e.mapping)
+			err = errors.Wrapf(err, "mapping cannot be inverted: '%s' is not a dataset path: %s", e.mapping)
 			return
 		}
 		inv.entries[i].mapping = MapFilterResultOk
@@ -187,43 +177,12 @@ func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
 	return inv, nil
 }
 
-// FIXME investigate whether we can support more...
-func (m DatasetMapFilter) Invert() (endpoint.FSMap, error) {
-
-	if m.filterMode {
-		return nil, errors.Errorf("can only invert mappings")
-	}
-
-	if len(m.entries) != 1 {
-		return nil, errors.Errorf("inversion of complicated mappings is not implemented") // FIXME
-	}
-
-	e := m.entries[0]
-
-	inv := &DatasetMapFilter{
-		make([]datasetMapFilterEntry, len(m.entries)),
-		false,
-	}
-	mp, err := zfs.NewDatasetPath(e.mapping)
-	if err != nil {
-		return nil, err
-	}
-
-	inv.entries[0] = datasetMapFilterEntry{
-		path:         mp,
-		mapping:      e.path.ToString(),
-		subtreeMatch: e.subtreeMatch,
-	}
-
-	return inv, nil
-}
-
 // Creates a new DatasetMapFilter in filter mode from a mapping
 // All accepting mapping results are mapped to accepting filter results
 // All rejecting mapping results are mapped to rejecting filter results
-func (m DatasetMapFilter) AsFilter() endpoint.FSFilter {
+func (m DatasetMapFilter) AsFilter() (f *DatasetMapFilter) {
 
-	f := &DatasetMapFilter{
+	f = &DatasetMapFilter{
 		make([]datasetMapFilterEntry, len(m.entries)),
 		true,
 	}
@@ -258,14 +217,16 @@ func (m DatasetMapFilter) parseDatasetFilterResult(result string) (pass bool, er
 	return false, fmt.Errorf("'%s' is not a valid filter result", result)
 }
 
-func DatasetMapFilterFromConfig(in map[string]bool) (f *DatasetMapFilter, err error) {
+func parseDatasetMapFilter(mi interface{}, filterMode bool) (f *DatasetMapFilter, err error) {
 
-	f = NewDatasetMapFilter(len(in), true)
+	var m map[string]string
-	for pathPattern, accept := range in {
+	if err = mapstructure.Decode(mi, &m); err != nil {
-		mapping := MapFilterResultOmit
+		err = fmt.Errorf("maps / filters must be specified as map[string]string: %s", err)
-		if accept {
+		return
-			mapping = MapFilterResultOk
+	}
-		}
+
+	f = NewDatasetMapFilter(len(m), filterMode)
+	for pathPattern, mapping := range m {
 		if err = f.Add(pathPattern, mapping); err != nil {
 			err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err)
 			return

cmd/config_parse.go

@@ -0,0 +1,294 @@
+package cmd
+
+import (
+	"io/ioutil"
+
+	"fmt"
+	yaml "github.com/go-yaml/yaml"
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"os"
+	"regexp"
+	"strconv"
+	"time"
+)
+
+var ConfigFileDefaultLocations []string = []string{
+	"/etc/zrepl/zrepl.yml",
+	"/usr/local/etc/zrepl/zrepl.yml",
+}
+
+const (
+	JobNameControl string = "control"
+)
+
+var ReservedJobNames []string = []string{
+	JobNameControl,
+}
+
+type ConfigParsingContext struct {
+	Global *Global
+}
+
+func ParseConfig(path string) (config *Config, err error) {
+
+	if path == "" {
+		// Try default locations
+		for _, l := range ConfigFileDefaultLocations {
+			stat, err := os.Stat(l)
+			if err != nil {
+				continue
+			}
+			if !stat.Mode().IsRegular() {
+				err = errors.Errorf("file at default location is not a regular file: %s", l)
+				continue
+			}
+			path = l
+			break
+		}
+	}
+
+	var i interface{}
+
+	var bytes []byte
+
+	if bytes, err = ioutil.ReadFile(path); err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+
+	if err = yaml.Unmarshal(bytes, &i); err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+
+	return parseConfig(i)
+}
+
+func parseConfig(i interface{}) (c *Config, err error) {
+
+	var asMap struct {
+		Global map[string]interface{}
+		Jobs   []map[string]interface{}
+	}
+	if err := mapstructure.Decode(i, &asMap); err != nil {
+		return nil, errors.Wrap(err, "config root must be a dict")
+	}
+
+	c = &Config{}
+
+	// Parse global with defaults
+	c.Global.Serve.Stdinserver.SockDir = "/var/run/zrepl/stdinserver"
+	c.Global.Control.Sockpath = "/var/run/zrepl/control"
+
+	err = mapstructure.Decode(asMap.Global, &c.Global)
+	if err != nil {
+		err = errors.Wrap(err, "mapstructure error on 'global' section: %s")
+		return
+	}
+
+	if c.Global.logging, err = parseLogging(asMap.Global["logging"]); err != nil {
+		return nil, errors.Wrap(err, "cannot parse logging section")
+	}
+
+	cpc := ConfigParsingContext{&c.Global}
+	jpc := JobParsingContext{cpc}
+
+	// Jobs
+	c.Jobs = make(map[string]Job, len(asMap.Jobs))
+	for i := range asMap.Jobs {
+		job, err := parseJob(jpc, asMap.Jobs[i])
+		if err != nil {
+			// Try to find its name
+			namei, ok := asMap.Jobs[i]["name"]
+			if !ok {
+				namei = fmt.Sprintf("<no name, entry #%d in list>", i)
+			}
+			err = errors.Wrapf(err, "cannot parse job '%v'", namei)
+			return nil, err
+		}
+		jn := job.JobName()
+		if _, ok := c.Jobs[jn]; ok {
+			err = errors.Errorf("duplicate job name: %s", jn)
+			return nil, err
+		}
+		c.Jobs[job.JobName()] = job
+	}
+
+	cj, err := NewControlJob(JobNameControl, jpc.Global.Control.Sockpath)
+	if err != nil {
+		err = errors.Wrap(err, "cannot create control job")
+		return
+	}
+	c.Jobs[JobNameControl] = cj
+
+	return c, nil
+
+}
+
+func extractStringField(i map[string]interface{}, key string, notempty bool) (field string, err error) {
+	vi, ok := i[key]
+	if !ok {
+		err = errors.Errorf("must have field '%s'", key)
+		return "", err
+	}
+	field, ok = vi.(string)
+	if !ok {
+		err = errors.Errorf("'%s' field must have type string", key)
+		return "", err
+	}
+	if notempty && len(field) <= 0 {
+		err = errors.Errorf("'%s' field must not be empty", key)
+		return "", err
+	}
+	return
+}
+
+type JobParsingContext struct {
+	ConfigParsingContext
+}
+
+func parseJob(c JobParsingContext, i map[string]interface{}) (j Job, err error) {
+
+	name, err := extractStringField(i, "name", true)
+	if err != nil {
+		return
+	}
+
+	for _, r := range ReservedJobNames {
+		if name == r {
+			err = errors.Errorf("job name '%s' is reserved", name)
+			return
+		}
+	}
+
+	jobtype, err := extractStringField(i, "type", true)
+	if err != nil {
+		return
+	}
+
+	switch jobtype {
+	case "pull":
+		return parsePullJob(c, name, i)
+	case "source":
+		return parseSourceJob(c, name, i)
+	case "local":
+		return parseLocalJob(c, name, i)
+	default:
+		return nil, errors.Errorf("unknown job type '%s'", jobtype)
+	}
+
+}
+
+func parseConnect(i map[string]interface{}) (c RWCConnecter, err error) {
+
+	t, err := extractStringField(i, "type", true)
+	if err != nil {
+		return nil, err
+	}
+
+	switch t {
+	case "ssh+stdinserver":
+		return parseSSHStdinserverConnecter(i)
+	default:
+		return nil, errors.Errorf("unknown connection type '%s'", t)
+	}
+
+}
+
+func parseInitialReplPolicy(v interface{}, defaultPolicy InitialReplPolicy) (p InitialReplPolicy, err error) {
+	s, ok := v.(string)
+	if !ok {
+		goto err
+	}
+
+	switch {
+	case s == "":
+		p = defaultPolicy
+	case s == "most_recent":
+		p = InitialReplPolicyMostRecent
+	case s == "all":
+		p = InitialReplPolicyAll
+	default:
+		goto err
+	}
+
+	return
+
+err:
+	err = errors.New(fmt.Sprintf("expected InitialReplPolicy, got %#v", v))
+	return
+}
+
+func parsePrunePolicy(v map[string]interface{}, willSeeBookmarks bool) (p PrunePolicy, err error) {
+
+	policyName, err := extractStringField(v, "policy", true)
+	if err != nil {
+		return
+	}
+
+	switch policyName {
+	case "grid":
+		return parseGridPrunePolicy(v, willSeeBookmarks)
+	case "noprune":
+		return NoPrunePolicy{}, nil
+	default:
+		err = errors.Errorf("unknown policy '%s'", policyName)
+		return
+	}
+}
+
+func parseAuthenticatedChannelListenerFactory(c JobParsingContext, v map[string]interface{}) (p AuthenticatedChannelListenerFactory, err error) {
+
+	t, err := extractStringField(v, "type", true)
+	if err != nil {
+		return nil, err
+	}
+
+	switch t {
+	case "stdinserver":
+		return parseStdinserverListenerFactory(c, v)
+	default:
+		err = errors.Errorf("unknown type '%s'", t)
+		return
+	}
+
+}
+
+var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
+
+func parsePostitiveDuration(e string) (d time.Duration, err error) {
+	comps := durationStringRegex.FindStringSubmatch(e)
+	if len(comps) != 3 {
+		err = fmt.Errorf("does not match regex: %s %#v", e, comps)
+		return
+	}
+
+	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+	if durationFactor <= 0 {
+		return 0, errors.New("duration must be positive integer")
+	}
+
+	var durationUnit time.Duration
+	switch comps[2] {
+	case "s":
+		durationUnit = time.Second
+	case "m":
+		durationUnit = time.Minute
+	case "h":
+		durationUnit = time.Hour
+	case "d":
+		durationUnit = 24 * time.Hour
+	case "w":
+		durationUnit = 24 * 7 * time.Hour
+	default:
+		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
+		return
+	}
+
+	d = time.Duration(durationFactor) * durationUnit
+	return
+}

cmd/config_prune_grid.go

@@ -0,0 +1,245 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/util"
+	"github.com/zrepl/zrepl/zfs"
+	"math"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type GridPrunePolicy struct {
+	RetentionGrid *util.RetentionGrid
+	MaxBookmarks  int
+}
+
+const GridPrunePolicyMaxBookmarksKeepAll = -1
+
+type retentionGridAdaptor struct {
+	zfs.FilesystemVersion
+}
+
+func (a retentionGridAdaptor) Date() time.Time {
+	return a.Creation
+}
+
+func (a retentionGridAdaptor) LessThan(b util.RetentionGridEntry) bool {
+	return a.CreateTXG < b.(retentionGridAdaptor).CreateTXG
+}
+
+// Prune filters snapshots with the retention grid.
+// Bookmarks are deleted such that KeepBookmarks are kept in the end.
+// The oldest bookmarks are removed first.
+func (p *GridPrunePolicy) Prune(_ *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error) {
+	skeep, sremove := p.pruneSnapshots(versions)
+	keep, remove = p.pruneBookmarks(skeep)
+	remove = append(remove, sremove...)
+	return keep, remove, nil
+}
+
+func (p *GridPrunePolicy) pruneSnapshots(versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion) {
+
+	// Build adaptors for retention grid
+	keep = []zfs.FilesystemVersion{}
+	adaptors := make([]util.RetentionGridEntry, 0)
+	for fsv := range versions {
+		if versions[fsv].Type != zfs.Snapshot {
+			keep = append(keep, versions[fsv])
+			continue
+		}
+		adaptors = append(adaptors, retentionGridAdaptor{versions[fsv]})
+	}
+
+	sort.SliceStable(adaptors, func(i, j int) bool {
+		return adaptors[i].LessThan(adaptors[j])
+	})
+	now := adaptors[len(adaptors)-1].Date()
+
+	// Evaluate retention grid
+	keepa, removea := p.RetentionGrid.FitEntries(now, adaptors)
+
+	// Revert adaptors
+	for i := range keepa {
+		keep = append(keep, keepa[i].(retentionGridAdaptor).FilesystemVersion)
+	}
+	remove = make([]zfs.FilesystemVersion, len(removea))
+	for i := range removea {
+		remove[i] = removea[i].(retentionGridAdaptor).FilesystemVersion
+	}
+	return
+
+}
+
+func (p *GridPrunePolicy) pruneBookmarks(versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion) {
+
+	if p.MaxBookmarks == GridPrunePolicyMaxBookmarksKeepAll {
+		return versions, []zfs.FilesystemVersion{}
+	}
+
+	keep = []zfs.FilesystemVersion{}
+	bookmarks := make([]zfs.FilesystemVersion, 0)
+	for fsv := range versions {
+		if versions[fsv].Type != zfs.Bookmark {
+			keep = append(keep, versions[fsv])
+			continue
+		}
+		bookmarks = append(bookmarks, versions[fsv])
+	}
+
+	if len(bookmarks) == 0 {
+		return keep, []zfs.FilesystemVersion{}
+	}
+	if len(bookmarks) < p.MaxBookmarks {
+		keep = append(keep, bookmarks...)
+		return keep, []zfs.FilesystemVersion{}
+	}
+
+	// NOTE: sorting descending by descending by createtxg <=> sorting ascending wrt creation time
+	sort.SliceStable(bookmarks, func(i, j int) bool {
+		return (bookmarks[i].CreateTXG > bookmarks[j].CreateTXG)
+	})
+
+	keep = append(keep, bookmarks[:p.MaxBookmarks]...)
+	remove = bookmarks[p.MaxBookmarks:]
+
+	return keep, remove
+}
+
+func parseGridPrunePolicy(e map[string]interface{}, willSeeBookmarks bool) (p *GridPrunePolicy, err error) {
+
+	const KeepBookmarksAllString = "all"
+	var i struct {
+		Grid          string
+		KeepBookmarks string `mapstructure:"keep_bookmarks"`
+	}
+
+	dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{Result: &i, WeaklyTypedInput: true})
+	if err != nil {
+		err = errors.Wrap(err, "mapstructure error")
+		return
+	}
+	if err = dec.Decode(e); err != nil {
+		err = errors.Wrapf(err, "mapstructure error")
+		return
+	}
+
+	// Parse grid
+	intervals, err := parseRetentionGridIntervalsString(i.Grid)
+	if err != nil {
+		err = fmt.Errorf("cannot parse retention grid: %s", err)
+		return
+	}
+	// Assert intervals are of increasing length (not necessarily required, but indicates config mistake)
+	lastDuration := time.Duration(0)
+	for i := range intervals {
+
+		if intervals[i].Length < lastDuration {
+			// If all intervals before were keep=all, this is ok
+			allPrevKeepCountAll := true
+			for j := i - 1; allPrevKeepCountAll && j >= 0; j-- {
+				allPrevKeepCountAll = intervals[j].KeepCount == util.RetentionGridKeepCountAll
+			}
+			if allPrevKeepCountAll {
+				goto isMonotonicIncrease
+			}
+			err = errors.New("retention grid interval length must be monotonically increasing")
+			return
+		}
+	isMonotonicIncrease:
+		lastDuration = intervals[i].Length
+
+	}
+
+	// Parse KeepBookmarks
+	keepBookmarks := 0
+	if i.KeepBookmarks == KeepBookmarksAllString || (i.KeepBookmarks == "" && !willSeeBookmarks) {
+		keepBookmarks = GridPrunePolicyMaxBookmarksKeepAll
+	} else {
+		i, err := strconv.ParseInt(i.KeepBookmarks, 10, 32)
+		if err != nil || i <= 0 || i > math.MaxInt32 {
+			return nil, errors.Errorf("keep_bookmarks must be positive integer or 'all'")
+		}
+		keepBookmarks = int(i)
+	}
+	return &GridPrunePolicy{
+		util.NewRetentionGrid(intervals),
+		keepBookmarks,
+	}, nil
+}
+
+var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
+
+func parseRetentionGridIntervalString(e string) (intervals []util.RetentionInterval, err error) {
+
+	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
+	if comps == nil {
+		err = fmt.Errorf("retention string does not match expected format")
+		return
+	}
+
+	times, err := strconv.Atoi(comps[1])
+	if err != nil {
+		return nil, err
+	} else if times <= 0 {
+		return nil, fmt.Errorf("contains factor <= 0")
+	}
+
+	duration, err := parsePostitiveDuration(comps[2])
+	if err != nil {
+		return nil, err
+	}
+
+	keepCount := 1
+	if comps[3] != "" {
+		// Decompose key=value, comma separated
+		// For now, only keep_count is supported
+		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
+		res := re.FindStringSubmatch(comps[4])
+		if res == nil || len(res) != 2 {
+			err = fmt.Errorf("interval parameter contains unknown parameters")
+			return
+		}
+		if res[1] == "all" {
+			keepCount = util.RetentionGridKeepCountAll
+		} else {
+			keepCount, err = strconv.Atoi(res[1])
+			if err != nil {
+				err = fmt.Errorf("cannot parse keep_count value")
+				return
+			}
+		}
+	}
+
+	intervals = make([]util.RetentionInterval, times)
+	for i := range intervals {
+		intervals[i] = util.RetentionInterval{
+			Length:    duration,
+			KeepCount: keepCount,
+		}
+	}
+
+	return
+
+}
+
+func parseRetentionGridIntervalsString(s string) (intervals []util.RetentionInterval, err error) {
+
+	ges := strings.Split(s, "|")
+	intervals = make([]util.RetentionInterval, 0, 7*len(ges))
+
+	for intervalIdx, e := range ges {
+		parsed, err := parseRetentionGridIntervalString(e)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
+		}
+		intervals = append(intervals, parsed...)
+	}
+
+	return
+}

cmd/config_prune_none.go

@@ -0,0 +1,11 @@
+package cmd
+
+import "github.com/zrepl/zrepl/zfs"
+
+type NoPrunePolicy struct{}
+
+func (p NoPrunePolicy) Prune(fs *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error) {
+	keep = versions
+	remove = []zfs.FilesystemVersion{}
+	return
+}

cmd/config_serve_stdinserver.go

@@ -0,0 +1,56 @@
+package cmd
+
+import (
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"github.com/problame/go-netssh"
+	"io"
+	"path"
+)
+
+type StdinserverListenerFactory struct {
+	ClientIdentity string `mapstructure:"client_identity"`
+	sockpath       string
+}
+
+func parseStdinserverListenerFactory(c JobParsingContext, i map[string]interface{}) (f *StdinserverListenerFactory, err error) {
+
+	f = &StdinserverListenerFactory{}
+
+	if err = mapstructure.Decode(i, f); err != nil {
+		return nil, errors.Wrap(err, "mapstructure error")
+	}
+	if !(len(f.ClientIdentity) > 0) {
+		err = errors.Errorf("must specify 'client_identity'")
+		return
+	}
+
+	f.sockpath = path.Join(c.Global.Serve.Stdinserver.SockDir, f.ClientIdentity)
+
+	return
+}
+
+func (f *StdinserverListenerFactory) Listen() (al AuthenticatedChannelListener, err error) {
+
+	if err = PreparePrivateSockpath(f.sockpath); err != nil {
+		return nil, err
+	}
+
+	l, err := netssh.Listen(f.sockpath)
+	if err != nil {
+		return nil, err
+	}
+	return StdinserverListener{l}, nil
+}
+
+type StdinserverListener struct {
+	l *netssh.Listener
+}
+
+func (l StdinserverListener) Accept() (ch io.ReadWriteCloser, err error) {
+	return l.l.Accept()
+}
+
+func (l StdinserverListener) Close() (err error) {
+	return l.l.Close()
+}

cmd/config_test.go

@@ -0,0 +1,222 @@
+package cmd
+
+import (
+	"testing"
+	"time"
+
+	"github.com/kr/pretty"
+	"github.com/stretchr/testify/assert"
+	"github.com/zrepl/zrepl/util"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
+
+	paths := []string{
+		"./sampleconf/localbackup/host1.yml",
+		"./sampleconf/pullbackup/backuphost.yml",
+		"./sampleconf/pullbackup/productionhost.yml",
+		"./sampleconf/random/debugging.yml",
+		"./sampleconf/random/logging.yml",
+	}
+
+	for _, p := range paths {
+
+		c, err := ParseConfig(p)
+		if err != nil {
+			t.Errorf("error parsing %s:\n%+v", p, err)
+		}
+
+		t.Logf("file: %s", p)
+		t.Log(pretty.Sprint(c))
+
+	}
+
+}
+
+func TestParseRetentionGridStringParsing(t *testing.T) {
+
+	intervals, err := parseRetentionGridIntervalsString("2x10m(keep=2) | 1x1h | 3x1w")
+
+	assert.Nil(t, err)
+	assert.Len(t, intervals, 6)
+	proto := util.RetentionInterval{
+		KeepCount: 2,
+		Length:    10 * time.Minute,
+	}
+	assert.EqualValues(t, proto, intervals[0])
+	assert.EqualValues(t, proto, intervals[1])
+
+	proto.KeepCount = 1
+	proto.Length = 1 * time.Hour
+	assert.EqualValues(t, proto, intervals[2])
+
+	proto.Length = 7 * 24 * time.Hour
+	assert.EqualValues(t, proto, intervals[3])
+	assert.EqualValues(t, proto, intervals[4])
+	assert.EqualValues(t, proto, intervals[5])
+
+	intervals, err = parseRetentionGridIntervalsString("|")
+	assert.Error(t, err)
+	intervals, err = parseRetentionGridIntervalsString("2x10m")
+	assert.NoError(t, err)
+
+	intervals, err = parseRetentionGridIntervalsString("1x10m(keep=all)")
+	assert.NoError(t, err)
+	assert.Len(t, intervals, 1)
+	assert.EqualValues(t, util.RetentionGridKeepCountAll, intervals[0].KeepCount)
+
+}
+
+func TestDatasetMapFilter(t *testing.T) {
+
+	expectMapping := func(m map[string]string, from, to string) {
+		dmf, err := parseDatasetMapFilter(m, false)
+		if err != nil {
+			t.Logf("expect test map to be valid: %s", err)
+			t.FailNow()
+		}
+		fromPath, err := zfs.NewDatasetPath(from)
+		if err != nil {
+			t.Logf("expect test from path to be valid: %s", err)
+			t.FailNow()
+		}
+
+		res, err := dmf.Map(fromPath)
+		if to == "" {
+			assert.Nil(t, res)
+			assert.Nil(t, err)
+			t.Logf("%s => NOT MAPPED", fromPath.ToString())
+			return
+		}
+
+		assert.Nil(t, err)
+		toPath, err := zfs.NewDatasetPath(to)
+		if err != nil {
+			t.Logf("expect test to path to be valid: %s", err)
+			t.FailNow()
+		}
+		assert.True(t, res.Equal(toPath))
+	}
+
+	expectFilter := func(m map[string]string, path string, pass bool) {
+		dmf, err := parseDatasetMapFilter(m, true)
+		if err != nil {
+			t.Logf("expect test filter to be valid: %s", err)
+			t.FailNow()
+		}
+		p, err := zfs.NewDatasetPath(path)
+		if err != nil {
+			t.Logf("expect test path to be valid: %s", err)
+			t.FailNow()
+		}
+		res, err := dmf.Filter(p)
+		assert.Nil(t, err)
+		assert.Equal(t, pass, res)
+	}
+
+	map1 := map[string]string{
+		"a/b/c<":     "root1",
+		"a/b<":       "root2",
+		"<":          "root3/b/c",
+		"b":          "!",
+		"a/b/c/d/e<": "!",
+		"q<":         "root4/1/2",
+	}
+
+	expectMapping(map1, "a/b/c", "root1")
+	expectMapping(map1, "a/b/c/d", "root1/d")
+	expectMapping(map1, "a/b/c/d/e", "")
+	expectMapping(map1, "a/b/e", "root2/e")
+	expectMapping(map1, "a/b", "root2")
+	expectMapping(map1, "x", "root3/b/c/x")
+	expectMapping(map1, "x/y", "root3/b/c/x/y")
+	expectMapping(map1, "q", "root4/1/2")
+	expectMapping(map1, "b", "")
+	expectMapping(map1, "q/r", "root4/1/2/r")
+
+	filter1 := map[string]string{
+		"<":    "!",
+		"a<":   "ok",
+		"a/b<": "!",
+	}
+
+	expectFilter(filter1, "b", false)
+	expectFilter(filter1, "a", true)
+	expectFilter(filter1, "a/d", true)
+	expectFilter(filter1, "a/b", false)
+	expectFilter(filter1, "a/b/c", false)
+
+	filter2 := map[string]string{}
+	expectFilter(filter2, "foo", false) // default to omit
+
+}
+
+func TestDatasetMapFilter_AsFilter(t *testing.T) {
+
+	mapspec := map[string]string{
+		"a/b/c<":     "root1",
+		"a/b<":       "root2",
+		"<":          "root3/b/c",
+		"b":          "!",
+		"a/b/c/d/e<": "!",
+		"q<":         "root4/1/2",
+	}
+
+	m, err := parseDatasetMapFilter(mapspec, false)
+	assert.Nil(t, err)
+
+	f := m.AsFilter()
+
+	t.Logf("Mapping:\n%s\nFilter:\n%s", pretty.Sprint(m), pretty.Sprint(f))
+
+	tf := func(f zfs.DatasetFilter, path string, pass bool) {
+		p, err := zfs.NewDatasetPath(path)
+		assert.Nil(t, err)
+		r, err := f.Filter(p)
+		assert.Nil(t, err)
+		assert.Equal(t, pass, r)
+	}
+
+	tf(f, "a/b/c", true)
+	tf(f, "a/b", true)
+	tf(f, "b", false)
+	tf(f, "a/b/c/d/e", false)
+	tf(f, "a/b/c/d/e/f", false)
+	tf(f, "a", true)
+
+}
+
+func TestDatasetMapFilter_InvertedFilter(t *testing.T) {
+	mapspec := map[string]string{
+		"a/b":      "1/2",
+		"a/b/c<":   "3",
+		"a/b/c/d<": "1/2/a",
+		"a/b/d":    "!",
+	}
+
+	m, err := parseDatasetMapFilter(mapspec, false)
+	assert.Nil(t, err)
+
+	inv, err := m.InvertedFilter()
+	assert.Nil(t, err)
+
+	t.Log(pretty.Sprint(inv))
+
+	expectMapping := func(m *DatasetMapFilter, ps string, expRes bool) {
+		p, err := zfs.NewDatasetPath(ps)
+		assert.Nil(t, err)
+		r, err := m.Filter(p)
+		assert.Nil(t, err)
+		assert.Equal(t, expRes, r)
+	}
+
+	expectMapping(inv, "4", false)
+	expectMapping(inv, "3", true)
+	expectMapping(inv, "3/x", true)
+	expectMapping(inv, "1", false)
+	expectMapping(inv, "1/2", true)
+	expectMapping(inv, "1/2/3", false)
+	expectMapping(inv, "1/2/a/b", true)
+
+}

cmd/control.go

@@ -0,0 +1,305 @@
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/dustin/go-humanize"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/logger"
+	"io"
+	golog "log"
+	"net"
+	"net/http"
+	"os"
+	"sort"
+	"strings"
+	"time"
+)
+
+var controlCmd = &cobra.Command{
+	Use:   "control",
+	Short: "control zrepl daemon",
+}
+
+var pprofCmd = &cobra.Command{
+	Use:   "pprof off | [on TCP_LISTEN_ADDRESS]",
+	Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
+	Run:   doControlPProf,
+	PreRunE: func(cmd *cobra.Command, args []string) error {
+		if cmd.Flags().NArg() < 1 {
+			goto enargs
+		}
+		switch cmd.Flags().Arg(0) {
+		case "on":
+			pprofCmdArgs.msg.Run = true
+			if cmd.Flags().NArg() != 2 {
+				return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
+			}
+			pprofCmdArgs.msg.HttpListenAddress = cmd.Flags().Arg(1)
+		case "off":
+			if cmd.Flags().NArg() != 1 {
+				goto enargs
+			}
+			pprofCmdArgs.msg.Run = false
+		}
+		return nil
+	enargs:
+		return errors.New("invalid number of positional arguments")
+
+	},
+}
+var pprofCmdArgs struct {
+	msg PprofServerControlMsg
+}
+
+var controlVersionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "print version of running zrepl daemon",
+	Run:   doControLVersionCmd,
+}
+
+var controlStatusCmdArgs struct {
+	format      string
+	level       logger.Level
+	onlyShowJob string
+}
+
+var controlStatusCmd = &cobra.Command{
+	Use:   "status [JOB_NAME]",
+	Short: "get current status",
+	Run:   doControlStatusCmd,
+}
+
+func init() {
+	RootCmd.AddCommand(controlCmd)
+	controlCmd.AddCommand(pprofCmd)
+	controlCmd.AddCommand(controlVersionCmd)
+	controlCmd.AddCommand(controlStatusCmd)
+	controlStatusCmd.Flags().StringVar(&controlStatusCmdArgs.format, "format", "human", "output format (human|raw)")
+	controlStatusCmdArgs.level = logger.Warn
+	controlStatusCmd.Flags().Var(&controlStatusCmdArgs.level, "level", "minimum log level to show")
+}
+
+func controlHttpClient() (client http.Client, err error) {
+
+	conf, err := ParseConfig(rootArgs.configFile)
+	if err != nil {
+		return http.Client{}, err
+	}
+
+	return http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", conf.Global.Control.Sockpath)
+			},
+		},
+	}, nil
+}
+
+func doControlPProf(cmd *cobra.Command, args []string) {
+
+	log := golog.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Printf("exiting after error")
+		os.Exit(1)
+	}
+
+	log.Printf("connecting to zrepl daemon")
+	httpc, err := controlHttpClient()
+	if err != nil {
+		log.Printf("error parsing config: %s", err)
+		die()
+	}
+
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(&pprofCmdArgs.msg); err != nil {
+		log.Printf("error marshaling request: %s", err)
+		die()
+	}
+	_, err = httpc.Post("http://unix"+ControlJobEndpointPProf, "application/json", &buf)
+	if err != nil {
+		log.Printf("error: %s", err)
+		die()
+	}
+
+	log.Printf("finished")
+}
+
+func doControLVersionCmd(cmd *cobra.Command, args []string) {
+
+	log := golog.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Printf("exiting after error")
+		os.Exit(1)
+	}
+
+	httpc, err := controlHttpClient()
+	if err != nil {
+		log.Printf("could not connect to daemon: %s", err)
+		die()
+	}
+
+	resp, err := httpc.Get("http://unix" + ControlJobEndpointVersion)
+	if err != nil {
+		log.Printf("error: %s", err)
+		die()
+	} else if resp.StatusCode != http.StatusOK {
+		var msg bytes.Buffer
+		io.CopyN(&msg, resp.Body, 4096)
+		log.Printf("error: %s", msg.String())
+		die()
+	}
+
+	var info ZreplVersionInformation
+	err = json.NewDecoder(resp.Body).Decode(&info)
+	if err != nil {
+		log.Printf("error unmarshaling response: %s", err)
+		die()
+	}
+
+	fmt.Println(info.String())
+
+}
+
+func doControlStatusCmd(cmd *cobra.Command, args []string) {
+
+	log := golog.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Print("exiting after error")
+		os.Exit(1)
+	}
+
+	if len(args) == 1 {
+		controlStatusCmdArgs.onlyShowJob = args[0]
+	} else if len(args) > 1 {
+		log.Print("can only specify one job as positional argument")
+		cmd.Usage()
+		die()
+	}
+
+	httpc, err := controlHttpClient()
+	if err != nil {
+		log.Printf("could not connect to daemon: %s", err)
+		die()
+	}
+
+	resp, err := httpc.Get("http://unix" + ControlJobEndpointStatus)
+	if err != nil {
+		log.Printf("error: %s", err)
+		die()
+	} else if resp.StatusCode != http.StatusOK {
+		var msg bytes.Buffer
+		io.CopyN(&msg, resp.Body, 4096)
+		log.Printf("error: %s", msg.String())
+		die()
+	}
+
+	var status DaemonStatus
+	err = json.NewDecoder(resp.Body).Decode(&status)
+	if err != nil {
+		log.Printf("error unmarshaling response: %s", err)
+		die()
+	}
+
+	switch controlStatusCmdArgs.format {
+	case "raw":
+		enc := json.NewEncoder(os.Stdout)
+		enc.SetIndent("", "  ")
+		if err := enc.Encode(status); err != nil {
+			log.Panic(err)
+		}
+	case "human":
+
+		formatter := HumanFormatter{}
+		formatter.SetMetadataFlags(MetadataAll)
+		formatter.SetIgnoreFields([]string{
+			logJobField,
+		})
+		jobNames := make([]string, 0, len(status.Jobs))
+		for name, _ := range status.Jobs {
+			jobNames = append(jobNames, name)
+		}
+		sort.Slice(jobNames, func(i, j int) bool {
+			return strings.Compare(jobNames[i], jobNames[j]) == -1
+		})
+		now := time.Now()
+		for _, name := range jobNames {
+
+			if controlStatusCmdArgs.onlyShowJob != "" && name != controlStatusCmdArgs.onlyShowJob {
+				continue
+			}
+
+			job := status.Jobs[name]
+			jobLogEntries := make([]logger.Entry, 0)
+			informAboutError := false
+
+			fmt.Printf("Job '%s':\n", name)
+			for _, task := range job.Tasks {
+
+				var header bytes.Buffer
+				fmt.Fprintf(&header, "  Task '%s': ", task.Name)
+				if !task.Idle {
+					fmt.Fprint(&header, strings.Join(task.ActivityStack, "."))
+				} else {
+					fmt.Fprint(&header, "<idle>")
+				}
+				fmt.Fprint(&header, " ")
+				const TASK_STALLED_HOLDOFF_DURATION = 10 * time.Second
+				sinceLastUpdate := now.Sub(task.LastUpdate)
+				if !task.Idle || task.ProgressRx != 0 || task.ProgressTx != 0 {
+					fmt.Fprintf(&header, "(%s / %s , Rx/Tx",
+						humanize.Bytes(uint64(task.ProgressRx)),
+						humanize.Bytes(uint64(task.ProgressTx)))
+					if task.Idle {
+						fmt.Fprint(&header, ", values from last run")
+					}
+					fmt.Fprint(&header, ")")
+				}
+				fmt.Fprint(&header, "\n")
+				if !task.Idle && !task.LastUpdate.IsZero() && sinceLastUpdate >= TASK_STALLED_HOLDOFF_DURATION {
+					informAboutError = true
+					fmt.Fprintf(&header, "    WARNING: last update %s ago at %s)",
+						sinceLastUpdate.String(),
+						task.LastUpdate.Format(HumanFormatterDateFormat))
+					fmt.Fprint(&header, "\n")
+				}
+				io.Copy(os.Stdout, &header)
+
+				jobLogEntries = append(jobLogEntries, task.LogEntries...)
+				informAboutError = informAboutError || task.MaxLogLevel >= logger.Warn
+			}
+
+			sort.Slice(jobLogEntries, func(i, j int) bool {
+				return jobLogEntries[i].Time.Before(jobLogEntries[j].Time)
+			})
+			if informAboutError {
+				fmt.Println("    WARNING: Some tasks encountered problems since the last time they left idle state:")
+				fmt.Println("             check the logs below or your log file for more information.")
+				fmt.Println("             Use the --level flag if you need debug information.")
+				fmt.Println()
+			}
+			for _, e := range jobLogEntries {
+				if e.Level < controlStatusCmdArgs.level {
+					continue
+				}
+				formatted, err := formatter.Format(&e)
+				if err != nil {
+					panic(err)
+				}
+				fmt.Printf("      %s\n", string(formatted))
+			}
+			fmt.Println()
+
+		}
+	default:
+		log.Printf("invalid output format '%s'", controlStatusCmdArgs.format)
+		die()
+	}
+
+}

cmd/control_pprof.go

@@ -1,4 +1,4 @@
-package daemon
+package cmd
 
 import (
 	"net/http"
@@ -7,12 +7,11 @@ import (
 	"context"
 	"net"
 	"net/http/pprof"
-
-	"github.com/zrepl/zrepl/daemon/job"
 )
 
-type pprofServer struct {
+type PProfServer struct {
 	cc       chan PprofServerControlMsg
+	state    PprofServerControlMsg
 	listener net.Listener
 }
 
@@ -23,9 +22,9 @@ type PprofServerControlMsg struct {
 	HttpListenAddress string
 }
 
-func NewPProfServer(ctx context.Context) *pprofServer {
+func NewPProfServer(ctx context.Context) *PProfServer {
 
-	s := &pprofServer{
+	s := &PProfServer{
 		cc: make(chan PprofServerControlMsg),
 	}
 
@@ -33,7 +32,7 @@ func NewPProfServer(ctx context.Context) *pprofServer {
 	return s
 }
 
-func (s *pprofServer) controlLoop(ctx context.Context) {
+func (s *PProfServer) controlLoop(ctx context.Context) {
 outer:
 	for {
 
@@ -64,14 +63,7 @@ outer:
 			mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
 			mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
 			mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
-			go func() {
+			go http.Serve(s.listener, mux)
-				err := http.Serve(s.listener, mux)
-				if ctx.Err() != nil {
-					return
-				} else if err != nil {
-					job.GetLogger(ctx).WithError(err).Error("pprof server serve error")
-				}
-			}()
 			continue
 		}
 
@@ -83,6 +75,6 @@ outer:
 	}
 }
 
-func (s *pprofServer) Control(msg PprofServerControlMsg) {
+func (s *PProfServer) Control(msg PprofServerControlMsg) {
 	s.cc <- msg
 }

cmd/daemon.go

@@ -0,0 +1,394 @@
+package cmd
+
+import (
+	"container/list"
+	"context"
+	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/logger"
+	"io"
+	"os"
+	"os/signal"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+)
+
+// daemonCmd represents the daemon command
+var daemonCmd = &cobra.Command{
+	Use:   "daemon",
+	Short: "start daemon",
+	Run:   doDaemon,
+}
+
+func init() {
+	RootCmd.AddCommand(daemonCmd)
+}
+
+type Job interface {
+	JobName() string
+	JobStart(ctxt context.Context)
+	JobStatus(ctxt context.Context) (*JobStatus, error)
+}
+
+func doDaemon(cmd *cobra.Command, args []string) {
+
+	conf, err := ParseConfig(rootArgs.configFile)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error parsing config: %s\n", err)
+		os.Exit(1)
+	}
+
+	log := logger.NewLogger(conf.Global.logging.Outlets, 1*time.Second)
+
+	log.Info(NewZreplVersionInformation().String())
+	log.Debug("starting daemon")
+	ctx := context.WithValue(context.Background(), contextKeyLog, log)
+	ctx = context.WithValue(ctx, contextKeyLog, log)
+
+	d := NewDaemon(conf)
+	d.Loop(ctx)
+
+}
+
+type contextKey string
+
+const (
+	contextKeyLog    contextKey = contextKey("log")
+	contextKeyDaemon contextKey = contextKey("daemon")
+)
+
+type Daemon struct {
+	conf      *Config
+	startedAt time.Time
+}
+
+func NewDaemon(initialConf *Config) *Daemon {
+	return &Daemon{conf: initialConf}
+}
+
+func (d *Daemon) Loop(ctx context.Context) {
+
+	d.startedAt = time.Now()
+
+	log := ctx.Value(contextKeyLog).(Logger)
+
+	ctx, cancel := context.WithCancel(ctx)
+	ctx = context.WithValue(ctx, contextKeyDaemon, d)
+
+	sigChan := make(chan os.Signal, 1)
+	finishs := make(chan Job)
+
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+
+	log.Info("starting jobs from config")
+	i := 0
+	for _, job := range d.conf.Jobs {
+		logger := log.WithField(logJobField, job.JobName())
+		logger.Info("starting")
+		i++
+		jobCtx := context.WithValue(ctx, contextKeyLog, logger)
+		go func(j Job) {
+			j.JobStart(jobCtx)
+			finishs <- j
+		}(job)
+	}
+
+	finishCount := 0
+outer:
+	for {
+		select {
+		case <-finishs:
+			finishCount++
+			if finishCount == len(d.conf.Jobs) {
+				log.Info("all jobs finished")
+				break outer
+			}
+
+		case sig := <-sigChan:
+			log.WithField("signal", sig).Info("received signal")
+			log.Info("cancelling all jobs")
+			cancel()
+		}
+	}
+
+	signal.Stop(sigChan)
+	cancel() // make go vet happy
+
+	log.Info("exiting")
+
+}
+
+// Representation of a Job's status that is composed of Tasks
+type JobStatus struct {
+	// Statuses of all tasks of this job
+	Tasks []*TaskStatus
+	// Error != "" if JobStatus() returned an error
+	JobStatusError string
+}
+
+// Representation of a Daemon's status that is composed of Jobs
+type DaemonStatus struct {
+	StartedAt time.Time
+	Jobs      map[string]*JobStatus
+}
+
+func (d *Daemon) Status() (s *DaemonStatus) {
+
+	s = &DaemonStatus{}
+	s.StartedAt = d.startedAt
+
+	s.Jobs = make(map[string]*JobStatus, len(d.conf.Jobs))
+
+	for name, j := range d.conf.Jobs {
+		status, err := j.JobStatus(context.TODO())
+		if err != nil {
+			s.Jobs[name] = &JobStatus{nil, err.Error()}
+			continue
+		}
+		s.Jobs[name] = status
+	}
+
+	return
+}
+
+// Representation of a Task's status
+type TaskStatus struct {
+	Name string
+	// Whether the task is idle.
+	Idle bool
+	// The stack of activities the task is currently executing.
+	// The first element is the root activity and equal to Name.
+	ActivityStack []string
+	// Number of bytes received by the task since it last left idle state.
+	ProgressRx int64
+	// Number of bytes sent by the task since it last left idle state.
+	ProgressTx int64
+	// Log entries emitted by the task since it last left idle state.
+	// Only contains the log entries emitted through the task's logger
+	// (provided by Task.Log()).
+	LogEntries []logger.Entry
+	// The maximum log level of LogEntries.
+	// Only valid if len(LogEntries) > 0.
+	MaxLogLevel logger.Level
+	// Last time something about the Task changed
+	LastUpdate time.Time
+}
+
+// An instance of Task tracks  a single thread of activity that is part of a Job.
+type Task struct {
+	// Stack of activities the task is currently in
+	// Members are instances of taskActivity
+	activities *list.List
+	// Last time activities was changed (not the activities inside, the list)
+	activitiesLastUpdate time.Time
+	// Protects Task members from modification
+	rwl sync.RWMutex
+}
+
+// Structure that describes the progress a Task has made
+type taskProgress struct {
+	rx         int64
+	tx         int64
+	lastUpdate time.Time
+	logEntries []logger.Entry
+	mtx        sync.RWMutex
+}
+
+func newTaskProgress() (p *taskProgress) {
+	return &taskProgress{
+		logEntries: make([]logger.Entry, 0),
+	}
+}
+
+func (p *taskProgress) UpdateIO(drx, dtx int64) {
+	p.mtx.Lock()
+	defer p.mtx.Unlock()
+	p.rx += drx
+	p.tx += dtx
+	p.lastUpdate = time.Now()
+}
+
+func (p *taskProgress) UpdateLogEntry(entry logger.Entry) {
+	p.mtx.Lock()
+	defer p.mtx.Unlock()
+	// FIXME: ensure maximum size (issue #48)
+	p.logEntries = append(p.logEntries, entry)
+	p.lastUpdate = time.Now()
+}
+
+func (p *taskProgress) DeepCopy() (out taskProgress) {
+	p.mtx.RLock()
+	defer p.mtx.RUnlock()
+	out.rx, out.tx = p.rx, p.tx
+	out.lastUpdate = p.lastUpdate
+	out.logEntries = make([]logger.Entry, len(p.logEntries))
+	for i := range p.logEntries {
+		out.logEntries[i] = p.logEntries[i]
+	}
+	return
+}
+
+// returns a copy of this taskProgress, the mutex carries no semantic value
+func (p *taskProgress) Read() (out taskProgress) {
+	p.mtx.RLock()
+	defer p.mtx.RUnlock()
+	return p.DeepCopy()
+}
+
+// Element of a Task's activity stack
+type taskActivity struct {
+	name   string
+	idle   bool
+	logger *logger.Logger
+	// The progress of the task that is updated by UpdateIO() and UpdateLogEntry()
+	//
+	// Progress happens on a task-level and is thus global to the task.
+	// That's why progress is just a pointer to the current taskProgress:
+	// we reset progress when leaving the idle root activity
+	progress *taskProgress
+}
+
+func NewTask(name string, lg *logger.Logger) *Task {
+	t := &Task{
+		activities: list.New(),
+	}
+	rootLogger := lg.ReplaceField(logTaskField, name).
+		WithOutlet(t, logger.Debug)
+	rootAct := &taskActivity{name, true, rootLogger, newTaskProgress()}
+	t.activities.PushFront(rootAct)
+	return t
+}
+
+// callers must hold t.rwl
+func (t *Task) cur() *taskActivity {
+	return t.activities.Front().Value.(*taskActivity)
+}
+
+// buildActivityStack returns the stack of activity names
+// t.rwl must be held, but the slice can be returned since strings are immutable
+func (t *Task) buildActivityStack() []string {
+	comps := make([]string, 0, t.activities.Len())
+	for e := t.activities.Back(); e != nil; e = e.Prev() {
+		act := e.Value.(*taskActivity)
+		comps = append(comps, act.name)
+	}
+	return comps
+}
+
+// Start a sub-activity.
+// Must always be matched with a call to t.Finish()
+// --- consider using defer for this purpose.
+func (t *Task) Enter(activity string) {
+	t.rwl.Lock()
+	defer t.rwl.Unlock()
+
+	prev := t.cur()
+	if prev.idle {
+		// reset progress when leaving idle task
+		// we leave the old progress dangling to have the user not worry about
+		prev.progress = newTaskProgress()
+	}
+	act := &taskActivity{activity, false, nil, prev.progress}
+	t.activities.PushFront(act)
+	stack := t.buildActivityStack()
+	activityField := strings.Join(stack, ".")
+	act.logger = prev.logger.ReplaceField(logTaskField, activityField)
+
+	t.activitiesLastUpdate = time.Now()
+}
+
+func (t *Task) UpdateProgress(dtx, drx int64) {
+	t.rwl.RLock()
+	p := t.cur().progress // protected by own rwlock
+	t.rwl.RUnlock()
+	p.UpdateIO(dtx, drx)
+}
+
+// Returns a wrapper io.Reader that updates this task's _current_ progress value.
+// Progress updates after this task resets its progress value are discarded.
+func (t *Task) ProgressUpdater(r io.Reader) *IOProgressUpdater {
+	t.rwl.RLock()
+	defer t.rwl.RUnlock()
+	return &IOProgressUpdater{r, t.cur().progress}
+}
+
+func (t *Task) Status() *TaskStatus {
+	t.rwl.RLock()
+	defer t.rwl.RUnlock()
+	// NOTE
+	// do not return any state in TaskStatus that is protected by t.rwl
+
+	cur := t.cur()
+	stack := t.buildActivityStack()
+	prog := cur.progress.Read()
+
+	var maxLevel logger.Level
+	for _, entry := range prog.logEntries {
+		if maxLevel < entry.Level {
+			maxLevel = entry.Level
+		}
+	}
+
+	lastUpdate := prog.lastUpdate
+	if lastUpdate.Before(t.activitiesLastUpdate) {
+		lastUpdate = t.activitiesLastUpdate
+	}
+
+	s := &TaskStatus{
+		Name:          stack[0],
+		ActivityStack: stack,
+		Idle:          cur.idle,
+		ProgressRx:    prog.rx,
+		ProgressTx:    prog.tx,
+		LogEntries:    prog.logEntries,
+		MaxLogLevel:   maxLevel,
+		LastUpdate:    lastUpdate,
+	}
+
+	return s
+}
+
+// Finish a sub-activity.
+// Corresponds to a preceding call to t.Enter()
+func (t *Task) Finish() {
+	t.rwl.Lock()
+	defer t.rwl.Unlock()
+	top := t.activities.Front()
+	if top.Next() == nil {
+		return // cannot remove root activity
+	}
+	t.activities.Remove(top)
+	t.activitiesLastUpdate = time.Now()
+
+}
+
+// Returns a logger derived from the logger passed to the constructor function.
+// The logger's task field contains the current activity stack joined by '.'.
+func (t *Task) Log() *logger.Logger {
+	t.rwl.RLock()
+	defer t.rwl.RUnlock()
+	// FIXME should influence TaskStatus's LastUpdate field
+	return t.cur().logger
+}
+
+// implement logger.Outlet interface
+func (t *Task) WriteEntry(entry logger.Entry) error {
+	t.rwl.RLock()
+	defer t.rwl.RUnlock()
+	t.cur().progress.UpdateLogEntry(entry)
+	return nil
+}
+
+type IOProgressUpdater struct {
+	r io.Reader
+	p *taskProgress
+}
+
+func (u *IOProgressUpdater) Read(p []byte) (n int, err error) {
+	n, err = u.r.Read(p)
+	u.p.UpdateIO(int64(n), 0)
+	return
+
+}

cmd/handler.go

@@ -0,0 +1,181 @@
+package cmd
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type DatasetMapping interface {
+	Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error)
+}
+
+type FilesystemRequest struct {
+	Roots []string // may be nil, indicating interest in all filesystems
+}
+
+type FilesystemVersionsRequest struct {
+	Filesystem *zfs.DatasetPath
+}
+
+type InitialTransferRequest struct {
+	Filesystem        *zfs.DatasetPath
+	FilesystemVersion zfs.FilesystemVersion
+}
+
+type IncrementalTransferRequest struct {
+	Filesystem *zfs.DatasetPath
+	From       zfs.FilesystemVersion
+	To         zfs.FilesystemVersion
+}
+
+type Handler struct {
+	logger Logger
+	dsf    zfs.DatasetFilter
+	fsvf   zfs.FilesystemVersionFilter
+}
+
+func NewHandler(logger Logger, dsfilter zfs.DatasetFilter, snapfilter zfs.FilesystemVersionFilter) (h Handler) {
+	return Handler{logger, dsfilter, snapfilter}
+}
+
+func registerEndpoints(server rpc.RPCServer, handler Handler) (err error) {
+	err = server.RegisterEndpoint("FilesystemRequest", handler.HandleFilesystemRequest)
+	if err != nil {
+		panic(err)
+	}
+	err = server.RegisterEndpoint("FilesystemVersionsRequest", handler.HandleFilesystemVersionsRequest)
+	if err != nil {
+		panic(err)
+	}
+	err = server.RegisterEndpoint("InitialTransferRequest", handler.HandleInitialTransferRequest)
+	if err != nil {
+		panic(err)
+	}
+	err = server.RegisterEndpoint("IncrementalTransferRequest", handler.HandleIncrementalTransferRequest)
+	if err != nil {
+		panic(err)
+	}
+	return nil
+}
+
+func (h Handler) HandleFilesystemRequest(r *FilesystemRequest, roots *[]*zfs.DatasetPath) (err error) {
+
+	log := h.logger.WithField("endpoint", "FilesystemRequest")
+
+	log.WithField("request", r).Debug("request")
+	log.WithField("dataset_filter", h.dsf).Debug("dsf")
+
+	allowed, err := zfs.ZFSListMapping(h.dsf)
+	if err != nil {
+		log.WithError(err).Error("error listing filesystems")
+		return
+	}
+
+	log.WithField("response", allowed).Debug("response")
+	*roots = allowed
+	return
+}
+
+func (h Handler) HandleFilesystemVersionsRequest(r *FilesystemVersionsRequest, versions *[]zfs.FilesystemVersion) (err error) {
+
+	log := h.logger.WithField("endpoint", "FilesystemVersionsRequest")
+
+	log.WithField("request", r).Debug("request")
+
+	// allowed to request that?
+	if h.pullACLCheck(r.Filesystem, nil); err != nil {
+		log.WithError(err).Warn("pull ACL check failed")
+		return
+	}
+
+	// find our versions
+	vs, err := zfs.ZFSListFilesystemVersions(r.Filesystem, h.fsvf)
+	if err != nil {
+		log.WithError(err).Error("cannot list filesystem versions")
+		return
+	}
+
+	log.WithField("response", vs).Debug("response")
+
+	*versions = vs
+	return
+
+}
+
+func (h Handler) HandleInitialTransferRequest(r *InitialTransferRequest, stream *io.Reader) (err error) {
+
+	log := h.logger.WithField("endpoint", "InitialTransferRequest")
+
+	log.WithField("request", r).Debug("request")
+	if err = h.pullACLCheck(r.Filesystem, &r.FilesystemVersion); err != nil {
+		log.WithError(err).Warn("pull ACL check failed")
+		return
+	}
+
+	log.Debug("invoking zfs send")
+
+	s, err := zfs.ZFSSend(r.Filesystem, &r.FilesystemVersion, nil)
+	if err != nil {
+		log.WithError(err).Error("cannot send filesystem")
+	}
+	*stream = s
+
+	return
+
+}
+
+func (h Handler) HandleIncrementalTransferRequest(r *IncrementalTransferRequest, stream *io.Reader) (err error) {
+
+	log := h.logger.WithField("endpoint", "IncrementalTransferRequest")
+	log.WithField("request", r).Debug("request")
+	if err = h.pullACLCheck(r.Filesystem, &r.From); err != nil {
+		log.WithError(err).Warn("pull ACL check failed")
+		return
+	}
+	if err = h.pullACLCheck(r.Filesystem, &r.To); err != nil {
+		log.WithError(err).Warn("pull ACL check failed")
+		return
+	}
+
+	log.Debug("invoking zfs send")
+
+	s, err := zfs.ZFSSend(r.Filesystem, &r.From, &r.To)
+	if err != nil {
+		log.WithError(err).Error("cannot send filesystem")
+	}
+
+	*stream = s
+	return
+
+}
+
+func (h Handler) pullACLCheck(p *zfs.DatasetPath, v *zfs.FilesystemVersion) (err error) {
+	var fsAllowed, vAllowed bool
+	fsAllowed, err = h.dsf.Filter(p)
+	if err != nil {
+		err = fmt.Errorf("error evaluating ACL: %s", err)
+		return
+	}
+	if !fsAllowed {
+		err = fmt.Errorf("ACL prohibits access to %s", p.ToString())
+		return
+	}
+	if v == nil {
+		return
+	}
+
+	vAllowed, err = h.fsvf.Filter(*v)
+	if err != nil {
+		err = errors.Wrap(err, "error evaluating version filter")
+		return
+	}
+	if !vAllowed {
+		err = fmt.Errorf("ACL prohibits access to %s", v.ToAbsPath(p))
+		return
+	}
+	return
+}

cmd/helpers.go

@@ -1,11 +1,10 @@
-package nethelpers
+package cmd
 
 import (
+	"github.com/pkg/errors"
 	"net"
 	"os"
 	"path/filepath"
-
-	"github.com/pkg/errors"
 )
 
 func PreparePrivateSockpath(sockpath string) error {

cmd/logging_formatters.go

@@ -1,18 +1,20 @@
-package logging
+package cmd
 
 import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"time"
-
-	"github.com/fatih/color"
 	"github.com/go-logfmt/logfmt"
 	"github.com/pkg/errors"
-
 	"github.com/zrepl/zrepl/logger"
+	"time"
 )
 
+type EntryFormatter interface {
+	SetMetadataFlags(flags MetadataFlags)
+	Format(e *logger.Entry) ([]byte, error)
+}
+
 const (
 	FieldLevel   = "level"
 	FieldMessage = "msg"
@@ -20,19 +22,13 @@ const (
 )
 
 const (
-	JobField    string = "job"
+	logJobField     string = "job"
-	SubsysField string = "subsystem"
+	logTaskField    string = "task"
-)
+	logFSField      string = "filesystem"
-
+	logMapFromField string = "map_from"
-type MetadataFlags int64
+	logMapToField   string = "map_to"
-
+	logIncFromField string = "inc_from"
-const (
+	logIncToField   string = "inc_to"
-	MetadataTime MetadataFlags = 1 << iota
-	MetadataLevel
-	MetadataColor
-
-	MetadataNone MetadataFlags = 0
-	MetadataAll  MetadataFlags = ^0
 )
 
 type NoFormatter struct{}
@@ -73,30 +69,40 @@ func (f *HumanFormatter) ignored(field string) bool {
 func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
 
 	var line bytes.Buffer
-	col := color.New()
-	if f.metadataFlags&MetadataColor != 0 {
-		col = e.Color()
-	}
 
 	if f.metadataFlags&MetadataTime != 0 {
 		fmt.Fprintf(&line, "%s ", e.Time.Format(HumanFormatterDateFormat))
 	}
 	if f.metadataFlags&MetadataLevel != 0 {
-		fmt.Fprintf(&line, "[%s]", col.Sprint(e.Level.Short()))
+		fmt.Fprintf(&line, "[%s]", e.Level.Short())
 	}
 
-	prefixFields := []string{JobField, SubsysField}
+	prefixFields := []string{logJobField, logTaskField, logFSField}
 	prefixed := make(map[string]bool, len(prefixFields)+2)
 	for _, field := range prefixFields {
 		val, ok := e.Fields[field].(string)
-		if !ok {
+		if ok {
-			continue
+			if !f.ignored(field) {
-		}
+				fmt.Fprintf(&line, "[%s]", val)
-		if !f.ignored(field) {
+				prefixed[field] = true
-			fmt.Fprintf(&line, "[%s]", col.Sprint(val))
+			}
-			prefixed[field] = true
+		} else {
+			break
 		}
 	}
+	// even more prefix fields
+	mapFrom, mapFromOk := e.Fields[logMapFromField].(string)
+	mapTo, mapToOk := e.Fields[logMapToField].(string)
+	if mapFromOk && mapToOk && !f.ignored(logMapFromField) && !f.ignored(logMapToField) {
+		fmt.Fprintf(&line, "[%s => %s]", mapFrom, mapTo)
+		prefixed[logMapFromField], prefixed[logMapToField] = true, true
+	}
+	incFrom, incFromOk := e.Fields[logIncFromField].(string)
+	incTo, incToOk := e.Fields[logIncToField].(string)
+	if incFromOk && incToOk && !f.ignored(logIncFromField) && !f.ignored(logMapToField) {
+		fmt.Fprintf(&line, "[%s => %s]", incFrom, incTo)
+		prefixed[logIncFromField], prefixed[logIncToField] = true, true
+	}
 
 	if line.Len() > 0 {
 		fmt.Fprint(&line, ": ")
@@ -104,11 +110,15 @@ func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
 	fmt.Fprint(&line, e.Message)
 
 	if len(e.Fields)-len(prefixed) > 0 {
+		fmt.Fprint(&line, " ")
+		enc := logfmt.NewEncoder(&line)
 		for field, value := range e.Fields {
 			if prefixed[field] || f.ignored(field) {
 				continue
 			}
-			fmt.Fprintf(&line, " %s=%q", col.Sprint(field), fmt.Sprint(value))
+			if err := logfmtTryEncodeKeyval(enc, field, value); err != nil {
+				return nil, err
+			}
 		}
 	}
 
@@ -161,21 +171,15 @@ func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
 	enc := logfmt.NewEncoder(&buf)
 
 	if f.metadataFlags&MetadataTime != 0 {
-		err := enc.EncodeKeyval(FieldTime, e.Time)
+		enc.EncodeKeyval(FieldTime, e.Time)
-		if err != nil {
-			return nil, errors.Wrap(err, "logfmt: encode time")
-		}
 	}
 	if f.metadataFlags&MetadataLevel != 0 {
-		err := enc.EncodeKeyval(FieldLevel, e.Level)
+		enc.EncodeKeyval(FieldLevel, e.Level)
-		if err != nil {
-			return nil, errors.Wrap(err, "logfmt: encode level")
-		}
 	}
 
 	// at least try and put job and task in front
 	prefixed := make(map[string]bool, 2)
-	prefix := []string{JobField, SubsysField}
+	prefix := []string{logJobField, logTaskField}
 	for _, pf := range prefix {
 		v, ok := e.Fields[pf]
 		if !ok {
@@ -187,10 +191,8 @@ func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
 		prefixed[pf] = true
 	}
 
-	err := enc.EncodeKeyval(FieldMessage, e.Message)
+	enc.EncodeKeyval(FieldMessage, e.Message)
-	if err != nil {
+
-		return nil, errors.Wrap(err, "logfmt: encode message")
-	}
 	for k, v := range e.Fields {
 		if !prefixed[k] {
 			if err := logfmtTryEncodeKeyval(enc, k, v); err != nil {
@@ -209,10 +211,7 @@ func logfmtTryEncodeKeyval(enc *logfmt.Encoder, field, value interface{}) error
 	case nil: // ok
 		return nil
 	case logfmt.ErrUnsupportedValueType:
-		err := enc.EncodeKeyval(field, fmt.Sprintf("<%T>", value))
+		enc.EncodeKeyval(field, fmt.Sprintf("<%T>", value))
-		if err != nil {
-			return errors.Wrap(err, "cannot encode unsuuported value type Go type")
-		}
 		return nil
 	}
 	return errors.Wrapf(err, "cannot encode field '%s'", field)

cmd/logging_outlets.go

@@ -1,39 +1,29 @@
-package logging
+package cmd
 
 import (
 	"bytes"
 	"context"
 	"crypto/tls"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/logger"
 	"io"
 	"log/syslog"
 	"net"
 	"time"
-
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/logger"
 )
 
-type EntryFormatter interface {
-	SetMetadataFlags(flags MetadataFlags)
-	Format(e *logger.Entry) ([]byte, error)
-}
-
 type WriterOutlet struct {
-	formatter EntryFormatter
+	Formatter EntryFormatter
-	writer    io.Writer
+	Writer    io.Writer
 }
 
 func (h WriterOutlet) WriteEntry(entry logger.Entry) error {
-	bytes, err := h.formatter.Format(&entry)
+	bytes, err := h.Formatter.Format(&entry)
 	if err != nil {
 		return err
 	}
-	_, err = h.writer.Write(bytes)
+	_, err = h.Writer.Write(bytes)
-	if err != nil {
+	h.Writer.Write([]byte("\n"))
-		return err
-	}
-	_, err = h.writer.Write([]byte("\n"))
 	return err
 }
 
@@ -97,10 +87,8 @@ func (h *TCPOutlet) outLoop(retryInterval time.Duration) {
 				conn = nil
 			}
 		}
-		err = conn.SetWriteDeadline(time.Now().Add(retryInterval))
+		conn.SetWriteDeadline(time.Now().Add(retryInterval))
-		if err == nil {
+		_, err = io.Copy(conn, msg)
-			_, err = io.Copy(conn, msg)
-		}
 		if err != nil {
 			retry = time.Now().Add(retryInterval)
 			conn.Close()
@@ -131,7 +119,6 @@ func (h *TCPOutlet) WriteEntry(e logger.Entry) error {
 type SyslogOutlet struct {
 	Formatter          EntryFormatter
 	RetryInterval      time.Duration
-	Facility           syslog.Priority
 	writer             *syslog.Writer
 	lastConnectAttempt time.Time
 }
@@ -150,7 +137,7 @@ func (o *SyslogOutlet) WriteEntry(entry logger.Entry) error {
 		if now.Sub(o.lastConnectAttempt) < o.RetryInterval {
 			return nil // not an error toward logger
 		}
-		o.writer, err = syslog.New(o.Facility, "zrepl")
+		o.writer, err = syslog.New(syslog.LOG_LOCAL0, "zrepl")
 		o.lastConnectAttempt = time.Now()
 		if err != nil {
 			o.writer = nil

cmd/main.go

@@ -0,0 +1,47 @@
+// zrepl replicates ZFS filesystems & volumes between pools
+//
+// Code Organization
+//
+// The cmd package uses github.com/spf13/cobra for its CLI.
+//
+// It combines the other packages in the zrepl project to implement zrepl functionality.
+//
+// Each subcommand's code is in the corresponding *.go file.
+// All other *.go files contain code shared by the subcommands.
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/logger"
+)
+
+//
+//type Logger interface {
+//	Printf(format string, v ...interface{})
+//}
+
+var (
+	zreplVersion string // set by build infrastructure
+)
+
+type Logger = *logger.Logger
+
+var RootCmd = &cobra.Command{
+	Use:   "zrepl",
+	Short: "ZFS dataset replication",
+	Long: `Replicate ZFS filesystems & volumes between pools:
+
+  - push & pull mode
+  - automatic snapshot creation & pruning
+  - local / over the network
+  - ACLs instead of blank SSH access`,
+}
+
+var rootArgs struct {
+	configFile string
+}
+
+func init() {
+	//cobra.OnInitialize(initConfig)
+	RootCmd.PersistentFlags().StringVar(&rootArgs.configFile, "config", "", "config file path")
+}

cmd/prune.go

@@ -0,0 +1,157 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"github.com/zrepl/zrepl/util"
+	"github.com/zrepl/zrepl/zfs"
+	"sync"
+	"time"
+)
+
+type Pruner struct {
+	task           *Task
+	Now            time.Time
+	DryRun         bool
+	DatasetFilter  zfs.DatasetFilter
+	SnapshotPrefix string
+	PrunePolicy    PrunePolicy
+}
+
+type PruneResult struct {
+	Filesystem *zfs.DatasetPath
+	All        []zfs.FilesystemVersion
+	Keep       []zfs.FilesystemVersion
+	Remove     []zfs.FilesystemVersion
+}
+
+// FIXME must not call p.task.Enter because it runs in parallel
+func (p *Pruner) filterFilesystems() (filesystems []*zfs.DatasetPath, stop bool) {
+	filesystems, err := zfs.ZFSListMapping(p.DatasetFilter)
+	if err != nil {
+		p.task.Log().WithError(err).Error("error applying filesystem filter")
+		return nil, true
+	}
+	if len(filesystems) <= 0 {
+		p.task.Log().Info("no filesystems matching filter")
+		return nil, true
+	}
+	return filesystems, false
+}
+
+// FIXME must not call p.task.Enter because it runs in parallel
+func (p *Pruner) filterVersions(fs *zfs.DatasetPath) (fsversions []zfs.FilesystemVersion, stop bool) {
+	log := p.task.Log().WithField(logFSField, fs.ToString())
+
+	filter := NewPrefixFilter(p.SnapshotPrefix)
+	fsversions, err := zfs.ZFSListFilesystemVersions(fs, filter)
+	if err != nil {
+		log.WithError(err).Error("error listing filesytem versions")
+		return nil, true
+	}
+	if len(fsversions) == 0 {
+		log.WithField("prefix", p.SnapshotPrefix).Info("no filesystem versions matching prefix")
+		return nil, true
+	}
+	return fsversions, false
+}
+
+// FIXME must not call p.task.Enter because it runs in parallel
+func (p *Pruner) pruneFilesystem(fs *zfs.DatasetPath, destroySemaphore util.Semaphore) (r PruneResult, valid bool) {
+	log := p.task.Log().WithField(logFSField, fs.ToString())
+
+	fsversions, stop := p.filterVersions(fs)
+	if stop {
+		return
+	}
+
+	keep, remove, err := p.PrunePolicy.Prune(fs, fsversions)
+	if err != nil {
+		log.WithError(err).Error("error evaluating prune policy")
+		return
+	}
+
+	log.WithField("fsversions", fsversions).
+		WithField("keep", keep).
+		WithField("remove", remove).
+		Debug("prune policy debug dump")
+
+	r = PruneResult{fs, fsversions, keep, remove}
+
+	var wg sync.WaitGroup
+	for v := range remove {
+		wg.Add(1)
+		go func(v zfs.FilesystemVersion) {
+			defer wg.Done()
+			// log fields
+			fields := make(map[string]interface{})
+			fields["version"] = v.ToAbsPath(fs)
+			timeSince := v.Creation.Sub(p.Now)
+			fields["age_ns"] = timeSince
+			const day time.Duration = 24 * time.Hour
+			days := timeSince / day
+			remainder := timeSince % day
+			fields["age_str"] = fmt.Sprintf("%dd%s", days, remainder)
+
+			log.WithFields(fields).Info("destroying version")
+			// echo what we'll do and exec zfs destroy if not dry run
+			// TODO special handling for EBUSY (zfs hold)
+			// TODO error handling for clones? just echo to cli, skip over, and exit with non-zero status code (we're idempotent)
+			if !p.DryRun {
+				destroySemaphore.Down()
+				err := zfs.ZFSDestroyFilesystemVersion(fs, v)
+				destroySemaphore.Up()
+				if err != nil {
+					log.WithFields(fields).WithError(err).Error("error destroying version")
+				}
+			}
+		}(remove[v])
+	}
+	wg.Wait()
+
+	return r, true
+}
+
+func (p *Pruner) Run(ctx context.Context) (r []PruneResult, err error) {
+	p.task.Enter("run")
+	defer p.task.Finish()
+
+	if p.DryRun {
+		p.task.Log().Info("doing dry run")
+	}
+
+	filesystems, stop := p.filterFilesystems()
+	if stop {
+		return
+	}
+
+	maxConcurrentDestroy := len(filesystems)
+	p.task.Log().WithField("max_concurrent_destroy", maxConcurrentDestroy).Info("begin concurrent destroy")
+	destroySem := util.NewSemaphore(maxConcurrentDestroy)
+
+	resChan := make(chan PruneResult, len(filesystems))
+	var wg sync.WaitGroup
+	for _, fs := range filesystems {
+		wg.Add(1)
+		go func(fs *zfs.DatasetPath) {
+			defer wg.Done()
+			res, ok := p.pruneFilesystem(fs, destroySem)
+			if ok {
+				resChan <- res
+			}
+
+		}(fs)
+	}
+	wg.Wait()
+	close(resChan)
+
+	p.task.Log().Info("destroys done")
+
+	r = make([]PruneResult, 0, len(filesystems))
+	for res := range resChan {
+		r = append(r, res)
+	}
+
+	return
+
+}

cmd/replication.go

@@ -0,0 +1,323 @@
+package cmd
+
+import (
+	"fmt"
+	"io"
+
+	"bytes"
+	"encoding/json"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type localPullACL struct{}
+
+func (a localPullACL) Filter(p *zfs.DatasetPath) (pass bool, err error) {
+	return true, nil
+}
+
+const DEFAULT_INITIAL_REPL_POLICY = InitialReplPolicyMostRecent
+
+type InitialReplPolicy string
+
+const (
+	InitialReplPolicyMostRecent InitialReplPolicy = "most_recent"
+	InitialReplPolicyAll        InitialReplPolicy = "all"
+)
+
+type Puller struct {
+	task              *Task
+	Remote            rpc.RPCClient
+	Mapping           DatasetMapping
+	InitialReplPolicy InitialReplPolicy
+}
+
+type remoteLocalMapping struct {
+	Remote *zfs.DatasetPath
+	Local  *zfs.DatasetPath
+}
+
+func (p *Puller) getRemoteFilesystems() (rfs []*zfs.DatasetPath, ok bool) {
+	p.task.Enter("fetch_remote_fs_list")
+	defer p.task.Finish()
+
+	fsr := FilesystemRequest{}
+	if err := p.Remote.Call("FilesystemRequest", &fsr, &rfs); err != nil {
+		p.task.Log().WithError(err).Error("cannot fetch remote filesystem list")
+		return nil, false
+	}
+	return rfs, true
+}
+
+func (p *Puller) buildReplMapping(remoteFilesystems []*zfs.DatasetPath) (replMapping map[string]remoteLocalMapping, ok bool) {
+	p.task.Enter("build_repl_mapping")
+	defer p.task.Finish()
+
+	replMapping = make(map[string]remoteLocalMapping, len(remoteFilesystems))
+	for fs := range remoteFilesystems {
+		var err error
+		var localFs *zfs.DatasetPath
+		localFs, err = p.Mapping.Map(remoteFilesystems[fs])
+		if err != nil {
+			err := fmt.Errorf("error mapping %s: %s", remoteFilesystems[fs], err)
+			p.task.Log().WithError(err).WithField(logMapFromField, remoteFilesystems[fs]).Error("cannot map")
+			return nil, false
+		}
+		if localFs == nil {
+			continue
+		}
+		p.task.Log().WithField(logMapFromField, remoteFilesystems[fs].ToString()).
+			WithField(logMapToField, localFs.ToString()).Debug("mapping")
+		m := remoteLocalMapping{remoteFilesystems[fs], localFs}
+		replMapping[m.Local.ToString()] = m
+	}
+	return replMapping, true
+}
+
+// returns true if the receiving filesystem (local side) exists and can have child filesystems
+func (p *Puller) replFilesystem(m remoteLocalMapping, localFilesystemState map[string]zfs.FilesystemState) (localExists bool) {
+
+	p.task.Enter("repl_fs")
+	defer p.task.Finish()
+	var err error
+	remote := p.Remote
+
+	log := p.task.Log().
+		WithField(logMapFromField, m.Remote.ToString()).
+		WithField(logMapToField, m.Local.ToString())
+
+	log.Debug("examining local filesystem state")
+	localState, localExists := localFilesystemState[m.Local.ToString()]
+	var versions []zfs.FilesystemVersion
+	switch {
+	case !localExists:
+		log.Info("local filesystem does not exist")
+	case localState.Placeholder:
+		log.Info("local filesystem is marked as placeholder")
+	default:
+		log.Debug("local filesystem exists")
+		log.Debug("requesting local filesystem versions")
+		if versions, err = zfs.ZFSListFilesystemVersions(m.Local, nil); err != nil {
+			log.WithError(err).Error("cannot get local filesystem versions")
+			return false
+		}
+	}
+
+	log.Info("requesting remote filesystem versions")
+	r := FilesystemVersionsRequest{
+		Filesystem: m.Remote,
+	}
+	var theirVersions []zfs.FilesystemVersion
+	if err = remote.Call("FilesystemVersionsRequest", &r, &theirVersions); err != nil {
+		log.WithError(err).Error("cannot get remote filesystem versions")
+		log.Warn("stopping replication for all filesystems mapped as children of receiving filesystem")
+		return false
+	}
+
+	log.Debug("computing diff between remote and local filesystem versions")
+	diff := zfs.MakeFilesystemDiff(versions, theirVersions)
+	log.WithField("diff", diff).Debug("diff between local and remote filesystem")
+
+	if localState.Placeholder && diff.Conflict != zfs.ConflictAllRight {
+		panic("internal inconsistency: local placeholder implies ConflictAllRight")
+	}
+
+	switch diff.Conflict {
+	case zfs.ConflictAllRight:
+
+		log.WithField("replication_policy", p.InitialReplPolicy).Info("performing initial sync, following policy")
+
+		if p.InitialReplPolicy != InitialReplPolicyMostRecent {
+			panic(fmt.Sprintf("policy '%s' not implemented", p.InitialReplPolicy))
+		}
+
+		snapsOnly := make([]zfs.FilesystemVersion, 0, len(diff.MRCAPathRight))
+		for s := range diff.MRCAPathRight {
+			if diff.MRCAPathRight[s].Type == zfs.Snapshot {
+				snapsOnly = append(snapsOnly, diff.MRCAPathRight[s])
+			}
+		}
+
+		if len(snapsOnly) < 1 {
+			log.Warn("cannot perform initial sync: no remote snapshots")
+			return false
+		}
+
+		r := InitialTransferRequest{
+			Filesystem:        m.Remote,
+			FilesystemVersion: snapsOnly[len(snapsOnly)-1],
+		}
+
+		log.WithField("version", r.FilesystemVersion).Debug("requesting snapshot stream")
+
+		var stream io.Reader
+
+		if err = remote.Call("InitialTransferRequest", &r, &stream); err != nil {
+			log.WithError(err).Error("cannot request initial transfer")
+			return false
+		}
+		log.Debug("received initial transfer request response")
+
+		log.Debug("invoke zfs receive")
+		recvArgs := []string{"-u"}
+		if localState.Placeholder {
+			log.Info("receive with forced rollback to replace placeholder filesystem")
+			recvArgs = append(recvArgs, "-F")
+		}
+		progressStream := p.task.ProgressUpdater(stream)
+		if err = zfs.ZFSRecv(m.Local, progressStream, recvArgs...); err != nil {
+			log.WithError(err).Error("cannot receive stream")
+			return false
+		}
+		log.Info("finished receiving stream") // TODO rx delta
+
+		// TODO unify with recv path of ConflictIncremental
+		log.Debug("configuring properties of received filesystem")
+		props := zfs.NewZFSProperties()
+		props.Set("readonly", "on")
+		if err = zfs.ZFSSet(m.Local, props); err != nil {
+			log.WithError(err).Error("cannot set readonly property")
+		}
+
+		log.Info("finished initial transfer")
+		return true
+
+	case zfs.ConflictIncremental:
+
+		if len(diff.IncrementalPath) < 2 {
+			log.Info("remote and local are in sync")
+			return true
+		}
+
+		log.Info("following incremental path from diff")
+		for i := 0; i < len(diff.IncrementalPath)-1; i++ {
+
+			from, to := diff.IncrementalPath[i], diff.IncrementalPath[i+1]
+
+			log, _ := log.WithField(logIncFromField, from.Name).WithField(logIncToField, to.Name), 0
+
+			log.Debug("requesting incremental snapshot stream")
+			r := IncrementalTransferRequest{
+				Filesystem: m.Remote,
+				From:       from,
+				To:         to,
+			}
+			var stream io.Reader
+			if err = remote.Call("IncrementalTransferRequest", &r, &stream); err != nil {
+				log.WithError(err).Error("cannot request incremental snapshot stream")
+				return false
+			}
+
+			log.Debug("invoking zfs receive")
+			progressStream := p.task.ProgressUpdater(stream)
+			// TODO protect against malicious incremental stream
+			if err = zfs.ZFSRecv(m.Local, progressStream); err != nil {
+				log.WithError(err).Error("cannot receive stream")
+				return false
+			}
+			log.Info("finished incremental transfer") // TODO increment rx
+
+		}
+		log.Info("finished following incremental path") // TODO path rx
+		return true
+
+	case zfs.ConflictNoCommonAncestor:
+		fallthrough
+	case zfs.ConflictDiverged:
+
+		var jsonDiff bytes.Buffer
+		if err := json.NewEncoder(&jsonDiff).Encode(diff); err != nil {
+			log.WithError(err).Error("cannot JSON-encode diff")
+			return false
+		}
+
+		var problem, resolution string
+
+		switch diff.Conflict {
+		case zfs.ConflictNoCommonAncestor:
+			problem = "remote and local filesystem have snapshots, but no common one"
+			resolution = "perform manual establish a common snapshot history"
+		case zfs.ConflictDiverged:
+			problem = "remote and local filesystem share a history but have diverged"
+			resolution = "perform manual replication or delete snapshots on the receiving" +
+				"side  to establish an incremental replication parse"
+		}
+
+		log.WithField("diff", jsonDiff.String()).
+			WithField("problem", problem).
+			WithField("resolution", resolution).
+			Error("manual conflict resolution required")
+
+		return false
+
+	}
+
+	panic("should not be reached")
+}
+
+func (p *Puller) Pull() {
+	p.task.Enter("run")
+	defer p.task.Finish()
+
+	p.task.Log().Info("request remote filesystem list")
+	remoteFilesystems, ok := p.getRemoteFilesystems()
+	if !ok {
+		return
+	}
+
+	p.task.Log().Debug("map remote filesystems to local paths and determine order for per-filesystem sync")
+	replMapping, ok := p.buildReplMapping(remoteFilesystems)
+	if !ok {
+
+	}
+
+	p.task.Log().Debug("build cache for already present local filesystem state")
+	p.task.Enter("cache_local_fs_state")
+	localFilesystemState, err := zfs.ZFSListFilesystemState()
+	p.task.Finish()
+	if err != nil {
+		p.task.Log().WithError(err).Error("cannot request local filesystem state")
+		return
+	}
+
+	localTraversal := zfs.NewDatasetPathForest()
+	for _, m := range replMapping {
+		localTraversal.Add(m.Local)
+	}
+
+	p.task.Log().Info("start per-filesystem sync")
+	localTraversal.WalkTopDown(func(v zfs.DatasetPathVisit) bool {
+
+		p.task.Enter("tree_walk")
+		defer p.task.Finish()
+
+		log := p.task.Log().WithField(logFSField, v.Path.ToString())
+
+		if v.FilledIn {
+			if _, exists := localFilesystemState[v.Path.ToString()]; exists {
+				// No need to verify if this is a placeholder or not. It is sufficient
+				// to know we can add child filesystems to it
+				return true
+			}
+			log.Debug("create placeholder filesystem")
+			p.task.Enter("create_placeholder")
+			err = zfs.ZFSCreatePlaceholderFilesystem(v.Path)
+			p.task.Finish()
+			if err != nil {
+				log.Error("cannot create placeholder filesystem")
+				return false
+			}
+			return true
+		}
+
+		m, ok := replMapping[v.Path.ToString()]
+		if !ok {
+			panic("internal inconsistency: replMapping should contain mapping for any path that was not filled in by WalkTopDown()")
+		}
+
+		return p.replFilesystem(m, localFilesystemState)
+	})
+
+	return
+
+}

cmd/sampleconf/localbackup/host1.yml

@@ -0,0 +1,29 @@
+jobs:
+- name: mirror_local
+  type: local
+
+  # snapshot the filesystems matched by the left-hand-side of the mapping
+  # every 10m with zrepl_ as prefix
+  mapping: {
+    "zroot/var/db<":    "storage/backups/local/zroot/var/db",
+    "zroot/usr/home<":  "storage/backups/local/zroot/usr/home",
+    "zroot/usr/home/paranoid":  "!", #don't backup paranoid user
+    "zroot/poudriere/ports<": "!", #don't backup the ports trees
+  }
+  snapshot_prefix: zrepl_
+  interval: 10m
+  initial_repl_policy: most_recent
+
+  # keep one hour of 10m interval snapshots of filesystems matched by
+  # the left-hand-side of the mapping
+  prune_lhs:
+    policy: grid
+    grid: 1x1h(keep=all)
+    keep_bookmarks: all
+
+  # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
+  prune_rhs:
+    policy: grid
+    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+
+

cmd/sampleconf/pullbackup/backuphost.yml

@@ -0,0 +1,27 @@
+jobs:
+- name: fullbackup_prod1
+  type: pull
+  # connect to remote using ssh / stdinserver command
+  connect:
+    type: ssh+stdinserver
+    host: prod1.example.com
+    user: root
+    port: 22
+    identity_file: /root/.ssh/id_ed25519
+
+  # pull (=ask for new snapshots) every 10m, prune afterwards
+  # this will leave us at most 10m behind production
+  interval: 10m
+
+  # pull all offered filesystems to storage/backups/zrepl/pull/prod1.example.com
+  mapping: {
+    "<":"storage/backups/zrepl/pull/prod1.example.com"
+  }
+  initial_repl_policy: most_recent
+
+  # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
+  snapshot_prefix: zrepl_
+  prune:
+    policy: grid
+    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+

cmd/sampleconf/pullbackup/productionhost.yml

@@ -0,0 +1,47 @@
+global:
+  serve:
+    stdinserver:
+      # Directory where AF_UNIX sockets for stdinserver command are placed.
+      #
+      # `zrepl stdinserver CLIENT_IDENTITY`
+      #   * connects to the socket in $sockdir/CLIENT_IDENTITY
+      #   * sends its stdin / stdout file descriptors to the `zrepl daemon` process (see cmsg(3))
+      #   * does nothing more
+      #
+      # This enables a setup where `zrepl daemon` is not directly exposed to the internet
+      # but instead all traffic is tunnelled through SSH.
+      # The server with the source job has an authorized_keys file entry for the public key
+      # used by the corresponding pull job
+      #
+      #   command="/mnt/zrepl stdinserver CLIENT_IDENTITY" ssh-ed25519 AAAAC3NzaC1E... zrepl@pullingserver
+      #
+      # Below is the default value.
+      sockdir: /var/run/zrepl/stdinserver
+
+jobs:
+
+- name: fullbackup_prod1
+  # expect remote to connect via ssh+stdinserver with fullbackup_prod1 as client_identity
+  type: source
+  serve:
+    type: stdinserver # see global.serve.stdinserver for explanation
+    client_identity: fullbackup_prod1
+
+  # snapshot these filesystems every 10m with zrepl_ as prefix
+  filesystems: {
+    "zroot/var/db<": "ok",
+    "zroot/usr/home<": "ok",
+    "zroot/var/tmp": "!", #don't backup /tmp
+  }
+  snapshot_prefix: zrepl_
+  interval: 10m
+
+
+  # keep 1 hour of snapshots (6 at 10m interval)
+  # and  one day of bookmarks in case pull doesn't work (link down, etc)
+  # => keep_bookmarks = 24h / interval = 24h / 10m = 144
+  prune:
+    policy: grid
+    grid: 1x1h(keep=all)
+    keep_bookmarks: 144
+

cmd/sampleconf/pushbackup/backuphost.yml

@@ -0,0 +1,20 @@
+jobs:
+- name: fullbackup_prod1
+
+  # expect remote to connect via ssh+stdinserver with fullbackup_prod1 as client_identity
+  type: push-sink
+  serve:
+    type: stdinserver
+    client_identity: fullbackup_prod1
+
+  # map all pushed datasets to storage/backups/zrepl/sink/prod1.example.com
+  mapping: {
+    "<":"storage/backups/zrepl/sink/prod1.example.com"
+  }
+
+  # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
+  prune:
+    policy: grid
+    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+
+

cmd/sampleconf/pushbackup/productionhost.yml

@@ -0,0 +1,26 @@
+jobs:
+- name: fullbackup_prod1
+
+  # connect to remote using ssh / stdinserver command
+  type: push
+  connect:
+      type: ssh+stdinserver
+      host: prod1.example.com
+      user: root
+      port: 22
+      identity_file: /root/.ssh/id_ed25519
+
+  # snapshot these datsets every 10m with zrepl_ as prefix
+  filesystems: {
+    "zroot/var/db<": "ok",
+    "zroot/usr/home<": "!",
+  }
+  snapshot_prefix: zrepl_
+  interval: 10m
+
+  # keep a one day window 10m interval snapshots in case push doesn't work (link down, etc)
+  # (we cannot keep more than one day because this host will run out of disk space)
+  prune:
+    policy: grid
+    grid: 1x1d(keep=all)
+

cmd/sampleconf/random/debugging.yml

@@ -0,0 +1,33 @@
+global:
+  serve:
+    stdinserver:
+      sockdir: /var/run/zrepl/stdinserver
+
+jobs:
+
+- name: debian2_pull
+  # JOB DEBUGGING OPTIONS
+  # should be equal for all job types, but each job implements the debugging itself
+  # => consult job documentation for supported options
+  debug:
+    conn: # debug the io.ReadWriteCloser connection
+      read_dump: /tmp/connlog_read   # dump results of Read() invocations to this file
+      write_dump: /tmp/connlog_write # dump results of Write() invocations to this file
+    rpc: # debug the RPC protocol implementation
+      log: true # log output from rpc layer to the job log
+
+  # ... just to make the unit tests pass.
+  # check other examples, e.g. localbackup or pullbackup for what the sutff below means
+  type: source
+  serve:
+    type: stdinserver
+    client_identity: debian2
+  filesystems: {
+    "pool1/db<": ok
+  }
+  snapshot_prefix: zrepl_
+  interval: 1s
+  prune:
+    policy: grid
+    grid: 1x10s(keep=all)
+    keep_bookmarks: all

cmd/sampleconf/random/logging.yml

@@ -0,0 +1,24 @@
+global:
+  logging:
+
+    - outlet: stdout
+      level: warn
+      format: human
+
+    - outlet: tcp
+      level: debug
+      format: json
+      net: tcp
+      address: 127.0.0.1:8080
+      retry_interval: 1s
+      tls: # if not specified, use plain TCP
+        ca: sampleconf/random/logging/logserver.crt
+        cert: sampleconf/random/logging/client.crt
+        key: sampleconf/random/logging/client.key
+
+    - outlet: syslog
+      level: debug
+      format: logfmt
+
+
+jobs: []

cmd/sampleconf/random/logging/client.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDIzCCAgsCAQEwDQYJKoZIhvcNAQELBQAwWTELMAkGA1UEBhMCQVUxEzARBgNV
+BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0
+ZDESMBAGA1UEAwwJbG9nc2VydmVyMB4XDTE3MDkyNDEyMzAzNloXDTE3MTAyNDEy
+MzAzNlowVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNV
+BAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xpZW50MIIB
+IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/xJTUlqApeJGzRD+w2J8sZS
+Bo+s+04T987L/M6gaCo8aDSTEb/ZH3XSoU5JEmO6kPpwNNapOsaEhTCjndZQdm5F
+uqiUtAg1uW0HCkBEIDkGr9bFHDKzpewGmmMgfQ2+hfiBR/4ZCrc/vd9P0W9BiWQS
+Dtc7p22XraWPVL8HlSz5K/Ih+V6i8O+kBltZkusiJh2bWPoRp/netiTZuc6du+Wp
+kpWp1OBaTU4GXIAlLj5afF14BBphRQK983Yhaz53BkA7OQ76XxowynMjmuLQVGmK
+f1R9zEJuohTX9XIr1tp/ueRHcS4Awk6LcNZUMCV6270FNSIw2f4hbOZvep+t2wID
+AQABMA0GCSqGSIb3DQEBCwUAA4IBAQACK3OeNzScpiNwz/jpg/usQzvXbZ/wDvml
+YLjtzn/A65ox8a8BhxvH1ydyoCM2YAGYX7+y7qXJnMgRO/v8565CQIVcznHhg9ST
+3828/WqZ3bXf2DV5GxKKQf7hPmBnyVUUhn/Ny91MECED27lZucWiX/bczN8ffDeh
+M3+ngezcJxsOBd4x0gLrqIJCoaFRSeepOaFEW6GHQ8loxE9GmA7FQd2phIpJHFSd
+Z7nQl7X5C1iN2OboEApJHwtmNVC45UlOpg53vo2sDTLhSfdogstiWi8x1HmvhIGM
+j3XHs0Illvo9OwVrmgUph8zQ7pvr/AFrTOIbhgzl/9uVUk5ApwFM
+-----END CERTIFICATE-----

cmd/sampleconf/random/logging/client.csr

@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIICmzCCAYMCAQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx
+ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xp
+ZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/xJTUlqApeJGzRD
++w2J8sZSBo+s+04T987L/M6gaCo8aDSTEb/ZH3XSoU5JEmO6kPpwNNapOsaEhTCj
+ndZQdm5FuqiUtAg1uW0HCkBEIDkGr9bFHDKzpewGmmMgfQ2+hfiBR/4ZCrc/vd9P
+0W9BiWQSDtc7p22XraWPVL8HlSz5K/Ih+V6i8O+kBltZkusiJh2bWPoRp/netiTZ
+uc6du+WpkpWp1OBaTU4GXIAlLj5afF14BBphRQK983Yhaz53BkA7OQ76XxowynMj
+muLQVGmKf1R9zEJuohTX9XIr1tp/ueRHcS4Awk6LcNZUMCV6270FNSIw2f4hbOZv
+ep+t2wIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAKnlr0Qs5KYF85u2YA7DJ5pL
+HwAx+qNoNbox5CS1aynrDBpDTWLaErviUJ+4WxRlRyTMEscMOIOKajbYhqqFmtGZ
+mu3SshZnFihErw8TOQMyU1LGGG+l6r+6ve5TciwJRLla2Y75z7izr6cyvQNRWdLr
+PvxL1/Yqr8LKha12+7o28R4SLf6/GY0GcedqoebRmtuwA/jES0PuGauEUD5lH4cj
+Me8sqRrB+IMHQ5j8hlJX4DbA8UQRUBL64sHkQzeQfWu+qkWmS5I19CFfLNrcH+OV
+yhyjGfN0q0jHyHdpckBhgzS7IIdo6P66AIlm4qpHM7Scra3JaGM7oaZPamJ6f8U=
+-----END CERTIFICATE REQUEST-----

cmd/sampleconf/random/logging/client.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC3/ElNSWoCl4kb
+NEP7DYnyxlIGj6z7ThP3zsv8zqBoKjxoNJMRv9kfddKhTkkSY7qQ+nA01qk6xoSF
+MKOd1lB2bkW6qJS0CDW5bQcKQEQgOQav1sUcMrOl7AaaYyB9Db6F+IFH/hkKtz+9
+30/Rb0GJZBIO1zunbZetpY9UvweVLPkr8iH5XqLw76QGW1mS6yImHZtY+hGn+d62
+JNm5zp275amSlanU4FpNTgZcgCUuPlp8XXgEGmFFAr3zdiFrPncGQDs5DvpfGjDK
+cyOa4tBUaYp/VH3MQm6iFNf1civW2n+55EdxLgDCTotw1lQwJXrbvQU1IjDZ/iFs
+5m96n63bAgMBAAECggEAF4om0sWe06ARwbJJNFjCGpa3LfG5/xk5Qs5pmPnS2iD1
+Q5veaTnzjKvlfA/pF3o9B4mTS59fXY7Cq8vSU0J1XwGy2DPzeqlGPmgtq2kXjkvd
+iCfhZj8ybvsoyR3/rSBSDRADcnOXPqC9fgyRSMmESBDOoql1D3HdIzF4ii46ySIU
+/XQvExS6NWifbP+Ue6DETV8NhreO5PqjeXLITQhhndtc8MDL/8eCNOyN8XjYIWKX
+smlBYtRQYOOY9BHOQgUn6yvPHrtKJNKci+qcQNvWir66mBhY1o40MH5wTIV+8yP2
+Vbm/VzoNKIYgeROsilBW7QTwGvkDn3R11zeTqfUNSQKBgQD0eFzhJAEZi4uBw6Tg
+NKmBC5Y1IHPOsb5gKPNz9Z9j4qYRDySgYl6ISk+2EdhgUCo1NmTk8EIPQjIerUVf
+S+EogFnpsj8U9LR3OM79DaGkNULxrHqhd209/g8DtVgk7yjkxL4vmVOv8qpHMp/7
+eWsylN7AOxj2RB/eXYQBPrw+jQKBgQDAqae9HasLmvpJ9ktTv30yZSKXC+LP4A0D
+RBBmx410VpPd4CvcpCJxXmjer6B7+9L1xHYP2pvsnMBid5i0knuvyK28dYy7fldl
+CzWvb+lqNA5YYPFXQED4oEdihlQczoI1Bm06SFizeAKD1Q9e2c+lgbR/51j8xuXi
+twvhMj/YBwKBgQCZw97/iQrcC2Zq7yiUEOuQjD4lGk1c83U/vGIsTJC9XcCAOFsc
+OeMlrD/oz96d7a4unBDn4qpaOJOXsfpRT0PGmrxy/jcpMiUUW/ntNpa11v5NTeQw
+DRL8DAFbnsNbL8Yz5f+Nps35fBNYBuKTZLJlNTfKByHTO9QjpAQ0WEZEvQKBgQCi
+Ovm83EuYVSKmvxcE6Tyx/8lVqTOO2Vn7wweQlD4/lVujvE0S2L8L+XSS9w5K+GzW
+eFz10p3zarbw80YJ30L5bSEmjVE43BUZR4woMzM4M6dUsiTm1HshIE2b4ALZ0uZ/
+Ye794ceXL9nmSrVLqFsaQZLNFPCwwYb4FiyRry9lZwKBgAO9VbWcN8SEeBDKo3z8
+yRbRTc6sI+AdKY44Dfx0tqOPmTjO3mE4X1GU4sbfD2Bvg3DdjwTuxxC/jHaKu0GG
+dTM0CbrZGbDAj7E87SOcN/PWEeBckSvuQq5H3DQfwIpTmlS1l5oZn9CxRGbLqC2G
+ifnel8XWUG0ROybsr1tk4mzW
+-----END PRIVATE KEY-----

cmd/sampleconf/random/logging/logserver.crt

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDiDCCAnCgAwIBAgIJALhp/WvTQeg/MA0GCSqGSIb3DQEBCwUAMFkxCzAJBgNV
+BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
+aWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMMCWxvZ3NlcnZlcjAeFw0xNzA5MjQxMjI3
+MDRaFw0yNzA5MjIxMjI3MDRaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21l
+LVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNV
+BAMMCWxvZ3NlcnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKs3
+TLYfXhV3hap71tOkhPQlM+m0EKRAo8Nua50Cci5UhDo4JkVpyYok1h+NFkqmjU2b
+IiIuGvsZZPOWYjbWWnSJE4+n5pBFBzcfNQ4d8xVxjANImFn6Tcehhj0WkbDIv/Ge
+364XUgywS7u3EGQj/FO7vZ8KHlUxBHNuPIOPHftwIVRyleh5K32UyBaSpSmnqGos
+rvI1byMuznavcZpOs4vlebZ+Jy6a20iKf9fj/0f0t0O+F5x3JIk07D3zSywhJ4RM
+M0mGIUmYXbh2SMh+f61KDZLDANpz/pMAPbUJe0mxEtBf0tnwK1gEqc3SLwA0EwiM
+8Hnn2iaH5Ln20UE3LOkCAwEAAaNTMFEwHQYDVR0OBBYEFDXoDcwx9SngzZcRYCeP
+BplBecfiMB8GA1UdIwQYMBaAFDXoDcwx9SngzZcRYCePBplBecfiMA8GA1UdEwEB
+/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBADyNvs4AA91x3gurQb1pcPVhK6nR
+mkYSTN1AsDKSRi/X2iCUmR7G7FlF7XW8mntTpHvVzcs+gr94WckH5wqEOA5iZnaw
+PXUWexmdXUge4hmC2q6kBQ5e2ykhSJMRVZXvOLZOZV9qitceamHESV1cKZSNMvZM
+aCSVA1RK61/nUzs04pVp5PFPv9gFxJp9ki39FYFdsgZmM5RZ5I/FqxxvTJzu4RnH
+VPjsMopzARYwJw6dV2bKdFSYOE8B/Vs3Yv0GxjrABw2ko4PkBPTjLIz22x6+Hd9r
+K9BQi4pVmQfvppF5+SORSftlHSS+N47b0DD1rW1f5R6QGi71dFuJGikOwvY=
+-----END CERTIFICATE-----

cmd/sampleconf/random/logging/logserver.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCrN0y2H14Vd4Wq
+e9bTpIT0JTPptBCkQKPDbmudAnIuVIQ6OCZFacmKJNYfjRZKpo1NmyIiLhr7GWTz
+lmI21lp0iROPp+aQRQc3HzUOHfMVcYwDSJhZ+k3HoYY9FpGwyL/xnt+uF1IMsEu7
+txBkI/xTu72fCh5VMQRzbjyDjx37cCFUcpXoeSt9lMgWkqUpp6hqLK7yNW8jLs52
+r3GaTrOL5Xm2ficumttIin/X4/9H9LdDvhecdySJNOw980ssISeETDNJhiFJmF24
+dkjIfn+tSg2SwwDac/6TAD21CXtJsRLQX9LZ8CtYBKnN0i8ANBMIjPB559omh+S5
+9tFBNyzpAgMBAAECggEBAIY8ZwJq+WKvQLb3POjWFf8so9TY/ispGrwAeJKy9j5o
+uPrERw0o8YBDfTVjclS43BQ6Srqtly3DLSjlgL8ps+WmCxYYN2ZpGE0ZRIl65bis
+O2/fnML+wbiAZTTD2xnVatfPDeP6GLQmDFpyHoHEzPIBQZvNXRbBxZGSnhMvQ/x7
+FhqSBQG4kf3b1XDCENIbFEVOBOCg7WtMiIgjEGS7QnW3I65/Zt+Ts1LXRZbz+6na
+Gmi0PGHA/oLUh1NRzsF4zuZn6fFzja5zw4mkt+JvCWEoxg1QhRAxRp6QQwmZ6MIc
+1rw1D4Z+c5UEKyqHeIwZj4M6UNPhCfTXVm47c9eSiGECgYEA4U8pB+7eRo2fqX0C
+nWsWMcmsULJvwplQnUSFenUayPn3E8ammS/ZBHksoKhj82vwIdDbtS1hQZn8Bzsi
+atc8au0wz0YRDcVDzHX4HknXVQayHtP/FTPeSr5hwpoY8vhEbySuxBTBkXCrp4dx
+u5ErfOiYEP3Q1ZvPRywelrATu20CgYEAwonV5dgOcen/4oAirlnvufc2NfqhAQwJ
+FJ/JSVMAcXxPYu3sZMv0dGWrX8mLc+P1+XMCuV/7eBM/vU2LbDzmpeUV8sJfB2jw
+wyKqKXZwBgeq60btriA4f+0ElwRGgU2KSiniUuuTX2JmyftFQx4cVAQRCFk27NY0
+09psSsYyre0CgYBo6unabdtH029EB5iOIW3GZXk+Yrk0TxyA/4WAjsOYTv5FUT4H
+G4bdVGf5sDBLDDpYJOAKsEUXvVLlMx5FzlCuIiGWg7QxS2jU7yJJSG1jhKixPlsM
+Toj3GUyAyC1SB1Ymw1g2qsuwpFzquGG3zFQJ6G3Xi7oRnmqZY+wik3+8yQKBgB11
+SdKYOPe++2SNCrNkIw0CBk9+OEs0S1u4Jn7X9sU4kbzlUlqhF89YZe8HUfqmlmTD
+qbHwet/f6lL8HxSw1Cxi2EP+cu1oUqz53tKQgL4pAxTFlNA9SND2Ty+fEh4aY8p/
+NSphSduzxuTnC8HyGVAPnZSqDcsnVLCP7r4T7TCxAoGAbJygkkk/gZ9pT4fZoIaq
+8CMR8FTfxtkwCuZsWccSMUOWtx9nqet3gbCpKHfyoYZiKB4ke+lnUz4uFS16Y3hG
+kN0hFfvfoNa8eB2Ox7vs60cMMfWJac0H7KSaDDy+EvbhE2KtQADT0eWxMyhzGR8p
+5CbIivB0QCjeQIA8dOQpE8E=
+-----END PRIVATE KEY-----

cmd/stdinserver.go

@@ -1,51 +1,55 @@
-package client
+package cmd
 
 import (
 	"os"
 
-	"github.com/problame/go-netssh"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-
 	"context"
-	"errors"
+	"github.com/problame/go-netssh"
+	"github.com/spf13/cobra"
 	"log"
 	"path"
 )
 
-var StdinserverCmd = &cli.Subcommand{
+var StdinserverCmd = &cobra.Command{
 	Use:   "stdinserver CLIENT_IDENTITY",
-	Short: "stdinserver transport mode (started from authorized_keys file as forced command)",
+	Short: "start in stdinserver mode (from authorized_keys file)",
-	Run: func(subcommand *cli.Subcommand, args []string) error {
+	Run:   cmdStdinServer,
-		return runStdinserver(subcommand.Config(), args)
-	},
 }
 
-func runStdinserver(config *config.Config, args []string) error {
+func init() {
+	RootCmd.AddCommand(StdinserverCmd)
+}
+
+func cmdStdinServer(cmd *cobra.Command, args []string) {
 
 	// NOTE: the netssh proxying protocol requires exiting with non-zero status if anything goes wrong
 	defer os.Exit(1)
 
 	log := log.New(os.Stderr, "", log.LUTC|log.Ldate|log.Ltime)
 
+	conf, err := ParseConfig(rootArgs.configFile)
+	if err != nil {
+		log.Printf("error parsing config: %s", err)
+		return
+	}
+
 	if len(args) != 1 || args[0] == "" {
-		err := errors.New("must specify client_identity as positional argument")
+		log.Print("must specify client_identity as positional argument")
-		return err
+		return
 	}
 
 	identity := args[0]
-	unixaddr := path.Join(config.Global.Serve.StdinServer.SockDir, identity)
+	unixaddr := path.Join(conf.Global.Serve.Stdinserver.SockDir, identity)
 
 	log.Printf("proxying client identity '%s' to zrepl daemon '%s'", identity, unixaddr)
 
 	ctx := netssh.ContextWithLog(context.TODO(), log)
 
-	err := netssh.Proxy(ctx, unixaddr)
+	err = netssh.Proxy(ctx, unixaddr)
 	if err == nil {
 		log.Print("proxying finished successfully, exiting with status 0")
 		os.Exit(0)
 	}
 	log.Printf("error proxying: %s", err)
-	return nil
+
 }

cmd/test.go

@@ -0,0 +1,215 @@
+package cmd
+
+import (
+	"os"
+
+	"bytes"
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/kr/pretty"
+	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/zfs"
+	"time"
+)
+
+var testCmd = &cobra.Command{
+	Use:              "test",
+	Short:            "test configuration",
+	PersistentPreRun: testCmdGlobalInit,
+}
+
+var testCmdGlobal struct {
+	log  Logger
+	conf *Config
+}
+
+var testConfigSyntaxCmd = &cobra.Command{
+	Use:   "config",
+	Short: "parse config file and dump parsed datastructure",
+	Run:   doTestConfig,
+}
+
+var testDatasetMapFilter = &cobra.Command{
+	Use:     "pattern jobname test/zfs/dataset/path",
+	Short:   "test dataset mapping / filter specified in config",
+	Example: ` zrepl test pattern my_pull_job tank/tmp`,
+	Run:     doTestDatasetMapFilter,
+}
+
+var testPrunePolicyArgs struct {
+	side        PrunePolicySide
+	showKept    bool
+	showRemoved bool
+}
+
+var testPrunePolicyCmd = &cobra.Command{
+	Use:   "prune jobname",
+	Short: "do a dry-run of the pruning part of a job",
+	Run:   doTestPrunePolicy,
+}
+
+func init() {
+	RootCmd.AddCommand(testCmd)
+	testCmd.AddCommand(testConfigSyntaxCmd)
+	testCmd.AddCommand(testDatasetMapFilter)
+
+	testPrunePolicyCmd.Flags().VarP(&testPrunePolicyArgs.side, "side", "s", "prune_lhs (left) or prune_rhs (right)")
+	testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showKept, "kept", false, "show kept snapshots")
+	testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showRemoved, "removed", true, "show removed snapshots")
+	testCmd.AddCommand(testPrunePolicyCmd)
+}
+
+func testCmdGlobalInit(cmd *cobra.Command, args []string) {
+
+	out := logger.NewOutlets()
+	out.Add(WriterOutlet{&NoFormatter{}, os.Stdout}, logger.Info)
+	log := logger.NewLogger(out, 1*time.Second)
+	testCmdGlobal.log = log
+
+	var err error
+	if testCmdGlobal.conf, err = ParseConfig(rootArgs.configFile); err != nil {
+		testCmdGlobal.log.Printf("error parsing config file: %s", err)
+		os.Exit(1)
+	}
+
+}
+
+func doTestConfig(cmd *cobra.Command, args []string) {
+
+	log, conf := testCmdGlobal.log, testCmdGlobal.conf
+
+	log.Printf("config ok")
+	log.Printf("%# v", pretty.Formatter(conf))
+	return
+}
+
+func doTestDatasetMapFilter(cmd *cobra.Command, args []string) {
+
+	log, conf := testCmdGlobal.log, testCmdGlobal.conf
+
+	if len(args) != 2 {
+		log.Printf("specify job name as first postitional argument, test input as second")
+		log.Printf(cmd.UsageString())
+		os.Exit(1)
+	}
+	n, i := args[0], args[1]
+
+	jobi, err := conf.LookupJob(n)
+	if err != nil {
+		log.Printf("%s", err)
+		os.Exit(1)
+	}
+
+	var mf *DatasetMapFilter
+	switch j := jobi.(type) {
+	case *PullJob:
+		mf = j.Mapping
+	case *SourceJob:
+		mf = j.Filesystems
+	case *LocalJob:
+		mf = j.Mapping
+	default:
+		panic("incomplete implementation")
+	}
+
+	ip, err := zfs.NewDatasetPath(i)
+	if err != nil {
+		log.Printf("cannot parse test input as ZFS dataset path: %s", err)
+		os.Exit(1)
+	}
+
+	if mf.filterMode {
+		pass, err := mf.Filter(ip)
+		if err != nil {
+			log.Printf("error evaluating filter: %s", err)
+			os.Exit(1)
+		}
+		log.Printf("filter result: %v", pass)
+	} else {
+		res, err := mf.Map(ip)
+		if err != nil {
+			log.Printf("error evaluating mapping: %s", err)
+			os.Exit(1)
+		}
+		toStr := "NO MAPPING"
+		if res != nil {
+			toStr = res.ToString()
+		}
+		log.Printf("%s => %s", ip.ToString(), toStr)
+
+	}
+
+}
+
+func doTestPrunePolicy(cmd *cobra.Command, args []string) {
+
+	log, conf := testCmdGlobal.log, testCmdGlobal.conf
+
+	if cmd.Flags().NArg() != 1 {
+		log.Printf("specify job name as first positional argument")
+		log.Printf(cmd.UsageString())
+		os.Exit(1)
+	}
+
+	jobname := cmd.Flags().Arg(0)
+	jobi, err := conf.LookupJob(jobname)
+	if err != nil {
+		log.Printf("%s", err)
+		os.Exit(1)
+	}
+
+	jobp, ok := jobi.(PruningJob)
+	if !ok {
+		log.Printf("job doesn't do any prunes")
+		os.Exit(0)
+	}
+
+	log.Printf("job dump:\n%s", pretty.Sprint(jobp))
+
+	task := NewTask("", log)
+	pruner, err := jobp.Pruner(task, testPrunePolicyArgs.side, true)
+	if err != nil {
+		log.Printf("cannot create test pruner: %s", err)
+		os.Exit(1)
+	}
+
+	log.Printf("start pruning")
+
+	ctx := context.WithValue(context.Background(), contextKeyLog, log)
+	result, err := pruner.Run(ctx)
+	if err != nil {
+		log.Printf("error running pruner: %s", err)
+		os.Exit(1)
+	}
+
+	sort.Slice(result, func(i, j int) bool {
+		return strings.Compare(result[i].Filesystem.ToString(), result[j].Filesystem.ToString()) == -1
+	})
+
+	var b bytes.Buffer
+	for _, r := range result {
+		fmt.Fprintf(&b, "%s\n", r.Filesystem.ToString())
+
+		if testPrunePolicyArgs.showKept {
+			fmt.Fprintf(&b, "\tkept:\n")
+			for _, v := range r.Keep {
+				fmt.Fprintf(&b, "\t- %s\n", v.Name)
+			}
+		}
+
+		if testPrunePolicyArgs.showRemoved {
+			fmt.Fprintf(&b, "\tremoved:\n")
+			for _, v := range r.Remove {
+				fmt.Fprintf(&b, "\t- %s\n", v.Name)
+			}
+		}
+
+	}
+
+	log.Printf("pruning result:\n%s", b.String())
+
+}

cmd/version.go

@@ -1,13 +1,24 @@
-package version
+package cmd
 
 import (
 	"fmt"
+	"github.com/spf13/cobra"
 	"runtime"
 )
 
-var (
+var versionCmd = &cobra.Command{
-	zreplVersion string // set by build infrastructure
+	Use:   "version",
-)
+	Short: "print version of zrepl binary (for running daemon 'zrepl control version' command)",
+	Run:   doVersion,
+}
+
+func init() {
+	RootCmd.AddCommand(versionCmd)
+}
+
+func doVersion(cmd *cobra.Command, args []string) {
+	fmt.Println(NewZreplVersionInformation().String())
+}
 
 type ZreplVersionInformation struct {
 	Version         string

config/config.go

@@ -1,583 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"io/ioutil"
-	"log/syslog"
-	"os"
-	"reflect"
-	"regexp"
-	"strconv"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/zrepl/yaml-config"
-)
-
-type Config struct {
-	Jobs   []JobEnum `yaml:"jobs"`
-	Global *Global   `yaml:"global,optional,fromdefaults"`
-}
-
-func (c *Config) Job(name string) (*JobEnum, error) {
-	for _, j := range c.Jobs {
-		if j.Name() == name {
-			return &j, nil
-		}
-	}
-	return nil, fmt.Errorf("job %q not defined in config", name)
-}
-
-type JobEnum struct {
-	Ret interface{}
-}
-
-func (j JobEnum) Name() string {
-	var name string
-	switch v := j.Ret.(type) {
-	case *SnapJob:
-		name = v.Name
-	case *PushJob:
-		name = v.Name
-	case *SinkJob:
-		name = v.Name
-	case *PullJob:
-		name = v.Name
-	case *SourceJob:
-		name = v.Name
-	default:
-		panic(fmt.Sprintf("unknown job type %T", v))
-	}
-	return name
-}
-
-type ActiveJob struct {
-	Type    string                `yaml:"type"`
-	Name    string                `yaml:"name"`
-	Connect ConnectEnum           `yaml:"connect"`
-	Pruning PruningSenderReceiver `yaml:"pruning"`
-	Debug   JobDebugSettings      `yaml:"debug,optional"`
-}
-
-type PassiveJob struct {
-	Type  string           `yaml:"type"`
-	Name  string           `yaml:"name"`
-	Serve ServeEnum        `yaml:"serve"`
-	Debug JobDebugSettings `yaml:"debug,optional"`
-}
-
-type SnapJob struct {
-	Type         string            `yaml:"type"`
-	Name         string            `yaml:"name"`
-	Pruning      PruningLocal      `yaml:"pruning"`
-	Debug        JobDebugSettings  `yaml:"debug,optional"`
-	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
-	Filesystems  FilesystemsFilter `yaml:"filesystems"`
-}
-
-type PushJob struct {
-	ActiveJob    `yaml:",inline"`
-	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
-	Filesystems  FilesystemsFilter `yaml:"filesystems"`
-}
-
-type PullJob struct {
-	ActiveJob `yaml:",inline"`
-	RootFS    string                   `yaml:"root_fs"`
-	Interval  PositiveDurationOrManual `yaml:"interval"`
-}
-
-type PositiveDurationOrManual struct {
-	Interval time.Duration
-	Manual   bool
-}
-
-var _ yaml.Unmarshaler = (*PositiveDurationOrManual)(nil)
-
-func (i *PositiveDurationOrManual) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	var s string
-	if err := u(&s, true); err != nil {
-		return err
-	}
-	switch s {
-	case "manual":
-		i.Manual = true
-		i.Interval = 0
-	case "":
-		return fmt.Errorf("value must not be empty")
-	default:
-		i.Manual = false
-		i.Interval, err = time.ParseDuration(s)
-		if err != nil {
-			return err
-		}
-		if i.Interval <= 0 {
-			return fmt.Errorf("value must be a positive duration, got %q", s)
-		}
-	}
-	return nil
-}
-
-type SinkJob struct {
-	PassiveJob `yaml:",inline"`
-	RootFS     string `yaml:"root_fs"`
-}
-
-type SourceJob struct {
-	PassiveJob   `yaml:",inline"`
-	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
-	Filesystems  FilesystemsFilter `yaml:"filesystems"`
-}
-
-type FilesystemsFilter map[string]bool
-
-type SnapshottingEnum struct {
-	Ret interface{}
-}
-
-type SnapshottingPeriodic struct {
-	Type     string        `yaml:"type"`
-	Prefix   string        `yaml:"prefix"`
-	Interval time.Duration `yaml:"interval,positive"`
-}
-
-type SnapshottingManual struct {
-	Type string `yaml:"type"`
-}
-
-type PruningSenderReceiver struct {
-	KeepSender   []PruningEnum `yaml:"keep_sender"`
-	KeepReceiver []PruningEnum `yaml:"keep_receiver"`
-}
-
-type PruningLocal struct {
-	Keep []PruningEnum `yaml:"keep"`
-}
-
-type LoggingOutletEnumList []LoggingOutletEnum
-
-func (l *LoggingOutletEnumList) SetDefault() {
-	def := `
-type: "stdout"
-time: true
-level: "warn"
-format: "human"
-`
-	s := &StdoutLoggingOutlet{}
-	err := yaml.UnmarshalStrict([]byte(def), &s)
-	if err != nil {
-		panic(err)
-	}
-	*l = []LoggingOutletEnum{LoggingOutletEnum{Ret: s}}
-}
-
-var _ yaml.Defaulter = &LoggingOutletEnumList{}
-
-type Global struct {
-	Logging    *LoggingOutletEnumList `yaml:"logging,optional,fromdefaults"`
-	Monitoring []MonitoringEnum       `yaml:"monitoring,optional"`
-	Control    *GlobalControl         `yaml:"control,optional,fromdefaults"`
-	Serve      *GlobalServe           `yaml:"serve,optional,fromdefaults"`
-}
-
-func Default(i interface{}) {
-	v := reflect.ValueOf(i)
-	if v.Kind() != reflect.Ptr {
-		panic(v)
-	}
-	y := `{}`
-	err := yaml.Unmarshal([]byte(y), v.Interface())
-	if err != nil {
-		panic(err)
-	}
-}
-
-type ConnectEnum struct {
-	Ret interface{}
-}
-
-type ConnectCommon struct {
-	Type string `yaml:"type"`
-}
-
-type TCPConnect struct {
-	ConnectCommon `yaml:",inline"`
-	Address       string        `yaml:"address"`
-	DialTimeout   time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
-}
-
-type TLSConnect struct {
-	ConnectCommon `yaml:",inline"`
-	Address       string        `yaml:"address"`
-	Ca            string        `yaml:"ca"`
-	Cert          string        `yaml:"cert"`
-	Key           string        `yaml:"key"`
-	ServerCN      string        `yaml:"server_cn"`
-	DialTimeout   time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
-}
-
-type SSHStdinserverConnect struct {
-	ConnectCommon        `yaml:",inline"`
-	Host                 string        `yaml:"host"`
-	User                 string        `yaml:"user"`
-	Port                 uint16        `yaml:"port"`
-	IdentityFile         string        `yaml:"identity_file"`
-	TransportOpenCommand []string      `yaml:"transport_open_command,optional"` //TODO unused
-	SSHCommand           string        `yaml:"ssh_command,optional"`            //TODO unused
-	Options              []string      `yaml:"options,optional"`
-	DialTimeout          time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
-}
-
-type LocalConnect struct {
-	ConnectCommon  `yaml:",inline"`
-	ListenerName   string `yaml:"listener_name"`
-	ClientIdentity string `yaml:"client_identity"`
-}
-
-type ServeEnum struct {
-	Ret interface{}
-}
-
-type ServeCommon struct {
-	Type string `yaml:"type"`
-}
-
-type TCPServe struct {
-	ServeCommon `yaml:",inline"`
-	Listen      string            `yaml:"listen"`
-	Clients     map[string]string `yaml:"clients"`
-}
-
-type TLSServe struct {
-	ServeCommon      `yaml:",inline"`
-	Listen           string        `yaml:"listen"`
-	Ca               string        `yaml:"ca"`
-	Cert             string        `yaml:"cert"`
-	Key              string        `yaml:"key"`
-	ClientCNs        []string      `yaml:"client_cns"`
-	HandshakeTimeout time.Duration `yaml:"handshake_timeout,zeropositive,default=10s"`
-}
-
-type StdinserverServer struct {
-	ServeCommon      `yaml:",inline"`
-	ClientIdentities []string `yaml:"client_identities"`
-}
-
-type LocalServe struct {
-	ServeCommon  `yaml:",inline"`
-	ListenerName string `yaml:"listener_name"`
-}
-
-type PruningEnum struct {
-	Ret interface{}
-}
-
-type PruneKeepNotReplicated struct {
-	Type                 string `yaml:"type"`
-	KeepSnapshotAtCursor bool   `yaml:"keep_snapshot_at_cursor,optional,default=true"`
-}
-
-type PruneKeepLastN struct {
-	Type  string `yaml:"type"`
-	Count int    `yaml:"count"`
-}
-
-type PruneKeepRegex struct { // FIXME rename to KeepRegex
-	Type   string `yaml:"type"`
-	Regex  string `yaml:"regex"`
-	Negate bool   `yaml:"negate,optional,default=false"`
-}
-
-type LoggingOutletEnum struct {
-	Ret interface{}
-}
-
-type LoggingOutletCommon struct {
-	Type   string `yaml:"type"`
-	Level  string `yaml:"level"`
-	Format string `yaml:"format"`
-}
-
-type StdoutLoggingOutlet struct {
-	LoggingOutletCommon `yaml:",inline"`
-	Time                bool `yaml:"time,default=true"`
-	Color               bool `yaml:"color,default=true"`
-}
-
-type SyslogLoggingOutlet struct {
-	LoggingOutletCommon `yaml:",inline"`
-	Facility            *SyslogFacility `yaml:"facility,optional,fromdefaults"`
-	RetryInterval       time.Duration   `yaml:"retry_interval,positive,default=10s"`
-}
-
-type TCPLoggingOutlet struct {
-	LoggingOutletCommon `yaml:",inline"`
-	Address             string               `yaml:"address"`
-	Net                 string               `yaml:"net,default=tcp"`
-	RetryInterval       time.Duration        `yaml:"retry_interval,positive,default=10s"`
-	TLS                 *TCPLoggingOutletTLS `yaml:"tls,optional"`
-}
-
-type TCPLoggingOutletTLS struct {
-	CA   string `yaml:"ca"`
-	Cert string `yaml:"cert"`
-	Key  string `yaml:"key"`
-}
-
-type MonitoringEnum struct {
-	Ret interface{}
-}
-
-type PrometheusMonitoring struct {
-	Type   string `yaml:"type"`
-	Listen string `yaml:"listen"`
-}
-
-type SyslogFacility syslog.Priority
-
-func (f *SyslogFacility) SetDefault() {
-	*f = SyslogFacility(syslog.LOG_LOCAL0)
-}
-
-var _ yaml.Defaulter = (*SyslogFacility)(nil)
-
-type GlobalControl struct {
-	SockPath string `yaml:"sockpath,default=/var/run/zrepl/control"`
-}
-
-type GlobalServe struct {
-	StdinServer *GlobalStdinServer `yaml:"stdinserver,optional,fromdefaults"`
-}
-
-type GlobalStdinServer struct {
-	SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
-}
-
-type JobDebugSettings struct {
-	Conn *struct {
-		ReadDump  string `yaml:"read_dump"`
-		WriteDump string `yaml:"write_dump"`
-	} `yaml:"conn,optional"`
-	RPCLog bool `yaml:"rpc_log,optional,default=false"`
-}
-
-func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) {
-	var in struct {
-		Type string
-	}
-	if err := u(&in, true); err != nil {
-		return nil, err
-	}
-	if in.Type == "" {
-		return nil, &yaml.TypeError{Errors: []string{"must specify type"}}
-	}
-
-	v, ok := types[in.Type]
-	if !ok {
-		return nil, &yaml.TypeError{Errors: []string{fmt.Sprintf("invalid type name %q", in.Type)}}
-	}
-	if err := u(v, false); err != nil {
-		return nil, err
-	}
-	return v, nil
-}
-
-func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"snap":   &SnapJob{},
-		"push":   &PushJob{},
-		"sink":   &SinkJob{},
-		"pull":   &PullJob{},
-		"source": &SourceJob{},
-	})
-	return
-}
-
-func (t *ConnectEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"tcp":             &TCPConnect{},
-		"tls":             &TLSConnect{},
-		"ssh+stdinserver": &SSHStdinserverConnect{},
-		"local":           &LocalConnect{},
-	})
-	return
-}
-
-func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"tcp":         &TCPServe{},
-		"tls":         &TLSServe{},
-		"stdinserver": &StdinserverServer{},
-		"local":       &LocalServe{},
-	})
-	return
-}
-
-func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"not_replicated": &PruneKeepNotReplicated{},
-		"last_n":         &PruneKeepLastN{},
-		"grid":           &PruneGrid{},
-		"regex":          &PruneKeepRegex{},
-	})
-	return
-}
-
-func (t *SnapshottingEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"periodic": &SnapshottingPeriodic{},
-		"manual":   &SnapshottingManual{},
-	})
-	return
-}
-
-func (t *LoggingOutletEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"stdout": &StdoutLoggingOutlet{},
-		"syslog": &SyslogLoggingOutlet{},
-		"tcp":    &TCPLoggingOutlet{},
-	})
-	return
-}
-
-func (t *MonitoringEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"prometheus": &PrometheusMonitoring{},
-	})
-	return
-}
-
-func (t *SyslogFacility) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	var s string
-	if err := u(&s, true); err != nil {
-		return err
-	}
-	var level syslog.Priority
-	switch s {
-	case "kern":
-		level = syslog.LOG_KERN
-	case "user":
-		level = syslog.LOG_USER
-	case "mail":
-		level = syslog.LOG_MAIL
-	case "daemon":
-		level = syslog.LOG_DAEMON
-	case "auth":
-		level = syslog.LOG_AUTH
-	case "syslog":
-		level = syslog.LOG_SYSLOG
-	case "lpr":
-		level = syslog.LOG_LPR
-	case "news":
-		level = syslog.LOG_NEWS
-	case "uucp":
-		level = syslog.LOG_UUCP
-	case "cron":
-		level = syslog.LOG_CRON
-	case "authpriv":
-		level = syslog.LOG_AUTHPRIV
-	case "ftp":
-		level = syslog.LOG_FTP
-	case "local0":
-		level = syslog.LOG_LOCAL0
-	case "local1":
-		level = syslog.LOG_LOCAL1
-	case "local2":
-		level = syslog.LOG_LOCAL2
-	case "local3":
-		level = syslog.LOG_LOCAL3
-	case "local4":
-		level = syslog.LOG_LOCAL4
-	case "local5":
-		level = syslog.LOG_LOCAL5
-	case "local6":
-		level = syslog.LOG_LOCAL6
-	case "local7":
-		level = syslog.LOG_LOCAL7
-	default:
-		return fmt.Errorf("invalid syslog level: %q", s)
-	}
-	*t = SyslogFacility(level)
-	return nil
-}
-
-var ConfigFileDefaultLocations = []string{
-	"/etc/zrepl/zrepl.yml",
-	"/usr/local/etc/zrepl/zrepl.yml",
-}
-
-func ParseConfig(path string) (i *Config, err error) {
-
-	if path == "" {
-		// Try default locations
-		for _, l := range ConfigFileDefaultLocations {
-			stat, statErr := os.Stat(l)
-			if statErr != nil {
-				continue
-			}
-			if !stat.Mode().IsRegular() {
-				err = errors.Errorf("file at default location is not a regular file: %s", l)
-				return
-			}
-			path = l
-			break
-		}
-	}
-
-	var bytes []byte
-
-	if bytes, err = ioutil.ReadFile(path); err != nil {
-		return
-	}
-
-	return ParseConfigBytes(bytes)
-}
-
-func ParseConfigBytes(bytes []byte) (*Config, error) {
-	var c *Config
-	if err := yaml.UnmarshalStrict(bytes, &c); err != nil {
-		return nil, err
-	}
-	if c == nil {
-		return nil, fmt.Errorf("config is empty or only consists of comments")
-	}
-	return c, nil
-}
-
-var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
-
-func parsePostitiveDuration(e string) (d time.Duration, err error) {
-	comps := durationStringRegex.FindStringSubmatch(e)
-	if len(comps) != 3 {
-		err = fmt.Errorf("does not match regex: %s %#v", e, comps)
-		return
-	}
-
-	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
-	if err != nil {
-		return 0, err
-	}
-	if durationFactor <= 0 {
-		return 0, errors.New("duration must be positive integer")
-	}
-
-	var durationUnit time.Duration
-	switch comps[2] {
-	case "s":
-		durationUnit = time.Second
-	case "m":
-		durationUnit = time.Minute
-	case "h":
-		durationUnit = time.Hour
-	case "d":
-		durationUnit = 24 * time.Hour
-	case "w":
-		durationUnit = 24 * 7 * time.Hour
-	default:
-		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
-		return
-	}
-
-	d = time.Duration(durationFactor) * durationUnit
-	return
-}

config/config_global_test.go

@@ -1,115 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"log/syslog"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"github.com/zrepl/yaml-config"
-)
-
-func testValidGlobalSection(t *testing.T, s string) *Config {
-	jobdef := `
-jobs:
-- name: dummyjob
-  type: sink
-  serve:
-    type: tcp
-    listen: ":2342"
-    clients: {
-      "10.0.0.1":"foo"
-    }
-  root_fs: zoot/foo
-`
-	_, err := ParseConfigBytes([]byte(jobdef))
-	require.NoError(t, err)
-	return testValidConfig(t, s+jobdef)
-}
-
-func TestOutletTypes(t *testing.T) {
-	conf := testValidGlobalSection(t, `
-global:
-  logging:
-  - type: stdout
-    level: debug
-    format: human
-  - type: syslog
-    level: info
-    retry_interval: 20s
-    format: human
-  - type: tcp
-    level: debug
-    format: json
-    address: logserver.example.com:1234
-  - type: tcp
-    level: debug
-    format: json
-    address: encryptedlogserver.example.com:1234
-    retry_interval: 20s 
-    tls:
-      ca: /etc/zrepl/log/ca.crt
-      cert: /etc/zrepl/log/key.pem
-      key: /etc/zrepl/log/cert.pem
-`)
-	assert.Equal(t, 4, len(*conf.Global.Logging))
-	assert.NotNil(t, (*conf.Global.Logging)[3].Ret.(*TCPLoggingOutlet).TLS)
-}
-
-func TestDefaultLoggingOutlet(t *testing.T) {
-	conf := testValidGlobalSection(t, "")
-	assert.Equal(t, 1, len(*conf.Global.Logging))
-	o := (*conf.Global.Logging)[0].Ret.(*StdoutLoggingOutlet)
-	assert.Equal(t, "warn", o.Level)
-	assert.Equal(t, "human", o.Format)
-}
-
-func TestPrometheusMonitoring(t *testing.T) {
-	conf := testValidGlobalSection(t, `
-global:
-  monitoring:
-    - type: prometheus
-      listen: ':9091'
-`)
-	assert.Equal(t, ":9091", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
-}
-
-func TestSyslogLoggingOutletFacility(t *testing.T) {
-	type SyslogFacilityPriority struct {
-		Facility string
-		Priority syslog.Priority
-	}
-	syslogFacilitiesPriorities := []SyslogFacilityPriority{
-		{"", syslog.LOG_LOCAL0}, // default
-		{"kern", syslog.LOG_KERN}, {"daemon", syslog.LOG_DAEMON}, {"auth", syslog.LOG_AUTH},
-		{"syslog", syslog.LOG_SYSLOG}, {"lpr", syslog.LOG_LPR}, {"news", syslog.LOG_NEWS},
-		{"uucp", syslog.LOG_UUCP}, {"cron", syslog.LOG_CRON}, {"authpriv", syslog.LOG_AUTHPRIV},
-		{"ftp", syslog.LOG_FTP}, {"local0", syslog.LOG_LOCAL0}, {"local1", syslog.LOG_LOCAL1},
-		{"local2", syslog.LOG_LOCAL2}, {"local3", syslog.LOG_LOCAL3}, {"local4", syslog.LOG_LOCAL4},
-		{"local5", syslog.LOG_LOCAL5}, {"local6", syslog.LOG_LOCAL6}, {"local7", syslog.LOG_LOCAL7},
-	}
-
-	for _, sFP := range syslogFacilitiesPriorities {
-		logcfg := fmt.Sprintf(`
-global:
-  logging:
-  - type: syslog
-    level: info
-    format: human
-    facility: %s
-`, sFP.Facility)
-		conf := testValidGlobalSection(t, logcfg)
-		assert.Equal(t, 1, len(*conf.Global.Logging))
-		assert.True(t, SyslogFacility(sFP.Priority) == *(*conf.Global.Logging)[0].Ret.(*SyslogLoggingOutlet).Facility)
-	}
-}
-
-func TestLoggingOutletEnumList_SetDefaults(t *testing.T) {
-	e := &LoggingOutletEnumList{}
-	var i yaml.Defaulter = e
-	require.NotPanics(t, func() {
-		i.SetDefault()
-		assert.Equal(t, "warn", (*e)[0].Ret.(*StdoutLoggingOutlet).Level)
-	})
-}

config/config_minimal_test.go

@@ -1,40 +0,0 @@
-package config
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestConfigEmptyFails(t *testing.T) {
-	conf, err := testConfig(t, "\n")
-	assert.Nil(t, conf)
-	assert.Error(t, err)
-}
-
-func TestJobsOnlyWorks(t *testing.T) {
-	testValidConfig(t, `
-jobs:
-- name: push
-  type: push
-  # snapshot the filesystems matched by the left-hand-side of the mapping
-  # every 10m with zrepl_ as prefix
-  connect:
-    type: tcp
-    address: localhost:2342
-  filesystems: {
-    "pool1/var/db<": true,
-    "pool1/usr/home<": true,
-    "pool1/usr/home/paranoid": false, #don't backup paranoid user
-    "pool1/poudriere/ports<": false #don't backup the ports trees
-  }
-  snapshotting:
-    type: manual
-  pruning:
-    keep_sender:
-    - type: not_replicated
-    keep_receiver:
-    - type: last_n 
-      count: 1
-`)
-}

config/config_positiveintervalormanual_test.go

@@ -1,41 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/zrepl/yaml-config"
-)
-
-func TestPositiveDurationOrManual(t *testing.T) {
-	cases := []struct {
-		Comment, Input string
-		Result         *PositiveDurationOrManual
-	}{
-		{"empty is error", "", nil},
-		{"negative is error", "-1s", nil},
-		{"zero seconds is error", "0s", nil},
-		{"zero is error", "0", nil},
-		{"non-manual is error", "something", nil},
-		{"positive seconds works", "1s", &PositiveDurationOrManual{Manual: false, Interval: 1 * time.Second}},
-		{"manual works", "manual", &PositiveDurationOrManual{Manual: true, Interval: 0}},
-	}
-	for _, tc := range cases {
-		t.Run(tc.Comment, func(t *testing.T) {
-			var out struct {
-				FieldName PositiveDurationOrManual `yaml:"fieldname"`
-			}
-			input := fmt.Sprintf("\nfieldname: %s\n", tc.Input)
-			err := yaml.UnmarshalStrict([]byte(input), &out)
-			if tc.Result == nil {
-				assert.Error(t, err)
-				t.Logf("%#v", out)
-			} else {
-				assert.Equal(t, *tc.Result, out.FieldName)
-			}
-		})
-	}
-
-}

config/config_snapshotting_test.go

@@ -1,58 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestSnapshotting(t *testing.T) {
-	tmpl := `
-jobs:
-- name: foo
-  type: push
-  connect:
-    type: local
-    listener_name: foo
-    client_identity: bar
-  filesystems: {"<": true}
-  %s
-  pruning:
-    keep_sender:
-    - type: last_n
-      count: 10
-    keep_receiver:
-    - type: last_n
-      count: 10
-`
-	manual := `
-  snapshotting:
-    type: manual
-`
-	periodic := `
-  snapshotting:
-    type: periodic
-    prefix: zrepl_
-    interval: 10m
-`
-
-	fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
-	var c *Config
-
-	t.Run("manual", func(t *testing.T) {
-		c = testValidConfig(t, fillSnapshotting(manual))
-		snm := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingManual)
-		assert.Equal(t, "manual", snm.Type)
-	})
-
-	t.Run("periodic", func(t *testing.T) {
-		c = testValidConfig(t, fillSnapshotting(periodic))
-		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
-		assert.Equal(t, "periodic", snp.Type)
-		assert.Equal(t, 10*time.Minute, snp.Interval)
-		assert.Equal(t, "zrepl_", snp.Prefix)
-	})
-
-}

config/config_test.go

@@ -1,67 +0,0 @@
-package config
-
-import (
-	"bytes"
-	"path"
-	"path/filepath"
-	"testing"
-	"text/template"
-
-	"github.com/kr/pretty"
-	"github.com/stretchr/testify/require"
-)
-
-func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
-	paths, err := filepath.Glob("./samples/*")
-	if err != nil {
-		t.Errorf("glob failed: %+v", err)
-	}
-
-	for _, p := range paths {
-
-		if path.Ext(p) != ".yml" {
-			t.Logf("skipping file %s", p)
-			continue
-		}
-
-		t.Run(p, func(t *testing.T) {
-			c, err := ParseConfig(p)
-			if err != nil {
-				t.Errorf("error parsing %s:\n%+v", p, err)
-			}
-
-			t.Logf("file: %s", p)
-			t.Log(pretty.Sprint(c))
-		})
-
-	}
-
-}
-
-// template must be a template/text template with a single '{{ . }}' as placehodler for val
-//nolint[:deadcode,unused]
-func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
-	tmp, err := template.New("master").Parse(tmpl)
-	if err != nil {
-		panic(err)
-	}
-	var buf bytes.Buffer
-	err = tmp.Execute(&buf, val)
-	if err != nil {
-		panic(err)
-	}
-	return testValidConfig(t, buf.String())
-}
-
-func testValidConfig(t *testing.T, input string) *Config {
-	t.Helper()
-	conf, err := testConfig(t, input)
-	require.NoError(t, err)
-	require.NotNil(t, conf)
-	return conf
-}
-
-func testConfig(t *testing.T, input string) (*Config, error) {
-	t.Helper()
-	return ParseConfigBytes([]byte(input))
-}

config/retentiongrid.go

@@ -1,119 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"regexp"
-	"strconv"
-	"strings"
-	"time"
-)
-
-type RetentionIntervalList []RetentionInterval
-
-type PruneGrid struct {
-	Type  string                `yaml:"type"`
-	Grid  RetentionIntervalList `yaml:"grid"`
-	Regex string                `yaml:"regex"`
-}
-
-type RetentionInterval struct {
-	length    time.Duration
-	keepCount int
-}
-
-func (i *RetentionInterval) Length() time.Duration {
-	return i.length
-}
-
-func (i *RetentionInterval) KeepCount() int {
-	return i.keepCount
-}
-
-const RetentionGridKeepCountAll int = -1
-
-func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	var in string
-	if err := u(&in, true); err != nil {
-		return err
-	}
-
-	intervals, err := parseRetentionGridIntervalsString(in)
-	if err != nil {
-		return err
-	}
-
-	*t = intervals
-
-	return nil
-}
-
-var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
-
-func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {
-
-	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
-	if comps == nil {
-		err = fmt.Errorf("retention string does not match expected format")
-		return
-	}
-
-	times, err := strconv.Atoi(comps[1])
-	if err != nil {
-		return nil, err
-	} else if times <= 0 {
-		return nil, fmt.Errorf("contains factor <= 0")
-	}
-
-	duration, err := parsePostitiveDuration(comps[2])
-	if err != nil {
-		return nil, err
-	}
-
-	keepCount := 1
-	if comps[3] != "" {
-		// Decompose key=value, comma separated
-		// For now, only keep_count is supported
-		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
-		res := re.FindStringSubmatch(comps[4])
-		if res == nil || len(res) != 2 {
-			err = fmt.Errorf("interval parameter contains unknown parameters")
-			return
-		}
-		if res[1] == "all" {
-			keepCount = RetentionGridKeepCountAll
-		} else {
-			keepCount, err = strconv.Atoi(res[1])
-			if err != nil {
-				err = fmt.Errorf("cannot parse keep_count value")
-				return
-			}
-		}
-	}
-
-	intervals = make([]RetentionInterval, times)
-	for i := range intervals {
-		intervals[i] = RetentionInterval{
-			length:    duration,
-			keepCount: keepCount,
-		}
-	}
-
-	return
-
-}
-
-func parseRetentionGridIntervalsString(s string) (intervals []RetentionInterval, err error) {
-
-	ges := strings.Split(s, "|")
-	intervals = make([]RetentionInterval, 0, 7*len(ges))
-
-	for intervalIdx, e := range ges {
-		parsed, err := parseRetentionGridIntervalString(e)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
-		}
-		intervals = append(intervals, parsed...)
-	}
-
-	return
-}

config/samples/local.yml

@@ -1,31 +0,0 @@
-
-jobs:
-  - type: sink
-    name: "local_sink"
-    root_fs: "storage/zrepl/sink"
-    serve:
-      type: local
-      listener_name: localsink
-
-  - type: push
-    name: "backup_system"
-    connect:
-      type: local
-      listener_name: localsink
-      client_identity: local_backup
-    filesystems: {
-      "system<": true,
-    }
-    snapshotting:
-      type: periodic
-      interval: 10m
-      prefix: zrepl_
-    pruning:
-      keep_sender:
-      - type: not_replicated
-      - type: last_n
-        count: 10
-      keep_receiver:
-      - type: grid
-        grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-        regex: "zrepl_.*"

config/samples/pull.yml

@@ -1,24 +0,0 @@
-jobs:
-- name: pull_servers
-  type: pull
-  connect:
-    type: tls
-    address: "server1.foo.bar:8888"
-    ca: "/certs/ca.crt"
-    cert: "/certs/cert.crt"
-    key: "/certs/key.pem"
-    server_cn: "server1"
-  root_fs: "pool2/backup_servers"
-  interval: 10m
-  pruning:
-    keep_sender:
-    - type: not_replicated
-    - type: last_n
-      count: 10
-    - type: grid
-      grid: 1x1h(keep=all) | 24x1h | 14x1d
-      regex: "zrepl_.*"
-    keep_receiver:
-    - type: grid
-      grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-      regex: "zrepl_.*"

config/samples/pull_ssh.yml

@@ -1,28 +0,0 @@
-jobs:
-
-- name: pull_servers
-  type: pull
-  connect:
-    type: ssh+stdinserver
-    host: app-srv.example.com
-    user: root
-    port: 22
-    identity_file: /etc/zrepl/ssh/identity
-    options: # optional, default [], `-o` arguments passed to ssh
-    - "Compression=on"
-  root_fs: "pool2/backup_servers"
-  interval: 10m
-  pruning:
-    keep_sender:
-    - type: not_replicated
-    - type: last_n
-      count: 10
-    - type: grid
-      grid: 1x1h(keep=all) | 24x1h | 14x1d
-      regex: "^zrepl_.*"
-    keep_receiver:
-    - type: regex
-      regex: keep_
-    - type: grid
-      grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-      regex: "^zrepl_.*"

config/samples/push.yml

@@ -1,24 +0,0 @@
-jobs:
-  - type: push
-    name: "push"
-    filesystems: {
-      "<": true,
-      "tmp": false
-    }
-    connect:
-      type: tcp
-      address: "backup-server.foo.bar:8888"
-    snapshotting:
-      type: manual
-    pruning:
-      keep_sender:
-        - type: not_replicated
-        - type: last_n
-          count: 10
-        - type: grid
-          grid: 1x1h(keep=all) | 24x1h | 14x1d
-          regex: "^zrepl_.*"
-      keep_receiver:
-        - type: grid
-          grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-          regex: "^zrepl_.*"

config/samples/sink.yml

@@ -1,13 +0,0 @@
-jobs:
-  - type: sink
-    name: "laptop_sink"
-    root_fs: "pool2/backup_laptops"
-    serve:
-      type: tls
-      listen: "192.168.122.189:8888"
-      ca: "ca.pem"
-      cert: "cert.pem"
-      key: "key.pem"
-      client_cns:
-        - "laptop1"
-        - "homeserver"

config/samples/snap.yml

@@ -1,14 +0,0 @@
-jobs:
-- name: snapjob
-  type: snap
-  filesystems: {
-    "tank/frequently_changed<": true,
-  }
-  snapshotting:
-    type: periodic
-    interval: 2m
-    prefix: zrepl_snapjob_
-  pruning:
-    keep:
-      - type: last_n
-        count: 60

config/samples/source.yml

@@ -1,17 +0,0 @@
-jobs:
-- name: pull_source
-  type: source
-  serve:
-    type: tcp
-    listen: "0.0.0.0:8888"
-    clients: {
-      "192.168.122.123" : "client1"
-    }
-  filesystems: {
-    "<": true,
-    "secret": false
-  }
-  snapshotting:
-    type: periodic
-    interval: 10m
-    prefix: zrepl_

config/samples/source_ssh.yml

@@ -1,17 +0,0 @@
-jobs:
-- name: pull_source
-  type: source
-  serve:
-    type: stdinserver
-    client_identities:
-      - "client1"
-      - "client2"
-  filesystems: {
-    "<": true,
-    "secret": false
-  }
-  snapshotting:
-    type: periodic
-    interval: 10m
-    prefix: zrepl_
-

daemon/control.go

@@ -1,272 +0,0 @@
-package daemon
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net"
-	"net/http"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/daemon/nethelpers"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/version"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type controlJob struct {
-	sockaddr *net.UnixAddr
-	jobs     *jobs
-}
-
-func newControlJob(sockpath string, jobs *jobs) (j *controlJob, err error) {
-	j = &controlJob{jobs: jobs}
-
-	j.sockaddr, err = net.ResolveUnixAddr("unix", sockpath)
-	if err != nil {
-		err = errors.Wrap(err, "cannot resolve unix address")
-		return
-	}
-
-	return
-}
-
-func (j *controlJob) Name() string { return jobNameControl }
-
-func (j *controlJob) Status() *job.Status { return &job.Status{Type: job.TypeInternal} }
-
-func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
-
-var promControl struct {
-	requestBegin    *prometheus.CounterVec
-	requestFinished *prometheus.HistogramVec
-}
-
-func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) {
-	promControl.requestBegin = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "zrepl",
-		Subsystem: "control",
-		Name:      "request_begin",
-		Help:      "number of request we started to handle",
-	}, []string{"endpoint"})
-
-	promControl.requestFinished = prometheus.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace: "zrepl",
-		Subsystem: "control",
-		Name:      "request_finished",
-		Help:      "time it took a request to finih",
-		Buckets:   []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3, 10e-3, 100e-3, 200e-3, 400e-3, 800e-3, 1, 10, 20},
-	}, []string{"endpoint"})
-	registerer.MustRegister(promControl.requestBegin)
-	registerer.MustRegister(promControl.requestFinished)
-}
-
-const (
-	ControlJobEndpointPProf   string = "/debug/pprof"
-	ControlJobEndpointVersion string = "/version"
-	ControlJobEndpointStatus  string = "/status"
-	ControlJobEndpointSignal  string = "/signal"
-)
-
-func (j *controlJob) Run(ctx context.Context) {
-
-	log := job.GetLogger(ctx)
-	defer log.Info("control job finished")
-
-	l, err := nethelpers.ListenUnixPrivate(j.sockaddr)
-	if err != nil {
-		log.WithError(err).Error("error listening")
-		return
-	}
-
-	pprofServer := NewPProfServer(ctx)
-
-	mux := http.NewServeMux()
-	mux.Handle(ControlJobEndpointPProf,
-		requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) {
-			var msg PprofServerControlMsg
-			err := decoder(&msg)
-			if err != nil {
-				return nil, errors.Errorf("decode failed")
-			}
-			pprofServer.Control(msg)
-			return struct{}{}, nil
-		}}})
-
-	mux.Handle(ControlJobEndpointVersion,
-		requestLogger{log: log, handler: jsonResponder{log, func() (interface{}, error) {
-			return version.NewZreplVersionInformation(), nil
-		}}})
-
-	mux.Handle(ControlJobEndpointStatus,
-		// don't log requests to status endpoint, too spammy
-		jsonResponder{log, func() (interface{}, error) {
-			s := j.jobs.status()
-			return s, nil
-		}})
-
-	mux.Handle(ControlJobEndpointSignal,
-		requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) {
-			type reqT struct {
-				Name string
-				Op   string
-			}
-			var req reqT
-			if decoder(&req) != nil {
-				return nil, errors.Errorf("decode failed")
-			}
-
-			var err error
-			switch req.Op {
-			case "wakeup":
-				err = j.jobs.wakeup(req.Name)
-			case "reset":
-				err = j.jobs.reset(req.Name)
-			default:
-				err = fmt.Errorf("operation %q is invalid", req.Op)
-			}
-
-			return struct{}{}, err
-		}}})
-	server := http.Server{
-		Handler: mux,
-		// control socket is local, 1s timeout should be more than sufficient, even on a loaded system
-		WriteTimeout: 1 * time.Second,
-		ReadTimeout:  1 * time.Second,
-	}
-
-outer:
-	for {
-
-		served := make(chan error)
-		go func() {
-			served <- server.Serve(l)
-			close(served)
-		}()
-
-		select {
-		case <-ctx.Done():
-			log.WithError(ctx.Err()).Info("context done")
-			err := server.Shutdown(context.Background())
-			if err != nil {
-				log.WithError(err).Error("cannot shutdown server")
-			}
-			break outer
-		case err = <-served:
-			if err != nil {
-				log.WithError(err).Error("error serving")
-				break outer
-			}
-		}
-
-	}
-
-}
-
-type jsonResponder struct {
-	log      Logger
-	producer func() (interface{}, error)
-}
-
-func (j jsonResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	logIoErr := func(err error) {
-		if err != nil {
-			j.log.WithError(err).Error("control handler io error")
-		}
-	}
-	res, err := j.producer()
-	if err != nil {
-		j.log.WithError(err).Error("control handler error")
-		w.WriteHeader(http.StatusInternalServerError)
-		_, err = io.WriteString(w, err.Error())
-		logIoErr(err)
-		return
-	}
-	var buf bytes.Buffer
-	err = json.NewEncoder(&buf).Encode(res)
-	if err != nil {
-		j.log.WithError(err).Error("control handler json marshal error")
-		w.WriteHeader(http.StatusInternalServerError)
-		_, err = io.WriteString(w, err.Error())
-	} else {
-		_, err = io.Copy(w, &buf)
-	}
-	logIoErr(err)
-}
-
-type jsonDecoder = func(interface{}) error
-
-type jsonRequestResponder struct {
-	log      Logger
-	producer func(decoder jsonDecoder) (interface{}, error)
-}
-
-func (j jsonRequestResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	logIoErr := func(err error) {
-		if err != nil {
-			j.log.WithError(err).Error("control handler io error")
-		}
-	}
-
-	var decodeError error
-	decoder := func(i interface{}) error {
-		err := json.NewDecoder(r.Body).Decode(&i)
-		decodeError = err
-		return err
-	}
-	res, producerErr := j.producer(decoder)
-
-	//If we had a decode error ignore output of producer and return error
-	if decodeError != nil {
-		w.WriteHeader(http.StatusBadRequest)
-		_, err := io.WriteString(w, decodeError.Error())
-		logIoErr(err)
-		return
-	}
-	if producerErr != nil {
-		j.log.WithError(producerErr).Error("control handler error")
-		w.WriteHeader(http.StatusInternalServerError)
-		_, err := io.WriteString(w, producerErr.Error())
-		logIoErr(err)
-		return
-	}
-
-	var buf bytes.Buffer
-	encodeErr := json.NewEncoder(&buf).Encode(res)
-	if encodeErr != nil {
-		j.log.WithError(producerErr).Error("control handler json marhsal error")
-		w.WriteHeader(http.StatusInternalServerError)
-		_, err := io.WriteString(w, encodeErr.Error())
-		logIoErr(err)
-	} else {
-		_, err := io.Copy(w, &buf)
-		logIoErr(err)
-	}
-}
-
-type requestLogger struct {
-	log         logger.Logger
-	handler     http.Handler
-	handlerFunc http.HandlerFunc
-}
-
-func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	log := l.log.WithField("method", r.Method).WithField("url", r.URL)
-	log.Debug("start")
-	promControl.requestBegin.WithLabelValues(r.URL.Path).Inc()
-	defer prometheus.NewTimer(promControl.requestFinished.WithLabelValues(r.URL.Path)).ObserveDuration()
-	if l.handlerFunc != nil {
-		l.handlerFunc(w, r)
-	} else if l.handler != nil {
-		l.handler.ServeHTTP(w, r)
-	} else {
-		log.Error("no handler or handlerFunc configured")
-	}
-	log.Debug("finish")
-}

daemon/daemon.go

@@ -1,222 +0,0 @@
-package daemon
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"os/signal"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/daemon/job/reset"
-	"github.com/zrepl/zrepl/daemon/job/wakeup"
-	"github.com/zrepl/zrepl/daemon/logging"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/version"
-)
-
-func Run(conf *config.Config) error {
-
-	ctx, cancel := context.WithCancel(context.Background())
-
-	defer cancel()
-	sigChan := make(chan os.Signal, 1)
-	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
-	go func() {
-		<-sigChan
-		cancel()
-	}()
-
-	outlets, err := logging.OutletsFromConfig(*conf.Global.Logging)
-	if err != nil {
-		return errors.Wrap(err, "cannot build logging from config")
-	}
-
-	confJobs, err := job.JobsFromConfig(conf)
-	if err != nil {
-		return errors.Wrap(err, "cannot build jobs from config")
-	}
-
-	log := logger.NewLogger(outlets, 1*time.Second)
-	log.Info(version.NewZreplVersionInformation().String())
-
-	for _, job := range confJobs {
-		if IsInternalJobName(job.Name()) {
-			panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME
-		}
-	}
-
-	ctx = job.WithLogger(ctx, log)
-
-	jobs := newJobs()
-
-	// start control socket
-	controlJob, err := newControlJob(conf.Global.Control.SockPath, jobs)
-	if err != nil {
-		panic(err) // FIXME
-	}
-	jobs.start(ctx, controlJob, true)
-
-	for i, jc := range conf.Global.Monitoring {
-		var (
-			job job.Job
-			err error
-		)
-		switch v := jc.Ret.(type) {
-		case *config.PrometheusMonitoring:
-			job, err = newPrometheusJobFromConfig(v)
-		default:
-			return errors.Errorf("unknown monitoring job #%d (type %T)", i, v)
-		}
-		if err != nil {
-			return errors.Wrapf(err, "cannot build monitorin gjob #%d", i)
-		}
-		jobs.start(ctx, job, true)
-	}
-
-	log.Info("starting daemon")
-
-	// start regular jobs
-	for _, j := range confJobs {
-		jobs.start(ctx, j, false)
-	}
-
-	select {
-	case <-jobs.wait():
-		log.Info("all jobs finished")
-	case <-ctx.Done():
-		log.WithError(ctx.Err()).Info("context finished")
-	}
-	log.Info("daemon exiting")
-	return nil
-}
-
-type jobs struct {
-	wg sync.WaitGroup
-
-	// m protects all fields below it
-	m       sync.RWMutex
-	wakeups map[string]wakeup.Func // by Job.Name
-	resets  map[string]reset.Func  // by Job.Name
-	jobs    map[string]job.Job
-}
-
-func newJobs() *jobs {
-	return &jobs{
-		wakeups: make(map[string]wakeup.Func),
-		resets:  make(map[string]reset.Func),
-		jobs:    make(map[string]job.Job),
-	}
-}
-
-const (
-	logJobField string = "job"
-)
-
-func (s *jobs) wait() <-chan struct{} {
-	ch := make(chan struct{})
-	go func() {
-		s.wg.Wait()
-	}()
-	return ch
-}
-
-func (s *jobs) status() map[string]*job.Status {
-	s.m.RLock()
-	defer s.m.RUnlock()
-
-	type res struct {
-		name   string
-		status *job.Status
-	}
-	var wg sync.WaitGroup
-	c := make(chan res, len(s.jobs))
-	for name, j := range s.jobs {
-		wg.Add(1)
-		go func(name string, j job.Job) {
-			defer wg.Done()
-			c <- res{name: name, status: j.Status()}
-		}(name, j)
-	}
-	wg.Wait()
-	close(c)
-	ret := make(map[string]*job.Status, len(s.jobs))
-	for res := range c {
-		ret[res.name] = res.status
-	}
-	return ret
-}
-
-func (s *jobs) wakeup(job string) error {
-	s.m.RLock()
-	defer s.m.RUnlock()
-
-	wu, ok := s.wakeups[job]
-	if !ok {
-		return errors.Errorf("Job %s does not exist", job)
-	}
-	return wu()
-}
-
-func (s *jobs) reset(job string) error {
-	s.m.RLock()
-	defer s.m.RUnlock()
-
-	wu, ok := s.resets[job]
-	if !ok {
-		return errors.Errorf("Job %s does not exist", job)
-	}
-	return wu()
-}
-
-const (
-	jobNamePrometheus = "_prometheus"
-	jobNameControl    = "_control"
-)
-
-func IsInternalJobName(s string) bool {
-	return strings.HasPrefix(s, "_")
-}
-
-func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
-	s.m.Lock()
-	defer s.m.Unlock()
-
-	jobLog := job.GetLogger(ctx).
-		WithField(logJobField, j.Name()).
-		WithOutlet(newPrometheusLogOutlet(j.Name()), logger.Debug)
-	jobName := j.Name()
-	if !internal && IsInternalJobName(jobName) {
-		panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName))
-	}
-	if internal && !IsInternalJobName(jobName) {
-		panic(fmt.Sprintf("internal job does not use internal job name %s", jobName))
-	}
-	if _, ok := s.jobs[jobName]; ok {
-		panic(fmt.Sprintf("duplicate job name %s", jobName))
-	}
-
-	j.RegisterMetrics(prometheus.DefaultRegisterer)
-
-	s.jobs[jobName] = j
-	ctx = job.WithLogger(ctx, jobLog)
-	ctx, wakeup := wakeup.Context(ctx)
-	ctx, resetFunc := reset.Context(ctx)
-	s.wakeups[jobName] = wakeup
-	s.resets[jobName] = resetFunc
-
-	s.wg.Add(1)
-	go func() {
-		defer s.wg.Done()
-		jobLog.Info("starting job")
-		defer jobLog.Info("job exited")
-		j.Run(ctx)
-	}()
-}

daemon/filters/fsvfilter.go

@@ -1,41 +0,0 @@
-package filters
-
-import (
-	"strings"
-
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type AnyFSVFilter struct{}
-
-func NewAnyFSVFilter() AnyFSVFilter {
-	return AnyFSVFilter{}
-}
-
-var _ zfs.FilesystemVersionFilter = AnyFSVFilter{}
-
-func (AnyFSVFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) {
-	return true, nil
-}
-
-type PrefixFilter struct {
-	prefix    string
-	fstype    zfs.VersionType
-	fstypeSet bool // optionals anyone?
-}
-
-var _ zfs.FilesystemVersionFilter = &PrefixFilter{}
-
-func NewPrefixFilter(prefix string) *PrefixFilter {
-	return &PrefixFilter{prefix: prefix}
-}
-
-func NewTypedPrefixFilter(prefix string, versionType zfs.VersionType) *PrefixFilter {
-	return &PrefixFilter{prefix, versionType, true}
-}
-
-func (f *PrefixFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) {
-	fstypeMatches := (!f.fstypeSet || t == f.fstype)
-	prefixMatches := strings.HasPrefix(name, f.prefix)
-	return fstypeMatches && prefixMatches, nil
-}

daemon/job/active.go

@@ -1,423 +0,0 @@
-package job
-
-import (
-	"context"
-	"sync"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/daemon/job/reset"
-	"github.com/zrepl/zrepl/daemon/job/wakeup"
-	"github.com/zrepl/zrepl/daemon/logging"
-	"github.com/zrepl/zrepl/daemon/pruner"
-	"github.com/zrepl/zrepl/daemon/snapper"
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/replication"
-	"github.com/zrepl/zrepl/replication/driver"
-	"github.com/zrepl/zrepl/replication/logic"
-	"github.com/zrepl/zrepl/replication/report"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/transport"
-	"github.com/zrepl/zrepl/transport/fromconfig"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type ActiveSide struct {
-	mode      activeMode
-	name      string
-	connecter transport.Connecter
-
-	prunerFactory *pruner.PrunerFactory
-
-	promRepStateSecs    *prometheus.HistogramVec // labels: state
-	promPruneSecs       *prometheus.HistogramVec // labels: prune_side
-	promBytesReplicated *prometheus.CounterVec   // labels: filesystem
-
-	tasksMtx sync.Mutex
-	tasks    activeSideTasks
-}
-
-//go:generate enumer -type=ActiveSideState
-type ActiveSideState int
-
-const (
-	ActiveSideReplicating ActiveSideState = 1 << iota
-	ActiveSidePruneSender
-	ActiveSidePruneReceiver
-	ActiveSideDone // also errors
-)
-
-type activeSideTasks struct {
-	state ActiveSideState
-
-	// valid for state ActiveSideReplicating, ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
-	replicationReport driver.ReportFunc
-	replicationCancel context.CancelFunc
-
-	// valid for state ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
-	prunerSender, prunerReceiver *pruner.Pruner
-
-	// valid for state ActiveSidePruneReceiver, ActiveSideDone
-	prunerSenderCancel, prunerReceiverCancel context.CancelFunc
-}
-
-func (a *ActiveSide) updateTasks(u func(*activeSideTasks)) activeSideTasks {
-	a.tasksMtx.Lock()
-	defer a.tasksMtx.Unlock()
-	copy := a.tasks
-	if u == nil {
-		return copy
-	}
-	u(&copy)
-	a.tasks = copy
-	return copy
-}
-
-type activeMode interface {
-	ConnectEndpoints(rpcLoggers rpc.Loggers, connecter transport.Connecter)
-	DisconnectEndpoints()
-	SenderReceiver() (logic.Sender, logic.Receiver)
-	Type() Type
-	RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{})
-	ResetConnectBackoff()
-}
-
-type modePush struct {
-	setupMtx sync.Mutex
-	sender   *endpoint.Sender
-	receiver *rpc.Client
-	fsfilter endpoint.FSFilter
-	snapper  *snapper.PeriodicOrManual
-}
-
-func (m *modePush) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Connecter) {
-	m.setupMtx.Lock()
-	defer m.setupMtx.Unlock()
-	if m.receiver != nil || m.sender != nil {
-		panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
-	}
-	m.sender = endpoint.NewSender(m.fsfilter)
-	m.receiver = rpc.NewClient(connecter, loggers)
-}
-
-func (m *modePush) DisconnectEndpoints() {
-	m.setupMtx.Lock()
-	defer m.setupMtx.Unlock()
-	m.receiver.Close()
-	m.sender = nil
-	m.receiver = nil
-}
-
-func (m *modePush) SenderReceiver() (logic.Sender, logic.Receiver) {
-	m.setupMtx.Lock()
-	defer m.setupMtx.Unlock()
-	return m.sender, m.receiver
-}
-
-func (m *modePush) Type() Type { return TypePush }
-
-func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
-	m.snapper.Run(ctx, wakeUpCommon)
-}
-
-func (m *modePush) ResetConnectBackoff() {
-	m.setupMtx.Lock()
-	defer m.setupMtx.Unlock()
-	if m.receiver != nil {
-		m.receiver.ResetConnectBackoff()
-	}
-}
-
-func modePushFromConfig(g *config.Global, in *config.PushJob) (*modePush, error) {
-	m := &modePush{}
-	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannnot build filesystem filter")
-	}
-	m.fsfilter = fsf
-
-	if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
-		return nil, errors.Wrap(err, "cannot build snapper")
-	}
-
-	return m, nil
-}
-
-type modePull struct {
-	setupMtx sync.Mutex
-	receiver *endpoint.Receiver
-	sender   *rpc.Client
-	rootFS   *zfs.DatasetPath
-	interval config.PositiveDurationOrManual
-}
-
-func (m *modePull) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Connecter) {
-	m.setupMtx.Lock()
-	defer m.setupMtx.Unlock()
-	if m.receiver != nil || m.sender != nil {
-		panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
-	}
-	m.receiver = endpoint.NewReceiver(m.rootFS, false)
-	m.sender = rpc.NewClient(connecter, loggers)
-}
-
-func (m *modePull) DisconnectEndpoints() {
-	m.setupMtx.Lock()
-	defer m.setupMtx.Unlock()
-	m.sender.Close()
-	m.sender = nil
-	m.receiver = nil
-}
-
-func (m *modePull) SenderReceiver() (logic.Sender, logic.Receiver) {
-	m.setupMtx.Lock()
-	defer m.setupMtx.Unlock()
-	return m.sender, m.receiver
-}
-
-func (*modePull) Type() Type { return TypePull }
-
-func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
-	if m.interval.Manual {
-		GetLogger(ctx).Info("manual pull configured, periodic pull disabled")
-		// "waiting for wakeups" is printed in common ActiveSide.do
-		return
-	}
-	t := time.NewTicker(m.interval.Interval)
-	defer t.Stop()
-	for {
-		select {
-		case <-t.C:
-			select {
-			case wakeUpCommon <- struct{}{}:
-			default:
-				GetLogger(ctx).
-					WithField("pull_interval", m.interval).
-					Warn("pull job took longer than pull interval")
-				wakeUpCommon <- struct{}{} // block anyways, to queue up the wakeup
-			}
-		case <-ctx.Done():
-			return
-		}
-	}
-}
-
-func (m *modePull) ResetConnectBackoff() {
-	m.setupMtx.Lock()
-	defer m.setupMtx.Unlock()
-	if m.sender != nil {
-		m.sender.ResetConnectBackoff()
-	}
-}
-
-func modePullFromConfig(g *config.Global, in *config.PullJob) (m *modePull, err error) {
-	m = &modePull{}
-	m.interval = in.Interval
-
-	m.rootFS, err = zfs.NewDatasetPath(in.RootFS)
-	if err != nil {
-		return nil, errors.New("RootFS is not a valid zfs filesystem path")
-	}
-	if m.rootFS.Length() <= 0 {
-		return nil, errors.New("RootFS must not be empty") // duplicates error check of receiver
-	}
-
-	return m, nil
-}
-
-func activeSide(g *config.Global, in *config.ActiveJob, mode activeMode) (j *ActiveSide, err error) {
-
-	j = &ActiveSide{mode: mode}
-	j.name = in.Name
-	j.promRepStateSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace:   "zrepl",
-		Subsystem:   "replication",
-		Name:        "state_time",
-		Help:        "seconds spent during replication",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
-	}, []string{"state"})
-	j.promBytesReplicated = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Namespace:   "zrepl",
-		Subsystem:   "replication",
-		Name:        "bytes_replicated",
-		Help:        "number of bytes replicated from sender to receiver per filesystem",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
-	}, []string{"filesystem"})
-
-	j.connecter, err = fromconfig.ConnecterFromConfig(g, in.Connect)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot build client")
-	}
-
-	j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace:   "zrepl",
-		Subsystem:   "pruning",
-		Name:        "time",
-		Help:        "seconds spent in pruner",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
-	}, []string{"prune_side"})
-	j.prunerFactory, err = pruner.NewPrunerFactory(in.Pruning, j.promPruneSecs)
-	if err != nil {
-		return nil, err
-	}
-
-	return j, nil
-}
-
-func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) {
-	registerer.MustRegister(j.promRepStateSecs)
-	registerer.MustRegister(j.promPruneSecs)
-	registerer.MustRegister(j.promBytesReplicated)
-}
-
-func (j *ActiveSide) Name() string { return j.name }
-
-type ActiveSideStatus struct {
-	Replication                    *report.Report
-	PruningSender, PruningReceiver *pruner.Report
-}
-
-func (j *ActiveSide) Status() *Status {
-	tasks := j.updateTasks(nil)
-
-	s := &ActiveSideStatus{}
-	t := j.mode.Type()
-	if tasks.replicationReport != nil {
-		s.Replication = tasks.replicationReport()
-	}
-	if tasks.prunerSender != nil {
-		s.PruningSender = tasks.prunerSender.Report()
-	}
-	if tasks.prunerReceiver != nil {
-		s.PruningReceiver = tasks.prunerReceiver.Report()
-	}
-	return &Status{Type: t, JobSpecific: s}
-}
-
-func (j *ActiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
-	pull, ok := j.mode.(*modePull)
-	if !ok {
-		_ = j.mode.(*modePush) // make sure we didn't introduce a new job type
-		return nil, false
-	}
-	return pull.rootFS.Copy(), true
-}
-
-func (j *ActiveSide) Run(ctx context.Context) {
-	log := GetLogger(ctx)
-	ctx = logging.WithSubsystemLoggers(ctx, log)
-
-	defer log.Info("job exiting")
-
-	periodicDone := make(chan struct{})
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-	go j.mode.RunPeriodic(ctx, periodicDone)
-
-	invocationCount := 0
-outer:
-	for {
-		log.Info("wait for wakeups")
-		select {
-		case <-ctx.Done():
-			log.WithError(ctx.Err()).Info("context")
-			break outer
-
-		case <-wakeup.Wait(ctx):
-			j.mode.ResetConnectBackoff()
-		case <-periodicDone:
-		}
-		invocationCount++
-		invLog := log.WithField("invocation", invocationCount)
-		j.do(WithLogger(ctx, invLog))
-	}
-}
-
-func (j *ActiveSide) do(ctx context.Context) {
-
-	log := GetLogger(ctx)
-	ctx = logging.WithSubsystemLoggers(ctx, log)
-	loggers := rpc.GetLoggersOrPanic(ctx) // filled by WithSubsystemLoggers
-	j.mode.ConnectEndpoints(loggers, j.connecter)
-	defer j.mode.DisconnectEndpoints()
-
-	// allow cancellation of an invocation (this function)
-	ctx, cancelThisRun := context.WithCancel(ctx)
-	defer cancelThisRun()
-	go func() {
-		select {
-		case <-reset.Wait(ctx):
-			log.Info("reset received, cancelling current invocation")
-			cancelThisRun()
-		case <-ctx.Done():
-		}
-	}()
-
-	sender, receiver := j.mode.SenderReceiver()
-
-	{
-		select {
-		case <-ctx.Done():
-			return
-		default:
-		}
-		ctx, repCancel := context.WithCancel(ctx)
-		var repWait driver.WaitFunc
-		j.updateTasks(func(tasks *activeSideTasks) {
-			// reset it
-			*tasks = activeSideTasks{}
-			tasks.replicationCancel = repCancel
-			tasks.replicationReport, repWait = replication.Do(
-				ctx, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver),
-			)
-			tasks.state = ActiveSideReplicating
-		})
-		log.Info("start replication")
-		repWait(true) // wait blocking
-		repCancel()   // always cancel to free up context resources
-	}
-
-	{
-		select {
-		case <-ctx.Done():
-			return
-		default:
-		}
-		ctx, senderCancel := context.WithCancel(ctx)
-		tasks := j.updateTasks(func(tasks *activeSideTasks) {
-			tasks.prunerSender = j.prunerFactory.BuildSenderPruner(ctx, sender, sender)
-			tasks.prunerSenderCancel = senderCancel
-			tasks.state = ActiveSidePruneSender
-		})
-		log.Info("start pruning sender")
-		tasks.prunerSender.Prune()
-		log.Info("finished pruning sender")
-		senderCancel()
-	}
-	{
-		select {
-		case <-ctx.Done():
-			return
-		default:
-		}
-		ctx, receiverCancel := context.WithCancel(ctx)
-		tasks := j.updateTasks(func(tasks *activeSideTasks) {
-			tasks.prunerReceiver = j.prunerFactory.BuildReceiverPruner(ctx, receiver, sender)
-			tasks.prunerReceiverCancel = receiverCancel
-			tasks.state = ActiveSidePruneReceiver
-		})
-		log.Info("start pruning receiver")
-		tasks.prunerReceiver.Prune()
-		log.Info("finished pruning receiver")
-		receiverCancel()
-	}
-
-	j.updateTasks(func(tasks *activeSideTasks) {
-		tasks.state = ActiveSideDone
-	})
-
-}

daemon/job/activesidestate_enumer.go

@@ -1,66 +0,0 @@
-// Code generated by "enumer -type=ActiveSideState"; DO NOT EDIT.
-
-package job
-
-import (
-	"fmt"
-)
-
-const (
-	_ActiveSideStateName_0 = "ActiveSideReplicatingActiveSidePruneSender"
-	_ActiveSideStateName_1 = "ActiveSidePruneReceiver"
-	_ActiveSideStateName_2 = "ActiveSideDone"
-)
-
-var (
-	_ActiveSideStateIndex_0 = [...]uint8{0, 21, 42}
-	_ActiveSideStateIndex_1 = [...]uint8{0, 23}
-	_ActiveSideStateIndex_2 = [...]uint8{0, 14}
-)
-
-func (i ActiveSideState) String() string {
-	switch {
-	case 1 <= i && i <= 2:
-		i -= 1
-		return _ActiveSideStateName_0[_ActiveSideStateIndex_0[i]:_ActiveSideStateIndex_0[i+1]]
-	case i == 4:
-		return _ActiveSideStateName_1
-	case i == 8:
-		return _ActiveSideStateName_2
-	default:
-		return fmt.Sprintf("ActiveSideState(%d)", i)
-	}
-}
-
-var _ActiveSideStateValues = []ActiveSideState{1, 2, 4, 8}
-
-var _ActiveSideStateNameToValueMap = map[string]ActiveSideState{
-	_ActiveSideStateName_0[0:21]:  1,
-	_ActiveSideStateName_0[21:42]: 2,
-	_ActiveSideStateName_1[0:23]:  4,
-	_ActiveSideStateName_2[0:14]:  8,
-}
-
-// ActiveSideStateString retrieves an enum value from the enum constants string name.
-// Throws an error if the param is not part of the enum.
-func ActiveSideStateString(s string) (ActiveSideState, error) {
-	if val, ok := _ActiveSideStateNameToValueMap[s]; ok {
-		return val, nil
-	}
-	return 0, fmt.Errorf("%s does not belong to ActiveSideState values", s)
-}
-
-// ActiveSideStateValues returns all values of the enum
-func ActiveSideStateValues() []ActiveSideState {
-	return _ActiveSideStateValues
-}
-
-// IsAActiveSideState returns "true" if the value is listed in the enum definition. "false" otherwise
-func (i ActiveSideState) IsAActiveSideState() bool {
-	for _, v := range _ActiveSideStateValues {
-		if i == v {
-			return true
-		}
-	}
-	return false
-}

daemon/job/build_jobs.go

@@ -1,120 +0,0 @@
-package job
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/config"
-)
-
-func JobsFromConfig(c *config.Config) ([]Job, error) {
-	js := make([]Job, len(c.Jobs))
-	for i := range c.Jobs {
-		j, err := buildJob(c.Global, c.Jobs[i])
-		if err != nil {
-			return nil, err
-		}
-		if j == nil || j.Name() == "" {
-			panic(fmt.Sprintf("implementation error: job builder returned nil job type %T", c.Jobs[i].Ret))
-		}
-		js[i] = j
-	}
-
-	// receiving-side root filesystems must not overlap
-	{
-		rfss := make([]string, 0, len(js))
-		for _, j := range js {
-			jrfs, ok := j.OwnedDatasetSubtreeRoot()
-			if !ok {
-				continue
-			}
-			rfss = append(rfss, jrfs.ToString())
-		}
-		if err := validateReceivingSidesDoNotOverlap(rfss); err != nil {
-			return nil, err
-		}
-	}
-
-	return js, nil
-}
-
-func buildJob(c *config.Global, in config.JobEnum) (j Job, err error) {
-	cannotBuildJob := func(e error, name string) (Job, error) {
-		return nil, errors.Wrapf(e, "cannot build job %q", name)
-	}
-	// FIXME prettify this
-	switch v := in.Ret.(type) {
-	case *config.SinkJob:
-		m, err := modeSinkFromConfig(c, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-		j, err = passiveSideFromConfig(c, &v.PassiveJob, m)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-	case *config.SourceJob:
-		m, err := modeSourceFromConfig(c, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-		j, err = passiveSideFromConfig(c, &v.PassiveJob, m)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-	case *config.SnapJob:
-		j, err = snapJobFromConfig(c, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-	case *config.PushJob:
-		m, err := modePushFromConfig(c, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-		j, err = activeSide(c, &v.ActiveJob, m)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-	case *config.PullJob:
-		m, err := modePullFromConfig(c, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-		j, err = activeSide(c, &v.ActiveJob, m)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-	default:
-		panic(fmt.Sprintf("implementation error: unknown job type %T", v))
-	}
-	return j, nil
-
-}
-
-func validateReceivingSidesDoNotOverlap(receivingRootFSs []string) error {
-	if len(receivingRootFSs) == 0 {
-		return nil
-	}
-	rfss := make([]string, len(receivingRootFSs))
-	copy(rfss, receivingRootFSs)
-	sort.Slice(rfss, func(i, j int) bool {
-		return strings.Compare(rfss[i], rfss[j]) == -1
-	})
-	// idea:
-	//   no path in rfss must be prefix of another
-	//
-	// rfss is now lexicographically sorted, which means that
-	// if i is prefix of j, i < j (in lexicographical order)
-	// thus,
-	// if any i is prefix of i+n (n >= 1), there is overlap
-	for i := 0; i < len(rfss)-1; i++ {
-		if strings.HasPrefix(rfss[i+1], rfss[i]) {
-			return fmt.Errorf("receiving jobs with overlapping root filesystems are forbidden")
-		}
-	}
-	return nil
-}

daemon/job/build_jobs_test.go

@@ -1,42 +0,0 @@
-package job
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
-	type testCase struct {
-		err   bool
-		input []string
-	}
-	tcs := []testCase{
-		{false, nil},
-		{false, []string{}},
-		{false, []string{""}}, // not our job to determine valid paths
-		{false, []string{"a"}},
-		{false, []string{"some/path"}},
-		{false, []string{"zroot/sink1", "zroot/sink2", "zroot/sink3"}},
-		{true, []string{"zroot/b", "zroot/b"}},
-		{true, []string{"zroot/foo", "zroot/foo/bar", "zroot/baz"}},
-		{false, []string{"a/x", "b/x"}},
-		{false, []string{"a", "b"}},
-		{true, []string{"a", "a"}},
-		{true, []string{"a/x/y", "a/x"}},
-		{true, []string{"a/x", "a/x/y"}},
-		{true, []string{"a/x", "b/x", "a/x/y"}},
-		{true, []string{"a", "a/b", "a/c", "a/b"}},
-		{true, []string{"a/b", "a/c", "a/b", "a/d", "a/c"}},
-	}
-
-	for _, tc := range tcs {
-		t.Logf("input: %v", tc.input)
-		err := validateReceivingSidesDoNotOverlap(tc.input)
-		if tc.err {
-			assert.Error(t, err)
-		} else {
-			assert.NoError(t, err)
-		}
-	}
-}

daemon/job/job.go

@@ -1,118 +0,0 @@
-package job
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type Logger = logger.Logger
-
-type contextKey int
-
-const (
-	contextKeyLog contextKey = iota
-)
-
-func GetLogger(ctx context.Context) Logger {
-	if l, ok := ctx.Value(contextKeyLog).(Logger); ok {
-		return l
-	}
-	return logger.NewNullLogger()
-}
-
-func WithLogger(ctx context.Context, l Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLog, l)
-}
-
-type Job interface {
-	Name() string
-	Run(ctx context.Context)
-	Status() *Status
-	RegisterMetrics(registerer prometheus.Registerer)
-	// Jobs that return a subtree of the dataset hierarchy
-	// must return the root of that subtree as rfs and ok = true
-	OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool)
-}
-
-type Type string
-
-const (
-	TypeInternal Type = "internal"
-	TypeSnap     Type = "snap"
-	TypePush     Type = "push"
-	TypeSink     Type = "sink"
-	TypePull     Type = "pull"
-	TypeSource   Type = "source"
-)
-
-type Status struct {
-	Type        Type
-	JobSpecific interface{}
-}
-
-func (s *Status) MarshalJSON() ([]byte, error) {
-	typeJson, err := json.Marshal(s.Type)
-	if err != nil {
-		return nil, err
-	}
-	jobJSON, err := json.Marshal(s.JobSpecific)
-	if err != nil {
-		return nil, err
-	}
-	m := map[string]json.RawMessage{
-		"type":         typeJson,
-		string(s.Type): jobJSON,
-	}
-	return json.Marshal(m)
-}
-
-func (s *Status) UnmarshalJSON(in []byte) (err error) {
-	var m map[string]json.RawMessage
-	if err := json.Unmarshal(in, &m); err != nil {
-		return err
-	}
-	tJSON, ok := m["type"]
-	if !ok {
-		return fmt.Errorf("field 'type' not found")
-	}
-	if err := json.Unmarshal(tJSON, &s.Type); err != nil {
-		return err
-	}
-	key := string(s.Type)
-	jobJSON, ok := m[key]
-	if !ok {
-		return fmt.Errorf("field '%s', not found", key)
-	}
-	switch s.Type {
-	case TypeSnap:
-		var st SnapJobStatus
-		err = json.Unmarshal(jobJSON, &st)
-		s.JobSpecific = &st
-
-	case TypePull:
-		fallthrough
-	case TypePush:
-		var st ActiveSideStatus
-		err = json.Unmarshal(jobJSON, &st)
-		s.JobSpecific = &st
-
-	case TypeSource:
-		fallthrough
-	case TypeSink:
-		var st PassiveStatus
-		err = json.Unmarshal(jobJSON, &st)
-		s.JobSpecific = &st
-
-	case TypeInternal:
-		// internal jobs do not report specifics
-	default:
-		err = fmt.Errorf("unknown job type '%s'", key)
-	}
-	return err
-}

daemon/job/passive.go

@@ -1,147 +0,0 @@
-package job
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/daemon/logging"
-	"github.com/zrepl/zrepl/daemon/snapper"
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/transport"
-	"github.com/zrepl/zrepl/transport/fromconfig"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type PassiveSide struct {
-	mode   passiveMode
-	name   string
-	listen transport.AuthenticatedListenerFactory
-}
-
-type passiveMode interface {
-	Handler() rpc.Handler
-	RunPeriodic(ctx context.Context)
-	Type() Type
-}
-
-type modeSink struct {
-	rootDataset *zfs.DatasetPath
-}
-
-func (m *modeSink) Type() Type { return TypeSink }
-
-func (m *modeSink) Handler() rpc.Handler {
-	return endpoint.NewReceiver(m.rootDataset, true)
-}
-
-func (m *modeSink) RunPeriodic(_ context.Context) {}
-
-func modeSinkFromConfig(g *config.Global, in *config.SinkJob) (m *modeSink, err error) {
-	m = &modeSink{}
-	m.rootDataset, err = zfs.NewDatasetPath(in.RootFS)
-	if err != nil {
-		return nil, errors.New("root dataset is not a valid zfs filesystem path")
-	}
-	if m.rootDataset.Length() <= 0 {
-		return nil, errors.New("root dataset must not be empty") // duplicates error check of receiver
-	}
-	return m, nil
-}
-
-type modeSource struct {
-	fsfilter zfs.DatasetFilter
-	snapper  *snapper.PeriodicOrManual
-}
-
-func modeSourceFromConfig(g *config.Global, in *config.SourceJob) (m *modeSource, err error) {
-	// FIXME exact dedup of modePush
-	m = &modeSource{}
-	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannnot build filesystem filter")
-	}
-	m.fsfilter = fsf
-
-	if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
-		return nil, errors.Wrap(err, "cannot build snapper")
-	}
-
-	return m, nil
-}
-
-func (m *modeSource) Type() Type { return TypeSource }
-
-func (m *modeSource) Handler() rpc.Handler {
-	return endpoint.NewSender(m.fsfilter)
-}
-
-func (m *modeSource) RunPeriodic(ctx context.Context) {
-	m.snapper.Run(ctx, nil)
-}
-
-func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, mode passiveMode) (s *PassiveSide, err error) {
-
-	s = &PassiveSide{mode: mode, name: in.Name}
-	if s.listen, err = fromconfig.ListenerFactoryFromConfig(g, in.Serve); err != nil {
-		return nil, errors.Wrap(err, "cannot build listener factory")
-	}
-
-	return s, nil
-}
-
-func (j *PassiveSide) Name() string { return j.name }
-
-type PassiveStatus struct{}
-
-func (s *PassiveSide) Status() *Status {
-	return &Status{Type: s.mode.Type()} // FIXME PassiveStatus
-}
-
-func (j *PassiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
-	sink, ok := j.mode.(*modeSink)
-	if !ok {
-		_ = j.mode.(*modeSource) // make sure we didn't introduce a new job type
-		return nil, false
-	}
-	return sink.rootDataset.Copy(), true
-}
-
-func (*PassiveSide) RegisterMetrics(registerer prometheus.Registerer) {}
-
-func (j *PassiveSide) Run(ctx context.Context) {
-
-	log := GetLogger(ctx)
-	defer log.Info("job exiting")
-	ctx = logging.WithSubsystemLoggers(ctx, log)
-	{
-		ctx, cancel := context.WithCancel(ctx) // shadowing
-		defer cancel()
-		go j.mode.RunPeriodic(ctx)
-	}
-
-	handler := j.mode.Handler()
-	if handler == nil {
-		panic(fmt.Sprintf("implementation error: j.mode.Handler() returned nil: %#v", j))
-	}
-
-	ctxInterceptor := func(handlerCtx context.Context) context.Context {
-		return logging.WithSubsystemLoggers(handlerCtx, log)
-	}
-
-	rpcLoggers := rpc.GetLoggersOrPanic(ctx) // WithSubsystemLoggers above
-	server := rpc.NewServer(handler, rpcLoggers, ctxInterceptor)
-
-	listener, err := j.listen()
-	if err != nil {
-		log.WithError(err).Error("cannot listen")
-		return
-	}
-
-	server.Serve(ctx, listener)
-}

daemon/job/reset/reset.go

@@ -1,35 +0,0 @@
-package reset
-
-import (
-	"context"
-	"errors"
-)
-
-type contextKey int
-
-const contextKeyReset contextKey = iota
-
-func Wait(ctx context.Context) <-chan struct{} {
-	wc, ok := ctx.Value(contextKeyReset).(chan struct{})
-	if !ok {
-		wc = make(chan struct{})
-	}
-	return wc
-}
-
-type Func func() error
-
-var AlreadyReset = errors.New("already reset")
-
-func Context(ctx context.Context) (context.Context, Func) {
-	wc := make(chan struct{})
-	wuf := func() error {
-		select {
-		case wc <- struct{}{}:
-			return nil
-		default:
-			return AlreadyReset
-		}
-	}
-	return context.WithValue(ctx, contextKeyReset, wc), wuf
-}

daemon/job/snapjob.go

@@ -1,168 +0,0 @@
-package job
-
-import (
-	"context"
-	"fmt"
-	"sort"
-
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/daemon/job/wakeup"
-	"github.com/zrepl/zrepl/daemon/logging"
-	"github.com/zrepl/zrepl/daemon/pruner"
-	"github.com/zrepl/zrepl/daemon/snapper"
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/replication/logic/pdu"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type SnapJob struct {
-	name     string
-	fsfilter zfs.DatasetFilter
-	snapper  *snapper.PeriodicOrManual
-
-	prunerFactory *pruner.LocalPrunerFactory
-
-	promPruneSecs *prometheus.HistogramVec // labels: prune_side
-
-	pruner *pruner.Pruner
-}
-
-func (j *SnapJob) Name() string { return j.name }
-
-func (j *SnapJob) Type() Type { return TypeSnap }
-
-func snapJobFromConfig(g *config.Global, in *config.SnapJob) (j *SnapJob, err error) {
-	j = &SnapJob{}
-	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannnot build filesystem filter")
-	}
-	j.fsfilter = fsf
-
-	if j.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
-		return nil, errors.Wrap(err, "cannot build snapper")
-	}
-	j.name = in.Name
-	j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace:   "zrepl",
-		Subsystem:   "pruning",
-		Name:        "time",
-		Help:        "seconds spent in pruner",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
-	}, []string{"prune_side"})
-	j.prunerFactory, err = pruner.NewLocalPrunerFactory(in.Pruning, j.promPruneSecs)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot build snapjob pruning rules")
-	}
-	return j, nil
-}
-
-func (j *SnapJob) RegisterMetrics(registerer prometheus.Registerer) {
-	registerer.MustRegister(j.promPruneSecs)
-}
-
-type SnapJobStatus struct {
-	Pruning *pruner.Report
-}
-
-func (j *SnapJob) Status() *Status {
-	s := &SnapJobStatus{}
-	t := j.Type()
-	if j.pruner != nil {
-		s.Pruning = j.pruner.Report()
-	}
-	return &Status{Type: t, JobSpecific: s}
-}
-
-func (j *SnapJob) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
-	return nil, false
-}
-
-func (j *SnapJob) Run(ctx context.Context) {
-	log := GetLogger(ctx)
-	ctx = logging.WithSubsystemLoggers(ctx, log)
-
-	defer log.Info("job exiting")
-
-	periodicDone := make(chan struct{})
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-	go j.snapper.Run(ctx, periodicDone)
-
-	invocationCount := 0
-outer:
-	for {
-		log.Info("wait for wakeups")
-		select {
-		case <-ctx.Done():
-			log.WithError(ctx.Err()).Info("context")
-			break outer
-
-		case <-wakeup.Wait(ctx):
-		case <-periodicDone:
-		}
-		invocationCount++
-		invLog := log.WithField("invocation", invocationCount)
-		j.doPrune(WithLogger(ctx, invLog))
-	}
-}
-
-// Adaptor that implements pruner.History around a pruner.Target.
-// The ReplicationCursor method is Get-op only and always returns
-// the filesystem's most recent version's GUID.
-//
-// TODO:
-// This is a work-around for the current package daemon/pruner
-// and package pruning.Snapshot limitation: they require the
-//  `Replicated` getter method be present, but obviously,
-// a local job like SnapJob can't deliver on that.
-// But the pruner.Pruner gives up on an FS if no replication
-// cursor is present, which is why this pruner returns the
-// most recent filesystem version.
-type alwaysUpToDateReplicationCursorHistory struct {
-	// the Target passed as Target to BuildLocalPruner
-	target pruner.Target
-}
-
-var _ pruner.History = (*alwaysUpToDateReplicationCursorHistory)(nil)
-
-func (h alwaysUpToDateReplicationCursorHistory) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
-	if req.GetGet() == nil {
-		return nil, fmt.Errorf("unsupported ReplicationCursor request: SnapJob only supports GETting a (faked) cursor")
-	}
-	fsvReq := &pdu.ListFilesystemVersionsReq{
-		Filesystem: req.GetFilesystem(),
-	}
-	res, err := h.target.ListFilesystemVersions(ctx, fsvReq)
-	if err != nil {
-		return nil, err
-	}
-	fsvs := res.GetVersions()
-	if len(fsvs) <= 0 {
-		return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Notexist{Notexist: true}}, nil
-	}
-	// always return must recent version
-	sort.Slice(fsvs, func(i, j int) bool {
-		return fsvs[i].CreateTXG < fsvs[j].CreateTXG
-	})
-	mostRecent := fsvs[len(fsvs)-1]
-	return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Guid{Guid: mostRecent.GetGuid()}}, nil
-}
-
-func (h alwaysUpToDateReplicationCursorHistory) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error) {
-	return h.target.ListFilesystems(ctx, req)
-}
-
-func (j *SnapJob) doPrune(ctx context.Context) {
-	log := GetLogger(ctx)
-	ctx = logging.WithSubsystemLoggers(ctx, log)
-	sender := endpoint.NewSender(j.fsfilter)
-	j.pruner = j.prunerFactory.BuildLocalPruner(ctx, sender, alwaysUpToDateReplicationCursorHistory{sender})
-	log.Info("start pruning")
-	j.pruner.Prune()
-	log.Info("finished pruning")
-}

daemon/job/wakeup/wakeup.go

@@ -1,35 +0,0 @@
-package wakeup
-
-import (
-	"context"
-	"errors"
-)
-
-type contextKey int
-
-const contextKeyWakeup contextKey = iota
-
-func Wait(ctx context.Context) <-chan struct{} {
-	wc, ok := ctx.Value(contextKeyWakeup).(chan struct{})
-	if !ok {
-		wc = make(chan struct{})
-	}
-	return wc
-}
-
-type Func func() error
-
-var AlreadyWokenUp = errors.New("already woken up")
-
-func Context(ctx context.Context) (context.Context, Func) {
-	wc := make(chan struct{})
-	wuf := func() error {
-		select {
-		case wc <- struct{}{}:
-			return nil
-		default:
-			return AlreadyWokenUp
-		}
-	}
-	return context.WithValue(ctx, contextKeyWakeup, wc), wuf
-}

daemon/logging/build_logging.go

@@ -1,231 +0,0 @@
-package logging
-
-import (
-	"context"
-	"crypto/tls"
-	"crypto/x509"
-	"log/syslog"
-	"os"
-
-	"github.com/mattn/go-isatty"
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/pruner"
-	"github.com/zrepl/zrepl/daemon/snapper"
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/replication/driver"
-	"github.com/zrepl/zrepl/replication/logic"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/rpc/transportmux"
-	"github.com/zrepl/zrepl/tlsconf"
-	"github.com/zrepl/zrepl/transport"
-)
-
-func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error) {
-
-	outlets := logger.NewOutlets()
-
-	if len(in) == 0 {
-		// Default config
-		out := WriterOutlet{&HumanFormatter{}, os.Stdout}
-		outlets.Add(out, logger.Warn)
-		return outlets, nil
-	}
-
-	var syslogOutlets, stdoutOutlets int
-	for lei, le := range in {
-
-		outlet, minLevel, err := parseOutlet(le)
-		if err != nil {
-			return nil, errors.Wrapf(err, "cannot parse outlet #%d", lei)
-		}
-		var _ logger.Outlet = WriterOutlet{}
-		var _ logger.Outlet = &SyslogOutlet{}
-		switch outlet.(type) {
-		case *SyslogOutlet:
-			syslogOutlets++
-		case WriterOutlet:
-			stdoutOutlets++
-		}
-
-		outlets.Add(outlet, minLevel)
-
-	}
-
-	if syslogOutlets > 1 {
-		return nil, errors.Errorf("can only define one 'syslog' outlet")
-	}
-	if stdoutOutlets > 1 {
-		return nil, errors.Errorf("can only define one 'stdout' outlet")
-	}
-
-	return outlets, nil
-
-}
-
-type Subsystem string
-
-const (
-	SubsysReplication  Subsystem = "repl"
-	SubsyEndpoint      Subsystem = "endpoint"
-	SubsysPruning      Subsystem = "pruning"
-	SubsysSnapshot     Subsystem = "snapshot"
-	SubsysTransport    Subsystem = "transport"
-	SubsysTransportMux Subsystem = "transportmux"
-	SubsysRPC          Subsystem = "rpc"
-	SubsysRPCControl   Subsystem = "rpc.ctrl"
-	SubsysRPCData      Subsystem = "rpc.data"
-)
-
-func WithSubsystemLoggers(ctx context.Context, log logger.Logger) context.Context {
-	ctx = logic.WithLogger(ctx, log.WithField(SubsysField, SubsysReplication))
-	ctx = driver.WithLogger(ctx, log.WithField(SubsysField, SubsysReplication))
-	ctx = endpoint.WithLogger(ctx, log.WithField(SubsysField, SubsyEndpoint))
-	ctx = pruner.WithLogger(ctx, log.WithField(SubsysField, SubsysPruning))
-	ctx = snapper.WithLogger(ctx, log.WithField(SubsysField, SubsysSnapshot))
-	ctx = transport.WithLogger(ctx, log.WithField(SubsysField, SubsysTransport))
-	ctx = transportmux.WithLogger(ctx, log.WithField(SubsysField, SubsysTransportMux))
-	ctx = rpc.WithLoggers(ctx,
-		rpc.Loggers{
-			General: log.WithField(SubsysField, SubsysRPC),
-			Control: log.WithField(SubsysField, SubsysRPCControl),
-			Data:    log.WithField(SubsysField, SubsysRPCData),
-		},
-	)
-	return ctx
-}
-
-func LogSubsystem(log logger.Logger, subsys Subsystem) logger.Logger {
-	return log.ReplaceField(SubsysField, subsys)
-}
-
-func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
-	var is string
-	switch j := i.(type) {
-	case string:
-		is = j
-	default:
-		return nil, errors.Errorf("invalid log format: wrong type: %T", i)
-	}
-
-	switch is {
-	case "human":
-		return &HumanFormatter{}, nil
-	case "logfmt":
-		return &LogfmtFormatter{}, nil
-	case "json":
-		return &JSONFormatter{}, nil
-	default:
-		return nil, errors.Errorf("invalid log format: '%s'", is)
-	}
-
-}
-
-func parseOutlet(in config.LoggingOutletEnum) (o logger.Outlet, level logger.Level, err error) {
-
-	parseCommon := func(common config.LoggingOutletCommon) (logger.Level, EntryFormatter, error) {
-		if common.Level == "" || common.Format == "" {
-			return 0, nil, errors.Errorf("must specify 'level' and 'format' field")
-		}
-
-		minLevel, err := logger.ParseLevel(common.Level)
-		if err != nil {
-			return 0, nil, errors.Wrap(err, "cannot parse 'level' field")
-		}
-		formatter, err := parseLogFormat(common.Format)
-		if err != nil {
-			return 0, nil, errors.Wrap(err, "cannot parse 'formatter' field")
-		}
-		return minLevel, formatter, nil
-	}
-
-	var f EntryFormatter
-
-	switch v := in.Ret.(type) {
-	case *config.StdoutLoggingOutlet:
-		level, f, err = parseCommon(v.LoggingOutletCommon)
-		if err != nil {
-			break
-		}
-		o, err = parseStdoutOutlet(v, f)
-	case *config.TCPLoggingOutlet:
-		level, f, err = parseCommon(v.LoggingOutletCommon)
-		if err != nil {
-			break
-		}
-		o, err = parseTCPOutlet(v, f)
-	case *config.SyslogLoggingOutlet:
-		level, f, err = parseCommon(v.LoggingOutletCommon)
-		if err != nil {
-			break
-		}
-		o, err = parseSyslogOutlet(v, f)
-	default:
-		panic(v)
-	}
-	return o, level, err
-}
-
-func parseStdoutOutlet(in *config.StdoutLoggingOutlet, formatter EntryFormatter) (WriterOutlet, error) {
-	flags := MetadataAll
-	writer := os.Stdout
-	if !isatty.IsTerminal(writer.Fd()) && !in.Time {
-		flags &= ^MetadataTime
-	}
-	if isatty.IsTerminal(writer.Fd()) && !in.Color {
-		flags &= ^MetadataColor
-	}
-
-	formatter.SetMetadataFlags(flags)
-	return WriterOutlet{
-		formatter,
-		os.Stdout,
-	}, nil
-}
-
-func parseTCPOutlet(in *config.TCPLoggingOutlet, formatter EntryFormatter) (out *TCPOutlet, err error) {
-	var tlsConfig *tls.Config
-	if in.TLS != nil {
-		tlsConfig, err = func(m *config.TCPLoggingOutletTLS, host string) (*tls.Config, error) {
-			clientCert, err := tls.LoadX509KeyPair(m.Cert, m.Key)
-			if err != nil {
-				return nil, errors.Wrap(err, "cannot load client cert")
-			}
-
-			var rootCAs *x509.CertPool
-			if m.CA == "" {
-				if rootCAs, err = x509.SystemCertPool(); err != nil {
-					return nil, errors.Wrap(err, "cannot open system cert pool")
-				}
-			} else {
-				rootCAs, err = tlsconf.ParseCAFile(m.CA)
-				if err != nil {
-					return nil, errors.Wrap(err, "cannot parse CA cert")
-				}
-			}
-			if rootCAs == nil {
-				panic("invariant violated")
-			}
-
-			return tlsconf.ClientAuthClient(host, rootCAs, clientCert)
-		}(in.TLS, in.Address)
-		if err != nil {
-			return nil, errors.New("cannot not parse TLS config in field 'tls'")
-		}
-	}
-
-	formatter.SetMetadataFlags(MetadataAll)
-	return NewTCPOutlet(formatter, in.Net, in.Address, tlsConfig, in.RetryInterval), nil
-
-}
-
-func parseSyslogOutlet(in *config.SyslogLoggingOutlet, formatter EntryFormatter) (out *SyslogOutlet, err error) {
-	out = &SyslogOutlet{}
-	out.Formatter = formatter
-	out.Formatter.SetMetadataFlags(MetadataNone)
-	out.Facility = syslog.Priority(*in.Facility)
-	out.RetryInterval = in.RetryInterval
-	return out, nil
-}

daemon/main.go

@@ -1,16 +0,0 @@
-package daemon
-
-import (
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/logger"
-)
-
-type Logger = logger.Logger
-
-var DaemonCmd = &cli.Subcommand{
-	Use:   "daemon",
-	Short: "run the zrepl daemon",
-	Run: func(subcommand *cli.Subcommand, args []string) error {
-		return Run(subcommand.Config())
-	},
-}

daemon/prometheus.go

@@ -1,95 +0,0 @@
-package daemon
-
-import (
-	"context"
-	"net"
-	"net/http"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/rpc/dataconn/frameconn"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type prometheusJob struct {
-	listen string
-}
-
-func newPrometheusJobFromConfig(in *config.PrometheusMonitoring) (*prometheusJob, error) {
-	if _, _, err := net.SplitHostPort(in.Listen); err != nil {
-		return nil, err
-	}
-	return &prometheusJob{in.Listen}, nil
-}
-
-var prom struct {
-	taskLogEntries *prometheus.CounterVec
-}
-
-func init() {
-	prom.taskLogEntries = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "zrepl",
-		Subsystem: "daemon",
-		Name:      "log_entries",
-		Help:      "number of log entries per job task and level",
-	}, []string{"zrepl_job", "level"})
-	prometheus.MustRegister(prom.taskLogEntries)
-}
-
-func (j *prometheusJob) Name() string { return jobNamePrometheus }
-
-func (j *prometheusJob) Status() *job.Status { return &job.Status{Type: job.TypeInternal} }
-
-func (j *prometheusJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
-
-func (j *prometheusJob) RegisterMetrics(registerer prometheus.Registerer) {}
-
-func (j *prometheusJob) Run(ctx context.Context) {
-
-	if err := zfs.PrometheusRegister(prometheus.DefaultRegisterer); err != nil {
-		panic(err)
-	}
-
-	if err := frameconn.PrometheusRegister(prometheus.DefaultRegisterer); err != nil {
-		panic(err)
-	}
-
-	log := job.GetLogger(ctx)
-
-	l, err := net.Listen("tcp", j.listen)
-	if err != nil {
-		log.WithError(err).Error("cannot listen")
-	}
-	go func() {
-		<-ctx.Done()
-		l.Close()
-	}()
-
-	mux := http.NewServeMux()
-	mux.Handle("/metrics", promhttp.Handler())
-
-	err = http.Serve(l, mux)
-	if err != nil && ctx.Err() == nil {
-		log.WithError(err).Error("error while serving")
-	}
-
-}
-
-type prometheusJobOutlet struct {
-	jobName string
-}
-
-var _ logger.Outlet = prometheusJobOutlet{}
-
-func newPrometheusLogOutlet(jobName string) prometheusJobOutlet {
-	return prometheusJobOutlet{jobName}
-}
-
-func (o prometheusJobOutlet) WriteEntry(entry logger.Entry) error {
-	prom.taskLogEntries.WithLabelValues(o.jobName, entry.Level.String()).Inc()
-	return nil
-}

daemon/pruner/pruner.go

@@ -1,582 +0,0 @@
-package pruner
-
-import (
-	"context"
-	"fmt"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/pruning"
-	"github.com/zrepl/zrepl/replication/logic/pdu"
-	"github.com/zrepl/zrepl/util/envconst"
-)
-
-// Try to keep it compatible with gitub.com/zrepl/zrepl/endpoint.Endpoint
-type History interface {
-	ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error)
-	ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
-}
-
-// Try to keep it compatible with gitub.com/zrepl/zrepl/endpoint.Endpoint
-type Target interface {
-	ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
-	ListFilesystemVersions(ctx context.Context, req *pdu.ListFilesystemVersionsReq) (*pdu.ListFilesystemVersionsRes, error)
-	DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error)
-}
-
-type Logger = logger.Logger
-
-type contextKey int
-
-const contextKeyLogger contextKey = 0
-
-func WithLogger(ctx context.Context, log Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLogger, log)
-}
-
-func GetLogger(ctx context.Context) Logger {
-	if l, ok := ctx.Value(contextKeyLogger).(Logger); ok {
-		return l
-	}
-	return logger.NewNullLogger()
-}
-
-type args struct {
-	ctx                            context.Context
-	target                         Target
-	receiver                       History
-	rules                          []pruning.KeepRule
-	retryWait                      time.Duration
-	considerSnapAtCursorReplicated bool
-	promPruneSecs                  prometheus.Observer
-}
-
-type Pruner struct {
-	args args
-
-	mtx sync.RWMutex
-
-	state State
-
-	// State PlanErr
-	err error
-
-	// State Exec
-	execQueue *execQueue
-}
-
-type PrunerFactory struct {
-	senderRules                    []pruning.KeepRule
-	receiverRules                  []pruning.KeepRule
-	retryWait                      time.Duration
-	considerSnapAtCursorReplicated bool
-	promPruneSecs                  *prometheus.HistogramVec
-}
-
-type LocalPrunerFactory struct {
-	keepRules     []pruning.KeepRule
-	retryWait     time.Duration
-	promPruneSecs *prometheus.HistogramVec
-}
-
-func NewLocalPrunerFactory(in config.PruningLocal, promPruneSecs *prometheus.HistogramVec) (*LocalPrunerFactory, error) {
-	rules, err := pruning.RulesFromConfig(in.Keep)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot build pruning rules")
-	}
-	for _, r := range in.Keep {
-		if _, ok := r.Ret.(*config.PruneKeepNotReplicated); ok {
-			// rule NotReplicated  for a local pruner doesn't make sense
-			// because no replication happens with that job type
-			return nil, fmt.Errorf("single-site pruner cannot support `not_replicated` keep rule")
-		}
-	}
-	f := &LocalPrunerFactory{
-		keepRules:     rules,
-		retryWait:     envconst.Duration("ZREPL_PRUNER_RETRY_INTERVAL", 10*time.Second),
-		promPruneSecs: promPruneSecs,
-	}
-	return f, nil
-}
-
-func NewPrunerFactory(in config.PruningSenderReceiver, promPruneSecs *prometheus.HistogramVec) (*PrunerFactory, error) {
-	keepRulesReceiver, err := pruning.RulesFromConfig(in.KeepReceiver)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot build receiver pruning rules")
-	}
-
-	keepRulesSender, err := pruning.RulesFromConfig(in.KeepSender)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot build sender pruning rules")
-	}
-
-	considerSnapAtCursorReplicated := false
-	for _, r := range in.KeepSender {
-		knr, ok := r.Ret.(*config.PruneKeepNotReplicated)
-		if !ok {
-			continue
-		}
-		considerSnapAtCursorReplicated = considerSnapAtCursorReplicated || !knr.KeepSnapshotAtCursor
-	}
-	f := &PrunerFactory{
-		senderRules:                    keepRulesSender,
-		receiverRules:                  keepRulesReceiver,
-		retryWait:                      envconst.Duration("ZREPL_PRUNER_RETRY_INTERVAL", 10*time.Second),
-		considerSnapAtCursorReplicated: considerSnapAtCursorReplicated,
-		promPruneSecs:                  promPruneSecs,
-	}
-	return f, nil
-}
-
-func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, receiver History) *Pruner {
-	p := &Pruner{
-		args: args{
-			WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "sender")),
-			target,
-			receiver,
-			f.senderRules,
-			f.retryWait,
-			f.considerSnapAtCursorReplicated,
-			f.promPruneSecs.WithLabelValues("sender"),
-		},
-		state: Plan,
-	}
-	return p
-}
-
-func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target, receiver History) *Pruner {
-	p := &Pruner{
-		args: args{
-			WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "receiver")),
-			target,
-			receiver,
-			f.receiverRules,
-			f.retryWait,
-			false, // senseless here anyways
-			f.promPruneSecs.WithLabelValues("receiver"),
-		},
-		state: Plan,
-	}
-	return p
-}
-
-func (f *LocalPrunerFactory) BuildLocalPruner(ctx context.Context, target Target, receiver History) *Pruner {
-	p := &Pruner{
-		args: args{
-			ctx,
-			target,
-			receiver,
-			f.keepRules,
-			f.retryWait,
-			false, // considerSnapAtCursorReplicated is not relevant for local pruning
-			f.promPruneSecs.WithLabelValues("local"),
-		},
-		state: Plan,
-	}
-	return p
-}
-
-//go:generate enumer -type=State
-type State int
-
-const (
-	Plan State = 1 << iota
-	PlanErr
-	Exec
-	ExecErr
-	Done
-)
-
-type updater func(func(*Pruner))
-
-func (p *Pruner) Prune() {
-	p.prune(p.args)
-}
-
-func (p *Pruner) prune(args args) {
-	u := func(f func(*Pruner)) {
-		p.mtx.Lock()
-		defer p.mtx.Unlock()
-		f(p)
-	}
-	// TODO support automatic retries
-	// It is advisable to merge this code with package replication/driver before
-	// That will likely require re-modelling struct fs like replication/driver.attempt,
-	// including figuring out how to resume a plan after being interrupted by network errors
-	// The non-retrying code in this package should move straight to replication/logic.
-	doOneAttempt(&args, u)
-}
-
-type Report struct {
-	State              string
-	Error              string
-	Pending, Completed []FSReport
-}
-
-type FSReport struct {
-	Filesystem                string
-	SnapshotList, DestroyList []SnapshotReport
-	SkipReason                FSSkipReason
-	LastError                 string
-}
-
-type SnapshotReport struct {
-	Name       string
-	Replicated bool
-	Date       time.Time
-}
-
-func (p *Pruner) Report() *Report {
-	p.mtx.Lock()
-	defer p.mtx.Unlock()
-
-	r := Report{State: p.state.String()}
-
-	if p.err != nil {
-		r.Error = p.err.Error()
-	}
-
-	if p.execQueue != nil {
-		r.Pending, r.Completed = p.execQueue.Report()
-	}
-
-	return &r
-}
-
-func (p *Pruner) State() State {
-	p.mtx.Lock()
-	defer p.mtx.Unlock()
-	return p.state
-}
-
-type fs struct {
-	path string
-
-	// permanent error during planning
-	planErr        error
-	planErrContext string
-
-	// if != "", the fs was skipped for planning and the field
-	// contains the reason
-	skipReason FSSkipReason
-
-	// snapshots presented by target
-	// (type snapshot)
-	snaps []pruning.Snapshot
-	// destroy list returned by pruning.PruneSnapshots(snaps)
-	// (type snapshot)
-	destroyList []pruning.Snapshot
-
-	mtx sync.RWMutex
-
-	// only during Exec state, also used by execQueue
-	execErrLast error
-}
-
-type FSSkipReason string
-
-const (
-	NotSkipped                   = ""
-	SkipPlaceholder              = "filesystem is placeholder"
-	SkipNoCorrespondenceOnSender = "filesystem has no correspondence on sender"
-)
-
-func (r FSSkipReason) NotSkipped() bool {
-	return r == NotSkipped
-}
-
-func (f *fs) Report() FSReport {
-	f.mtx.Lock()
-	defer f.mtx.Unlock()
-
-	r := FSReport{}
-	r.Filesystem = f.path
-	r.SkipReason = f.skipReason
-	if !r.SkipReason.NotSkipped() {
-		return r
-	}
-
-	if f.planErr != nil {
-		r.LastError = f.planErr.Error()
-	} else if f.execErrLast != nil {
-		r.LastError = f.execErrLast.Error()
-	}
-
-	r.SnapshotList = make([]SnapshotReport, len(f.snaps))
-	for i, snap := range f.snaps {
-		r.SnapshotList[i] = snap.(snapshot).Report()
-	}
-
-	r.DestroyList = make([]SnapshotReport, len(f.destroyList))
-	for i, snap := range f.destroyList {
-		r.DestroyList[i] = snap.(snapshot).Report()
-	}
-
-	return r
-}
-
-type snapshot struct {
-	replicated bool
-	date       time.Time
-	fsv        *pdu.FilesystemVersion
-}
-
-func (s snapshot) Report() SnapshotReport {
-	return SnapshotReport{
-		Name:       s.Name(),
-		Replicated: s.Replicated(),
-		Date:       s.Date(),
-	}
-}
-
-var _ pruning.Snapshot = snapshot{}
-
-func (s snapshot) Name() string { return s.fsv.Name }
-
-func (s snapshot) Replicated() bool { return s.replicated }
-
-func (s snapshot) Date() time.Time { return s.date }
-
-func doOneAttempt(a *args, u updater) {
-
-	ctx, target, receiver := a.ctx, a.target, a.receiver
-
-	sfssres, err := receiver.ListFilesystems(ctx, &pdu.ListFilesystemReq{})
-	if err != nil {
-		u(func(p *Pruner) {
-			p.state = PlanErr
-			p.err = err
-		})
-		return
-	}
-	sfss := make(map[string]*pdu.Filesystem)
-	for _, sfs := range sfssres.GetFilesystems() {
-		sfss[sfs.GetPath()] = sfs
-	}
-
-	tfssres, err := target.ListFilesystems(ctx, &pdu.ListFilesystemReq{})
-	if err != nil {
-		u(func(p *Pruner) {
-			p.state = PlanErr
-			p.err = err
-		})
-		return
-	}
-	tfss := tfssres.GetFilesystems()
-
-	pfss := make([]*fs, len(tfss))
-tfss_loop:
-	for i, tfs := range tfss {
-
-		l := GetLogger(ctx).WithField("fs", tfs.Path)
-		l.Debug("plan filesystem")
-
-		pfs := &fs{
-			path: tfs.Path,
-		}
-		pfss[i] = pfs
-
-		if tfs.GetIsPlaceholder() {
-			pfs.skipReason = SkipPlaceholder
-			l.WithField("skip_reason", pfs.skipReason).Debug("skipping filesystem")
-			continue
-		} else if sfs := sfss[tfs.GetPath()]; sfs == nil {
-			pfs.skipReason = SkipNoCorrespondenceOnSender
-			l.WithField("skip_reason", pfs.skipReason).WithField("sfs", sfs.GetPath()).Debug("skipping filesystem")
-			continue
-		}
-
-		pfsPlanErrAndLog := func(err error, message string) {
-			t := fmt.Sprintf("%T", err)
-			pfs.planErr = err
-			pfs.planErrContext = message
-			l.WithField("orig_err_type", t).WithError(err).Error(fmt.Sprintf("%s: plan error, skipping filesystem", message))
-		}
-
-		tfsvsres, err := target.ListFilesystemVersions(ctx, &pdu.ListFilesystemVersionsReq{Filesystem: tfs.Path})
-		if err != nil {
-			pfsPlanErrAndLog(err, "cannot list filesystem versions")
-			continue tfss_loop
-		}
-		tfsvs := tfsvsres.GetVersions()
-		// no progress here since we could run in a live-lock (must have used target AND receiver before progress)
-
-		pfs.snaps = make([]pruning.Snapshot, 0, len(tfsvs))
-
-		rcReq := &pdu.ReplicationCursorReq{
-			Filesystem: tfs.Path,
-			Op: &pdu.ReplicationCursorReq_Get{
-				Get: &pdu.ReplicationCursorReq_GetOp{},
-			},
-		}
-		rc, err := receiver.ReplicationCursor(ctx, rcReq)
-		if err != nil {
-			pfsPlanErrAndLog(err, "cannot get replication cursor bookmark")
-			continue tfss_loop
-		}
-		if rc.GetNotexist() {
-			err := errors.New("replication cursor bookmark does not exist (one successful replication is required before pruning works)")
-			pfsPlanErrAndLog(err, "")
-			continue tfss_loop
-		}
-
-		// scan from older to newer, all snapshots older than cursor are interpreted as replicated
-		sort.Slice(tfsvs, func(i, j int) bool {
-			return tfsvs[i].CreateTXG < tfsvs[j].CreateTXG
-		})
-
-		haveCursorSnapshot := false
-		for _, tfsv := range tfsvs {
-			if tfsv.Type != pdu.FilesystemVersion_Snapshot {
-				continue
-			}
-			if tfsv.Guid == rc.GetGuid() {
-				haveCursorSnapshot = true
-			}
-		}
-		preCursor := haveCursorSnapshot
-		for _, tfsv := range tfsvs {
-			if tfsv.Type != pdu.FilesystemVersion_Snapshot {
-				continue
-			}
-			creation, err := tfsv.CreationAsTime()
-			if err != nil {
-				err := fmt.Errorf("%s: %s", tfsv.RelName(), err)
-				pfsPlanErrAndLog(err, "fs version with invalid creation date")
-				continue tfss_loop
-			}
-			// note that we cannot use CreateTXG because target and receiver could be on different pools
-			atCursor := tfsv.Guid == rc.GetGuid()
-			preCursor = preCursor && !atCursor
-			pfs.snaps = append(pfs.snaps, snapshot{
-				replicated: preCursor || (a.considerSnapAtCursorReplicated && atCursor),
-				date:       creation,
-				fsv:        tfsv,
-			})
-		}
-		if preCursor {
-			pfsPlanErrAndLog(fmt.Errorf("replication cursor not found in prune target filesystem versions"), "")
-			continue tfss_loop
-		}
-
-		// Apply prune rules
-		pfs.destroyList = pruning.PruneSnapshots(pfs.snaps, a.rules)
-	}
-
-	u(func(pruner *Pruner) {
-		pruner.execQueue = newExecQueue(len(pfss))
-		for _, pfs := range pfss {
-			pruner.execQueue.Put(pfs, nil, false)
-		}
-		pruner.state = Exec
-	})
-
-	for {
-		var pfs *fs
-		u(func(pruner *Pruner) {
-			pfs = pruner.execQueue.Pop()
-		})
-		if pfs == nil {
-			break
-		}
-		doOneAttemptExec(a, u, pfs)
-	}
-
-	var rep *Report
-	{
-		// must not hold lock for report
-		var pruner *Pruner
-		u(func(p *Pruner) {
-			pruner = p
-		})
-		rep = pruner.Report()
-	}
-	u(func(p *Pruner) {
-		if len(rep.Pending) > 0 {
-			panic("queue should not have pending items at this point")
-		}
-		hadErr := false
-		for _, fsr := range rep.Completed {
-			hadErr = hadErr || fsr.SkipReason.NotSkipped() && fsr.LastError != ""
-		}
-		if hadErr {
-			p.state = ExecErr
-		} else {
-			p.state = Done
-		}
-	})
-
-}
-
-// attempts to exec pfs, puts it back into the queue with the result
-func doOneAttemptExec(a *args, u updater, pfs *fs) {
-
-	destroyList := make([]*pdu.FilesystemVersion, len(pfs.destroyList))
-	for i := range destroyList {
-		destroyList[i] = pfs.destroyList[i].(snapshot).fsv
-		GetLogger(a.ctx).
-			WithField("fs", pfs.path).
-			WithField("destroy_snap", destroyList[i].Name).
-			Debug("policy destroys snapshot")
-	}
-	req := pdu.DestroySnapshotsReq{
-		Filesystem: pfs.path,
-		Snapshots:  destroyList,
-	}
-	GetLogger(a.ctx).WithField("fs", pfs.path).Debug("destroying snapshots")
-	res, err := a.target.DestroySnapshots(a.ctx, &req)
-	if err != nil {
-		u(func(pruner *Pruner) {
-			pruner.execQueue.Put(pfs, err, false)
-		})
-		return
-	}
-	// check if all snapshots were destroyed
-	destroyResults := make(map[string]*pdu.DestroySnapshotRes)
-	for _, fsres := range res.Results {
-		destroyResults[fsres.Snapshot.Name] = fsres
-	}
-	err = nil
-	destroyFails := make([]*pdu.DestroySnapshotRes, 0)
-	for _, reqDestroy := range destroyList {
-		res, ok := destroyResults[reqDestroy.Name]
-		if !ok {
-			err = fmt.Errorf("missing destroy-result for %s", reqDestroy.RelName())
-			break
-		} else if res.Error != "" {
-			destroyFails = append(destroyFails, res)
-		}
-	}
-	if err == nil && len(destroyFails) > 0 {
-		names := make([]string, len(destroyFails))
-		pairs := make([]string, len(destroyFails))
-		allSame := true
-		lastMsg := destroyFails[0].Error
-		for i := 0; i < len(destroyFails); i++ {
-			allSame = allSame && destroyFails[i].Error == lastMsg
-			relname := destroyFails[i].Snapshot.RelName()
-			names[i] = relname
-			pairs[i] = fmt.Sprintf("(%s: %s)", relname, destroyFails[i].Error)
-		}
-		if allSame {
-			err = fmt.Errorf("destroys failed %s: %s",
-				strings.Join(names, ", "), lastMsg)
-		} else {
-			err = fmt.Errorf("destroys failed: %s", strings.Join(pairs, ", "))
-		}
-	}
-	u(func(pruner *Pruner) {
-		pruner.execQueue.Put(pfs, err, err == nil)
-	})
-	if err != nil {
-		GetLogger(a.ctx).WithError(err).Error("target could not destroy snapshots")
-		return
-	}
-}

daemon/pruner/pruner_queue.go

@@ -1,82 +0,0 @@
-package pruner
-
-import (
-	"sort"
-	"strings"
-	"sync"
-)
-
-type execQueue struct {
-	mtx                sync.Mutex
-	pending, completed []*fs
-}
-
-func newExecQueue(cap int) *execQueue {
-	q := execQueue{
-		pending:   make([]*fs, 0, cap),
-		completed: make([]*fs, 0, cap),
-	}
-	return &q
-}
-
-func (q *execQueue) Report() (pending, completed []FSReport) {
-	q.mtx.Lock()
-	defer q.mtx.Unlock()
-
-	pending = make([]FSReport, len(q.pending))
-	for i, fs := range q.pending {
-		pending[i] = fs.Report()
-	}
-	completed = make([]FSReport, len(q.completed))
-	for i, fs := range q.completed {
-		completed[i] = fs.Report()
-	}
-
-	return pending, completed
-}
-
-func (q *execQueue) HasCompletedFSWithErrors() bool {
-	q.mtx.Lock()
-	defer q.mtx.Unlock()
-	for _, fs := range q.completed {
-		if fs.execErrLast != nil {
-			return true
-		}
-	}
-	return false
-}
-
-func (q *execQueue) Pop() *fs {
-	if len(q.pending) == 0 {
-		return nil
-	}
-	fs := q.pending[0]
-	q.pending = q.pending[1:]
-	return fs
-}
-
-func (q *execQueue) Put(fs *fs, err error, done bool) {
-	fs.mtx.Lock()
-	fs.execErrLast = err
-	if done || err != nil {
-		fs.mtx.Unlock()
-		q.mtx.Lock()
-		q.completed = append(q.completed, fs)
-		q.mtx.Unlock()
-		return
-	}
-	fs.mtx.Unlock()
-
-	q.mtx.Lock()
-	// inefficient priority q
-	q.pending = append(q.pending, fs)
-	sort.SliceStable(q.pending, func(i, j int) bool {
-		q.pending[i].mtx.Lock()
-		defer q.pending[i].mtx.Unlock()
-		q.pending[j].mtx.Lock()
-		defer q.pending[j].mtx.Unlock()
-		return strings.Compare(q.pending[i].path, q.pending[j].path) == -1
-	})
-	q.mtx.Unlock()
-
-}

daemon/pruner/state_enumer.go

@@ -1,71 +0,0 @@
-// Code generated by "enumer -type=State"; DO NOT EDIT.
-
-package pruner
-
-import (
-	"fmt"
-)
-
-const (
-	_StateName_0 = "PlanPlanErr"
-	_StateName_1 = "Exec"
-	_StateName_2 = "ExecErr"
-	_StateName_3 = "Done"
-)
-
-var (
-	_StateIndex_0 = [...]uint8{0, 4, 11}
-	_StateIndex_1 = [...]uint8{0, 4}
-	_StateIndex_2 = [...]uint8{0, 7}
-	_StateIndex_3 = [...]uint8{0, 4}
-)
-
-func (i State) String() string {
-	switch {
-	case 1 <= i && i <= 2:
-		i -= 1
-		return _StateName_0[_StateIndex_0[i]:_StateIndex_0[i+1]]
-	case i == 4:
-		return _StateName_1
-	case i == 8:
-		return _StateName_2
-	case i == 16:
-		return _StateName_3
-	default:
-		return fmt.Sprintf("State(%d)", i)
-	}
-}
-
-var _StateValues = []State{1, 2, 4, 8, 16}
-
-var _StateNameToValueMap = map[string]State{
-	_StateName_0[0:4]:  1,
-	_StateName_0[4:11]: 2,
-	_StateName_1[0:4]:  4,
-	_StateName_2[0:7]:  8,
-	_StateName_3[0:4]:  16,
-}
-
-// StateString retrieves an enum value from the enum constants string name.
-// Throws an error if the param is not part of the enum.
-func StateString(s string) (State, error) {
-	if val, ok := _StateNameToValueMap[s]; ok {
-		return val, nil
-	}
-	return 0, fmt.Errorf("%s does not belong to State values", s)
-}
-
-// StateValues returns all values of the enum
-func StateValues() []State {
-	return _StateValues
-}
-
-// IsAState returns "true" if the value is listed in the enum definition. "false" otherwise
-func (i State) IsAState() bool {
-	for _, v := range _StateValues {
-		if i == v {
-			return true
-		}
-	}
-	return false
-}

daemon/snapper/snapper.go

@@ -1,389 +0,0 @@
-package snapper
-
-import (
-	"context"
-	"fmt"
-	"sort"
-	"sync"
-	"time"
-
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-//go:generate stringer -type=SnapState
-type SnapState uint
-
-const (
-	SnapPending SnapState = 1 << iota
-	SnapStarted
-	SnapDone
-	SnapError
-)
-
-type snapProgress struct {
-	state SnapState
-
-	// SnapStarted, SnapDone, SnapError
-	name    string
-	startAt time.Time
-
-	// SnapDone
-	doneAt time.Time
-
-	// SnapErr
-	err error
-}
-
-type args struct {
-	ctx            context.Context
-	log            Logger
-	prefix         string
-	interval       time.Duration
-	fsf            *filters.DatasetMapFilter
-	snapshotsTaken chan<- struct{}
-}
-
-type Snapper struct {
-	args args
-
-	mtx   sync.Mutex
-	state State
-
-	// set in state Plan, used in Waiting
-	lastInvocation time.Time
-
-	// valid for state Snapshotting
-	plan map[*zfs.DatasetPath]snapProgress
-
-	// valid for state SyncUp and Waiting
-	sleepUntil time.Time
-
-	// valid for state Err
-	err error
-}
-
-//go:generate stringer -type=State
-type State uint
-
-const (
-	SyncUp State = 1 << iota
-	SyncUpErrWait
-	Planning
-	Snapshotting
-	Waiting
-	ErrorWait
-	Stopped
-)
-
-func (s State) sf() state {
-	m := map[State]state{
-		SyncUp:        syncUp,
-		SyncUpErrWait: wait,
-		Planning:      plan,
-		Snapshotting:  snapshot,
-		Waiting:       wait,
-		ErrorWait:     wait,
-		Stopped:       nil,
-	}
-	return m[s]
-}
-
-type updater func(u func(*Snapper)) State
-type state func(a args, u updater) state
-
-type contextKey int
-
-const (
-	contextKeyLog contextKey = 0
-)
-
-type Logger = logger.Logger
-
-func WithLogger(ctx context.Context, log Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLog, log)
-}
-
-func getLogger(ctx context.Context) Logger {
-	if log, ok := ctx.Value(contextKeyLog).(Logger); ok {
-		return log
-	}
-	return logger.NewNullLogger()
-}
-
-func PeriodicFromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in *config.SnapshottingPeriodic) (*Snapper, error) {
-	if in.Prefix == "" {
-		return nil, errors.New("prefix must not be empty")
-	}
-	if in.Interval <= 0 {
-		return nil, errors.New("interval must be positive")
-	}
-
-	args := args{
-		prefix:   in.Prefix,
-		interval: in.Interval,
-		fsf:      fsf,
-		// ctx and log is set in Run()
-	}
-
-	return &Snapper{state: SyncUp, args: args}, nil
-}
-
-func (s *Snapper) Run(ctx context.Context, snapshotsTaken chan<- struct{}) {
-
-	getLogger(ctx).Debug("start")
-	defer getLogger(ctx).Debug("stop")
-
-	s.args.snapshotsTaken = snapshotsTaken
-	s.args.ctx = ctx
-	s.args.log = getLogger(ctx)
-
-	u := func(u func(*Snapper)) State {
-		s.mtx.Lock()
-		defer s.mtx.Unlock()
-		if u != nil {
-			u(s)
-		}
-		return s.state
-	}
-
-	var st state = syncUp
-
-	for st != nil {
-		pre := u(nil)
-		st = st(s.args, u)
-		post := u(nil)
-		getLogger(ctx).
-			WithField("transition", fmt.Sprintf("%s=>%s", pre, post)).
-			Debug("state transition")
-
-	}
-
-}
-
-func onErr(err error, u updater) state {
-	return u(func(s *Snapper) {
-		s.err = err
-		preState := s.state
-		switch s.state {
-		case SyncUp:
-			s.state = SyncUpErrWait
-		case Planning:
-			fallthrough
-		case Snapshotting:
-			s.state = ErrorWait
-		}
-		s.args.log.WithError(err).WithField("pre_state", preState).WithField("post_state", s.state).Error("snapshotting error")
-	}).sf()
-}
-
-func onMainCtxDone(ctx context.Context, u updater) state {
-	return u(func(s *Snapper) {
-		s.err = ctx.Err()
-		s.state = Stopped
-	}).sf()
-}
-
-func syncUp(a args, u updater) state {
-	u(func(snapper *Snapper) {
-		snapper.lastInvocation = time.Now()
-	})
-	fss, err := listFSes(a.ctx, a.fsf)
-	if err != nil {
-		return onErr(err, u)
-	}
-	syncPoint, err := findSyncPoint(a.log, fss, a.prefix, a.interval)
-	if err != nil {
-		return onErr(err, u)
-	}
-	u(func(s *Snapper) {
-		s.sleepUntil = syncPoint
-	})
-	t := time.NewTimer(time.Until(syncPoint))
-	defer t.Stop()
-	select {
-	case <-t.C:
-		return u(func(s *Snapper) {
-			s.state = Planning
-		}).sf()
-	case <-a.ctx.Done():
-		return onMainCtxDone(a.ctx, u)
-	}
-}
-
-func plan(a args, u updater) state {
-	u(func(snapper *Snapper) {
-		snapper.lastInvocation = time.Now()
-	})
-	fss, err := listFSes(a.ctx, a.fsf)
-	if err != nil {
-		return onErr(err, u)
-	}
-
-	plan := make(map[*zfs.DatasetPath]snapProgress, len(fss))
-	for _, fs := range fss {
-		plan[fs] = snapProgress{state: SnapPending}
-	}
-	return u(func(s *Snapper) {
-		s.state = Snapshotting
-		s.plan = plan
-	}).sf()
-}
-
-func snapshot(a args, u updater) state {
-
-	var plan map[*zfs.DatasetPath]snapProgress
-	u(func(snapper *Snapper) {
-		plan = snapper.plan
-	})
-
-	hadErr := false
-	// TODO channel programs -> allow a little jitter?
-	for fs, progress := range plan {
-		suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
-		snapname := fmt.Sprintf("%s%s", a.prefix, suffix)
-
-		l := a.log.
-			WithField("fs", fs.ToString()).
-			WithField("snap", snapname)
-
-		u(func(snapper *Snapper) {
-			progress.name = snapname
-			progress.startAt = time.Now()
-			progress.state = SnapStarted
-		})
-
-		l.Debug("create snapshot")
-		err := zfs.ZFSSnapshot(fs, snapname, false)
-		if err != nil {
-			hadErr = true
-			l.WithError(err).Error("cannot create snapshot")
-		}
-		doneAt := time.Now()
-
-		u(func(snapper *Snapper) {
-			progress.doneAt = doneAt
-			progress.state = SnapDone
-			if err != nil {
-				progress.state = SnapError
-				progress.err = err
-			}
-		})
-	}
-
-	select {
-	case a.snapshotsTaken <- struct{}{}:
-	default:
-		if a.snapshotsTaken != nil {
-			a.log.Warn("callback channel is full, discarding snapshot update event")
-		}
-	}
-
-	return u(func(snapper *Snapper) {
-		if hadErr {
-			snapper.state = ErrorWait
-			snapper.err = errors.New("one or more snapshots could not be created, check logs for details")
-		} else {
-			snapper.state = Waiting
-		}
-	}).sf()
-}
-
-func wait(a args, u updater) state {
-	var sleepUntil time.Time
-	u(func(snapper *Snapper) {
-		lastTick := snapper.lastInvocation
-		snapper.sleepUntil = lastTick.Add(a.interval)
-		sleepUntil = snapper.sleepUntil
-		log := a.log.WithField("sleep_until", sleepUntil).WithField("duration", a.interval)
-		logFunc := log.Debug
-		if snapper.state == ErrorWait || snapper.state == SyncUpErrWait {
-			logFunc = log.Error
-		}
-		logFunc("enter wait-state after error")
-	})
-
-	t := time.NewTimer(time.Until(sleepUntil))
-	defer t.Stop()
-
-	select {
-	case <-t.C:
-		return u(func(snapper *Snapper) {
-			snapper.state = Planning
-		}).sf()
-	case <-a.ctx.Done():
-		return onMainCtxDone(a.ctx, u)
-	}
-}
-
-func listFSes(ctx context.Context, mf *filters.DatasetMapFilter) (fss []*zfs.DatasetPath, err error) {
-	return zfs.ZFSListMapping(ctx, mf)
-}
-
-func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval time.Duration) (syncPoint time.Time, err error) {
-	type snapTime struct {
-		ds   *zfs.DatasetPath
-		time time.Time
-	}
-
-	if len(fss) == 0 {
-		return time.Now(), nil
-	}
-
-	snaptimes := make([]snapTime, 0, len(fss))
-
-	now := time.Now()
-
-	log.Debug("examine filesystem state")
-	for _, d := range fss {
-
-		l := log.WithField("fs", d.ToString())
-
-		fsvs, err := zfs.ZFSListFilesystemVersions(d, filters.NewTypedPrefixFilter(prefix, zfs.Snapshot))
-		if err != nil {
-			l.WithError(err).Error("cannot list filesystem versions")
-			continue
-		}
-		if len(fsvs) <= 0 {
-			l.WithField("prefix", prefix).Debug("no filesystem versions with prefix")
-			continue
-		}
-
-		// Sort versions by creation
-		sort.SliceStable(fsvs, func(i, j int) bool {
-			return fsvs[i].CreateTXG < fsvs[j].CreateTXG
-		})
-
-		latest := fsvs[len(fsvs)-1]
-		l.WithField("creation", latest.Creation).
-			Debug("found latest snapshot")
-
-		since := now.Sub(latest.Creation)
-		if since < 0 {
-			l.WithField("snapshot", latest.Name).
-				WithField("creation", latest.Creation).
-				Error("snapshot is from the future")
-			continue
-		}
-		next := now
-		if since < interval {
-			next = latest.Creation.Add(interval)
-		}
-		snaptimes = append(snaptimes, snapTime{d, next})
-	}
-
-	if len(snaptimes) == 0 {
-		snaptimes = append(snaptimes, snapTime{nil, now})
-	}
-
-	sort.Slice(snaptimes, func(i, j int) bool {
-		return snaptimes[i].time.Before(snaptimes[j].time)
-	})
-
-	return snaptimes[0].time, nil
-
-}

daemon/snapper/snapper_all.go

@@ -1,40 +0,0 @@
-package snapper
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/filters"
-)
-
-// FIXME: properly abstract snapshotting:
-//   - split up things that trigger snapshotting from the mechanism
-//     - timer-based trigger (periodic)
-//     - call from control socket (manual)
-//     - mixed modes?
-//   - support a `zrepl snapshot JOBNAME` subcommand for config.SnapshottingManual
-type PeriodicOrManual struct {
-	s *Snapper
-}
-
-func (s *PeriodicOrManual) Run(ctx context.Context, wakeUpCommon chan<- struct{}) {
-	if s.s != nil {
-		s.s.Run(ctx, wakeUpCommon)
-	}
-}
-
-func FromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in config.SnapshottingEnum) (*PeriodicOrManual, error) {
-	switch v := in.Ret.(type) {
-	case *config.SnapshottingPeriodic:
-		snapper, err := PeriodicFromConfig(g, fsf, v)
-		if err != nil {
-			return nil, err
-		}
-		return &PeriodicOrManual{snapper}, nil
-	case *config.SnapshottingManual:
-		return &PeriodicOrManual{}, nil
-	default:
-		return nil, fmt.Errorf("unknown snapshotting type %T", v)
-	}
-}

daemon/snapper/snapstate_string.go

@@ -1,29 +0,0 @@
-// Code generated by "stringer -type=SnapState"; DO NOT EDIT.
-
-package snapper
-
-import "strconv"
-
-const (
-	_SnapState_name_0 = "SnapPendingSnapStarted"
-	_SnapState_name_1 = "SnapDone"
-	_SnapState_name_2 = "SnapError"
-)
-
-var (
-	_SnapState_index_0 = [...]uint8{0, 11, 22}
-)
-
-func (i SnapState) String() string {
-	switch {
-	case 1 <= i && i <= 2:
-		i -= 1
-		return _SnapState_name_0[_SnapState_index_0[i]:_SnapState_index_0[i+1]]
-	case i == 4:
-		return _SnapState_name_1
-	case i == 8:
-		return _SnapState_name_2
-	default:
-		return "SnapState(" + strconv.FormatInt(int64(i), 10) + ")"
-	}
-}

daemon/snapper/state_string.go

@@ -1,38 +0,0 @@
-// Code generated by "stringer -type=State"; DO NOT EDIT.
-
-package snapper
-
-import "strconv"
-
-const (
-	_State_name_0 = "SyncUpSyncUpErrWait"
-	_State_name_1 = "Planning"
-	_State_name_2 = "Snapshotting"
-	_State_name_3 = "Waiting"
-	_State_name_4 = "ErrorWait"
-	_State_name_5 = "Stopped"
-)
-
-var (
-	_State_index_0 = [...]uint8{0, 6, 19}
-)
-
-func (i State) String() string {
-	switch {
-	case 1 <= i && i <= 2:
-		i -= 1
-		return _State_name_0[_State_index_0[i]:_State_index_0[i+1]]
-	case i == 4:
-		return _State_name_1
-	case i == 8:
-		return _State_name_2
-	case i == 16:
-		return _State_name_3
-	case i == 32:
-		return _State_name_4
-	case i == 64:
-		return _State_name_5
-	default:
-		return "State(" + strconv.FormatInt(int64(i), 10) + ")"
-	}
-}

dist/grafana/grafana-prometheus-zrepl-0.1.json

@@ -1,595 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "$$hashKey": "object:3351",
-        "builtIn": 1,
-        "datasource": "-- Grafana --",
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "gnetId": null,
-  "graphTooltip": 1,
-  "id": 7,
-  "iteration": 1552746418826,
-  "links": [],
-  "panels": [
-    {
-      "content": "# zrepl Prometheus Metrics\n\nzrepl exposes Prometheus metrics and ships with this Grafana dashboard.\nThe exported metrics are suitable for health checks:\n\n* The log should generally be warning & error-free\n  * The `Log Messages that require attention` graph visualizes log message counts indicating problems.\n* The number of goroutines should not grow unboundedly over time.\n  * During replication, the number of goroutines can be way higher than during idle time.\n  * If the goroutine count grows with each replication, there is clearly a goroutine leak. Please open a bug report.\n* The sys memory consumption should not grow unboundedly over time.\n  * Note that the Go runtime pre-allocates some of its heap from the OS.\n  * zrepl actually uses much less memory than allocated from the OS.\n  * Since Go 1.11, Go pre-allocates more aggressively.\n* Monitor that some data is replicated, although that metric does not guarantee that replication was successful.\n\n**In general, note that the exported metrics are not stable unless declared otherwise.**",
-      "gridPos": {
-        "h": 9,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "id": 35,
-      "mode": "markdown",
-      "title": "Panel Title",
-      "type": "text"
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": null,
-      "fill": 1,
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 0,
-        "y": 9
-      },
-      "id": 15,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
-      "percentage": false,
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": true,
-      "steppedLine": false,
-      "targets": [
-        {
-          "$$hashKey": "object:3436",
-          "expr": "up{job='$prom_job_name'}",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "zrepl Instances Up",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": "5",
-          "min": "0",
-          "show": true
-        },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ]
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": null,
-      "fill": 1,
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 12,
-        "y": 9
-      },
-      "id": 22,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
-      "percentage": false,
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "expr": "increase(zrepl_daemon_log_entries{job='$prom_job_name',level=~'warn|error'}[$__interval])",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "Log Messages that require attention",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": "0",
-          "show": true
-        },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ]
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": null,
-      "fill": 1,
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 0,
-        "y": 18
-      },
-      "id": 33,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
-      "percentage": false,
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [
-        {
-          "alias": "/replicated bytes in last.*/",
-          "yaxis": 2
-        }
-      ],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "expr": "sum(rate(zrepl_replication_bytes_replicated{job='$prom_job_name'}[$__interval])) by (zrepl_job)",
-          "format": "time_series",
-          "hide": false,
-          "interval": "",
-          "intervalFactor": 1,
-          "legendFormat": "replication data rate zrepl_job={{zrepl_job}}",
-          "refId": "A"
-        },
-        {
-          "expr": "sum(increase(zrepl_replication_bytes_replicated{job='$prom_job_name'}[10m])) by (zrepl_job)",
-          "format": "time_series",
-          "hide": false,
-          "interval": "",
-          "intervalFactor": 1,
-          "legendFormat": "replicated bytes in last 10min zrepl_job={{zrepl_job}}",
-          "refId": "B"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "Replication Data Rate and Volume(integrates last 10min)",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "Bps",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "format": "bytes",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ]
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": null,
-      "fill": 1,
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 12,
-        "y": 18
-      },
-      "id": 23,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
-      "percentage": false,
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "expr": "sum(increase(zrepl_daemon_log_entries{job='$prom_job_name',zrepl_job=~\"^[^_].*\"}[$__interval])) by (instance,zrepl_job)",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "Log Activity (without internal jobs)",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": "0",
-          "show": true
-        },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ]
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": null,
-      "fill": 1,
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 0,
-        "y": 27
-      },
-      "id": 17,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
-      "percentage": false,
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "$$hashKey": "object:3535",
-          "expr": "go_memstats_sys_bytes{job='$prom_job_name'}",
-          "format": "time_series",
-          "hide": false,
-          "intervalFactor": 1,
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "Memory Allocated by the Go runtime from the OS (should not grow unboundedly)",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "bytes",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": "0",
-          "show": true
-        },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ]
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": null,
-      "fill": 1,
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 12,
-        "y": 27
-      },
-      "id": 19,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
-      "percentage": false,
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "expr": "go_goroutines{job='$prom_job_name'}",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "number of goroutines (should not grow unboundedly)",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": "0",
-          "show": true
-        },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ]
-    }
-  ],
-  "refresh": "1m",
-  "schemaVersion": 16,
-  "style": "dark",
-  "tags": [],
-  "templating": {
-    "list": [
-      {
-        "allValue": null,
-        "current": {
-          "text": "zrepl",
-          "value": "zrepl"
-        },
-        "datasource": "prometheus",
-        "hide": 0,
-        "includeAll": false,
-        "label": "Prometheus Job Name",
-        "multi": false,
-        "name": "prom_job_name",
-        "options": [],
-        "query": "label_values(up, job)",
-        "refresh": 1,
-        "regex": "",
-        "sort": 1,
-        "tagValuesQuery": "",
-        "tags": [],
-        "tagsQuery": "",
-        "type": "query",
-        "useTags": false
-      }
-    ]
-  },
-  "time": {
-    "from": "now-2d",
-    "to": "now"
-  },
-  "timepicker": {
-    "refresh_intervals": [
-      "5s",
-      "10s",
-      "30s",
-      "1m",
-      "5m",
-      "15m",
-      "30m",
-      "1h",
-      "2h",
-      "1d"
-    ],
-    "time_options": [
-      "5m",
-      "15m",
-      "1h",
-      "6h",
-      "12h",
-      "24h",
-      "2d",
-      "7d",
-      "30d"
-    ]
-  },
-  "timezone": "",
-  "title": "zrepl 0.1",
-  "uid": "xTljn4qmk",
-  "version": 6
-}

dist/systemd/zrepl.service

@@ -1,41 +0,0 @@
-[Unit]
-Description=zrepl daemon
-Documentation=https://zrepl.github.io
-
-[Service]
-Type=simple
-ExecStart=/usr/local/bin/zrepl --config /etc/zrepl/zrepl.yml daemon
-RuntimeDirectory=zrepl
-RuntimeDirectoryMode=0700
-
-ProtectSystem=strict
-#PrivateDevices=yes # TODO ZFS needs access to /dev/zfs, could we limit this?
-ProtectKernelTunables=yes
-ProtectControlGroups=yes
-PrivateTmp=yes
-#PrivateUsers=yes # TODO Does not work, why?
-ProtectKernelModules=true
-RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
-RestrictNamespaces=true
-RestrictRealtime=yes
-SystemCallArchitectures=native
-
-# BEGIN ProtectHome
-ProtectHome=read-only   # DEBIAN STRETCH
-# ProtectHome=tmpfs     # FEDORA 28 / 29
-# END ProtectHome
-
-# BEGIN SystemCallFilter
-## BEGIN DEBIAN STRETCH
-SystemCallFilter=~ @mount @cpu-emulation @keyring @module @obsolete @privileged @raw-io @debug @clock @resources
-## END DEBIAN STRETCH
-## BEGIN FEDORA 28/29 
-## Syscall blacklist (should be fairly stable)
-#SystemCallFilter=~ @mount @aio @cpu-emulation @keyring @memlock @module @obsolete @privileged @raw-io @reboot @setuid @swap @sync @timer @debug @clock @chown @resources
-## Syscall whitelist (not sure how stable)
-#SystemCallFilter=@default @file-system @process @basic-io @ipc @network-io @signal @io-event brk mprotect sched_getaffinity ioctl getrandom
-## END END FEDORA 28/29
-# END SystemCallFilter
-
-[Install]
-WantedBy=multi-user.target