build: use go 1.19 for testing & release builds

New docker image since the old one was deprecated, according
to https://discuss.circleci.com/t/go-lang-docker-image-circleci-golang-1-19-is-missing/44961

.circleci/config.yml

@@ -0,0 +1,379 @@
+version: 2.1
+
+commands:
+  setup-home-local-bin:
+    steps:
+      - run:
+          shell: /bin/bash -euo pipefail
+          command: |
+            mkdir -p "$HOME/.local/bin"
+            line='export PATH="$HOME/.local/bin:$PATH"'
+            if grep "$line" $BASH_ENV >/dev/null; then
+              echo "$line" >> $BASH_ENV
+            fi
+
+  invoke-lazy-sh:
+    parameters:
+      subcommand:
+        type: string
+    steps:
+      - run:
+          environment:
+            TERM: xterm
+          command: ./lazy.sh <<parameters.subcommand>>
+
+  apt-update-and-install-common-deps:
+    steps:
+      - run: sudo apt update && sudo apt install gawk make
+
+  restore-cache-gomod:
+    steps:
+      - restore_cache:
+          key: go-mod-v4-{{ checksum "go.sum" }}
+  save-cache-gomod:
+    steps:
+      - save_cache:
+          key: go-mod-v4-{{ checksum "go.sum" }}
+          paths:
+            - "/go/pkg/mod"
+
+  install-godep:
+    steps:
+      - apt-update-and-install-common-deps
+      - invoke-lazy-sh:
+          subcommand: godep
+
+  install-docdep:
+    steps:
+      - apt-update-and-install-common-deps
+      - run: sudo apt install python3 python3-pip libgirepository1.0-dev
+      - invoke-lazy-sh:
+          subcommand: docdep
+
+  download-and-install-minio-client:
+    steps:
+      - setup-home-local-bin
+      - restore_cache:
+          key: minio-client-v2
+      - run:
+          shell: /bin/bash -eo pipefail
+          command: |
+            if which mc; then exit 0; fi
+            sudo curl -sSL https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2020-08-20T00-23-01Z \
+              -o "$HOME/.local/bin/mc"
+            sudo chmod +x "$HOME/.local/bin/mc"
+      - save_cache:
+          key: minio-client-v2
+          paths:
+            - "$HOME/.local/bin/mc"
+
+  upload-minio:
+    parameters:
+      src:
+        type: string
+      dst:
+        type: string
+    steps:
+      - run:
+          shell: /bin/bash -eo pipefail
+          when: always
+          command: |
+            if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
+              echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
+              exit 0
+            fi
+            set -u # from now on
+
+            mc config host add --api s3v4 zrepl-minio https://minio.cschwarz.com ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
+
+            # keep in sync with set-github-minio-status
+            jobprefix=zrepl-ci-artifacts/${CIRCLE_SHA1}-pipeline-<<pipeline.number>>/${CIRCLE_JOB}
+
+            # Upload artifacts
+            mkdir -p ./artifacts
+            mc cp -r <<parameters.src>> "zrepl-minio/$jobprefix/<<parameters.dst>>"
+
+  set-github-minio-status:
+    parameters:
+      context:
+        type: string
+      description:
+        type: string
+      minio-dst:
+        type: string
+    steps:
+      - run:
+          shell: /bin/bash -eo pipefail
+          command: |
+            if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
+              echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
+              exit 0
+            fi
+            set -u # from now on
+
+            # keep in sync with with upload-minio command
+            jobprefix=zrepl-ci-artifacts/${CIRCLE_SHA1}-pipeline-<<pipeline.number>>/${CIRCLE_JOB}
+            # Push Artifact Link to GitHub
+            REPO="zrepl/zrepl"
+            COMMIT="${CIRCLE_SHA1}"
+            JOB_NAME="${CIRCLE_JOB}"
+            CONTEXT="<<parameters.context>>"
+            DESCRIPTION="<<parameters.description>>"
+            TARGETURL=https://minio.cschwarz.com/minio/"$jobprefix"/"<<parameters.minio-dst>>"
+            curl "https://api.github.com/repos/$REPO/statuses/$COMMIT" \
+              -H "Content-Type: application/json" \
+              -H "Authorization: token $GITHUB_COMMIT_STATUS_TOKEN" \
+              -X POST \
+              -d '{"context":"'"$CONTEXT"'", "state": "success", "description":"'"$DESCRIPTION"'", "target_url":"'"$TARGETURL"'"}'
+
+
+  trigger-pipeline:
+    parameters:
+      body_no_shell_subst:
+        type: string
+    steps:
+      - run: |
+          curl -X POST https://circleci.com/api/v2/project/github/zrepl/zrepl/pipeline \
+            -H 'Content-Type: application/json' \
+            -H 'Accept: application/json' \
+            -H "Circle-Token: $ZREPL_BOT_CIRCLE_TOKEN" \
+            --data '<<parameters.body_no_shell_subst>>'
+
+parameters:
+  do_ci:
+    type: boolean
+    default: true
+
+  do_release:
+    type: boolean
+    default: false
+
+  release_docker_baseimage_tag:
+    type: string
+    default: "1.19"
+
+workflows:
+  version: 2
+
+  ci:
+    when: << pipeline.parameters.do_ci >>
+    jobs:
+      - quickcheck-docs
+      - quickcheck-go: &quickcheck-go-smoketest
+          name: quickcheck-go-amd64-linux-1.19
+          goversion: &latest-go-release "1.19"
+          goos: linux
+          goarch: amd64
+      - test-go-on-latest-go-release:
+          goversion: *latest-go-release
+      - quickcheck-go:
+          requires:
+            - quickcheck-go-amd64-linux-1.19 #quickcheck-go-smoketest.name
+          matrix: &quickcheck-go-matrix
+            alias: quickcheck-go-matrix
+            parameters:
+              goversion: [*latest-go-release, "1.12"]
+              goos: ["linux", "freebsd"]
+              goarch: ["amd64", "arm64"]
+            exclude:
+              # don't re-do quickcheck-go-smoketest
+              - goversion: *latest-go-release
+                goos: linux
+                goarch: amd64
+              # not supported by Go 1.12
+              - goversion: "1.12"
+                goos: freebsd
+                goarch: arm64
+
+  release:
+    when: << pipeline.parameters.do_release >>
+    jobs:
+      - release-build
+      - release-deb:
+          requires:
+            - release-build
+      - release-rpm:
+          requires:
+            - release-build
+      - release-upload:
+          requires:
+            - release-build
+            - release-deb
+            - release-rpm
+
+  periodic:
+    triggers:
+      - schedule:
+          cron: "00 17 * * *"
+          filters:
+            branches:
+              only:
+                - master
+                - stable
+                - problame/circleci-build
+    jobs:
+      - periodic-full-pipeline-run
+
+  zrepl.github.io:
+    jobs:
+      - publish-zrepl-github-io:
+          filters:
+            branches:
+              only:
+                - stable
+
+jobs:
+  quickcheck-docs:
+    docker:
+      - image: cimg/base:2020.08
+    steps:
+      - checkout
+      - install-docdep
+      - run: make docs
+
+      - download-and-install-minio-client
+      - upload-minio:
+          src: artifacts
+          dst: ""
+      - set-github-minio-status:
+          context: artifacts/${CIRCLE_JOB}
+          description: artifacts of CI job ${CIRCLE_JOB}
+          minio-dst: ""
+
+  quickcheck-go:
+    parameters:
+      goversion:
+        type: string
+      goos:
+        type: string
+      goarch:
+        type: string
+    docker:
+      - image: cimg/go:<<parameters.goversion>>
+    environment:
+      GOOS: <<parameters.goos>>
+      GOARCH: <<parameters.goarch>>
+
+    steps:
+      - checkout
+
+      - restore-cache-gomod
+      - run: go mod download
+      - run: cd build && go mod download
+      - save-cache-gomod
+
+      - install-godep
+      - run: make formatcheck
+      - run: make generate-platform-test-list
+      - run: make zrepl-bin test-platform-bin
+      - run: make vet
+      - run: make lint
+
+      - download-and-install-minio-client
+      - run: rm -f artifacts/generate-platform-test-list
+      - store_artifacts:
+          path: artifacts
+      - upload-minio:
+          src: artifacts
+          dst: ""
+      - set-github-minio-status:
+          context: artifacts/${CIRCLE_JOB}
+          description: artifacts of CI job ${CIRCLE_JOB}
+          minio-dst: ""
+
+  test-go-on-latest-go-release:
+    parameters:
+      goversion:
+        type: string
+    docker:
+      - image: cimg/go:<<parameters.goversion>>
+    steps:
+      - checkout
+      - restore-cache-gomod
+      - run: make test-go
+      # don't save-cache-gomod here, test-go doesn't pull all the dependencies
+
+  release-build:
+    machine:
+      image: ubuntu-2004:202201-02
+    steps:
+      - checkout
+      - run: make release-docker RELEASE_DOCKER_BASEIMAGE_TAG=<<pipeline.parameters.release_docker_baseimage_tag>>
+      - persist_to_workspace:
+          root: .
+          paths: [.]
+  release-deb:
+    machine:
+      image: ubuntu-2004:202201-02
+    steps:
+      - attach_workspace:
+          at: .
+      - run: make debs-docker
+      - persist_to_workspace:
+          root: .
+          paths:
+            - "artifacts/*.deb"
+
+  release-rpm:
+    machine:
+      image: ubuntu-2004:202201-02
+    steps:
+      - attach_workspace:
+          at: .
+      - run: make rpms-docker
+      - persist_to_workspace:
+          root: .
+          paths:
+            - "artifacts/*.rpm"
+
+  release-upload:
+    docker:
+      - image: cimg/base:2020.08
+    steps:
+      - attach_workspace:
+          at: .
+      - store_artifacts:
+          path: artifacts
+      - download-and-install-minio-client
+      - upload-minio:
+          src: artifacts
+          dst: ""
+      - set-github-minio-status:
+          context: artifacts/release
+          description: CI-generated release artifacts
+          minio-dst: ""
+
+  periodic-full-pipeline-run:
+    docker:
+      - image: cimg/base:2020.08
+    steps:
+      - trigger-pipeline:
+          body_no_shell_subst: '{"branch":"<<pipeline.git.branch>>", "parameters": { "do_ci": true, "do_release": true }}'
+
+  publish-zrepl-github-io:
+    docker:
+      - image: cimg/python:3.7
+    steps:
+      - checkout
+      - invoke-lazy-sh:
+          subcommand: docdep
+      - run:
+          command: |
+            git config --global user.email "zreplbot@cschwarz.com"
+            git config --global user.name "zrepl-github-io-ci"
+
+      # https://circleci.com/docs/2.0/add-ssh-key/#adding-multiple-keys-with-blank-hostnames
+      - run: ssh-add -D
+      # the default circleci ssh config only additional ssh keys for Host !github.com
+      - run:
+          command: |
+            cat > ~/.ssh/config \<<EOF
+            Host *
+              IdentityFile /home/circleci/.ssh/id_rsa_458e62c517f6c480e40452126ce47421
+            EOF
+      - add_ssh_keys:
+          fingerprints:
+            # deploy key for zrepl.github.io
+            - "45:8e:62:c5:17:f6:c4:80:e4:04:52:12:6c:e4:74:21"
+
+      - run: bash -x docs/publish.sh -c -a

.circleci/trigger_debian_binary_packaging_workflow.bash

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+COMMIT="$1"
+GO_VERSION="$2"
+
+curl -v -X POST https://api.github.com/repos/zrepl/debian-binary-packaging/dispatches \
+    -H 'Accept: application/vnd.github.v3+json' \
+    -H "Authorization: token $GITHUB_ACCESS_TOKEN" \
+    --data '{"event_type": "push", "client_payload": { "zrepl_main_repo_commit": "'"$COMMIT"'", "go_version": "'"$GO_VERSION"'" }}'

.github/FUNDING.yml

@@ -0,0 +1,5 @@
+github: problame
+patreon: zrepl
+liberapay: zrepl
+custom: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96
+ko_fi: problame

.golangci.yml

@@ -0,0 +1,16 @@
+linters:
+  enable:
+  - goimports
+
+issues:
+  exclude-rules:
+    - path: _test\.go
+      linters:
+        - errcheck
+    # Disable staticcheck 'Empty body in an if or else branch' as it's useful
+    # to put a comment into an empty else-clause that explains why whatever
+    # is done in the if-caluse is not necessary if the condition is false.
+    - linters:
+        - staticcheck
+      text: "SA9003:"
+

Gopkg.lock

@@ -1,123 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/dustin/go-humanize"
-  packages = ["."]
-  revision = "bb3d318650d48840a39aa21a027c6630e198e626"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/ftrvxmtrx/fd"
-  packages = ["."]
-  revision = "c6d800382fff6dc1412f34269f71b7f83bd059ad"
-
-[[projects]]
-  name = "github.com/go-logfmt/logfmt"
-  packages = ["."]
-  revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5"
-  version = "v0.3.0"
-
-[[projects]]
-  branch = "v2"
-  name = "github.com/go-yaml/yaml"
-  packages = ["."]
-  revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/jinzhu/copier"
-  packages = ["."]
-  revision = "db4671f3a9b8df855e993f7c94ec5ef1ffb0a23b"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/kr/logfmt"
-  packages = ["."]
-  revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/kr/pretty"
-  packages = ["."]
-  revision = "cfb55aafdaf3ec08f0db22699ab822c50091b1c4"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/kr/text"
-  packages = ["."]
-  revision = "7cafcd837844e784b526369c9bce262804aebc60"
-
-[[projects]]
-  name = "github.com/mattn/go-isatty"
-  packages = ["."]
-  revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
-  version = "v0.0.3"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/mitchellh/mapstructure"
-  packages = ["."]
-  revision = "d0303fe809921458f417bcf828397a65db30a7e4"
-
-[[projects]]
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  name = "github.com/pmezard/go-difflib"
-  packages = ["difflib"]
-  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/problame/go-netssh"
-  packages = ["."]
-  revision = "53a2e445f8ace7ec678f2d8cdd9c1428dfef4562"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/problame/go-rwccmd"
-  packages = ["."]
-  revision = "391d2c78c8404a9683d79f75dd24ab53040f89f7"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/spf13/cobra"
-  packages = ["."]
-  revision = "b78744579491c1ceeaaa3b40205e56b0591b93a3"
-
-[[projects]]
-  name = "github.com/spf13/pflag"
-  packages = ["."]
-  revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/stretchr/testify"
-  packages = ["assert"]
-  revision = "69483b4bd14f5845b5a1e55bca19e954e827f1d0"
-  version = "v1.1.4"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sys"
-  packages = ["unix"]
-  revision = "bf42f188b9bc6f2cf5b8ee5a912ef1aedd0eba4c"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  inputs-digest = "fcfc66322b5f9dbe329411b7688a4e9dcebe3c529c1dadf4f78bcdb2bcfb8fbf"
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,53 +0,0 @@
-ignored = [ "github.com/inconshreveable/mousetrap" ]
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/ftrvxmtrx/fd"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/jinzhu/copier"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/kr/pretty"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/mitchellh/go-homedir"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/mitchellh/mapstructure"
-
-[[constraint]]
-  name = "github.com/pkg/errors"
-  version = "0.8.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/spf13/cobra"
-
-[[constraint]]
-  name = "github.com/spf13/viper"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "github.com/stretchr/testify"
-  version = "1.1.4"
-
-[[constraint]]
-  branch = "v2"
-  name = "github.com/go-yaml/yaml"
-
-[[constraint]]
-  name = "github.com/go-logfmt/logfmt"
-  version = "*"
-
-[[constraint]]
-  name = "github.com/problame/go-rwccmd"
-  branch = "master"
-
-[[constraint]]
-  name = "github.com/problame/go-netssh"
-  branch = "master"

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2017 Christian Schwarz
+Copyright (c) 2017-2018 Christian Schwarz
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,106 +1,344 @@
-.PHONY: generate build test vet cover release docs docs-clean clean vendordeps
+.PHONY: generate build test vet cover release docs docs-clean clean format lint platformtest
-.DEFAULT_GOAL := build
+.PHONY: release bins-all release-noarch
-
+.DEFAULT_GOAL := zrepl-bin
-ROOT := github.com/zrepl/zrepl
-SUBPKGS := cmd logger rpc util zfs
-
-_TESTPKGS := $(ROOT) $(foreach p,$(SUBPKGS),$(ROOT)/$(p))
 
 ARTIFACTDIR := artifacts
 
-ifndef ZREPL_VERSION
+ifdef ZREPL_VERSION
-    ZREPL_VERSION := $(shell git describe --dirty 2>/dev/null || echo "ZREPL_BUILD_INVALID_VERSION" )
+    _ZREPL_VERSION := $(ZREPL_VERSION)
-    ifeq ($(ZREPL_VERSION),ZREPL_BUILD_INVALID_VERSION) # can't use .SHELLSTATUS because Debian Stretch is still on gmake 4.1
+endif
+ifndef _ZREPL_VERSION
+    _ZREPL_VERSION := $(shell git describe --always --dirty 2>/dev/null || echo "ZREPL_BUILD_INVALID_VERSION" )
+    ifeq ($(_ZREPL_VERSION),ZREPL_BUILD_INVALID_VERSION) # can't use .SHELLSTATUS because Debian Stretch is still on gmake 4.1
         $(error cannot infer variable ZREPL_VERSION using git and variable is not overriden by make invocation)
     endif
 endif
-GO_LDFLAGS := "-X github.com/zrepl/zrepl/cmd.zreplVersion=$(ZREPL_VERSION)"
 
-GO_BUILD := go build -ldflags $(GO_LDFLAGS)
+GO := go
+GOOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOOS"')
+GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"')
+GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"')
+GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"')
+GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
+GO_ENV_VARS := GO111MODULE=on
+GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
+GO_MOD_READONLY := -mod=readonly
+GO_EXTRA_BUILDFLAGS :=
+GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
+GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
+GOLANGCI_LINT := golangci-lint
+GOCOVMERGE := gocovmerge
+RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.19
+RELEASE_DOCKER_BASEIMAGE ?= golang:$(RELEASE_DOCKER_BASEIMAGE_TAG)
 
-RELEASE_BINS := $(ARTIFACTDIR)/zrepl-freebsd-amd64 $(ARTIFACTDIR)/zrepl-linux-amd64
+ifneq ($(GOARM),)
-RELEASE_NOARCH := $(ARTIFACTDIR)/zrepl-noarch.tar
+	ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM)
-THIS_PLATFORM_RELEASE_BIN := $(shell bash -c 'source <(go env) && echo "zrepl-$${GOOS}-$${GOARCH}"' )
+else
+	ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)
+endif
 
-vendordeps:
+.PHONY: printvars
-	dep ensure -v -vendor-only
+printvars:
-
+	@echo GOOS=$(GOOS)
-generate: #not part of the build, must do that manually
+	@echo GOARCH=$(GOARCH)
-	@for pkg in $(_TESTPKGS); do\
+	@echo GOARM=$(GOARM)
-		go generate "$$pkg" || exit 1; \
-	done;
-
-build:
-	@echo "INFO: In case of missing dependencies, run 'make vendordeps'"
-	$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl"
-
-test:
-	@for pkg in $(_TESTPKGS); do \
-		echo "Testing $$pkg"; \
-		go test "$$pkg" || exit 1; \
-	done;
-
-vet:
-	@for pkg in $(_TESTPKGS); do \
-		echo "Vetting $$pkg"; \
-		go vet "$$pkg" || exit 1; \
-	done;
-
-cover: artifacts
-	@for pkg in $(_TESTPKGS); do \
-		profile="$(ARTIFACTDIR)/cover-$$(basename $$pkg).out"; \
-		go test -coverprofile "$$profile" $$pkg || exit 1; \
-		if [ -f "$$profile" ]; then \
-   			go tool cover -html="$$profile" -o "$${profile}.html" || exit 2; \
-		fi; \
-	done;
-
-$(ARTIFACTDIR):
-	mkdir -p "$@"
-
-$(ARTIFACTDIR)/docs: $(ARTIFACTDIR)
-	mkdir -p "$@"
-
-$(ARTIFACTDIR)/bash_completion: $(RELEASE_BINS)
-	artifacts/$(THIS_PLATFORM_RELEASE_BIN) bashcomp "$@"
-
-$(ARTIFACTDIR)/go_version.txt:
-	go version > $@
-
-docs: $(ARTIFACTDIR)/docs
-	make -C docs \
-		html \
-		BUILDDIR=../artifacts/docs \
-
-docs-clean:
-	make -C docs \
-		clean \
-		BUILDDIR=../artifacts/docs
 
 
-.PHONY: $(RELEASE_BINS)
+##################### PRODUCING A RELEASE #############
-# TODO: two wildcards possible
+.PHONY: release wrapup-and-checksum check-git-clean sign clean
-$(RELEASE_BINS): $(ARTIFACTDIR)/zrepl-%-amd64: generate $(ARTIFACTDIR) vet test
-	@echo "INFO: In case of missing dependencies, run 'make vendordeps'"
-	GOOS=$* GOARCH=amd64   $(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$*-amd64"
 
-$(RELEASE_NOARCH): docs $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_version.txt
+release: clean
+	# no cross-platform support for target test
+	$(MAKE) test-go
+	$(MAKE) bins-all
+	$(MAKE) noarch
+	$(MAKE) wrapup-and-checksum
+	$(MAKE) check-git-clean
+ifeq (SIGN, 1)
+	$(MAKE) sign
+endif
+	@echo "ZREPL RELEASE ARTIFACTS AVAILABLE IN artifacts/release"
+
+release-docker: $(ARTIFACTDIR)
+	sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > artifacts/release-docker.Dockerfile
+	docker build -t zrepl_release --pull -f artifacts/release-docker.Dockerfile .
+	docker run --rm -i -v $(CURDIR):/src -u $$(id -u):$$(id -g) \
+		zrepl_release \
+		make release GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+
+debs-docker:
+	$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=deb
+rpms-docker:
+	$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=rpm
+_debs_or_rpms_docker: # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
+	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=amd64
+	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=arm64
+	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=arm GOARM=7
+	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=386
+
+rpm: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
+	$(eval _ZREPL_RPM_VERSION := $(subst -,.,$(_ZREPL_VERSION)))
+	$(eval _ZREPL_RPM_TOPDIR_ABS := $(CURDIR)/$(ARTIFACTDIR)/rpmbuild)
+	rm -rf "$(_ZREPL_RPM_TOPDIR_ABS)"
+	mkdir "$(_ZREPL_RPM_TOPDIR_ABS)"
+	mkdir -p "$(_ZREPL_RPM_TOPDIR_ABS)"/{SPECS,RPMS,BUILD,BUILDROOT}
+	sed "s/^Version:.*/Version:          $(_ZREPL_RPM_VERSION)/g" \
+	    packaging/rpm/zrepl.spec >  $(_ZREPL_RPM_TOPDIR_ABS)/SPECS/zrepl.spec
+
+	# see /usr/lib/rpm/platform
+ifeq ($(GOARCH),amd64)
+	$(eval _ZREPL_RPMBUILD_TARGET := x86_64)
+else ifeq ($(GOARCH), 386)
+	$(eval _ZREPL_RPMBUILD_TARGET := i386)
+else ifeq ($(GOARCH), arm64)
+	$(eval _ZREPL_RPMBUILD_TARGET := aarch64)
+else ifeq ($(GOARCH), arm)
+	$(eval _ZREPL_RPMBUILD_TARGET := armv7hl)
+else
+	$(eval _ZREPL_RPMBUILD_TARGET := $(GOARCH))
+endif
+	rpmbuild \
+		--build-in-place \
+		--define "_sourcedir $(CURDIR)" \
+		--define "_topdir $(_ZREPL_RPM_TOPDIR_ABS)" \
+		--define "_zrepl_binary_filename zrepl-$(ZREPL_TARGET_TUPLE)" \
+		--target $(_ZREPL_RPMBUILD_TARGET) \
+		-bb "$(_ZREPL_RPM_TOPDIR_ABS)"/SPECS/zrepl.spec
+	cp "$(_ZREPL_RPM_TOPDIR_ABS)"/RPMS/$(_ZREPL_RPMBUILD_TARGET)/zrepl-$(_ZREPL_RPM_VERSION)*.rpm $(ARTIFACTDIR)/
+
+rpm-docker:
+	docker build -t zrepl_rpm_pkg --pull -f packaging/rpm/Dockerfile .
+	docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
+		zrepl_rpm_pkg \
+		make rpm GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+
+deb: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
+
+	cp packaging/deb/debian/changelog.template packaging/deb/debian/changelog
+	sed -i 's/DATE_DASH_R_OUTPUT/$(shell date -R)/' packaging/deb/debian/changelog
+	VERSION="$(subst -,.,$(_ZREPL_VERSION))"; \
+		export VERSION="$${VERSION#v}"; \
+		sed -i 's/VERSION/'"$$VERSION"'/' packaging/deb/debian/changelog
+
+ifeq ($(GOARCH), arm)
+	$(eval  DEB_HOST_ARCH := armhf)
+else ifeq ($(GOARCH), 386)
+	$(eval DEB_HOST_ARCH := i386)
+else
+	$(eval DEB_HOST_ARCH := $(GOARCH))
+endif
+
+	export ZREPL_DPKG_ZREPL_BINARY_FILENAME=zrepl-$(ZREPL_TARGET_TUPLE); \
+		dpkg-buildpackage -b --no-sign --host-arch $(DEB_HOST_ARCH)
+	cp ../*.deb artifacts/
+
+deb-docker:
+	docker build -t zrepl_debian_pkg --pull -f packaging/deb/Dockerfile .
+	docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
+		zrepl_debian_pkg \
+		make deb GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+
+# expects `release` target to have run before
+NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
+wrapup-and-checksum:
+	rm -f $(NOARCH_TARBALL)
 	tar --mtime='1970-01-01' --sort=name \
-		--transform 's/$(ARTIFACTDIR)/zrepl-$(ZREPL_VERSION)-noarch/' \
+		--transform 's/$(ARTIFACTDIR)/zrepl-$(_ZREPL_VERSION)-noarch/' \
-		-acf $@ \
+		--transform 's#dist#zrepl-$(_ZREPL_VERSION)-noarch/dist#' \
+		--transform 's#config/samples#zrepl-$(_ZREPL_VERSION)-noarch/config#' \
+		-acf $(NOARCH_TARBALL) \
 		$(ARTIFACTDIR)/docs/html \
 		$(ARTIFACTDIR)/bash_completion \
-		$(ARTIFACTDIR)/go_version.txt
+		$(ARTIFACTDIR)/_zrepl.zsh_completion \
-
+		$(ARTIFACTDIR)/go_env.txt \
-release: $(RELEASE_BINS) $(RELEASE_NOARCH)
+		dist \
+		config/samples
 	rm -rf "$(ARTIFACTDIR)/release"
 	mkdir -p "$(ARTIFACTDIR)/release"
-	cp $^ "$(ARTIFACTDIR)/release"
+	cp -l $(ARTIFACTDIR)/zrepl-* \
+		$(ARTIFACTDIR)/platformtest-* \
+		"$(ARTIFACTDIR)/release"
 	cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
-	@if echo "$(ZREPL_VERSION)" | grep dirty > /dev/null; then\
+
-		echo '[WARN] Do not publish the artifacts, make variable ZREPL_VERSION=$(ZREPL_VERSION) indicates they are dirty!'; \
+check-git-clean:
-		exit 1; \
+	@# note that we use ZREPL_VERSION and not _ZREPL_VERSION because we want to detect the override
-	fi
+	@if git describe --always --dirty 2>/dev/null | grep dirty >/dev/null; then \
+        echo '[INFO] either git reports checkout is dirty or git is not installed or this is not a git checkout'; \
+		if [ "$(ZREPL_VERSION)" = "" ]; then \
+			echo '[WARN] git checkout is dirty and make variable ZREPL_VERSION was not used to override'; \
+			git status; \
+			echo "git diff:";  \
+			git diff | cat; \
+			exit 1; \
+		fi; \
+	fi;
+
+tag-release:
+	test -n "$(ZREPL_TAG_VERSION)" || exit 1
+	git tag -u E27CA5FC -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)"
+
+sign:
+	gpg -u "89BC 5D89 C845 568B F578  B306 CDBD 8EC8 E27C A5FC" \
+		--armor \
+		--detach-sign $(ARTIFACTDIR)/release/sha512sum.txt
 
 clean: docs-clean
 	rm -rf "$(ARTIFACTDIR)"
+
+##################### BINARIES #####################
+.PHONY: bins-all lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin generate-platform-test-list
+
+BINS_ALL_TARGETS := zrepl-bin test-platform-bin vet lint
+GO_SUPPORTS_ILLUMOS := $(shell $(GO) version | gawk -F '.' '/^go version /{split($$0, comps, " "); split(comps[3], v, "."); if (v[1] == "go1" && v[2] >= 13) { print "illumos"; } else { print "noillumos"; }}')
+bins-all:
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=amd64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=386
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=arm	  GOARM=7
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=arm64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=amd64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm     GOARM=7
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=386
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=darwin    GOARCH=amd64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=solaris   GOARCH=amd64
+ifeq ($(GO_SUPPORTS_ILLUMOS), illumos)
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=illumos   GOARCH=amd64
+else ifeq ($(GO_SUPPORTS_ILLUMOS), noillumos)
+	@echo "SKIPPING ILLUMOS BUILD BECAUSE GO VERSION DOESN'T SUPPORT IT"
+else
+	@echo "CANNOT DETERMINE WHETHER GO VERSION SUPPORTS GOOS=illumos"; exit 1
+endif
+
+lint:
+	$(GO_ENV_VARS) $(GOLANGCI_LINT) run ./...
+
+vet:
+	$(GO_ENV_VARS) $(GO) vet $(GO_BUILDFLAGS) ./...
+
+test-go: $(ARTIFACTDIR)
+	rm -f "$(ARTIFACTDIR)/gotest.cover"
+ifeq ($(COVER),1)
+	$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
+		-coverpkg github.com/zrepl/zrepl/... \
+		-covermode atomic \
+		-coverprofile "$(ARTIFACTDIR)/gotest.cover" \
+		./...
+else
+	$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
+		./...
+endif
+
+zrepl-bin:
+	$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(ZREPL_TARGET_TUPLE)"
+
+generate-platform-test-list:
+	$(GO_BUILD) -o $(ARTIFACTDIR)/generate-platform-test-list ./platformtest/tests/gen
+
+COVER_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-cover-$(ZREPL_TARGET_TUPLE)
+cover-platform-bin:
+	$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
+		-c -o "$(COVER_PLATFORM_BIN_PATH)" \
+		-covermode=atomic -cover -coverpkg github.com/zrepl/zrepl/... \
+		./platformtest/harness
+cover-platform:
+	# do not track dependency on cover-platform-bin to allow build of binary outside of test VM
+	export _TEST_PLATFORM_CMD="$(COVER_PLATFORM_BIN_PATH) \
+		-test.coverprofile \"$(ARTIFACTDIR)/platformtest.cover\" \
+		-test.v \
+		__DEVEL--i-heard-you-like-tests"; \
+		$(MAKE) _test-or-cover-platform-impl
+
+TEST_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)
+test-platform-bin:
+	$(GO_BUILD) -o "$(TEST_PLATFORM_BIN_PATH)" ./platformtest/harness
+test-platform:
+	export _TEST_PLATFORM_CMD="\"$(TEST_PLATFORM_BIN_PATH)\""; \
+		$(MAKE) _test-or-cover-platform-impl
+
+ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
+ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
+ZREPL_PLATFORMTEST_MOUNTPOINT := /tmp/zreplplatformtest.pool
+ZREPL_PLATFORMTEST_ZFS_LOG := /tmp/zreplplatformtest.zfs.log
+# ZREPL_PLATFORMTEST_STOP_AND_KEEP := -failure.stop-and-keep-pool
+ZREPL_PLATFORMTEST_ARGS :=
+_test-or-cover-platform-impl: $(ARTIFACTDIR)
+ifndef _TEST_PLATFORM_CMD
+	$(error _TEST_PLATFORM_CMD is undefined, caller 'cover-platform' or 'test-platform' should have defined it)
+endif
+	rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
+	rm -f "$(ARTIFACTDIR)/platformtest.cover"
+	platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
+		$(_TEST_PLATFORM_CMD) \
+		-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
+		-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
+		-mountpoint "$(ZREPL_PLATFORMTEST_MOUNTPOINT)" \
+		$(ZREPL_PLATFORMTEST_STOP_AND_KEEP) \
+		$(ZREPL_PLATFORMTEST_ARGS)
+
+cover-merge: $(ARTIFACTDIR)
+	$(GOCOVMERGE) $(ARTIFACTDIR)/platformtest.cover $(ARTIFACTDIR)/gotest.cover > $(ARTIFACTDIR)/merged.cover
+cover-html: cover-merge
+	$(GO) tool cover -html "$(ARTIFACTDIR)/merged.cover" -o "$(ARTIFACTDIR)/merged.cover.html"
+
+cover-full:
+	test "$$(id -u)" = "0" || echo "MUST RUN AS ROOT" 1>&2
+	$(MAKE) test-go COVER=1
+	$(MAKE) cover-platform-bin
+	$(MAKE) cover-platform
+	$(MAKE) cover-html
+
+##################### DEV TARGETS #####################
+# not part of the build, must do that manually
+.PHONY: generate formatcheck format
+
+generate: generate-platform-test-list
+	protoc -I=replication/logic/pdu --go_out=replication/logic/pdu --go-grpc_out=replication/logic/pdu replication/logic/pdu/pdu.proto
+	protoc -I=rpc/grpcclientidentity/example --go_out=rpc/grpcclientidentity/example/pdu --go-grpc_out=rpc/grpcclientidentity/example/pdu rpc/grpcclientidentity/example/grpcauth.proto
+	$(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./...
+
+GOIMPORTS := goimports -srcdir . -local 'github.com/zrepl/zrepl'
+FINDSRCFILES := find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go'
+
+formatcheck:
+	@# goimports doesn't have a knob to exit with non-zero status code if formatting is needed
+	@# see https://go-review.googlesource.com/c/tools/+/237378
+	@ affectedfiles=$$($(GOIMPORTS) -l $(shell $(FINDSRCFILES)) | tee /dev/stderr | wc -l); test "$$affectedfiles" = 0
+
+format:
+	@ $(GOIMPORTS) -w -d $(shell  $(FINDSRCFILES))
+
+##################### NOARCH #####################
+.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
+
+
+$(ARTIFACTDIR):
+	mkdir -p "$@"
+$(ARTIFACTDIR)/docs: $(ARTIFACTDIR)
+	mkdir -p "$@"
+
+noarch: $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs
+	# pass
+
+$(ARTIFACTDIR)/bash_completion:
+	$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
+	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion bash "$@"
+
+$(ARTIFACTDIR)/_zrepl.zsh_completion:
+	$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
+	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion zsh "$@"
+
+$(ARTIFACTDIR)/go_env.txt:
+	$(GO_ENV_VARS) $(GO) env > $@
+	$(GO) version >> $@
+
+docs: $(ARTIFACTDIR)/docs
+	# https://www.sphinx-doc.org/en/master/man/sphinx-build.html
+	$(MAKE) -C docs \
+		html \
+		BUILDDIR=../artifacts/docs \
+		SPHINXOPTS="-W --keep-going -n"
+
+docs-clean:
+	$(MAKE) -C docs \
+		clean \
+		BUILDDIR=../artifacts/docs

README.md

@@ -1,5 +1,15 @@
+[![GitHub license](https://img.shields.io/github/license/zrepl/zrepl.svg)](https://github.com/zrepl/zrepl/blob/master/LICENSE)
+[![Language: Go](https://img.shields.io/badge/lang-Go-6ad7e5.svg)](https://golang.org/)
+[![User Docs](https://img.shields.io/badge/docs-web-blue.svg)](https://zrepl.github.io)
+[![Support me on Patreon](https://img.shields.io/badge/dynamic/json?color=yellow&label=Patreon&query=data.attributes.patron_count&url=https%3A%2F%2Fwww.patreon.com%2Fapi%2Fcampaigns%2F3095079)](https://patreon.com/zrepl)
+[![Donate via GitHub Sponsors](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&style=flat&color=yellow)](https://github.com/sponsors/problame)
+[![Donate via Liberapay](https://img.shields.io/liberapay/patrons/zrepl.svg?logo=liberapay)](https://liberapay.com/zrepl/donate)
+[![Donate via PayPal](https://img.shields.io/badge/donate-paypal-yellow.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
+[![Twitter](https://img.shields.io/twitter/url/https/github.com/zrepl/zrepl.svg?style=social)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl)
+[![Chat](https://img.shields.io/badge/chat-matrix-blue.svg)](https://matrix.to/#/#zrepl:matrix.org)
+
 # zrepl
-zrepl is a ZFS filesystem backup & replication solution written in Go.
+zrepl is a one-stop ZFS backup & replication solution.
 
 ## User Documentation
 
@@ -12,52 +22,92 @@ zrepl is a ZFS filesystem backup & replication solution written in Go.
 
 ## Feature Requests
 
-1. Does you feature request require default values / some kind of configuration?
+1. Does your feature request require default values / some kind of configuration?
    If so, think of an expressive configuration example.
 2. Think of at least one use case that generalizes from your concrete application.
 3. Open an issue on GitHub with example conf & use case attached.
+4. **Optional**: [Post a bounty](https://www.bountysource.com/teams/zrepl) on the issue, or [contact Christian Schwarz](https://cschwarz.com) for contract work.
 
 The above does not apply if you already implemented everything.
 Check out the *Coding Workflow* section below for details.
 
-## Package Maintainer Information
+## Building, Releasing, Downstream-Packaging
 
-* Follow the steps in `docs/installation.rst -> Compiling from Source` and read the Makefile / shell scripts used in this process.
+This section provides an overview of the zrepl build & release process.
-* Make sure your distro is compatible with the paths in `docs/installation.rst`.
+Check out `docs/installation/compile-from-source.rst` for build-from-source instructions.
+
+### Overview
+
+zrepl is written in [Go](https://golang.org) and uses [Go modules](https://github.com/golang/go/wiki/Modules) to manage dependencies.
+The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
+
+Install **build dependencies** using  `./lazy.sh devsetup`.
+`lazy.sh` uses `python3-pip` to fetch the build dependencies for the docs - you might want to use a [venv](https://docs.python.org/3/library/venv.html).
+If you just want to install the Go dependencies, run `./lazy.sh godep`.
+
+The **test suite** is split into pure **Go tests** (`make test-go`) and **platform tests** that interact with ZFS and thus generally **require root privileges** (`sudo make test-platform`).
+Platform tests run on their own pool with the name `zreplplatformtest`, which is created using the file vdev in `/tmp`.
+
+For a full **code coverage** profile, run `make test-go COVER=1 && sudo make test-platform && make cover-merge`.
+An HTML report can be generated using `make cover-html`.
+
+**Code generation** is triggered by `make generate`. Generated code is committed to the source tree.
+
+### Build & Release Process
+
+**The `Makefile` is catering to the needs of developers & CI, not distro packagers**.
+It provides phony targets for
+* local development (building, running tests, etc)
+* building a release in Docker (used by the CI & release management)
+* building .deb and .rpm packages out of the release artifacts.
+
+**Build tooling & dependencies** are documented as code in `lazy.sh`.
+Go dependencies are then fetched by the go command and pip dependencies are pinned through a `requirements.txt`.
+
+**We use CircleCI for continuous integration**.
+There are two workflows:
+
+* `ci` runs for every commit / branch / tag pushed to GitHub.
+  It is supposed to run very fast (<5min and provides quick feedback to developers).
+  It runs formatting checks, lints and tests on the most important OSes / architectures.
+  Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
+
+* `release` runs
+  * on manual triggers through the CircleCI API (in order to produce a release)
+  * periodically on `master`
+  Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
+
+**Releases** are issued via Git tags + GitHub Releases feature.
+The procedure to issue a release is as follows:
+* Issue the source release:
+  * Git tag the release on the `master` branch.
+  * Push the tag.
+  * Run `./docs/publish.sh` to re-build & push zrepl.github.io.
+* Issue the official binary release:
+  * Run the `release` pipeline (triggered via CircleCI API)
+  * Download the artifacts to the release manager's machine.
+  * Create a GitHub release, edit the changelog, upload all the release artifacts, including .rpm and .deb files.
+  * Issue the GitHub release.
+  * Add the .rpm and .deb files to the official zrepl repos, publish those.
+
+**Official binary releases are not re-built when Go receives an update. If the Go update is critical to zrepl (e.g. a Go security update that affects zrepl), we'd issue a new source release**.
+The rationale for this is that whereas distros provide a mechanism for this (`$zrepl_source_release-$distro_package_revision`), GitHub Releases doesn't which means we'd need to update the existing GitHub release's assets, which nobody would notice (no RSS feed updates, etc.).
+Downstream packagers can read the changelog to determine whether they want to push that minor release into their distro or simply skip it.
+
+### Additional Notes to Distro Package Maintainers
+
+* Run the platform tests (Docs -> Usage -> Platform Tests) **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
 * Ship a default config that adheres to your distro's `hier` and logging system.
 * Ship a service manager file and _please_ try to upstream it to this repository.
-* Use `make release ZREPL_VERSION='mydistro-1.2.3_1'`
+  * `dist/systemd` contains a Systemd unit template.
-    * Your distro's name and any versioning supplemental to zrepl's (e.g. package revision) should be in this string
+* Ship other material provided in `./dist`, e.g. in `/usr/share/zrepl/`.
+* Have a look at the `Makefile`'s `ZREPL_VERSION` variable and how it passed to Go's `ldFlags`.
+  This is how `zrepl version` knows what version number to show.
+  Your build system should set the `ldFlags` flags appropriately and add a prefix or suffix that indicates that the given zrepl binary is a distro build, not an official one.
 * Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
 
-## Developer Documentation
 
-First, use `./lazy.sh devsetup` to install build dependencies and read `docs/installation.rst -> Compiling from Source`.
+## Contributing Code
-
-### Overall Architecture
-
-The application architecture is documented as part of the user docs in the *Implementation* section (`docs/content/impl`).
-Make sure to develop an understanding how zrepl is typically used by studying the user docs first.
-
-### Project Structure
-
-```
-├── cmd
-│   ├── sampleconf          # example configuration
-├── docs                    # sphinx-based documentation
-│   ├── **/*.rst            # documentation in reStructuredText
-│   ├── sphinxconf
-│   │   └── conf.py         # sphinx config (see commit 445a280 why its not in docs/)
-│   ├── requirements.txt    # pip3 requirements to build documentation
-│   ├── publish.sh          # shell script for automated rendering & deploy to zrepl.github.io repo
-│   ├── public_git          # checkout of zrepl.github.io managed by above shell script
-├── logger                  # logger package used by zrepl
-├── rpc                     # rpc protocol implementation
-├── util
-└── zfs                     # ZFS wrappers, filesystemm diffing
-```
-
-### Coding Workflow
 
 * Open an issue when starting to hack on a new feature
 * Commits should reference the issue they are related to
@@ -67,9 +117,6 @@ Make sure to develop an understanding how zrepl is typically used by studying th
 
 Backward-incompatible changes must be documented in the git commit message and are listed in `docs/changelog.rst`.
 
-* Config-breaking changes must contain a line `BREAK CONFIG` in the commit message
-* Other breaking changes must contain a line `BREAK` in the commit message
-
 ### Glossary & Naming Inconsistencies
 
 In ZFS, *dataset* refers to the objects *filesystem*, *ZVOL* and *snapshot*. <br />
@@ -86,4 +133,3 @@ variables and types are often named *dataset* when they in fact refer to a *file
 There will not be a big refactoring (an attempt was made, but it's destroying too much history without much gain).
 
 However, new contributions & patches should fix naming without further notice in the commit message.
-

build.Dockerfile

@@ -1,17 +1,30 @@
-FROM golang:latest
+FROM !SUBSTITUTED_BY_MAKEFILE
 
 RUN apt-get update && apt-get install -y \
-    python3-pip
+    python3-pip \
+    unzip \
+    gawk
+
+ADD build.installprotoc.bash ./
+RUN bash build.installprotoc.bash
 
 ADD lazy.sh /tmp/lazy.sh
 ADD docs/requirements.txt /tmp/requirements.txt
 ENV ZREPL_LAZY_DOCS_REQPATH=/tmp/requirements.txt
-RUN /tmp/lazy.sh devsetup
+RUN /tmp/lazy.sh docdep
 
 # prepare volume mount of git checkout to /zrepl
-RUN mkdir -p /go/src/github.com/zrepl/zrepl
+RUN mkdir -p /src/github.com/zrepl/zrepl
-RUN chmod -R 0777 /go
 RUN mkdir -p /.cache && chmod -R 0777 /.cache
-WORKDIR /go/src/github.com/zrepl/zrepl
 
+# $GOPATH is /go
+# Go 1.12 doesn't use modules within GOPATH, but 1.13 and later do
+# => store source outside of GOPATH
+WORKDIR /src
+
+# Install build tools (e.g. protobuf generator, stringer) into $GOPATH/bin
+ADD build/ /tmp/build
+RUN /tmp/lazy.sh godep
+
+RUN chmod -R 0777 /go
 

build.installprotoc.bash

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+
+MACH=$(uname -m)
+MACH="${MACH/aarch64/aarch_64}"
+
+VERSION=3.6.1
+FILENAME=protoc-"$VERSION"-linux-"$MACH".zip
+
+if [ -e "$FILENAME" ]; then
+    echo "$FILENAME" already exists 1>&2
+    exit 1
+fi
+
+wget https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
+
+stat "$FILENAME"
+
+sha256sum -c --ignore-missing <<EOF
+6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807  protoc-3.6.1-linux-x86_64.zip
+af8e5aaaf39ddec62ec8dd2be1b8d9602c6da66564883a16393ade5f71170922  protoc-3.6.1-linux-aarch_64.zip
+EOF
+
+unzip -d /usr "$FILENAME"

build/go.mod

@@ -0,0 +1,13 @@
+module github.com/zrepl/zrepl/build
+
+go 1.12
+
+require (
+	github.com/alvaroloes/enumer v1.1.1
+	github.com/golangci/golangci-lint v1.50.1
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad
+	golang.org/x/tools v0.2.0
+	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 // indirect
+	google.golang.org/protobuf v1.28.0
+)

build/tools.go

@@ -0,0 +1,15 @@
+//go:build tools
+// +build tools
+
+package main
+
+// the lines are parsed by lazy.sh, do not edit
+import (
+	_ "github.com/alvaroloes/enumer"
+	_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
+	_ "github.com/wadey/gocovmerge"
+	_ "golang.org/x/tools/cmd/goimports"
+	_ "golang.org/x/tools/cmd/stringer"
+	_ "google.golang.org/grpc/cmd/protoc-gen-go-grpc"
+	_ "google.golang.org/protobuf/cmd/protoc-gen-go"
+)

cli/cli.go

@@ -0,0 +1,155 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+
+	"github.com/zrepl/zrepl/config"
+)
+
+var rootArgs struct {
+	configPath string
+}
+
+var rootCmd = &cobra.Command{
+	Use:   "zrepl",
+	Short: "One-stop ZFS replication solution",
+}
+
+func init() {
+	rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
+}
+
+var genCompletionCmd = &cobra.Command{
+	Use:   "gencompletion",
+	Short: "generate shell auto-completions",
+}
+
+type completionCmdInfo struct {
+	genFunc func(outpath string) error
+	help    string
+}
+
+var completionCmdMap = map[string]completionCmdInfo{
+	"zsh": {
+		rootCmd.GenZshCompletionFile,
+		"  save to file `_zrepl` in your zsh's $fpath",
+	},
+	"bash": {
+		rootCmd.GenBashCompletionFile,
+		"  save to a path and source that path in your .bashrc",
+	},
+}
+
+func init() {
+	for sh, info := range completionCmdMap {
+		sh, info := sh, info
+		genCompletionCmd.AddCommand(&cobra.Command{
+			Use:     fmt.Sprintf("%s path/to/out/file", sh),
+			Short:   fmt.Sprintf("generate %s completions", sh),
+			Example: info.help,
+			Run: func(cmd *cobra.Command, args []string) {
+				if len(args) != 1 {
+					fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
+					err := cmd.Usage()
+					if err != nil {
+						panic(err)
+					}
+					os.Exit(1)
+				}
+				if err := info.genFunc(args[0]); err != nil {
+					fmt.Fprintf(os.Stderr, "error generating %s completion: %s", sh, err)
+					os.Exit(1)
+				}
+			},
+		})
+	}
+	rootCmd.AddCommand(genCompletionCmd)
+}
+
+type Subcommand struct {
+	Use              string
+	Short            string
+	Example          string
+	NoRequireConfig  bool
+	Run              func(ctx context.Context, subcommand *Subcommand, args []string) error
+	SetupFlags       func(f *pflag.FlagSet)
+	SetupSubcommands func() []*Subcommand
+
+	config    *config.Config
+	configErr error
+}
+
+func (s *Subcommand) ConfigParsingError() error {
+	return s.configErr
+}
+
+func (s *Subcommand) Config() *config.Config {
+	if !s.NoRequireConfig && s.config == nil {
+		panic("command that requires config is running and has no config set")
+	}
+	return s.config
+}
+
+func (s *Subcommand) run(cmd *cobra.Command, args []string) {
+	s.tryParseConfig()
+	ctx := context.Background()
+	endTask := trace.WithTaskFromStackUpdateCtx(&ctx)
+	defer endTask()
+	err := s.Run(ctx, s, args)
+	endTask()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+		os.Exit(1)
+	}
+}
+
+func (s *Subcommand) tryParseConfig() {
+	config, err := config.ParseConfig(rootArgs.configPath)
+	s.configErr = err
+	if err != nil {
+		if s.NoRequireConfig {
+			// doesn't matter
+			return
+		} else {
+			fmt.Fprintf(os.Stderr, "could not parse config: %s\n", err)
+			os.Exit(1)
+		}
+	}
+	s.config = config
+}
+
+func AddSubcommand(s *Subcommand) {
+	addSubcommandToCobraCmd(rootCmd, s)
+}
+
+func addSubcommandToCobraCmd(c *cobra.Command, s *Subcommand) {
+	cmd := cobra.Command{
+		Use:     s.Use,
+		Short:   s.Short,
+		Example: s.Example,
+	}
+	if s.SetupSubcommands == nil {
+		cmd.Run = s.run
+	} else {
+		for _, sub := range s.SetupSubcommands() {
+			addSubcommandToCobraCmd(&cmd, sub)
+		}
+	}
+	if s.SetupFlags != nil {
+		s.SetupFlags(cmd.Flags())
+	}
+	c.AddCommand(&cmd)
+}
+
+func Run() {
+	if err := rootCmd.Execute(); err != nil {
+		os.Exit(1)
+	}
+}

client/configcheck.go

@@ -0,0 +1,131 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/kr/pretty"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+	"github.com/zrepl/yaml-config"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/logger"
+)
+
+var configcheckArgs struct {
+	format        string
+	what          string
+	skipCertCheck bool
+}
+
+var ConfigcheckCmd = &cli.Subcommand{
+	Use:   "configcheck",
+	Short: "check if config can be parsed without errors",
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]")
+		f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]")
+		f.BoolVar(&configcheckArgs.skipCertCheck, "skip-cert-check", false, "skip checking cert files")
+	},
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		formatMap := map[string]func(interface{}){
+			"": func(i interface{}) {},
+			"pretty": func(i interface{}) {
+				if _, err := pretty.Println(i); err != nil {
+					panic(err)
+				}
+			},
+			"json": func(i interface{}) {
+				if err := json.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
+					panic(err)
+				}
+			},
+			"yaml": func(i interface{}) {
+				if err := yaml.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
+					panic(err)
+				}
+			},
+		}
+
+		formatter, ok := formatMap[configcheckArgs.format]
+		if !ok {
+			return fmt.Errorf("unsupported --format %q", configcheckArgs.format)
+		}
+
+		var hadErr bool
+
+		parseFlags := config.ParseFlagsNone
+
+		if configcheckArgs.skipCertCheck {
+			parseFlags |= config.ParseFlagsNoCertCheck
+		}
+
+		// further: try to build jobs
+		confJobs, err := job.JobsFromConfig(subcommand.Config(), parseFlags)
+
+		if err != nil {
+			err := errors.Wrap(err, "cannot build jobs from config")
+			if configcheckArgs.what == "jobs" {
+				return err
+			} else {
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				confJobs = nil
+				hadErr = true
+			}
+		}
+
+		// further: try to build logging outlets
+		outlets, err := logging.OutletsFromConfig(*subcommand.Config().Global.Logging)
+		if err != nil {
+			err := errors.Wrap(err, "cannot build logging from config")
+			if configcheckArgs.what == "logging" {
+				return err
+			} else {
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				outlets = nil
+				hadErr = true
+			}
+		}
+
+		whatMap := map[string]func(){
+			"all": func() {
+				o := struct {
+					config  *config.Config
+					jobs    []job.Job
+					logging *logger.Outlets
+				}{
+					subcommand.Config(),
+					confJobs,
+					outlets,
+				}
+				formatter(o)
+			},
+			"config": func() {
+				formatter(subcommand.Config())
+			},
+			"jobs": func() {
+				formatter(confJobs)
+			},
+			"logging": func() {
+				formatter(outlets)
+			},
+		}
+
+		wf, ok := whatMap[configcheckArgs.what]
+		if !ok {
+			return fmt.Errorf("unsupported --format %q", configcheckArgs.what)
+		}
+		wf()
+
+		if hadErr {
+			return fmt.Errorf("config parsing failed")
+		} else {
+			return nil
+		}
+	},
+}

client/jsonclient.go

@@ -0,0 +1,49 @@
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net"
+	"net/http"
+
+	"github.com/pkg/errors"
+)
+
+func controlHttpClient(sockpath string) (client http.Client, err error) {
+	return http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", sockpath)
+			},
+		},
+	}, nil
+}
+
+func jsonRequestResponse(c http.Client, endpoint string, req interface{}, res interface{}) error {
+	var buf bytes.Buffer
+	encodeErr := json.NewEncoder(&buf).Encode(req)
+	if encodeErr != nil {
+		return encodeErr
+	}
+
+	resp, err := c.Post("http://unix"+endpoint, "application/json", &buf)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		var msg bytes.Buffer
+		_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
+		return errors.Errorf("%s", msg.String())
+	}
+
+	decodeError := json.NewDecoder(resp.Body).Decode(&res)
+	if decodeError != nil {
+		return decodeError
+	}
+
+	return nil
+}

client/migrate.go

@@ -0,0 +1,268 @@
+package client
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/fatih/color"
+	"github.com/kr/pretty"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+)
+
+var (
+	MigrateCmd = &cli.Subcommand{
+		Use:   "migrate",
+		Short: "perform migration of the on-disk / zfs properties",
+		SetupSubcommands: func() []*cli.Subcommand {
+			return migrations
+		},
+	}
+)
+
+var migrations = []*cli.Subcommand{
+	&cli.Subcommand{
+		Use: "0.0.X:0.1:placeholder",
+		Run: doMigratePlaceholder0_1,
+		SetupFlags: func(f *pflag.FlagSet) {
+			f.BoolVar(&migratePlaceholder0_1Args.dryRun, "dry-run", false, "dry run")
+		},
+	},
+	&cli.Subcommand{
+		Use: "replication-cursor:v1-v2",
+		Run: doMigrateReplicationCursor,
+		SetupFlags: func(f *pflag.FlagSet) {
+			f.BoolVar(&migrateReplicationCursorArgs.dryRun, "dry-run", false, "dry run")
+		},
+	},
+}
+
+var migratePlaceholder0_1Args struct {
+	dryRun bool
+}
+
+func doMigratePlaceholder0_1(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	if len(args) != 0 {
+		return fmt.Errorf("migration does not take arguments, got %v", args)
+	}
+
+	cfg := sc.Config()
+
+	allFSS, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
+	if err != nil {
+		return errors.Wrap(err, "cannot list filesystems")
+	}
+
+	type workItem struct {
+		jobName string
+		rootFS  *zfs.DatasetPath
+		fss     []*zfs.DatasetPath
+	}
+	var wis []workItem
+	for i, j := range cfg.Jobs {
+		var rfsS string
+		switch job := j.Ret.(type) {
+		case *config.SinkJob:
+			rfsS = job.RootFS
+		case *config.PullJob:
+			rfsS = job.RootFS
+		default:
+			fmt.Printf("ignoring job %q (%d/%d, type %T)\n", j.Name(), i, len(cfg.Jobs), j.Ret)
+			continue
+		}
+		rfs, err := zfs.NewDatasetPath(rfsS)
+		if err != nil {
+			return errors.Wrapf(err, "root fs for job %q is not a valid dataset path", j.Name())
+		}
+		var fss []*zfs.DatasetPath
+		for _, fs := range allFSS {
+			if fs.HasPrefix(rfs) {
+				fss = append(fss, fs)
+			}
+		}
+		wis = append(wis, workItem{j.Name(), rfs, fss})
+	}
+
+	for _, wi := range wis {
+		fmt.Printf("job %q => migrate filesystems below root_fs %q\n", wi.jobName, wi.rootFS.ToString())
+		if len(wi.fss) == 0 {
+			fmt.Printf("\tno filesystems\n")
+			continue
+		}
+		for _, fs := range wi.fss {
+			fmt.Printf("\t%q ... ", fs.ToString())
+			r, err := zfs.ZFSMigrateHashBasedPlaceholderToCurrent(ctx, fs, migratePlaceholder0_1Args.dryRun)
+			if err != nil {
+				fmt.Printf("error: %s\n", err)
+			} else if !r.NeedsModification {
+				fmt.Printf("unchanged (placeholder=%v)\n", r.OriginalState.IsPlaceholder)
+			} else {
+				fmt.Printf("migrate (placeholder=%v) (old value = %q)\n",
+					r.OriginalState.IsPlaceholder, r.OriginalState.RawLocalPropertyValue)
+			}
+		}
+	}
+
+	return nil
+}
+
+var migrateReplicationCursorArgs struct {
+	dryRun bool
+}
+
+var bold = color.New(color.Bold)
+var succ = color.New(color.FgGreen)
+var fail = color.New(color.FgRed)
+
+var migrateReplicationCursorSkipSentinel = fmt.Errorf("skipping this filesystem")
+
+func doMigrateReplicationCursor(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	if len(args) != 0 {
+		return fmt.Errorf("migration does not take arguments, got %v", args)
+	}
+
+	cfg := sc.Config()
+	jobs, err := job.JobsFromConfig(cfg, config.ParseFlagsNone)
+	if err != nil {
+		fmt.Printf("cannot parse config:\n%s\n\n", err)
+		fmt.Printf("NOTE: this migration was released together with a change in job name requirements.\n")
+		return fmt.Errorf("exiting migration after error")
+	}
+
+	v1cursorJobs := make([]job.Job, 0, len(cfg.Jobs))
+	for i, j := range cfg.Jobs {
+		if jobs[i].Name() != j.Name() {
+			panic("implementation error")
+		}
+		switch j.Ret.(type) {
+		case *config.PushJob:
+			v1cursorJobs = append(v1cursorJobs, jobs[i])
+		case *config.SourceJob:
+			v1cursorJobs = append(v1cursorJobs, jobs[i])
+		default:
+			fmt.Printf("ignoring job %q (%d/%d, type %T), not supposed to create v1 replication cursors\n", j.Name(), i, len(cfg.Jobs), j.Ret)
+			continue
+		}
+	}
+
+	// scan all filesystems for v1 replication cursors
+
+	fss, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
+	if err != nil {
+		return errors.Wrap(err, "list filesystems")
+	}
+
+	var hadError bool
+	for _, fs := range fss {
+
+		bold.Printf("INSPECT FILESYSTEM %q\n", fs.ToString())
+
+		err := doMigrateReplicationCursorFS(ctx, v1cursorJobs, fs)
+		if err == migrateReplicationCursorSkipSentinel {
+			bold.Printf("FILESYSTEM SKIPPED\n")
+		} else if err != nil {
+			hadError = true
+			fail.Printf("MIGRATION FAILED: %s\n", err)
+		} else {
+			succ.Printf("FILESYSTEM %q COMPLETE\n", fs.ToString())
+		}
+	}
+
+	if hadError {
+		fail.Printf("\n\none or more filesystems could not be migrated, please inspect output and or re-run migration")
+		return errors.Errorf("")
+	}
+	return nil
+}
+
+func doMigrateReplicationCursorFS(ctx context.Context, v1CursorJobs []job.Job, fs *zfs.DatasetPath) error {
+
+	var owningJob job.Job = nil
+	for _, job := range v1CursorJobs {
+		conf := job.SenderConfig()
+		if conf == nil {
+			continue
+		}
+		pass, err := conf.FSF.Filter(fs)
+		if err != nil {
+			return errors.Wrapf(err, "filesystem filter error in job %q for fs %q", job.Name(), fs.ToString())
+		}
+		if !pass {
+			continue
+		}
+		if owningJob != nil {
+			return errors.Errorf("jobs %q and %q both match %q\ncannot attribute replication cursor to either one", owningJob.Name(), job.Name(), fs)
+		}
+		owningJob = job
+	}
+	if owningJob == nil {
+		fmt.Printf("no job's Filesystems filter matches\n")
+		return migrateReplicationCursorSkipSentinel
+	}
+	fmt.Printf("identified owning job %q\n", owningJob.Name())
+
+	bookmarks, err := zfs.ZFSListFilesystemVersions(ctx, fs, zfs.ListFilesystemVersionsOptions{
+		Types: zfs.Bookmarks,
+	})
+	if err != nil {
+		return errors.Wrapf(err, "list filesystem versions of %q", fs.ToString())
+	}
+
+	var oldCursor *zfs.FilesystemVersion
+	for i, fsv := range bookmarks {
+		_, _, err := endpoint.ParseReplicationCursorBookmarkName(fsv.ToAbsPath(fs))
+		if err != endpoint.ErrV1ReplicationCursor {
+			continue
+		}
+
+		if oldCursor != nil {
+			fmt.Printf("unexpected v1 replication cursor candidate: %q", fsv.ToAbsPath(fs))
+			return errors.Wrap(err, "multiple filesystem versions identified as v1 replication cursors")
+		}
+
+		oldCursor = &bookmarks[i]
+
+	}
+
+	if oldCursor == nil {
+		bold.Printf("no v1 replication cursor found for filesystem %q\n", fs.ToString())
+		return migrateReplicationCursorSkipSentinel
+	}
+
+	fmt.Printf("found v1 replication cursor:\n%s\n", pretty.Sprint(oldCursor))
+
+	mostRecentNew, err := endpoint.GetMostRecentReplicationCursorOfJob(ctx, fs.ToString(), owningJob.SenderConfig().JobID)
+	if err != nil {
+		return errors.Wrapf(err, "get most recent v2 replication cursor")
+	}
+
+	if mostRecentNew == nil {
+		return errors.Errorf("no v2 replication cursor found for job %q on filesystem %q", owningJob.SenderConfig().JobID, fs.ToString())
+	}
+
+	fmt.Printf("most recent v2 replication cursor:\n%#v", oldCursor)
+
+	if !(mostRecentNew.CreateTXG >= oldCursor.CreateTXG) {
+		return errors.Errorf("v1 replication cursor createtxg is higher than v2 cursor's, skipping this filesystem")
+	}
+
+	fmt.Printf("determined that v2 cursor is bookmark of same or newer version than v1 cursor\n")
+	fmt.Printf("destroying v1 cursor %q\n", oldCursor.ToAbsPath(fs))
+
+	if migrateReplicationCursorArgs.dryRun {
+		succ.Printf("DRY RUN\n")
+	} else {
+		if err := zfs.ZFSDestroyFilesystemVersion(ctx, fs, oldCursor); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

client/migrate_test.go

@@ -0,0 +1,16 @@
+package client
+
+import "testing"
+
+func TestMigrationsUnambiguousNames(t *testing.T) {
+	names := make(map[string]bool)
+	for _, mig := range migrations {
+		if _, ok := names[mig.Use]; ok {
+			t.Errorf("duplicate migration name %q", mig.Use)
+			t.FailNow()
+			return
+		} else {
+			names[mig.Use] = true
+		}
+	}
+}

client/pprof.go

@@ -0,0 +1,10 @@
+package client
+
+import "github.com/zrepl/zrepl/cli"
+
+var PprofCmd = &cli.Subcommand{
+	Use: "pprof",
+	SetupSubcommands: func() []*cli.Subcommand {
+		return []*cli.Subcommand{PprofListenCmd, pprofActivityTraceCmd}
+	},
+}

client/pprof_activity_trace.go

@@ -0,0 +1,44 @@
+package client
+
+import (
+	"context"
+	"io"
+	"log"
+	"os"
+
+	"golang.org/x/net/websocket"
+
+	"github.com/zrepl/zrepl/cli"
+)
+
+var pprofActivityTraceCmd = &cli.Subcommand{
+	Use:   "activity-trace ZREPL_PPROF_HOST:ZREPL_PPROF_PORT",
+	Short: "attach to zrepl daemon with activated pprof listener and dump an activity-trace to stdout",
+	Run:   runPProfActivityTrace,
+}
+
+func runPProfActivityTrace(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+	log := log.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Printf("exiting after error")
+		os.Exit(1)
+	}
+
+	if len(args) != 1 {
+		log.Printf("exactly one positional argument is required")
+		die()
+	}
+
+	url := "ws://" + args[0] + "/debug/zrepl/activity-trace" // FIXME dont' repeat that
+
+	log.Printf("attaching to activity trace stream %s", url)
+	ws, err := websocket.Dial(url, "", url)
+	if err != nil {
+		log.Printf("error: %s", err)
+		die()
+	}
+
+	_, err = io.Copy(os.Stdout, ws)
+	return err
+}

client/pprof_listen.go

@@ -0,0 +1,68 @@
+package client
+
+import (
+	"context"
+	"errors"
+	"log"
+	"os"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+)
+
+var pprofListenCmd struct {
+	daemon.PprofServerControlMsg
+}
+
+var PprofListenCmd = &cli.Subcommand{
+	Use:   "listen off | [on TCP_LISTEN_ADDRESS]",
+	Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		if len(args) < 1 {
+			goto enargs
+		}
+		switch args[0] {
+		case "on":
+			pprofListenCmd.Run = true
+			if len(args) != 2 {
+				return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
+			}
+			pprofListenCmd.HttpListenAddress = args[1]
+		case "off":
+			if len(args) != 1 {
+				goto enargs
+			}
+			pprofListenCmd.Run = false
+		}
+
+		RunPProf(subcommand.Config())
+		return nil
+	enargs:
+		return errors.New("invalid number of positional arguments")
+
+	},
+}
+
+func RunPProf(conf *config.Config) {
+	log := log.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Printf("exiting after error")
+		os.Exit(1)
+	}
+
+	log.Printf("connecting to zrepl daemon")
+
+	httpc, err := controlHttpClient(conf.Global.Control.SockPath)
+	if err != nil {
+		log.Printf("error creating http client: %s", err)
+		die()
+	}
+	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofListenCmd.PprofServerControlMsg, struct{}{})
+	if err != nil {
+		log.Printf("error sending control message: %s", err)
+		die()
+	}
+	log.Printf("finished")
+}

client/signal.go

@@ -0,0 +1,42 @@
+package client
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+)
+
+var SignalCmd = &cli.Subcommand{
+	Use:   "signal [wakeup|reset] JOB",
+	Short: "wake up a job from wait state or abort its current invocation",
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		return runSignalCmd(subcommand.Config(), args)
+	},
+}
+
+func runSignalCmd(config *config.Config, args []string) error {
+	if len(args) != 2 {
+		return errors.Errorf("Expected 2 arguments: [wakeup|reset] JOB")
+	}
+
+	httpc, err := controlHttpClient(config.Global.Control.SockPath)
+	if err != nil {
+		return err
+	}
+
+	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointSignal,
+		struct {
+			Name string
+			Op   string
+		}{
+			Name: args[1],
+			Op:   args[0],
+		},
+		struct{}{},
+	)
+	return err
+}

client/status/client/client.go

@@ -0,0 +1,111 @@
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net"
+	"net/http"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/daemon"
+)
+
+type Client struct {
+	h *http.Client
+}
+
+func New(network, addr string) (*Client, error) {
+	httpc, err := makeControlHttpClient(func(_ context.Context) (net.Conn, error) { return net.Dial(network, addr) })
+	if err != nil {
+		return nil, err
+	}
+	return &Client{httpc}, nil
+}
+
+func (c *Client) Status() (s daemon.Status, _ error) {
+	err := jsonRequestResponse(c.h, daemon.ControlJobEndpointStatus,
+		struct{}{},
+		&s,
+	)
+	return s, err
+}
+
+func (c *Client) StatusRaw() ([]byte, error) {
+	var r json.RawMessage
+	err := jsonRequestResponse(c.h, daemon.ControlJobEndpointStatus, struct{}{}, &r)
+	if err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+
+func (c *Client) signal(job, sig string) error {
+	return jsonRequestResponse(c.h, daemon.ControlJobEndpointSignal,
+		struct {
+			Name string
+			Op   string
+		}{
+			Name: job,
+			Op:   sig,
+		},
+		struct{}{},
+	)
+}
+
+func (c *Client) SignalWakeup(job string) error {
+	return c.signal(job, "wakeup")
+}
+
+func (c *Client) SignalReset(job string) error {
+	return c.signal(job, "reset")
+}
+
+func makeControlHttpClient(dialfunc func(context.Context) (net.Conn, error)) (client *http.Client, err error) {
+	return &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+				return dialfunc(ctx)
+			},
+		},
+	}, nil
+}
+
+func jsonRequestResponse(c *http.Client, endpoint string, req interface{}, res interface{}) error {
+	var buf bytes.Buffer
+	encodeErr := json.NewEncoder(&buf).Encode(req)
+	if encodeErr != nil {
+		return encodeErr
+	}
+
+	hreq, err := http.NewRequest("POST", "http://unix"+endpoint, &buf)
+	if err != nil {
+		return err
+	}
+	hreq.Header.Set("Content-Type", "application/json")
+	// Prevent EOF errors when client request frequency and server keepalive close are at the same time.
+	// Found this by watching http.Server.ConnState changes, then found
+	// https://stackoverflow.com/questions/17714494/golang-http-request-results-in-eof-errors-when-making-multiple-requests-successi
+	// Note: The issue seems even more prounounced with local TCP sockets than unix domain sockets. So, I used that for debugging.
+	hreq.Close = true
+	resp, err := c.Do(hreq)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		var msg bytes.Buffer
+		_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
+		return errors.Errorf("%s", msg.String())
+	}
+
+	decodeError := json.NewDecoder(resp.Body).Decode(&res)
+	if decodeError != nil {
+		return decodeError
+	}
+
+	return nil
+}

client/status/status.go

@@ -0,0 +1,97 @@
+package status
+
+import (
+	"context"
+	"os"
+	"time"
+
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/client/status/client"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+	"github.com/zrepl/zrepl/util/choices"
+)
+
+type Client interface {
+	Status() (daemon.Status, error)
+	StatusRaw() ([]byte, error)
+	SignalWakeup(job string) error
+	SignalReset(job string) error
+}
+
+type statusFlags struct {
+	Mode  choices.Choices
+	Job   string
+	Delay time.Duration
+}
+
+var statusv2Flags statusFlags
+
+type statusv2Mode int
+
+const (
+	StatusV2ModeInteractive statusv2Mode = 1 + iota
+	StatusV2ModeDump
+	StatusV2ModeRaw
+	StatusV2ModeLegacy
+)
+
+var Subcommand = &cli.Subcommand{
+	Use:   "status",
+	Short: "retrieve & display daemon status information",
+	SetupFlags: func(f *pflag.FlagSet) {
+		statusv2Flags.Mode.Init(
+			"interactive", StatusV2ModeInteractive,
+			"dump", StatusV2ModeDump,
+			"raw", StatusV2ModeRaw,
+			"legacy", StatusV2ModeLegacy,
+		)
+		statusv2Flags.Mode.SetTypeString("mode")
+		statusv2Flags.Mode.SetDefaultValue(StatusV2ModeInteractive)
+		f.Var(&statusv2Flags.Mode, "mode", statusv2Flags.Mode.Usage())
+		f.StringVar(&statusv2Flags.Job, "job", "", "only show specified job (works in \"dump\" and \"interactive\" mode)")
+		f.DurationVarP(&statusv2Flags.Delay, "delay", "d", 1*time.Second, "use -d 3s for 3 seconds delay (minimum delay is 1s)")
+	},
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		return runStatusV2Command(ctx, subcommand.Config(), args)
+	},
+}
+
+func runStatusV2Command(ctx context.Context, config *config.Config, args []string) error {
+
+	c, err := client.New("unix", config.Global.Control.SockPath)
+	if err != nil {
+		return errors.Wrapf(err, "connect to daemon socket at %q", config.Global.Control.SockPath)
+	}
+
+	mode := statusv2Flags.Mode.Value().(statusv2Mode)
+
+	if !isatty.IsTerminal(os.Stdout.Fd()) && mode != StatusV2ModeDump && mode != StatusV2ModeRaw {
+		dumpmode, err := statusv2Flags.Mode.InputForChoice(StatusV2ModeDump)
+		if err != nil {
+			panic(err)
+		}
+		rawmode, err := statusv2Flags.Mode.InputForChoice(StatusV2ModeRaw)
+		if err != nil {
+			panic(err)
+		}
+		return errors.Errorf("error: stdout is not a tty, please use --mode %s or --mode %s", dumpmode, rawmode)
+	}
+
+	switch mode {
+	case StatusV2ModeInteractive:
+		return interactive(c, statusv2Flags)
+	case StatusV2ModeDump:
+		return dump(c, statusv2Flags.Job)
+	case StatusV2ModeRaw:
+		return raw(c)
+	case StatusV2ModeLegacy:
+		return legacy(c, statusv2Flags)
+	default:
+		panic("unreachable")
+	}
+}

client/status/status_dump.go

@@ -0,0 +1,70 @@
+package status
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/gdamore/tcell"
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/client/status/viewmodel"
+)
+
+func dump(c Client, job string) error {
+	s, err := c.Status()
+	if err != nil {
+		return err
+	}
+
+	if job != "" {
+		if _, ok := s.Jobs[job]; !ok {
+			return errors.Errorf("job %q not found", job)
+		}
+	}
+
+	width := (1 << 31) - 1
+	wrap := false
+	hline := strings.Repeat("-", 80)
+	if isatty.IsTerminal(os.Stdout.Fd()) {
+		wrap = true
+		screen, err := tcell.NewScreen()
+		if err != nil {
+			return errors.Wrap(err, "get terminal dimensions")
+		}
+		if err := screen.Init(); err != nil {
+			return errors.Wrap(err, "init screen")
+		}
+		width, _ = screen.Size()
+		screen.Fini()
+		hline = strings.Repeat("-", width)
+	}
+
+	m := viewmodel.New()
+	params := viewmodel.Params{
+		Report:                  s.Jobs,
+		ReportFetchError:        nil,
+		SelectedJob:             nil,
+		FSFilter:                func(s string) bool { return true },
+		DetailViewWidth:         width,
+		DetailViewWrap:          wrap,
+		ShortKeybindingOverview: "",
+	}
+	m.Update(params)
+	for _, j := range m.Jobs() {
+		if job != "" && j.Name() != job {
+			continue
+		}
+		params.SelectedJob = j
+		m.Update(params)
+		fmt.Println(m.SelectedJob().FullDescription())
+		if job != "" {
+			return nil
+		} else {
+			fmt.Println(hline)
+		}
+	}
+
+	return nil
+}

client/status/status_interactive.go

@@ -0,0 +1,347 @@
+package status
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gdamore/tcell/v2"
+	tview "gitlab.com/tslocum/cview"
+
+	"github.com/zrepl/zrepl/client/status/viewmodel"
+)
+
+func interactive(c Client, flag statusFlags) error {
+
+	// Set this so we don't overwrite the default terminal colors
+	// See https://github.com/rivo/tview/blob/master/styles.go
+	tview.Styles.PrimitiveBackgroundColor = tcell.ColorDefault
+	tview.Styles.ContrastBackgroundColor = tcell.ColorDefault
+	tview.Styles.PrimaryTextColor = tcell.ColorDefault
+	tview.Styles.BorderColor = tcell.ColorDefault
+	app := tview.NewApplication()
+
+	jobDetailSplit := tview.NewFlex()
+	jobMenu := tview.NewTreeView()
+	jobMenuRoot := tview.NewTreeNode("jobs")
+	jobMenuRoot.SetSelectable(true)
+	jobMenu.SetRoot(jobMenuRoot)
+	jobMenu.SetCurrentNode(jobMenuRoot)
+	jobMenu.SetSelectedTextColor(tcell.ColorGreen)
+	jobTextDetail := tview.NewTextView()
+	jobTextDetail.SetWrap(false)
+
+	jobMenu.SetBorder(true)
+	jobTextDetail.SetBorder(true)
+
+	toolbarSplit := tview.NewFlex()
+	toolbarSplit.SetDirection(tview.FlexRow)
+	inputBarContainer := tview.NewFlex()
+	fsFilterInput := tview.NewInputField()
+	fsFilterInput.SetBorder(false)
+	fsFilterInput.SetFieldBackgroundColor(tcell.ColorDefault)
+	inputBarLabel := tview.NewTextView()
+	inputBarLabel.SetText("[::b]FILTER ")
+	inputBarLabel.SetDynamicColors(true)
+	inputBarContainer.AddItem(inputBarLabel, 7, 1, false)
+	inputBarContainer.AddItem(fsFilterInput, 0, 10, false)
+	toolbarSplit.AddItem(inputBarContainer, 1, 0, false)
+	toolbarSplit.AddItem(jobDetailSplit, 0, 10, false)
+
+	bottombar := tview.NewFlex()
+	bottombar.SetDirection(tview.FlexColumn)
+	bottombarDateView := tview.NewTextView()
+	bottombar.AddItem(bottombarDateView, len(time.Now().String()), 0, false)
+	bottomBarStatus := tview.NewTextView()
+	bottomBarStatus.SetDynamicColors(true)
+	bottomBarStatus.SetTextAlign(tview.AlignRight)
+	bottombar.AddItem(bottomBarStatus, 0, 10, false)
+	toolbarSplit.AddItem(bottombar, 1, 0, false)
+
+	tabbableWithJobMenu := []tview.Primitive{jobMenu, jobTextDetail, fsFilterInput}
+	tabbableWithoutJobMenu := []tview.Primitive{jobTextDetail, fsFilterInput}
+	var tabbable []tview.Primitive
+	tabbableActiveIndex := 0
+	tabbableRedraw := func() {
+		if len(tabbable) == 0 {
+			app.SetFocus(nil)
+			return
+		}
+		if tabbableActiveIndex >= len(tabbable) {
+			app.SetFocus(tabbable[0])
+			return
+		}
+		app.SetFocus(tabbable[tabbableActiveIndex])
+	}
+	tabbableCycle := func() {
+		if len(tabbable) == 0 {
+			return
+		}
+		tabbableActiveIndex = (tabbableActiveIndex + 1) % len(tabbable)
+		app.SetFocus(tabbable[tabbableActiveIndex])
+		tabbableRedraw()
+	}
+
+	jobMenuVisisble := false
+	reconfigureJobDetailSplit := func(setJobMenuVisible bool) {
+		if jobMenuVisisble == setJobMenuVisible {
+			return
+		}
+		jobMenuVisisble = setJobMenuVisible
+		if setJobMenuVisible {
+			jobDetailSplit.RemoveItem(jobTextDetail)
+			jobDetailSplit.AddItem(jobMenu, 0, 1, true)
+			jobDetailSplit.AddItem(jobTextDetail, 0, 5, false)
+			tabbable = tabbableWithJobMenu
+		} else {
+			jobDetailSplit.RemoveItem(jobMenu)
+			tabbable = tabbableWithoutJobMenu
+		}
+		tabbableRedraw()
+	}
+
+	showModal := func(m *tview.Modal, modalDoneFunc func(idx int, label string)) {
+		preModalFocus := app.GetFocus()
+		m.SetDoneFunc(func(idx int, label string) {
+			if modalDoneFunc != nil {
+				modalDoneFunc(idx, label)
+			}
+			app.SetRoot(toolbarSplit, true)
+			app.SetFocus(preModalFocus)
+			app.Draw()
+		})
+		app.SetRoot(m, true)
+		app.Draw()
+	}
+
+	app.SetRoot(toolbarSplit, true)
+	// initial focus
+	tabbableActiveIndex = len(tabbable)
+	tabbableCycle()
+	reconfigureJobDetailSplit(true)
+
+	m := viewmodel.New()
+	params := &viewmodel.Params{
+		Report:                  nil,
+		SelectedJob:             nil,
+		FSFilter:                func(_ string) bool { return true },
+		DetailViewWidth:         100,
+		DetailViewWrap:          false,
+		ShortKeybindingOverview: "[::b]Q[::-] quit  [::b]<TAB>[::-] switch panes [::b]W[::-] wrap lines  [::b]Shift+M[::-] toggle navbar  [::b]Shift+S[::-] signal job [::b]</>[::-] filter filesystems",
+	}
+	paramsMtx := &sync.Mutex{}
+	var redraw func()
+	viewmodelupdate := func(cb func(*viewmodel.Params)) {
+		paramsMtx.Lock()
+		defer paramsMtx.Unlock()
+		cb(params)
+		m.Update(*params)
+	}
+	redraw = func() {
+		jobs := m.Jobs()
+		if flag.Job != "" {
+			job_found := false
+			for _, job := range jobs {
+				if strings.Compare(flag.Job, job.Name()) == 0 {
+					jobs = []*viewmodel.Job{job}
+					job_found = true
+					break
+				}
+			}
+			if !job_found {
+				jobs = nil
+			}
+		}
+		redrawJobsList := false
+		var selectedJobN *tview.TreeNode
+		if len(jobMenuRoot.GetChildren()) == len(jobs) {
+			for i, jobN := range jobMenuRoot.GetChildren() {
+				if jobN.GetReference().(*viewmodel.Job) != jobs[i] {
+					redrawJobsList = true
+					break
+				}
+				if jobN.GetReference().(*viewmodel.Job) == m.SelectedJob() {
+					selectedJobN = jobN
+				}
+			}
+		} else {
+			redrawJobsList = true
+		}
+		if redrawJobsList {
+			selectedJobN = nil
+			children := make([]*tview.TreeNode, len(jobs))
+			for i := range jobs {
+				jobN := tview.NewTreeNode(jobs[i].JobTreeTitle())
+				jobN.SetReference(jobs[i])
+				jobN.SetSelectable(true)
+				children[i] = jobN
+				jobN.SetSelectedFunc(func() {
+					viewmodelupdate(func(p *viewmodel.Params) {
+						p.SelectedJob = jobN.GetReference().(*viewmodel.Job)
+					})
+				})
+				if jobs[i] == m.SelectedJob() {
+					selectedJobN = jobN
+				}
+			}
+			jobMenuRoot.SetChildren(children)
+		}
+
+		if selectedJobN != nil && jobMenu.GetCurrentNode() != selectedJobN {
+			jobMenu.SetCurrentNode(selectedJobN)
+		} else if selectedJobN == nil {
+			// select something, otherwise selection breaks (likely bug in tview)
+			jobMenu.SetCurrentNode(jobMenuRoot)
+		}
+
+		if selJ := m.SelectedJob(); selJ != nil {
+			jobTextDetail.SetText(selJ.FullDescription())
+		} else {
+			jobTextDetail.SetText("please select a job")
+		}
+
+		bottombardatestring := m.DateString()
+		bottombarDateView.SetText(bottombardatestring)
+		bottombar.ResizeItem(bottombarDateView, len(bottombardatestring), 0)
+
+		bottomBarStatus.SetText(m.BottomBarStatus())
+
+		app.Draw()
+
+	}
+
+	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				app.Suspend(func() {
+					panic(err)
+				})
+			}
+		}()
+		for {
+			st, err := c.Status()
+			viewmodelupdate(func(p *viewmodel.Params) {
+				p.Report = st.Jobs
+				p.ReportFetchError = err
+			})
+			app.QueueUpdateDraw(redraw)
+
+			time.Sleep(flag.Delay)
+		}
+	}()
+
+	jobMenu.SetChangedFunc(func(jobN *tview.TreeNode) {
+		viewmodelupdate(func(p *viewmodel.Params) {
+			p.SelectedJob, _ = jobN.GetReference().(*viewmodel.Job)
+		})
+		redraw()
+		jobTextDetail.ScrollToBeginning()
+	})
+	jobMenu.SetSelectedFunc(func(jobN *tview.TreeNode) {
+		app.SetFocus(jobTextDetail)
+	})
+
+	app.SetBeforeDrawFunc(func(screen tcell.Screen) bool {
+		viewmodelupdate(func(p *viewmodel.Params) {
+			_, _, p.DetailViewWidth, _ = jobTextDetail.GetInnerRect()
+		})
+		return false
+	})
+
+	app.SetInputCapture(func(e *tcell.EventKey) *tcell.EventKey {
+		if e.Key() == tcell.KeyTab {
+			tabbableCycle()
+			return nil
+		}
+
+		if e.Key() == tcell.KeyRune && app.GetFocus() == fsFilterInput {
+			return e
+		}
+
+		if e.Key() == tcell.KeyRune && e.Rune() == '/' {
+			if app.GetFocus() != fsFilterInput {
+				app.SetFocus(fsFilterInput)
+			}
+			return e
+		}
+
+		if e.Key() == tcell.KeyRune && e.Rune() == 'M' {
+			reconfigureJobDetailSplit(!jobMenuVisisble)
+			return nil
+		}
+
+		if e.Key() == tcell.KeyRune && e.Rune() == 'q' {
+			app.Stop()
+		}
+
+		if e.Key() == tcell.KeyRune && e.Rune() == 'S' {
+			job, ok := jobMenu.GetCurrentNode().GetReference().(*viewmodel.Job)
+			if !ok {
+				return nil
+			}
+			signals := []string{"wakeup", "reset"}
+			clientFuncs := []func(job string) error{c.SignalWakeup, c.SignalReset}
+			sigMod := tview.NewModal()
+			sigMod.SetBackgroundColor(tcell.ColorDefault)
+			sigMod.SetBorder(true)
+			sigMod.GetForm().SetButtonTextColorFocused(tcell.ColorGreen)
+			sigMod.AddButtons(signals)
+			sigMod.SetText(fmt.Sprintf("Send a signal to job %q", job.Name()))
+			showModal(sigMod, func(idx int, _ string) {
+				go func() {
+					if idx == -1 {
+						return
+					}
+					err := clientFuncs[idx](job.Name())
+					if err != nil {
+						app.QueueUpdate(func() {
+							me := tview.NewModal()
+							me.SetText(fmt.Sprintf("signal error: %s", err))
+							me.AddButtons([]string{"Close"})
+							showModal(me, nil)
+						})
+					}
+				}()
+			})
+		}
+
+		return e
+	})
+
+	fsFilterInput.SetChangedFunc(func(searchterm string) {
+		viewmodelupdate(func(p *viewmodel.Params) {
+			p.FSFilter = func(fs string) bool {
+				r, err := regexp.Compile(searchterm)
+				if err != nil {
+					return true
+				}
+				return r.MatchString(fs)
+			}
+		})
+		redraw()
+		jobTextDetail.ScrollToBeginning()
+	})
+	fsFilterInput.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+		if event.Key() == tcell.KeyEnter {
+			app.SetFocus(jobTextDetail)
+			return nil
+		}
+		return event
+	})
+
+	jobTextDetail.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+		if event.Key() == tcell.KeyRune && event.Rune() == 'w' {
+			// toggle wrapping
+			viewmodelupdate(func(p *viewmodel.Params) {
+				p.DetailViewWrap = !p.DetailViewWrap
+			})
+			redraw()
+			return nil
+		}
+		return event
+	})
+
+	return app.Run()
+}

client/status/status_legacy.go

@@ -0,0 +1,128 @@
+package status
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gdamore/tcell/v2"
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+	tview "gitlab.com/tslocum/cview"
+
+	"github.com/zrepl/zrepl/client/status/viewmodel"
+)
+
+func legacy(c Client, flag statusFlags) error {
+
+	// Set this so we don't overwrite the default terminal colors
+	// See https://github.com/rivo/tview/blob/master/styles.go
+	tview.Styles.PrimitiveBackgroundColor = tcell.ColorDefault
+	tview.Styles.ContrastBackgroundColor = tcell.ColorDefault
+	tview.Styles.PrimaryTextColor = tcell.ColorDefault
+	tview.Styles.BorderColor = tcell.ColorDefault
+	app := tview.NewApplication()
+
+	textView := tview.NewTextView()
+	textView.SetWrap(true)
+	textView.SetScrollable(true) // so that it allows us to set scroll position
+	textView.SetScrollBarVisibility(tview.ScrollBarNever)
+
+	app.SetRoot(textView, true)
+
+	width := (1 << 31) - 1
+	wrap := false
+	if isatty.IsTerminal(os.Stdout.Fd()) {
+		wrap = true
+		screen, err := tcell.NewScreen()
+		if err != nil {
+			return errors.Wrap(err, "get terminal dimensions")
+		}
+		if err := screen.Init(); err != nil {
+			return errors.Wrap(err, "init screen")
+		}
+		width, _ = screen.Size()
+		screen.Fini()
+	}
+
+	paramsMtx := &sync.Mutex{}
+	params := viewmodel.Params{
+		Report:                  nil,
+		ReportFetchError:        nil,
+		SelectedJob:             nil,
+		FSFilter:                func(s string) bool { return true },
+		DetailViewWidth:         width,
+		DetailViewWrap:          wrap,
+		ShortKeybindingOverview: "",
+	}
+
+	redraw := func() {
+		textView.Clear()
+
+		paramsMtx.Lock()
+		defer paramsMtx.Unlock()
+
+		if params.ReportFetchError != nil {
+			fmt.Fprintln(textView, params.ReportFetchError.Error())
+		} else if params.Report != nil {
+			m := viewmodel.New()
+			m.Update(params)
+			for _, j := range m.Jobs() {
+				if flag.Job != "" && j.Name() != flag.Job {
+					continue
+				}
+				params.SelectedJob = j
+				m.Update(params)
+				fmt.Fprintln(textView, m.SelectedJob().FullDescription())
+				if flag.Job != "" {
+					break
+				} else {
+					hline := strings.Repeat("-", params.DetailViewWidth)
+					fmt.Fprintln(textView, hline)
+				}
+			}
+		} else {
+			fmt.Fprintln(textView, "waiting for request results")
+		}
+		textView.ScrollToBeginning()
+	}
+
+	app.SetBeforeDrawFunc(func(screen tcell.Screen) bool {
+		// sync resizes to `params`
+		paramsMtx.Lock()
+		_, _, newWidth, _ := textView.GetInnerRect()
+		if newWidth != params.DetailViewWidth {
+			params.DetailViewWidth = newWidth
+			app.QueueUpdateDraw(redraw)
+		}
+		paramsMtx.Unlock()
+
+		textView.ScrollToBeginning() // has the effect of inhibiting user scrolls
+		return false
+	})
+
+	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				app.Suspend(func() {
+					panic(err)
+				})
+			}
+		}()
+		for {
+			st, err := c.Status()
+			paramsMtx.Lock()
+			params.Report = st.Jobs
+			params.ReportFetchError = err
+			paramsMtx.Unlock()
+
+			app.QueueUpdateDraw(redraw)
+
+			time.Sleep(flag.Delay)
+		}
+	}()
+
+	return app.Run()
+}

client/status/status_raw.go

@@ -0,0 +1,18 @@
+package status
+
+import (
+	"bytes"
+	"io"
+	"os"
+)
+
+func raw(c Client) error {
+	b, err := c.StatusRaw()
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(os.Stdout, bytes.NewReader(b)); err != nil {
+		return err
+	}
+	return nil
+}

client/status/viewmodel/bytecountbinary.go

@@ -0,0 +1,26 @@
+package viewmodel
+
+import (
+	"fmt"
+	"math"
+)
+
+func ByteCountBinaryUint(b uint64) string {
+	if b > math.MaxInt64 {
+		panic(b)
+	}
+	return ByteCountBinary(int64(b))
+}
+
+func ByteCountBinary(b int64) string {
+	const unit = 1024
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := unit, 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
+}

client/status/viewmodel/bytesprogresshistory.go

@@ -0,0 +1,49 @@
+package viewmodel
+
+import "time"
+
+type byteProgressMeasurement struct {
+	time time.Time
+	val  uint64
+}
+
+type bytesProgressHistory struct {
+	last        *byteProgressMeasurement // pointer as poor man's optional
+	changeCount int
+	lastChange  time.Time
+}
+
+func (p *bytesProgressHistory) Update(currentVal uint64) (bytesPerSecondAvg int64, changeCount int) {
+
+	if p.last == nil {
+		p.last = &byteProgressMeasurement{
+			time: time.Now(),
+			val:  currentVal,
+		}
+		return 0, 0
+	}
+
+	if p.last.val != currentVal {
+		p.changeCount++
+		p.lastChange = time.Now()
+	}
+
+	if time.Since(p.lastChange) > 3*time.Second {
+		p.last = nil
+		return 0, 0
+	}
+
+	var deltaV int64
+	if currentVal >= p.last.val {
+		deltaV = int64(currentVal - p.last.val)
+	} else {
+		deltaV = -int64(p.last.val - currentVal)
+	}
+	deltaT := time.Since(p.last.time)
+	rate := float64(deltaV) / deltaT.Seconds()
+
+	p.last.time = time.Now()
+	p.last.val = currentVal
+
+	return int64(rate), p.changeCount
+}

client/status/viewmodel/render.go

@@ -0,0 +1,656 @@
+package viewmodel
+
+import (
+	"fmt"
+	"math"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/go-playground/validator/v10"
+	yaml "github.com/zrepl/yaml-config"
+
+	"github.com/zrepl/zrepl/client/status/viewmodel/stringbuilder"
+	"github.com/zrepl/zrepl/daemon"
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/daemon/pruner"
+	"github.com/zrepl/zrepl/daemon/snapper"
+	"github.com/zrepl/zrepl/replication/report"
+)
+
+type M struct {
+	jobs            map[string]*Job
+	jobsList        []*Job
+	selectedJob     *Job
+	dateString      string
+	bottomBarStatus string
+}
+
+type Job struct {
+	// long-lived
+	name         string
+	byteProgress *bytesProgressHistory
+
+	lastStatus      *job.Status
+	fulldescription string
+}
+
+func New() *M {
+	return &M{
+		jobs:        make(map[string]*Job),
+		jobsList:    make([]*Job, 0),
+		selectedJob: nil,
+	}
+}
+
+type FilterFunc func(string) bool
+
+type Params struct {
+	Report                  map[string]*job.Status
+	ReportFetchError        error
+	SelectedJob             *Job
+	FSFilter                FilterFunc `validate:"required"`
+	DetailViewWidth         int        `validate:"gte=1"`
+	DetailViewWrap          bool
+	ShortKeybindingOverview string
+}
+
+var validate = validator.New()
+
+func (m *M) Update(p Params) {
+
+	if err := validate.Struct(p); err != nil {
+		panic(err)
+	}
+
+	if p.ReportFetchError != nil {
+		m.bottomBarStatus = fmt.Sprintf("[red::]status fetch: %s", p.ReportFetchError)
+	} else {
+		m.bottomBarStatus = p.ShortKeybindingOverview
+		for jobname, st := range p.Report {
+			// TODO handle job renames & deletions
+			j, ok := m.jobs[jobname]
+			if !ok {
+				j = &Job{
+					name:         jobname,
+					byteProgress: &bytesProgressHistory{},
+				}
+				m.jobs[jobname] = j
+				m.jobsList = append(m.jobsList, j)
+			}
+			j.lastStatus = st
+		}
+	}
+
+	// filter out internal jobs
+	var jobsList []*Job
+	for _, j := range m.jobsList {
+		if daemon.IsInternalJobName(j.name) {
+			continue
+		}
+		jobsList = append(jobsList, j)
+	}
+	m.jobsList = jobsList
+
+	// determinism!
+	sort.Slice(m.jobsList, func(i, j int) bool {
+		return strings.Compare(m.jobsList[i].name, m.jobsList[j].name) < 0
+	})
+
+	// try to not lose the selected job
+	m.selectedJob = nil
+	for _, j := range m.jobsList {
+		j.updateFullDescription(p)
+		if j == p.SelectedJob {
+			m.selectedJob = j
+		}
+	}
+
+	m.dateString = time.Now().Format(time.RFC3339)
+
+}
+
+func (m *M) BottomBarStatus() string { return m.bottomBarStatus }
+
+func (m *M) Jobs() []*Job { return m.jobsList }
+
+// may be nil
+func (m *M) SelectedJob() *Job { return m.selectedJob }
+
+func (m *M) DateString() string { return m.dateString }
+
+func (j *Job) updateFullDescription(p Params) {
+	width := p.DetailViewWidth
+	if !p.DetailViewWrap {
+		width = 10000000 // FIXME
+	}
+	b := stringbuilder.New(stringbuilder.Config{
+		IndentMultiplier: 3,
+		Width:            width,
+	})
+	drawJob(b, j.name, j.lastStatus, j.byteProgress, p.FSFilter)
+	j.fulldescription = b.String()
+}
+
+func (j *Job) JobTreeTitle() string {
+	return j.name
+}
+
+func (j *Job) FullDescription() string {
+	return j.fulldescription
+}
+
+func (j *Job) Name() string {
+	return j.name
+}
+
+func drawJob(t *stringbuilder.B, name string, v *job.Status, history *bytesProgressHistory, fsfilter FilterFunc) {
+
+	t.Printf("Job: %s\n", name)
+	t.Printf("Type: %s\n\n", v.Type)
+
+	if v.Type == job.TypePush || v.Type == job.TypePull {
+		activeStatus, ok := v.JobSpecific.(*job.ActiveSideStatus)
+		if !ok || activeStatus == nil {
+			t.Printf("ActiveSideStatus is null")
+			t.Newline()
+			return
+		}
+
+		t.Printf("Replication:")
+		t.AddIndentAndNewline(1)
+		renderReplicationReport(t, activeStatus.Replication, history, fsfilter)
+		t.AddIndentAndNewline(-1)
+
+		t.Printf("Pruning Sender:")
+		t.AddIndentAndNewline(1)
+		renderPrunerReport(t, activeStatus.PruningSender, fsfilter)
+		t.AddIndentAndNewline(-1)
+
+		t.Printf("Pruning Receiver:")
+		t.AddIndentAndNewline(1)
+		renderPrunerReport(t, activeStatus.PruningReceiver, fsfilter)
+		t.AddIndentAndNewline(-1)
+
+		if v.Type == job.TypePush {
+			t.Printf("Snapshotting:")
+			t.AddIndentAndNewline(1)
+			renderSnapperReport(t, activeStatus.Snapshotting, fsfilter)
+			t.AddIndentAndNewline(-1)
+		}
+
+	} else if v.Type == job.TypeSnap {
+		snapStatus, ok := v.JobSpecific.(*job.SnapJobStatus)
+		if !ok || snapStatus == nil {
+			t.Printf("SnapJobStatus is null")
+			t.Newline()
+			return
+		}
+		t.Printf("Pruning snapshots:")
+		t.AddIndentAndNewline(1)
+		renderPrunerReport(t, snapStatus.Pruning, fsfilter)
+		t.AddIndentAndNewline(-1)
+		t.Printf("Snapshotting:")
+		t.AddIndentAndNewline(1)
+		renderSnapperReport(t, snapStatus.Snapshotting, fsfilter)
+		t.AddIndentAndNewline(-1)
+	} else if v.Type == job.TypeSource {
+
+		st := v.JobSpecific.(*job.PassiveStatus)
+		t.Printf("Snapshotting:\n")
+		t.AddIndent(1)
+		renderSnapperReport(t, st.Snapper, fsfilter)
+		t.AddIndentAndNewline(-1)
+
+	} else {
+		t.Printf("No status representation for job type '%s', dumping as YAML", v.Type)
+		t.Newline()
+		asYaml, err := yaml.Marshal(v.JobSpecific)
+		if err != nil {
+			t.Printf("Error marshaling status to YAML: %s", err)
+			t.Newline()
+			return
+		}
+		t.Write(string(asYaml))
+		t.Newline()
+	}
+}
+
+func printFilesystemStatus(t *stringbuilder.B, rep *report.FilesystemReport, maxFS int) {
+
+	expected, replicated, containsInvalidSizeEstimates := rep.BytesSum()
+	sizeEstimationImpreciseNotice := ""
+	if containsInvalidSizeEstimates {
+		sizeEstimationImpreciseNotice = " (some steps lack size estimation)"
+	}
+	if rep.CurrentStep < len(rep.Steps) && rep.Steps[rep.CurrentStep].Info.BytesExpected == 0 {
+		sizeEstimationImpreciseNotice = " (step lacks size estimation)"
+	}
+
+	userVisisbleCurrentStep, userVisibleTotalSteps := rep.CurrentStep, len(rep.Steps)
+	// `.CurrentStep` is == len(rep.Steps) if all steps are done.
+	// Until then, it's an index into .Steps that starts at 0.
+	// For the user, we want it to start at 1.
+	if rep.CurrentStep >= len(rep.Steps) {
+		// rep.CurrentStep is what we want to show.
+		// We check for >= and not == for robustness.
+	} else {
+		// We're not done yet, so, make step count start at 1
+		// (The `.State` is included in the output, indicating we're not done yet)
+		userVisisbleCurrentStep = rep.CurrentStep + 1
+	}
+	status := fmt.Sprintf("%s (step %d/%d, %s/%s)%s",
+		strings.ToUpper(string(rep.State)),
+		userVisisbleCurrentStep, userVisibleTotalSteps,
+		ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected),
+		sizeEstimationImpreciseNotice,
+	)
+
+	activeIndicator := " "
+	if rep.BlockedOn == report.FsBlockedOnNothing &&
+		(rep.State == report.FilesystemPlanning || rep.State == report.FilesystemStepping) {
+		activeIndicator = "*"
+	}
+	t.AddIndent(1)
+	t.Printf("%s %s %s ",
+		activeIndicator,
+		stringbuilder.RightPad(rep.Info.Name, maxFS, " "),
+		status)
+
+	next := ""
+	if err := rep.Error(); err != nil {
+		next = err.Err
+	} else if rep.State != report.FilesystemDone {
+		if nextStep := rep.NextStep(); nextStep != nil {
+			if nextStep.IsIncremental() {
+				next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
+			} else {
+				next = fmt.Sprintf("next: full send %s", nextStep.Info.To)
+			}
+			attribs := []string{}
+
+			if nextStep.Info.Resumed {
+				attribs = append(attribs, "resumed")
+			}
+
+			if len(attribs) > 0 {
+				next += fmt.Sprintf(" (%s)", strings.Join(attribs, ", "))
+			}
+		} else {
+			next = "" // individual FSes may still be in planning state
+		}
+
+	}
+	t.Printf("%s", next)
+
+	t.AddIndent(-1)
+	t.Newline()
+}
+
+func renderReplicationReport(t *stringbuilder.B, rep *report.Report, history *bytesProgressHistory, fsfilter FilterFunc) {
+	if rep == nil {
+		t.Printf("...\n")
+		return
+	}
+
+	if rep.WaitReconnectError != nil {
+		t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: %s", rep.WaitReconnectError)
+		t.Newline()
+	}
+	if !rep.WaitReconnectSince.IsZero() {
+		delta := time.Until(rep.WaitReconnectUntil).Round(time.Second)
+		if rep.WaitReconnectUntil.IsZero() || delta > 0 {
+			var until string
+			if rep.WaitReconnectUntil.IsZero() {
+				until = "waiting indefinitely"
+			} else {
+				until = fmt.Sprintf("hard fail in %s @ %s", delta, rep.WaitReconnectUntil)
+			}
+			t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnecting with exponential backoff (since %s) (%s)",
+				rep.WaitReconnectSince, until)
+		} else {
+			t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnects reached hard-fail timeout @ %s", rep.WaitReconnectUntil)
+		}
+		t.Newline()
+	}
+
+	// TODO visualize more than the latest attempt by folding all attempts into one
+	if len(rep.Attempts) == 0 {
+		t.Printf("no attempts made yet")
+		return
+	} else {
+		t.Printf("Attempt #%d", len(rep.Attempts))
+		if len(rep.Attempts) > 1 {
+			t.Printf(". Previous attempts failed with the following statuses:")
+			t.AddIndentAndNewline(1)
+			for i, a := range rep.Attempts[:len(rep.Attempts)-1] {
+				t.PrintfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)\n", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt))
+			}
+			t.AddIndentAndNewline(-1)
+		} else {
+			t.Newline()
+		}
+	}
+
+	latest := rep.Attempts[len(rep.Attempts)-1]
+	sort.Slice(latest.Filesystems, func(i, j int) bool {
+		return latest.Filesystems[i].Info.Name < latest.Filesystems[j].Info.Name
+	})
+
+	// apply filter
+	filtered := make([]*report.FilesystemReport, 0, len(latest.Filesystems))
+	for _, fs := range latest.Filesystems {
+		if !fsfilter(fs.Info.Name) {
+			continue
+		}
+		filtered = append(filtered, fs)
+	}
+	latest.Filesystems = filtered
+
+	t.Printf("Status: %s", latest.State)
+	t.Newline()
+	if !latest.FinishAt.IsZero() {
+		t.Printf("Last Run: %s (lasted %s)\n", latest.FinishAt.Round(time.Second), latest.FinishAt.Sub(latest.StartAt).Round(time.Second))
+	} else {
+		t.Printf("Started: %s (lasting %s)\n", latest.StartAt.Round(time.Second), time.Since(latest.StartAt).Round(time.Second))
+	}
+
+	if latest.State == report.AttemptPlanningError {
+		t.Printf("Problem: ")
+		t.PrintfDrawIndentedAndWrappedIfMultiline("%s", latest.PlanError)
+		t.Newline()
+	} else if latest.State == report.AttemptFanOutError {
+		t.Printf("Problem: one or more of the filesystems encountered errors")
+		t.Newline()
+	}
+
+	if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
+		// Draw global progress bar
+		// Progress: [---------------]
+		expected, replicated, containsInvalidSizeEstimates := latest.BytesSum()
+		rate, changeCount := history.Update(replicated)
+		eta := time.Duration(0)
+		if rate > 0 {
+			eta = time.Duration((float64(expected)-float64(replicated))/float64(rate)) * time.Second
+		}
+
+		t.Write("Progress: ")
+		t.DrawBar(50, replicated, expected, changeCount)
+		t.Write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected), ByteCountBinary(rate)))
+		if eta != 0 {
+			t.Write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
+		}
+		t.Newline()
+		if containsInvalidSizeEstimates {
+			t.Write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
+			t.Newline()
+		}
+
+		if len(latest.Filesystems) == 0 {
+			t.Write("NOTE: no filesystems were considered for replication!")
+			t.Newline()
+		}
+
+		var maxFSLen int
+		for _, fs := range latest.Filesystems {
+			if len(fs.Info.Name) > maxFSLen {
+				maxFSLen = len(fs.Info.Name)
+			}
+		}
+		for _, fs := range latest.Filesystems {
+			printFilesystemStatus(t, fs, maxFSLen)
+		}
+
+	}
+}
+
+func humanizeDuration(duration time.Duration) string {
+	days := int64(duration.Hours() / 24)
+	hours := int64(math.Mod(duration.Hours(), 24))
+	minutes := int64(math.Mod(duration.Minutes(), 60))
+	seconds := int64(math.Mod(duration.Seconds(), 60))
+
+	var parts []string
+
+	force := false
+	chunks := []int64{days, hours, minutes, seconds}
+	for i, chunk := range chunks {
+		if force || chunk > 0 {
+			padding := 0
+			if force {
+				padding = 2
+			}
+			parts = append(parts, fmt.Sprintf("%*d%c", padding, chunk, "dhms"[i]))
+			force = true
+		}
+	}
+
+	return strings.Join(parts, " ")
+}
+
+func renderPrunerReport(t *stringbuilder.B, r *pruner.Report, fsfilter FilterFunc) {
+	if r == nil {
+		t.Printf("...\n")
+		return
+	}
+
+	state, err := pruner.StateString(r.State)
+	if err != nil {
+		t.Printf("Status: %q (parse error: %q)\n", r.State, err)
+		return
+	}
+
+	t.Printf("Status: %s", state)
+	t.Newline()
+
+	if r.Error != "" {
+		t.Printf("Error: %s\n", r.Error)
+	}
+
+	type commonFS struct {
+		*pruner.FSReport
+		completed bool
+	}
+	all := make([]commonFS, 0, len(r.Pending)+len(r.Completed))
+	for i := range r.Pending {
+		all = append(all, commonFS{&r.Pending[i], false})
+	}
+	for i := range r.Completed {
+		all = append(all, commonFS{&r.Completed[i], true})
+	}
+
+	// filter all
+	filtered := make([]commonFS, 0, len(all))
+	for _, fs := range all {
+		if fsfilter(fs.FSReport.Filesystem) {
+			filtered = append(filtered, fs)
+		}
+	}
+	all = filtered
+
+	switch state {
+	case pruner.Plan:
+		fallthrough
+	case pruner.PlanErr:
+		return
+	}
+
+	if len(all) == 0 {
+		t.Printf("nothing to do\n")
+		return
+	}
+
+	var totalDestroyCount, completedDestroyCount int
+	var maxFSname int
+	for _, fs := range all {
+		totalDestroyCount += len(fs.DestroyList)
+		if fs.completed {
+			completedDestroyCount += len(fs.DestroyList)
+		}
+		if maxFSname < len(fs.Filesystem) {
+			maxFSname = len(fs.Filesystem)
+		}
+	}
+
+	// global progress bar
+	progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
+	t.Write("Progress: ")
+	t.Write("[")
+	t.Write(stringbuilder.Times("=", progress))
+	t.Write(">")
+	t.Write(stringbuilder.Times("-", 80-progress))
+	t.Write("]")
+	t.Printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
+	t.Newline()
+
+	sort.SliceStable(all, func(i, j int) bool {
+		return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
+	})
+
+	// Draw a table-like representation of 'all'
+	for _, fs := range all {
+		t.Write(stringbuilder.RightPad(fs.Filesystem, maxFSname, " "))
+		t.Write(" ")
+		if !fs.SkipReason.NotSkipped() {
+			t.Printf("skipped: %s\n", fs.SkipReason)
+			continue
+		}
+		if fs.LastError != "" {
+			if strings.ContainsAny(fs.LastError, "\r\n") {
+				t.Printf("ERROR:")
+				t.PrintfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError)
+			} else {
+				t.PrintfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError)
+			}
+			t.Newline()
+			continue
+		}
+
+		pruneRuleActionStr := fmt.Sprintf("(destroy %d of %d snapshots)",
+			len(fs.DestroyList), len(fs.SnapshotList))
+
+		if fs.completed {
+			t.Printf("Completed  %s\n", pruneRuleActionStr)
+			continue
+		}
+
+		t.Write("Pending    ") // whitespace is padding 10
+		if len(fs.DestroyList) == 1 {
+			t.Write(fs.DestroyList[0].Name)
+		} else {
+			t.Write(pruneRuleActionStr)
+		}
+		t.Newline()
+	}
+
+}
+
+func renderSnapperReport(t *stringbuilder.B, r *snapper.Report, fsfilter FilterFunc) {
+	if r == nil {
+		t.Printf("<no snapshotting report available>\n")
+		return
+	}
+	t.Printf("Type: %s\n", r.Type)
+	if r.Periodic != nil {
+		renderSnapperReportPeriodic(t, r.Periodic, fsfilter)
+	} else if r.Cron != nil {
+		renderSnapperReportCron(t, r.Cron, fsfilter)
+	} else {
+		t.Printf("<no details available>")
+	}
+}
+
+func renderSnapperReportPeriodic(t *stringbuilder.B, r *snapper.PeriodicReport, fsfilter FilterFunc) {
+	t.Printf("Status: %s", r.State)
+	t.Newline()
+
+	if r.Error != "" {
+		t.Printf("Error: %s\n", r.Error)
+	}
+	if !r.SleepUntil.IsZero() {
+		t.Printf("Sleep until: %s\n", r.SleepUntil)
+	}
+
+	renderSnapperPlanReportFilesystem(t, r.Progress, fsfilter)
+}
+
+func renderSnapperReportCron(t *stringbuilder.B, r *snapper.CronReport, fsfilter FilterFunc) {
+	t.Printf("State: %s\n", r.State)
+
+	now := time.Now()
+	if r.WakeupTime.After(now) {
+		t.Printf("Sleep until: %s (%s remaining)\n", r.WakeupTime, r.WakeupTime.Sub(now).Round(time.Second))
+	} else {
+		t.Printf("Started: %s (lasting %s)\n", r.WakeupTime, now.Sub(r.WakeupTime).Round(time.Second))
+	}
+
+	renderSnapperPlanReportFilesystem(t, r.Progress, fsfilter)
+}
+
+func renderSnapperPlanReportFilesystem(t *stringbuilder.B, fss []*snapper.ReportFilesystem, fsfilter FilterFunc) {
+	sort.Slice(fss, func(i, j int) bool {
+		return strings.Compare(fss[i].Path, fss[j].Path) == -1
+	})
+
+	dur := func(d time.Duration) string {
+		return d.Round(100 * time.Millisecond).String()
+	}
+
+	type row struct {
+		path, state, duration, remainder, hookReport string
+	}
+	var widths struct {
+		path, state, duration int
+	}
+	rows := make([]*row, 0, len(fss))
+	for _, fs := range fss {
+		if !fsfilter(fs.Path) {
+			continue
+		}
+		r := &row{
+			path:  fs.Path,
+			state: fs.State.String(),
+		}
+		if fs.HooksHadError {
+			r.hookReport = fs.Hooks // FIXME render here, not in daemon
+		}
+		switch fs.State {
+		case snapper.SnapPending:
+			r.duration = "..."
+			r.remainder = ""
+		case snapper.SnapStarted:
+			r.duration = dur(time.Since(fs.StartAt))
+			r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
+		case snapper.SnapDone:
+			fallthrough
+		case snapper.SnapError:
+			r.duration = dur(fs.DoneAt.Sub(fs.StartAt))
+			r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
+		}
+		rows = append(rows, r)
+		if len(r.path) > widths.path {
+			widths.path = len(r.path)
+		}
+		if len(r.state) > widths.state {
+			widths.state = len(r.state)
+		}
+		if len(r.duration) > widths.duration {
+			widths.duration = len(r.duration)
+		}
+	}
+
+	for _, r := range rows {
+		path := stringbuilder.RightPad(r.path, widths.path, " ")
+		state := stringbuilder.RightPad(r.state, widths.state, " ")
+		duration := stringbuilder.RightPad(r.duration, widths.duration, " ")
+		t.Printf("%s %s %s", path, state, duration)
+		t.PrintfDrawIndentedAndWrappedIfMultiline(" %s", r.remainder)
+		if r.hookReport != "" {
+			t.AddIndent(1)
+			t.Newline()
+			t.Printf("%s", r.hookReport)
+			t.AddIndent(-1)
+		}
+		t.Newline()
+	}
+}

client/status/viewmodel/stringbuilder/stringbuilder.go

@@ -0,0 +1,119 @@
+package stringbuilder
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/go-playground/validator/v10"
+)
+
+type B struct {
+	// const
+	indentMultiplier int
+
+	// mut
+	sb     *strings.Builder
+	indent int
+	width  int
+	x, y   int
+}
+
+type Config struct {
+	IndentMultiplier int `validate:"gte=1"`
+	Width            int `validate:"gte=1"`
+}
+
+var validate = validator.New()
+
+func New(config Config) *B {
+
+	if err := validate.Struct(config); err != nil {
+		panic(err)
+	}
+
+	return &B{sb: &strings.Builder{}, width: config.Width, indentMultiplier: config.IndentMultiplier}
+}
+
+func (b *B) String() string { return b.sb.String() }
+
+func (w *B) Newline() {
+	w.Write("\n")
+}
+
+func (w *B) PrintfDrawIndentedAndWrappedIfMultiline(format string, args ...interface{}) {
+	whole := fmt.Sprintf(format, args...)
+	if strings.ContainsAny(whole, "\n\r") {
+		w.AddIndent(1)
+		defer w.AddIndent(-1)
+	}
+	w.Write(whole)
+}
+
+func (w *B) Printf(format string, args ...interface{}) {
+	whole := fmt.Sprintf(format, args...)
+	w.Write(whole)
+}
+
+func (t *B) AddIndent(delta int) {
+	t.indent += delta * t.indentMultiplier
+}
+
+func (t *B) AddIndentAndNewline(delta int) {
+	t.indent += delta * t.indentMultiplier
+	t.Write("\n")
+}
+
+func (w *B) Write(s string) {
+	for _, c := range s {
+		if c == '\n' {
+			fmt.Fprint(w.sb, "\n")
+			w.x = 0
+			fmt.Fprint(w.sb, Times(" ", w.indent-w.x))
+			w.x = w.indent
+			w.y++
+			continue
+		}
+		if w.x >= w.width {
+			fmt.Fprint(w.sb, "\n")
+			w.x = 0
+			fmt.Fprint(w.sb, Times(" ", w.indent-w.x))
+			w.x = w.indent
+		}
+		fmt.Fprintf(w.sb, "%c", c)
+		w.x++
+	}
+}
+
+func Times(str string, n int) (out string) {
+	for i := 0; i < n; i++ {
+		out += str
+	}
+	return
+}
+
+func RightPad(str string, length int, pad string) string {
+	if len(str) > length {
+		return str[:length]
+	}
+	return str + strings.Repeat(pad, length-len(str))
+}
+
+// changeCount = 0 indicates stall / no progress
+func (w *B) DrawBar(length int, bytes, totalBytes uint64, changeCount int) {
+	const arrowPositions = `>\|/`
+	var completedLength int
+	if totalBytes > 0 {
+		completedLength = int(uint64(length) * bytes / totalBytes)
+		if completedLength > length {
+			completedLength = length
+		}
+	} else if totalBytes == bytes {
+		completedLength = length
+	}
+
+	w.Write("[")
+	w.Write(Times("=", completedLength))
+	w.Write(string(arrowPositions[changeCount%len(arrowPositions)]))
+	w.Write(Times("-", length-completedLength))
+	w.Write("]")
+}

client/stdinserver.go

@@ -1,55 +1,51 @@
-package cmd
+package client
 
 import (
 	"os"
 
-	"context"
 	"github.com/problame/go-netssh"
-	"github.com/spf13/cobra"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+
+	"context"
+	"errors"
 	"log"
 	"path"
 )
 
-var StdinserverCmd = &cobra.Command{
+var StdinserverCmd = &cli.Subcommand{
 	Use:   "stdinserver CLIENT_IDENTITY",
-	Short: "start in stdinserver mode (from authorized_keys file)",
+	Short: "stdinserver transport mode (started from authorized_keys file as forced command)",
-	Run:   cmdStdinServer,
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		return runStdinserver(subcommand.Config(), args)
+	},
 }
 
-func init() {
+func runStdinserver(config *config.Config, args []string) error {
-	RootCmd.AddCommand(StdinserverCmd)
-}
-
-func cmdStdinServer(cmd *cobra.Command, args []string) {
 
 	// NOTE: the netssh proxying protocol requires exiting with non-zero status if anything goes wrong
 	defer os.Exit(1)
 
 	log := log.New(os.Stderr, "", log.LUTC|log.Ldate|log.Ltime)
 
-	conf, err := ParseConfig(rootArgs.configFile)
-	if err != nil {
-		log.Printf("error parsing config: %s", err)
-		return
-	}
-
 	if len(args) != 1 || args[0] == "" {
-		log.Print("must specify client_identity as positional argument")
+		err := errors.New("must specify client_identity as positional argument")
-		return
+		return err
 	}
 
 	identity := args[0]
-	unixaddr := path.Join(conf.Global.Serve.Stdinserver.SockDir, identity)
+	unixaddr := path.Join(config.Global.Serve.StdinServer.SockDir, identity)
 
 	log.Printf("proxying client identity '%s' to zrepl daemon '%s'", identity, unixaddr)
 
 	ctx := netssh.ContextWithLog(context.TODO(), log)
 
-	err = netssh.Proxy(ctx, unixaddr)
+	err := netssh.Proxy(ctx, unixaddr)
 	if err == nil {
 		log.Print("proxying finished successfully, exiting with status 0")
 		os.Exit(0)
 	}
 	log.Printf("error proxying: %s", err)
-
+	return nil
 }

client/testcmd.go

@@ -0,0 +1,220 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var TestCmd = &cli.Subcommand{
+	Use: "test",
+	SetupSubcommands: func() []*cli.Subcommand {
+		return []*cli.Subcommand{testFilter, testPlaceholder, testDecodeResumeToken}
+	},
+}
+
+var testFilterArgs struct {
+	job   string
+	all   bool
+	input string
+}
+
+var testFilter = &cli.Subcommand{
+	Use:   "filesystems --job JOB [--all | --input INPUT]",
+	Short: "test filesystems filter specified in push or source job",
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&testFilterArgs.job, "job", "", "the name of the push or source job")
+		f.StringVar(&testFilterArgs.input, "input", "", "a filesystem name to test against the job's filters")
+		f.BoolVar(&testFilterArgs.all, "all", false, "test all local filesystems")
+	},
+	Run: runTestFilterCmd,
+}
+
+func runTestFilterCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+
+	if testFilterArgs.job == "" {
+		return fmt.Errorf("must specify --job flag")
+	}
+	if !(testFilterArgs.all != (testFilterArgs.input != "")) { // xor
+		return fmt.Errorf("must set one: --all or --input")
+	}
+
+	conf := subcommand.Config()
+
+	var confFilter config.FilesystemsFilter
+	job, err := conf.Job(testFilterArgs.job)
+	if err != nil {
+		return err
+	}
+	switch j := job.Ret.(type) {
+	case *config.SourceJob:
+		confFilter = j.Filesystems
+	case *config.PushJob:
+		confFilter = j.Filesystems
+	case *config.SnapJob:
+		confFilter = j.Filesystems
+	default:
+		return fmt.Errorf("job type %T does not have filesystems filter", j)
+	}
+
+	f, err := filters.DatasetMapFilterFromConfig(confFilter)
+	if err != nil {
+		return fmt.Errorf("filter invalid: %s", err)
+	}
+
+	var fsnames []string
+	if testFilterArgs.input != "" {
+		fsnames = []string{testFilterArgs.input}
+	} else {
+		out, err := zfs.ZFSList(ctx, []string{"name"})
+		if err != nil {
+			return fmt.Errorf("could not list ZFS filesystems: %s", err)
+		}
+		for _, row := range out {
+
+			fsnames = append(fsnames, row[0])
+		}
+	}
+
+	fspaths := make([]*zfs.DatasetPath, len(fsnames))
+	for i, fsname := range fsnames {
+		path, err := zfs.NewDatasetPath(fsname)
+		if err != nil {
+			return err
+		}
+		fspaths[i] = path
+	}
+
+	hadFilterErr := false
+	for _, in := range fspaths {
+		var res string
+		var errStr string
+		pass, err := f.Filter(in)
+		if err != nil {
+			res = "ERROR"
+			errStr = err.Error()
+			hadFilterErr = true
+		} else if pass {
+			res = "ACCEPT"
+		} else {
+			res = "REJECT"
+		}
+		fmt.Printf("%s\t%s\t%s\n", res, in.ToString(), errStr)
+	}
+
+	if hadFilterErr {
+		return fmt.Errorf("filter errors occurred")
+	}
+	return nil
+}
+
+var testPlaceholderArgs struct {
+	ds  string
+	all bool
+}
+
+var testPlaceholder = &cli.Subcommand{
+	Use:   "placeholder [--all | --dataset DATASET]",
+	Short: fmt.Sprintf("list received placeholder filesystems (zfs property %q)", zfs.PlaceholderPropertyName),
+	Example: `
+	placeholder --all
+	placeholder --dataset path/to/sink/clientident/fs`,
+	NoRequireConfig: true,
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&testPlaceholderArgs.ds, "dataset", "", "dataset path (not required to exist)")
+		f.BoolVar(&testPlaceholderArgs.all, "all", false, "list tab-separated placeholder status of all filesystems")
+	},
+	Run: runTestPlaceholder,
+}
+
+func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+
+	var checkDPs []*zfs.DatasetPath
+	var datasetWasExplicitArgument bool
+
+	// all actions first
+	if testPlaceholderArgs.all {
+		datasetWasExplicitArgument = false
+		out, err := zfs.ZFSList(ctx, []string{"name"})
+		if err != nil {
+			return errors.Wrap(err, "could not list ZFS filesystems")
+		}
+		for _, row := range out {
+			dp, err := zfs.NewDatasetPath(row[0])
+			if err != nil {
+				panic(err)
+			}
+			checkDPs = append(checkDPs, dp)
+		}
+	} else {
+		datasetWasExplicitArgument = true
+		dp, err := zfs.NewDatasetPath(testPlaceholderArgs.ds)
+		if err != nil {
+			return err
+		}
+		if dp.Empty() {
+			return fmt.Errorf("must specify --dataset DATASET or --all")
+		}
+		checkDPs = append(checkDPs, dp)
+	}
+
+	fmt.Printf("IS_PLACEHOLDER\tDATASET\tzrepl:placeholder\n")
+	for _, dp := range checkDPs {
+		ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, dp)
+		if err != nil {
+			return errors.Wrap(err, "cannot get placeholder state")
+		}
+		if !ph.FSExists {
+			if datasetWasExplicitArgument {
+				return errors.Errorf("filesystem %q does not exist", ph.FS)
+			} else {
+				// got deleted between ZFSList and ZFSGetFilesystemPlaceholderState
+				continue
+			}
+		}
+		is := "yes"
+		if !ph.IsPlaceholder {
+			is = "no"
+		}
+		fmt.Printf("%s\t%s\t%s\n", is, dp.ToString(), ph.RawLocalPropertyValue)
+	}
+	return nil
+}
+
+var testDecodeResumeTokenArgs struct {
+	token string
+}
+
+var testDecodeResumeToken = &cli.Subcommand{
+	Use:   "decoderesumetoken --token TOKEN",
+	Short: "decode resume token",
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&testDecodeResumeTokenArgs.token, "token", "", "the resume token obtained from the receive_resume_token property")
+	},
+	Run: runTestDecodeResumeTokenCmd,
+}
+
+func runTestDecodeResumeTokenCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+	if testDecodeResumeTokenArgs.token == "" {
+		return fmt.Errorf("token argument must be specified")
+	}
+	token, err := zfs.ParseResumeToken(ctx, testDecodeResumeTokenArgs.token)
+	if err != nil {
+		return err
+	}
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", " ")
+	if err := enc.Encode(&token); err != nil {
+		panic(err)
+	}
+	return nil
+}

client/version.go

@@ -0,0 +1,75 @@
+package client
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+	"github.com/zrepl/zrepl/version"
+)
+
+var versionArgs struct {
+	Show      string
+	Config    *config.Config
+	ConfigErr error
+}
+
+var VersionCmd = &cli.Subcommand{
+	Use:             "version",
+	Short:           "print version of zrepl binary and running daemon",
+	NoRequireConfig: true,
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)")
+	},
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		versionArgs.Config = subcommand.Config()
+		versionArgs.ConfigErr = subcommand.ConfigParsingError()
+		return runVersionCmd()
+	},
+}
+
+func runVersionCmd() error {
+	args := versionArgs
+
+	if args.Show != "daemon" && args.Show != "client" && args.Show != "" {
+		return fmt.Errorf("show flag must be 'client' or 'server' or be left empty")
+	}
+
+	var clientVersion, daemonVersion *version.ZreplVersionInformation
+	if args.Show == "client" || args.Show == "" {
+		clientVersion = version.NewZreplVersionInformation()
+		fmt.Printf("client: %s\n", clientVersion.String())
+	}
+	if args.Show == "daemon" || args.Show == "" {
+
+		if args.ConfigErr != nil {
+			return fmt.Errorf("config parsing error: %s", args.ConfigErr)
+		}
+
+		httpc, err := controlHttpClient(args.Config.Global.Control.SockPath)
+		if err != nil {
+			return fmt.Errorf("server: error: %s\n", err)
+		}
+
+		var info version.ZreplVersionInformation
+		err = jsonRequestResponse(httpc, daemon.ControlJobEndpointVersion, "", &info)
+		if err != nil {
+			return fmt.Errorf("server: error: %s\n", err)
+		}
+		daemonVersion = &info
+		fmt.Printf("server: %s\n", daemonVersion.String())
+	}
+
+	if args.Show == "" {
+		if clientVersion.Version != daemonVersion.Version {
+			fmt.Fprintf(os.Stderr, "WARNING: client version != daemon version, restart zrepl daemon\n")
+		}
+	}
+
+	return nil
+}

client/zfsabstractions.go

@@ -0,0 +1,147 @@
+package client
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var (
+	ZFSAbstractionsCmd = &cli.Subcommand{
+		Use:   "zfs-abstraction",
+		Short: "manage abstractions that zrepl builds on top of ZFS",
+		SetupSubcommands: func() []*cli.Subcommand {
+			return []*cli.Subcommand{
+				zabsCmdList,
+				zabsCmdReleaseAll,
+				zabsCmdReleaseStale,
+				zabsCmdCreate,
+			}
+		},
+	}
+)
+
+// a common set of CLI flags that map to the fields of an
+// endpoint.ListZFSHoldsAndBookmarksQuery
+type zabsFilterFlags struct {
+	Filesystems FilesystemsFilterFlag
+	Job         JobIDFlag
+	Types       AbstractionTypesFlag
+	Concurrency int64
+}
+
+// produce a query from the CLI flags
+func (f zabsFilterFlags) Query() (endpoint.ListZFSHoldsAndBookmarksQuery, error) {
+	q := endpoint.ListZFSHoldsAndBookmarksQuery{
+		FS:          f.Filesystems.FlagValue(),
+		What:        f.Types.FlagValue(),
+		JobID:       f.Job.FlagValue(),
+		Concurrency: f.Concurrency,
+	}
+	return q, q.Validate()
+}
+
+func (f *zabsFilterFlags) registerZabsFilterFlags(s *pflag.FlagSet, verb string) {
+	// Note: the default value is defined in the .FlagValue methods
+	s.Var(&f.Filesystems, "fs", fmt.Sprintf("only %s holds on the specified filesystem [default: all filesystems] [comma-separated list of <dataset-pattern>:<ok|!> pairs]", verb))
+	s.Var(&f.Job, "job", fmt.Sprintf("only %s holds created by the specified job [default: any job]", verb))
+
+	variants := make([]string, 0, len(endpoint.AbstractionTypesAll))
+	for v := range endpoint.AbstractionTypesAll {
+		variants = append(variants, string(v))
+	}
+	sort.Strings(variants)
+	variantsJoined := strings.Join(variants, "|")
+	s.Var(&f.Types, "type", fmt.Sprintf("only %s holds of the specified type [default: all] [comma-separated list of %s]", verb, variantsJoined))
+
+	s.Int64VarP(&f.Concurrency, "concurrency", "p", 1, "number of concurrently queried filesystems")
+}
+
+type JobIDFlag struct{ J *endpoint.JobID }
+
+func (f *JobIDFlag) Set(s string) error {
+	if len(s) == 0 {
+		*f = JobIDFlag{J: nil}
+		return nil
+	}
+
+	jobID, err := endpoint.MakeJobID(s)
+	if err != nil {
+		return err
+	}
+	*f = JobIDFlag{J: &jobID}
+	return nil
+}
+func (f JobIDFlag) Type() string               { return "job-ID" }
+func (f JobIDFlag) String() string             { return fmt.Sprint(f.J) }
+func (f JobIDFlag) FlagValue() *endpoint.JobID { return f.J }
+
+type AbstractionTypesFlag map[endpoint.AbstractionType]bool
+
+func (f *AbstractionTypesFlag) Set(s string) error {
+	ats, err := endpoint.AbstractionTypeSetFromStrings(strings.Split(s, ","))
+	if err != nil {
+		return err
+	}
+	*f = AbstractionTypesFlag(ats)
+	return nil
+}
+func (f AbstractionTypesFlag) Type() string { return "abstraction-type" }
+func (f AbstractionTypesFlag) String() string {
+	return endpoint.AbstractionTypeSet(f).String()
+}
+func (f AbstractionTypesFlag) FlagValue() map[endpoint.AbstractionType]bool {
+	if len(f) > 0 {
+		return f
+	}
+	return endpoint.AbstractionTypesAll
+}
+
+type FilesystemsFilterFlag struct {
+	F endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter
+}
+
+func (flag *FilesystemsFilterFlag) Set(s string) error {
+	mappings := strings.Split(s, ",")
+	if len(mappings) == 1 && !strings.Contains(mappings[0], ":") {
+		flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &mappings[0],
+		}
+		return nil
+	}
+
+	f := filters.NewDatasetMapFilter(len(mappings), true)
+	for _, m := range mappings {
+		thisMappingErr := fmt.Errorf("expecting comma-separated list of <dataset-pattern>:<ok|!> pairs, got %q", m)
+		lhsrhs := strings.SplitN(m, ":", 2)
+		if len(lhsrhs) != 2 {
+			return thisMappingErr
+		}
+		err := f.Add(lhsrhs[0], lhsrhs[1])
+		if err != nil {
+			return fmt.Errorf("%s: %s", thisMappingErr, err)
+		}
+	}
+	flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+		Filter: f,
+	}
+	return nil
+}
+func (flag FilesystemsFilterFlag) Type() string { return "filesystem filter spec" }
+func (flag FilesystemsFilterFlag) String() string {
+	return fmt.Sprintf("%v", flag.F)
+}
+func (flag FilesystemsFilterFlag) FlagValue() endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter {
+	var z FilesystemsFilterFlag
+	if flag == z {
+		return endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{Filter: zfs.NoFilter()}
+	}
+	return flag.F
+}

client/zfsabstractions_create.go

@@ -0,0 +1,14 @@
+package client
+
+import "github.com/zrepl/zrepl/cli"
+
+var zabsCmdCreate = &cli.Subcommand{
+	Use:             "create",
+	NoRequireConfig: true,
+	Short:           `create zrepl ZFS abstractions (mostly useful for debugging & development, users should not need to use this command)`,
+	SetupSubcommands: func() []*cli.Subcommand {
+		return []*cli.Subcommand{
+			zabsCmdCreateStepHold,
+		}
+	},
+}

client/zfsabstractions_create_step_hold.go

@@ -0,0 +1,58 @@
+package client
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var zabsCreateStepHoldFlags struct {
+	target string
+	jobid  JobIDFlag
+}
+
+var zabsCmdCreateStepHold = &cli.Subcommand{
+	Use:             "step",
+	Run:             doZabsCreateStep,
+	NoRequireConfig: true,
+	Short:           `create a step hold or bookmark`,
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVarP(&zabsCreateStepHoldFlags.target, "target", "t", "", "snapshot to be held / bookmark to be held")
+		f.VarP(&zabsCreateStepHoldFlags.jobid, "jobid", "j", "jobid for which the hold is installed")
+	},
+}
+
+func doZabsCreateStep(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	if len(args) > 0 {
+		return errors.New("subcommand takes no arguments")
+	}
+
+	f := &zabsCreateStepHoldFlags
+
+	fs, _, _, err := zfs.DecomposeVersionString(f.target)
+	if err != nil {
+		return errors.Wrapf(err, "%q invalid target", f.target)
+	}
+
+	if f.jobid.FlagValue() == nil {
+		return errors.Errorf("jobid must be set")
+	}
+
+	v, err := zfs.ZFSGetFilesystemVersion(ctx, f.target)
+	if err != nil {
+		return errors.Wrapf(err, "get info about target %q", f.target)
+	}
+
+	step, err := endpoint.HoldStep(ctx, fs, v, *f.jobid.FlagValue())
+	if err != nil {
+		return errors.Wrap(err, "create step hold")
+	}
+	fmt.Println(step.String())
+	return nil
+}

client/zfsabstractions_list.go

@@ -0,0 +1,100 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/util/chainlock"
+)
+
+var zabsListFlags struct {
+	Filter zabsFilterFlags
+	Json   bool
+}
+
+var zabsCmdList = &cli.Subcommand{
+	Use:             "list",
+	Short:           `list zrepl ZFS abstractions`,
+	Run:             doZabsList,
+	NoRequireConfig: true,
+	SetupFlags: func(f *pflag.FlagSet) {
+		zabsListFlags.Filter.registerZabsFilterFlags(f, "list")
+		f.BoolVar(&zabsListFlags.Json, "json", false, "emit JSON")
+	},
+}
+
+func doZabsList(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsListFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	abstractions, errors, drainDone, err := endpoint.ListAbstractionsStreamed(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+	defer drainDone()
+
+	var line chainlock.L
+	var wg sync.WaitGroup
+	defer wg.Wait()
+
+	// print results
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		enc := json.NewEncoder(os.Stdout)
+		for a := range abstractions {
+			func() {
+				defer line.Lock().Unlock()
+				if zabsListFlags.Json {
+					enc.SetIndent("", "  ")
+					if err := enc.Encode(a); err != nil {
+						panic(err)
+					}
+					fmt.Println()
+				} else {
+					fmt.Println(a)
+				}
+			}()
+		}
+	}()
+
+	// print errors to stderr
+	errorColor := color.New(color.FgRed)
+	var errorsSlice []endpoint.ListAbstractionsError
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for err := range errors {
+			func() {
+				defer line.Lock().Unlock()
+				errorsSlice = append(errorsSlice, err)
+				errorColor.Fprintf(os.Stderr, "%s\n", err)
+			}()
+		}
+	}()
+	wg.Wait()
+	if len(errorsSlice) > 0 {
+		errorColor.Add(color.Bold).Fprintf(os.Stderr, "there were errors in listing the abstractions")
+		return fmt.Errorf("")
+	} else {
+		return nil
+	}
+
+}

client/zfsabstractions_release.go

@@ -0,0 +1,143 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+)
+
+// shared between release-all and release-step
+var zabsReleaseFlags struct {
+	Filter zabsFilterFlags
+	Json   bool
+	DryRun bool
+}
+
+func registerZabsReleaseFlags(s *pflag.FlagSet) {
+	zabsReleaseFlags.Filter.registerZabsFilterFlags(s, "release")
+	s.BoolVar(&zabsReleaseFlags.Json, "json", false, "emit json instead of pretty-printed")
+	s.BoolVar(&zabsReleaseFlags.DryRun, "dry-run", false, "do a dry-run")
+}
+
+var zabsCmdReleaseAll = &cli.Subcommand{
+	Use:             "release-all",
+	Run:             doZabsReleaseAll,
+	NoRequireConfig: true,
+	Short:           `(DANGEROUS) release ALL zrepl ZFS abstractions (mostly useful for uninstalling zrepl completely or for "de-zrepl-ing" a filesystem)`,
+	SetupFlags:      registerZabsReleaseFlags,
+}
+
+var zabsCmdReleaseStale = &cli.Subcommand{
+	Use:             "release-stale",
+	Run:             doZabsReleaseStale,
+	NoRequireConfig: true,
+	Short:           `release stale zrepl ZFS abstractions (useful if zrepl has a bug and does not do it by itself)`,
+	SetupFlags:      registerZabsReleaseFlags,
+}
+
+func doZabsReleaseAll(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsReleaseFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	abstractions, listErrors, err := endpoint.ListAbstractions(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+	if len(listErrors) > 0 {
+		color.New(color.FgRed).Fprintf(os.Stderr, "there were errors in listing the abstractions:\n%s\n", listErrors)
+		// proceed anyways with rest of abstractions
+	}
+
+	return doZabsRelease_Common(ctx, abstractions)
+}
+
+func doZabsReleaseStale(ctx context.Context, sc *cli.Subcommand, args []string) error {
+
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsReleaseFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	stalenessInfo, err := endpoint.ListStale(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+
+	return doZabsRelease_Common(ctx, stalenessInfo.Stale)
+}
+
+func doZabsRelease_Common(ctx context.Context, destroy []endpoint.Abstraction) error {
+
+	if zabsReleaseFlags.DryRun {
+		if zabsReleaseFlags.Json {
+			m, err := json.MarshalIndent(destroy, "", "  ")
+			if err != nil {
+				panic(err)
+			}
+			if _, err := os.Stdout.Write(m); err != nil {
+				panic(err)
+			}
+			fmt.Println()
+		} else {
+			for _, a := range destroy {
+				fmt.Printf("would destroy %s\n", a)
+			}
+		}
+		return nil
+	}
+
+	outcome := endpoint.BatchDestroy(ctx, destroy)
+	hadErr := false
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	colorErr := color.New(color.FgRed)
+	printfSuccess := color.New(color.FgGreen).FprintfFunc()
+	printfSection := color.New(color.Bold).FprintfFunc()
+
+	for res := range outcome {
+		hadErr = hadErr || res.DestroyErr != nil
+		if zabsReleaseFlags.Json {
+			err := enc.Encode(res)
+			if err != nil {
+				colorErr.Fprintf(os.Stderr, "cannot marshal there were errors in destroying the abstractions")
+			}
+		} else {
+			printfSection(os.Stdout, "destroy %s ...", res.Abstraction)
+			if res.DestroyErr != nil {
+				colorErr.Fprintf(os.Stdout, " failed:\n%s\n", res.DestroyErr)
+			} else {
+				printfSuccess(os.Stdout, " OK\n")
+			}
+		}
+	}
+
+	if hadErr {
+		colorErr.Add(color.Bold).Fprintf(os.Stderr, "there were errors in destroying the abstractions")
+		return fmt.Errorf("")
+	} else {
+		return nil
+	}
+}

cmd/autosnap.go

@@ -1,201 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"github.com/zrepl/zrepl/zfs"
-	"sort"
-	"time"
-)
-
-type IntervalAutosnap struct {
-	task             *Task
-	DatasetFilter    zfs.DatasetFilter
-	Prefix           string
-	SnapshotInterval time.Duration
-}
-
-func (a *IntervalAutosnap) filterFilesystems() (fss []*zfs.DatasetPath, stop bool) {
-	a.task.Enter("filter_filesystems")
-	defer a.task.Finish()
-	fss, err := zfs.ZFSListMapping(a.DatasetFilter)
-	stop = err != nil
-	if err != nil {
-		a.task.Log().WithError(err).Error("cannot list datasets")
-	}
-	if len(fss) == 0 {
-		a.task.Log().Warn("no filesystem matching filesystem filter")
-	}
-	return fss, stop
-}
-
-func (a *IntervalAutosnap) findSyncPoint(fss []*zfs.DatasetPath) (syncPoint time.Time, err error) {
-	a.task.Enter("find_sync_point")
-	defer a.task.Finish()
-	type snapTime struct {
-		ds   *zfs.DatasetPath
-		time time.Time
-	}
-
-	if len(fss) == 0 {
-		return time.Now(), nil
-	}
-
-	snaptimes := make([]snapTime, 0, len(fss))
-
-	now := time.Now()
-
-	a.task.Log().Debug("examine filesystem state")
-	for _, d := range fss {
-
-		l := a.task.Log().WithField(logFSField, d.ToString())
-
-		fsvs, err := zfs.ZFSListFilesystemVersions(d, NewPrefixFilter(a.Prefix))
-		if err != nil {
-			l.WithError(err).Error("cannot list filesystem versions")
-			continue
-		}
-		if len(fsvs) <= 0 {
-			l.WithField("prefix", a.Prefix).Info("no filesystem versions with prefix")
-			continue
-		}
-
-		// Sort versions by creation
-		sort.SliceStable(fsvs, func(i, j int) bool {
-			return fsvs[i].CreateTXG < fsvs[j].CreateTXG
-		})
-
-		latest := fsvs[len(fsvs)-1]
-		l.WithField("creation", latest.Creation).
-			Debug("found latest snapshot")
-
-		since := now.Sub(latest.Creation)
-		if since < 0 {
-			l.WithField("snapshot", latest.Name).
-				WithField("creation", latest.Creation).
-				Error("snapshot is from the future")
-			continue
-		}
-		next := now
-		if since < a.SnapshotInterval {
-			next = latest.Creation.Add(a.SnapshotInterval)
-		}
-		snaptimes = append(snaptimes, snapTime{d, next})
-	}
-
-	if len(snaptimes) == 0 {
-		snaptimes = append(snaptimes, snapTime{nil, now})
-	}
-
-	sort.Slice(snaptimes, func(i, j int) bool {
-		return snaptimes[i].time.Before(snaptimes[j].time)
-	})
-
-	return snaptimes[0].time, nil
-
-}
-
-func (a *IntervalAutosnap) waitForSyncPoint(ctx context.Context, syncPoint time.Time) {
-	a.task.Enter("wait_sync_point")
-	defer a.task.Finish()
-
-	const LOG_TIME_FMT string = time.ANSIC
-
-	a.task.Log().WithField("sync_point", syncPoint.Format(LOG_TIME_FMT)).
-		Info("wait for sync point")
-
-	select {
-	case <-ctx.Done():
-		a.task.Log().WithError(ctx.Err()).Info("context done")
-		return
-	case <-time.After(syncPoint.Sub(time.Now())):
-	}
-}
-
-func (a *IntervalAutosnap) syncUpRun(ctx context.Context, didSnaps chan struct{}) (stop bool) {
-	a.task.Enter("sync_up")
-	defer a.task.Finish()
-
-	fss, stop := a.filterFilesystems()
-	if stop {
-		return true
-	}
-
-	syncPoint, err := a.findSyncPoint(fss)
-	if err != nil {
-		return true
-	}
-
-	a.waitForSyncPoint(ctx, syncPoint)
-
-	a.task.Log().Debug("snapshot all filesystems to enable further snaps in lockstep")
-	a.doSnapshots(didSnaps)
-	return false
-}
-
-func (a *IntervalAutosnap) Run(ctx context.Context, didSnaps chan struct{}) {
-
-	if a.syncUpRun(ctx, didSnaps) {
-		a.task.Log().Error("stoppping autosnap after error in sync up")
-		return
-	}
-
-	// task drops back to idle here
-
-	a.task.Log().Debug("setting up ticker in SnapshotInterval")
-	ticker := time.NewTicker(a.SnapshotInterval)
-	for {
-		select {
-		case <-ctx.Done():
-			ticker.Stop()
-			a.task.Log().WithError(ctx.Err()).Info("context done")
-			return
-
-		case <-ticker.C:
-			a.doSnapshots(didSnaps)
-		}
-	}
-
-}
-
-func (a *IntervalAutosnap) doSnapshots(didSnaps chan struct{}) {
-
-	a.task.Enter("do_snapshots")
-	defer a.task.Finish()
-
-	// don't cache the result from previous run in case the user added
-	// a new dataset in the meantime
-	ds, stop := a.filterFilesystems()
-	if stop {
-		return
-	}
-
-	// TODO channel programs -> allow a little jitter?
-	for _, d := range ds {
-		suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
-		snapname := fmt.Sprintf("%s%s", a.Prefix, suffix)
-
-		l := a.task.Log().WithField(logFSField, d.ToString()).
-			WithField("snapname", snapname)
-
-		l.Info("create snapshot")
-		err := zfs.ZFSSnapshot(d, snapname, false)
-		if err != nil {
-			a.task.Log().WithError(err).Error("cannot create snapshot")
-		}
-
-		l.Info("create corresponding bookmark")
-		err = zfs.ZFSBookmark(d, snapname, snapname)
-		if err != nil {
-			a.task.Log().WithError(err).Error("cannot create bookmark")
-		}
-
-	}
-
-	select {
-	case didSnaps <- struct{}{}:
-	default:
-		a.task.Log().Error("warning: callback channel is full, discarding")
-	}
-
-}

cmd/bashcomp.go

@@ -1,29 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-
-	"github.com/spf13/cobra"
-	"os"
-)
-
-var bashcompCmd = &cobra.Command{
-	Use:   "bashcomp path/to/out/file",
-	Short: "generate bash completions",
-	Run: func(cmd *cobra.Command, args []string) {
-		if len(args) != 1 {
-			fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
-			cmd.Usage()
-			os.Exit(1)
-		}
-		if err := RootCmd.GenBashCompletionFile(args[0]); err != nil {
-			fmt.Fprintf(os.Stderr, "error generating bash completion: %s", err)
-			os.Exit(1)
-		}
-	},
-	Hidden: true,
-}
-
-func init() {
-	RootCmd.AddCommand(bashcompCmd)
-}

cmd/config.go

@@ -1,102 +0,0 @@
-package cmd
-
-import (
-	"io"
-
-	"fmt"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type Config struct {
-	Global Global
-	Jobs   map[string]Job
-}
-
-func (c *Config) LookupJob(name string) (j Job, err error) {
-	j, ok := c.Jobs[name]
-	if !ok {
-		return nil, errors.Errorf("job '%s' is not defined", name)
-	}
-	return j, nil
-}
-
-type Global struct {
-	Serve struct {
-		Stdinserver struct {
-			SockDir string
-		}
-	}
-	Control struct {
-		Sockpath string
-	}
-	logging *LoggingConfig
-}
-
-type JobDebugSettings struct {
-	Conn struct {
-		ReadDump  string `mapstructure:"read_dump"`
-		WriteDump string `mapstructure:"write_dump"`
-	}
-	RPC struct {
-		Log bool
-	}
-}
-
-type RWCConnecter interface {
-	Connect() (io.ReadWriteCloser, error)
-}
-type AuthenticatedChannelListenerFactory interface {
-	Listen() (AuthenticatedChannelListener, error)
-}
-
-type AuthenticatedChannelListener interface {
-	Accept() (ch io.ReadWriteCloser, err error)
-	Close() (err error)
-}
-
-type SSHStdinServerConnectDescr struct {
-}
-
-type PrunePolicy interface {
-	// Prune filters versions and decide which to keep and which to remove.
-	// Prune **does not** implement the actual removal of the versions.
-	Prune(fs *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error)
-}
-
-type PruningJob interface {
-	Pruner(task *Task, side PrunePolicySide, dryRun bool) (Pruner, error)
-}
-
-// A type for constants describing different prune policies of a PruningJob
-// This is mostly a special-case for LocalJob, which is the only job that has two prune policies
-// instead of one.
-// It implements github.com/spf13/pflag.Value to be used as CLI flag for the test subcommand
-type PrunePolicySide string
-
-const (
-	PrunePolicySideDefault PrunePolicySide = ""
-	PrunePolicySideLeft    PrunePolicySide = "left"
-	PrunePolicySideRight   PrunePolicySide = "right"
-)
-
-func (s *PrunePolicySide) String() string {
-	return string(*s)
-}
-
-func (s *PrunePolicySide) Set(news string) error {
-	p := PrunePolicySide(news)
-	switch p {
-	case PrunePolicySideRight:
-		fallthrough
-	case PrunePolicySideLeft:
-		*s = p
-	default:
-		return errors.Errorf("must be either %s or %s", PrunePolicySideLeft, PrunePolicySideRight)
-	}
-	return nil
-}
-
-func (s *PrunePolicySide) Type() string {
-	return fmt.Sprintf("%s | %s", PrunePolicySideLeft, PrunePolicySideRight)
-}

cmd/config_connect.go

@@ -1,66 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"io"
-
-	"context"
-	"github.com/jinzhu/copier"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/problame/go-netssh"
-	"time"
-)
-
-type SSHStdinserverConnecter struct {
-	Host                 string
-	User                 string
-	Port                 uint16
-	IdentityFile         string   `mapstructure:"identity_file"`
-	TransportOpenCommand []string `mapstructure:"transport_open_command"`
-	SSHCommand           string   `mapstructure:"ssh_command"`
-	Options              []string
-	DialTimeout          string `mapstructure:"dial_timeout"`
-	dialTimeout          time.Duration
-}
-
-func parseSSHStdinserverConnecter(i map[string]interface{}) (c *SSHStdinserverConnecter, err error) {
-
-	c = &SSHStdinserverConnecter{}
-	if err = mapstructure.Decode(i, c); err != nil {
-		err = errors.New(fmt.Sprintf("could not parse ssh transport: %s", err))
-		return nil, err
-	}
-
-	if c.DialTimeout != "" {
-		c.dialTimeout, err = time.ParseDuration(c.DialTimeout)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot parse dial_timeout")
-		}
-	} else {
-		c.dialTimeout = 10 * time.Second
-	}
-
-	// TODO assert fields are filled
-	return
-
-}
-
-func (c *SSHStdinserverConnecter) Connect() (rwc io.ReadWriteCloser, err error) {
-
-	var endpoint netssh.Endpoint
-	if err = copier.Copy(&endpoint, c); err != nil {
-		return nil, errors.WithStack(err)
-	}
-	var dialCtx context.Context
-	dialCtx, dialCancel := context.WithTimeout(context.TODO(), c.dialTimeout) // context.TODO tied to error handling below
-	defer dialCancel()
-	if rwc, err = netssh.Dial(dialCtx, endpoint); err != nil {
-		if err == context.DeadlineExceeded {
-			err = errors.Errorf("dial_timeout of %s exceeded", c.dialTimeout)
-		}
-		err = errors.WithStack(err)
-		return
-	}
-	return
-}

cmd/config_fsvfilter.go

@@ -1,36 +0,0 @@
-package cmd
-
-import (
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/zfs"
-	"strings"
-)
-
-type PrefixFilter struct {
-	prefix    string
-	fstype    zfs.VersionType
-	fstypeSet bool // optionals anyone?
-}
-
-func NewPrefixFilter(prefix string) *PrefixFilter {
-	return &PrefixFilter{prefix: prefix}
-}
-
-func NewTypedPrefixFilter(prefix string, versionType zfs.VersionType) *PrefixFilter {
-	return &PrefixFilter{prefix, versionType, true}
-}
-
-func parseSnapshotPrefix(i string) (p string, err error) {
-	if len(i) <= 0 {
-		err = errors.Errorf("snapshot prefix must not be empty string")
-		return
-	}
-	p = i
-	return
-}
-
-func (f *PrefixFilter) Filter(fsv zfs.FilesystemVersion) (accept bool, err error) {
-	fstypeMatches := (!f.fstypeSet || fsv.Type == f.fstype)
-	prefixMatches := strings.HasPrefix(fsv.Name, f.prefix)
-	return fstypeMatches && prefixMatches, nil
-}

cmd/config_job_control.go

@@ -1,143 +0,0 @@
-package cmd
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"github.com/pkg/errors"
-	"io"
-	"net"
-	"net/http"
-)
-
-type ControlJob struct {
-	Name     string
-	sockaddr *net.UnixAddr
-}
-
-func NewControlJob(name, sockpath string) (j *ControlJob, err error) {
-	j = &ControlJob{Name: name}
-
-	j.sockaddr, err = net.ResolveUnixAddr("unix", sockpath)
-	if err != nil {
-		err = errors.Wrap(err, "cannot resolve unix address")
-		return
-	}
-
-	return
-}
-
-func (j *ControlJob) JobName() string {
-	return j.Name
-}
-
-func (j *ControlJob) JobStatus(ctx context.Context) (*JobStatus, error) {
-	return &JobStatus{Tasks: nil}, nil
-}
-
-const (
-	ControlJobEndpointPProf   string = "/debug/pprof"
-	ControlJobEndpointVersion string = "/version"
-	ControlJobEndpointStatus  string = "/status"
-)
-
-func (j *ControlJob) JobStart(ctx context.Context) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-	defer log.Info("control job finished")
-
-	daemon := ctx.Value(contextKeyDaemon).(*Daemon)
-
-	l, err := ListenUnixPrivate(j.sockaddr)
-	if err != nil {
-		log.WithError(err).Error("error listening")
-		return
-	}
-
-	pprofServer := NewPProfServer(ctx)
-
-	mux := http.NewServeMux()
-	mux.Handle(ControlJobEndpointPProf, requestLogger{log: log, handlerFunc: func(w http.ResponseWriter, r *http.Request) {
-		var msg PprofServerControlMsg
-		err := json.NewDecoder(r.Body).Decode(&msg)
-		if err != nil {
-			log.WithError(err).Error("bad pprof request from client")
-			w.WriteHeader(http.StatusBadRequest)
-		}
-		pprofServer.Control(msg)
-		w.WriteHeader(200)
-	}})
-	mux.Handle(ControlJobEndpointVersion,
-		requestLogger{log: log, handler: jsonResponder{func() (interface{}, error) {
-			return NewZreplVersionInformation(), nil
-		}}})
-	mux.Handle(ControlJobEndpointStatus,
-		requestLogger{log: log, handler: jsonResponder{func() (interface{}, error) {
-			return daemon.Status(), nil
-		}}})
-	server := http.Server{Handler: mux}
-
-outer:
-	for {
-
-		served := make(chan error)
-		go func() {
-			served <- server.Serve(l)
-			close(served)
-		}()
-
-		select {
-		case <-ctx.Done():
-			log.WithError(ctx.Err()).Info("context done")
-			server.Shutdown(context.Background())
-			break outer
-		case err = <-served:
-			if err != nil {
-				log.WithError(err).Error("error serving")
-				break outer
-			}
-		}
-
-	}
-
-}
-
-type jsonResponder struct {
-	producer func() (interface{}, error)
-}
-
-func (j jsonResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	res, err := j.producer()
-	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		io.WriteString(w, err.Error())
-		return
-	}
-	var buf bytes.Buffer
-	err = json.NewEncoder(&buf).Encode(res)
-	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		io.WriteString(w, err.Error())
-	} else {
-		io.Copy(w, &buf)
-	}
-}
-
-type requestLogger struct {
-	log         Logger
-	handler     http.Handler
-	handlerFunc http.HandlerFunc
-}
-
-func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	log := l.log.WithField("method", r.Method).WithField("url", r.URL)
-	log.Info("start")
-	if l.handlerFunc != nil {
-		l.handlerFunc(w, r)
-	} else if l.handler != nil {
-		l.handler.ServeHTTP(w, r)
-	} else {
-		log.Error("no handler or handlerFunc configured")
-	}
-	log.Info("finish")
-}

cmd/config_job_local.go

@@ -1,214 +0,0 @@
-package cmd
-
-import (
-	"time"
-
-	"context"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/zfs"
-	"sync"
-)
-
-type LocalJob struct {
-	Name              string
-	Mapping           *DatasetMapFilter
-	SnapshotPrefix    string
-	Interval          time.Duration
-	InitialReplPolicy InitialReplPolicy
-	PruneLHS          PrunePolicy
-	PruneRHS          PrunePolicy
-	Debug             JobDebugSettings
-	snapperTask       *Task
-	mainTask          *Task
-	handlerTask       *Task
-	pruneRHSTask      *Task
-	pruneLHSTask      *Task
-}
-
-func parseLocalJob(c JobParsingContext, name string, i map[string]interface{}) (j *LocalJob, err error) {
-
-	var asMap struct {
-		Mapping           map[string]string
-		SnapshotPrefix    string `mapstructure:"snapshot_prefix"`
-		Interval          string
-		InitialReplPolicy string                 `mapstructure:"initial_repl_policy"`
-		PruneLHS          map[string]interface{} `mapstructure:"prune_lhs"`
-		PruneRHS          map[string]interface{} `mapstructure:"prune_rhs"`
-		Debug             map[string]interface{}
-	}
-
-	if err = mapstructure.Decode(i, &asMap); err != nil {
-		err = errors.Wrap(err, "mapstructure error")
-		return nil, err
-	}
-
-	j = &LocalJob{Name: name}
-
-	if j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false); err != nil {
-		return
-	}
-
-	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
-		return
-	}
-
-	if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
-		err = errors.Wrap(err, "cannot parse interval")
-		return
-	}
-
-	if j.InitialReplPolicy, err = parseInitialReplPolicy(asMap.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY); err != nil {
-		return
-	}
-
-	if j.PruneLHS, err = parsePrunePolicy(asMap.PruneLHS, true); err != nil {
-		err = errors.Wrap(err, "cannot parse 'prune_lhs'")
-		return
-	}
-	if j.PruneRHS, err = parsePrunePolicy(asMap.PruneRHS, false); err != nil {
-		err = errors.Wrap(err, "cannot parse 'prune_rhs'")
-		return
-	}
-
-	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
-		err = errors.Wrap(err, "cannot parse 'debug'")
-		return
-	}
-
-	return
-}
-
-func (j *LocalJob) JobName() string {
-	return j.Name
-}
-
-func (j *LocalJob) JobStart(ctx context.Context) {
-
-	rootLog := ctx.Value(contextKeyLog).(Logger)
-
-	j.snapperTask = NewTask("snapshot", rootLog)
-	j.mainTask = NewTask("main", rootLog)
-	j.handlerTask = NewTask("handler", rootLog)
-	j.pruneRHSTask = NewTask("prune_rhs", rootLog)
-	j.pruneLHSTask = NewTask("prune_lhs", rootLog)
-
-	local := rpc.NewLocalRPC()
-	// Allow access to any dataset since we control what mapping
-	// is passed to the pull routine.
-	// All local datasets will be passed to its Map() function,
-	// but only those for which a mapping exists will actually be pulled.
-	// We can pay this small performance penalty for now.
-	handler := NewHandler(j.handlerTask.Log(), localPullACL{}, NewPrefixFilter(j.SnapshotPrefix))
-
-	registerEndpoints(local, handler)
-
-	snapper := IntervalAutosnap{
-		task:             j.snapperTask,
-		DatasetFilter:    j.Mapping.AsFilter(),
-		Prefix:           j.SnapshotPrefix,
-		SnapshotInterval: j.Interval,
-	}
-
-	plhs, err := j.Pruner(j.pruneLHSTask, PrunePolicySideLeft, false)
-	if err != nil {
-		rootLog.WithError(err).Error("error creating lhs pruner")
-		return
-	}
-	prhs, err := j.Pruner(j.pruneRHSTask, PrunePolicySideRight, false)
-	if err != nil {
-		rootLog.WithError(err).Error("error creating rhs pruner")
-		return
-	}
-
-	didSnaps := make(chan struct{})
-	go snapper.Run(ctx, didSnaps)
-
-outer:
-	for {
-
-		select {
-		case <-ctx.Done():
-			j.mainTask.Log().WithError(ctx.Err()).Info("context")
-			break outer
-		case <-didSnaps:
-			j.mainTask.Log().Debug("finished taking snapshots")
-			j.mainTask.Log().Info("starting replication procedure")
-		}
-
-		j.mainTask.Log().Debug("replicating from lhs to rhs")
-		j.mainTask.Enter("replicate")
-		puller := Puller{j.mainTask, local, j.Mapping, j.InitialReplPolicy}
-		puller.Pull()
-		j.mainTask.Finish()
-
-		// use a ctx as soon as Pull gains ctx support
-		select {
-		case <-ctx.Done():
-			break outer
-		default:
-		}
-
-		var wg sync.WaitGroup
-
-		j.mainTask.Log().Info("pruning lhs")
-		wg.Add(1)
-		go func() {
-			plhs.Run(ctx)
-			wg.Done()
-		}()
-
-		j.mainTask.Log().Info("pruning rhs")
-		wg.Add(1)
-		go func() {
-			prhs.Run(ctx)
-			wg.Done()
-		}()
-
-		wg.Wait()
-
-	}
-
-}
-
-func (j *LocalJob) JobStatus(ctxt context.Context) (*JobStatus, error) {
-	return &JobStatus{Tasks: []*TaskStatus{
-		j.snapperTask.Status(),
-		j.pruneLHSTask.Status(),
-		j.pruneRHSTask.Status(),
-		j.mainTask.Status(),
-	}}, nil
-}
-
-func (j *LocalJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) {
-
-	var dsfilter zfs.DatasetFilter
-	var pp PrunePolicy
-	switch side {
-	case PrunePolicySideLeft:
-		pp = j.PruneLHS
-		dsfilter = j.Mapping.AsFilter()
-	case PrunePolicySideRight:
-		pp = j.PruneRHS
-		dsfilter, err = j.Mapping.InvertedFilter()
-		if err != nil {
-			err = errors.Wrap(err, "cannot invert mapping for prune_rhs")
-			return
-		}
-	default:
-		err = errors.Errorf("must be either left or right side")
-		return
-	}
-
-	p = Pruner{
-		task,
-		time.Now(),
-		dryRun,
-		dsfilter,
-		j.SnapshotPrefix,
-		pp,
-	}
-
-	return
-}

cmd/config_job_pull.go

@@ -1,195 +0,0 @@
-package cmd
-
-import (
-	"time"
-
-	"context"
-	"fmt"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/util"
-)
-
-type PullJob struct {
-	Name     string
-	Connect  RWCConnecter
-	Interval time.Duration
-	Mapping  *DatasetMapFilter
-	// constructed from mapping during parsing
-	pruneFilter       *DatasetMapFilter
-	SnapshotPrefix    string
-	InitialReplPolicy InitialReplPolicy
-	Prune             PrunePolicy
-	Debug             JobDebugSettings
-
-	task *Task
-}
-
-func parsePullJob(c JobParsingContext, name string, i map[string]interface{}) (j *PullJob, err error) {
-
-	var asMap struct {
-		Connect           map[string]interface{}
-		Interval          string
-		Mapping           map[string]string
-		InitialReplPolicy string `mapstructure:"initial_repl_policy"`
-		Prune             map[string]interface{}
-		SnapshotPrefix    string `mapstructure:"snapshot_prefix"`
-		Debug             map[string]interface{}
-	}
-
-	if err = mapstructure.Decode(i, &asMap); err != nil {
-		err = errors.Wrap(err, "mapstructure error")
-		return nil, err
-	}
-
-	j = &PullJob{Name: name}
-
-	j.Connect, err = parseSSHStdinserverConnecter(asMap.Connect)
-	if err != nil {
-		err = errors.Wrap(err, "cannot parse 'connect'")
-		return nil, err
-	}
-
-	if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
-		err = errors.Wrap(err, "cannot parse 'interval'")
-		return nil, err
-	}
-
-	j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false)
-	if err != nil {
-		err = errors.Wrap(err, "cannot parse 'mapping'")
-		return nil, err
-	}
-
-	if j.pruneFilter, err = j.Mapping.InvertedFilter(); err != nil {
-		err = errors.Wrap(err, "cannot automatically invert 'mapping' for prune job")
-		return nil, err
-	}
-
-	j.InitialReplPolicy, err = parseInitialReplPolicy(asMap.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY)
-	if err != nil {
-		err = errors.Wrap(err, "cannot parse 'initial_repl_policy'")
-		return
-	}
-
-	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
-		return
-	}
-
-	if j.Prune, err = parsePrunePolicy(asMap.Prune, false); err != nil {
-		err = errors.Wrap(err, "cannot parse prune policy")
-		return
-	}
-
-	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
-		err = errors.Wrap(err, "cannot parse 'debug'")
-		return
-	}
-
-	return
-}
-
-func (j *PullJob) JobName() string {
-	return j.Name
-}
-
-func (j *PullJob) JobStart(ctx context.Context) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-	defer log.Info("exiting")
-	j.task = NewTask("main", log)
-
-	// j.task is idle here idle here
-
-	ticker := time.NewTicker(j.Interval)
-	for {
-		j.doRun(ctx)
-		select {
-		case <-ctx.Done():
-			j.task.Log().WithError(ctx.Err()).Info("context")
-			return
-		case <-ticker.C:
-		}
-	}
-}
-
-func (j *PullJob) doRun(ctx context.Context) {
-
-	j.task.Enter("run")
-	defer j.task.Finish()
-
-	j.task.Log().Info("connecting")
-	rwc, err := j.Connect.Connect()
-	if err != nil {
-		j.task.Log().WithError(err).Error("error connecting")
-		return
-	}
-
-	rwc, err = util.NewReadWriteCloserLogger(rwc, j.Debug.Conn.ReadDump, j.Debug.Conn.WriteDump)
-	if err != nil {
-		return
-	}
-
-	client := rpc.NewClient(rwc)
-	if j.Debug.RPC.Log {
-		client.SetLogger(j.task.Log(), true)
-	}
-
-	j.task.Enter("pull")
-	puller := Puller{j.task, client, j.Mapping, j.InitialReplPolicy}
-	puller.Pull()
-	closeRPCWithTimeout(j.task, client, time.Second*1, "")
-	rwc.Close()
-	j.task.Finish()
-
-	j.task.Enter("prune")
-	pruner, err := j.Pruner(j.task, PrunePolicySideDefault, false)
-	if err != nil {
-		j.task.Log().WithError(err).Error("error creating pruner")
-	} else {
-		pruner.Run(ctx)
-	}
-	j.task.Finish()
-
-}
-
-func (j *PullJob) JobStatus(ctxt context.Context) (*JobStatus, error) {
-	return &JobStatus{Tasks: []*TaskStatus{j.task.Status()}}, nil
-}
-
-func (j *PullJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) {
-	p = Pruner{
-		task,
-		time.Now(),
-		dryRun,
-		j.pruneFilter,
-		j.SnapshotPrefix,
-		j.Prune,
-	}
-	return
-}
-
-func closeRPCWithTimeout(task *Task, remote rpc.RPCClient, timeout time.Duration, goodbye string) {
-
-	task.Log().Info("closing rpc connection")
-
-	ch := make(chan error)
-	go func() {
-		ch <- remote.Close()
-		close(ch)
-	}()
-
-	var err error
-	select {
-	case <-time.After(timeout):
-		err = fmt.Errorf("timeout exceeded (%s)", timeout)
-	case closeRequestErr := <-ch:
-		err = closeRequestErr
-	}
-
-	if err != nil {
-		task.Log().WithError(err).Error("error closing connection")
-	}
-	return
-}

cmd/config_job_source.go

@@ -1,250 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"io"
-	"time"
-
-	mapstructure "github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/util"
-)
-
-type SourceJob struct {
-	Name           string
-	Serve          AuthenticatedChannelListenerFactory
-	Filesystems    *DatasetMapFilter
-	SnapshotPrefix string
-	Interval       time.Duration
-	Prune          PrunePolicy
-	Debug          JobDebugSettings
-	serveTask      *Task
-	autosnapTask   *Task
-	pruneTask      *Task
-}
-
-func parseSourceJob(c JobParsingContext, name string, i map[string]interface{}) (j *SourceJob, err error) {
-
-	var asMap struct {
-		Serve          map[string]interface{}
-		Filesystems    map[string]string
-		SnapshotPrefix string `mapstructure:"snapshot_prefix"`
-		Interval       string
-		Prune          map[string]interface{}
-		Debug          map[string]interface{}
-	}
-
-	if err = mapstructure.Decode(i, &asMap); err != nil {
-		err = errors.Wrap(err, "mapstructure error")
-		return nil, err
-	}
-
-	j = &SourceJob{Name: name}
-
-	if j.Serve, err = parseAuthenticatedChannelListenerFactory(c, asMap.Serve); err != nil {
-		return
-	}
-
-	if j.Filesystems, err = parseDatasetMapFilter(asMap.Filesystems, true); err != nil {
-		return
-	}
-
-	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
-		return
-	}
-
-	if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
-		err = errors.Wrap(err, "cannot parse 'interval'")
-		return
-	}
-
-	if j.Prune, err = parsePrunePolicy(asMap.Prune, true); err != nil {
-		err = errors.Wrap(err, "cannot parse 'prune'")
-		return
-	}
-
-	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
-		err = errors.Wrap(err, "cannot parse 'debug'")
-		return
-	}
-
-	return
-}
-
-func (j *SourceJob) JobName() string {
-	return j.Name
-}
-
-func (j *SourceJob) JobStart(ctx context.Context) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-	defer log.Info("exiting")
-
-	j.autosnapTask = NewTask("autosnap", log)
-	j.pruneTask = NewTask("prune", log)
-	j.serveTask = NewTask("serve", log)
-
-	a := IntervalAutosnap{j.autosnapTask, j.Filesystems, j.SnapshotPrefix, j.Interval}
-	p, err := j.Pruner(j.pruneTask, PrunePolicySideDefault, false)
-
-	if err != nil {
-		log.WithError(err).Error("error creating pruner")
-		return
-	}
-
-	didSnaps := make(chan struct{})
-
-	go j.serve(ctx, j.serveTask)
-	go a.Run(ctx, didSnaps)
-
-outer:
-	for {
-		select {
-		case <-ctx.Done():
-			break outer
-		case <-didSnaps:
-			log.Info("starting pruner")
-			p.Run(ctx)
-			log.Info("pruner done")
-		}
-	}
-	log.WithError(ctx.Err()).Info("context")
-
-}
-
-func (j *SourceJob) JobStatus(ctxt context.Context) (*JobStatus, error) {
-	return &JobStatus{
-		Tasks: []*TaskStatus{
-			j.autosnapTask.Status(),
-			j.pruneTask.Status(),
-			j.serveTask.Status(),
-		}}, nil
-}
-
-func (j *SourceJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) {
-	p = Pruner{
-		task,
-		time.Now(),
-		dryRun,
-		j.Filesystems,
-		j.SnapshotPrefix,
-		j.Prune,
-	}
-	return
-}
-
-func (j *SourceJob) serve(ctx context.Context, task *Task) {
-
-	listener, err := j.Serve.Listen()
-	if err != nil {
-		task.Log().WithError(err).Error("error listening")
-		return
-	}
-
-	type rwcChanMsg struct {
-		rwc io.ReadWriteCloser
-		err error
-	}
-	rwcChan := make(chan rwcChanMsg)
-
-	// Serve connections until interrupted or error
-outer:
-	for {
-
-		go func() {
-			rwc, err := listener.Accept()
-			if err != nil {
-				rwcChan <- rwcChanMsg{rwc, err}
-				close(rwcChan)
-				return
-			}
-			rwcChan <- rwcChanMsg{rwc, err}
-		}()
-
-		select {
-
-		case rwcMsg := <-rwcChan:
-
-			if rwcMsg.err != nil {
-				task.Log().WithError(err).Error("error accepting connection")
-				break outer
-			}
-
-			j.handleConnection(rwcMsg.rwc, task)
-
-		case <-ctx.Done():
-			task.Log().WithError(ctx.Err()).Info("context")
-			break outer
-
-		}
-
-	}
-
-	task.Enter("close_listener")
-	defer task.Finish()
-	err = listener.Close()
-	if err != nil {
-		task.Log().WithError(err).Error("error closing listener")
-	}
-
-	return
-
-}
-
-func (j *SourceJob) handleConnection(rwc io.ReadWriteCloser, task *Task) {
-
-	task.Enter("handle_connection")
-	defer task.Finish()
-
-	task.Log().Info("handling client connection")
-
-	rwc, err := util.NewReadWriteCloserLogger(rwc, j.Debug.Conn.ReadDump, j.Debug.Conn.WriteDump)
-	if err != nil {
-		panic(err)
-	}
-
-	// construct connection handler
-	handler := NewHandler(task.Log(), j.Filesystems, NewPrefixFilter(j.SnapshotPrefix))
-
-	// handle connection
-	rpcServer := rpc.NewServer(rwc)
-	if j.Debug.RPC.Log {
-		rpclog := task.Log().WithField("subsystem", "rpc")
-		rpcServer.SetLogger(rpclog, true)
-	}
-	registerEndpoints(rpcServer, handler)
-	if err = rpcServer.Serve(); err != nil {
-		task.Log().WithError(err).Error("error serving connection")
-	}
-
-	// wait for client to close connection
-	// FIXME: we cannot just close it like we would to with a TCP socket because
-	// FIXME: go-nettsh's Close() may overtake the remaining data in the pipe
-	const CLIENT_HANGUP_TIMEOUT = 1 * time.Second
-	task.Log().
-		WithField("timeout", CLIENT_HANGUP_TIMEOUT).
-		Debug("waiting for client to hang up")
-
-	wchan := make(chan error)
-	go func() {
-		var pseudo [1]byte
-		_, err := io.ReadFull(rwc, pseudo[:])
-		wchan <- err
-	}()
-	var werr error
-	select {
-	case werr = <-wchan:
-		// all right
-	case <-time.After(CLIENT_HANGUP_TIMEOUT):
-		werr = errors.New("client did not close connection within timeout")
-	}
-	if werr != nil && werr != io.EOF {
-		task.Log().WithError(werr).
-			Error("error waiting for client to hang up")
-	}
-	task.Log().Info("closing client connection")
-	if err = rwc.Close(); err != nil {
-		task.Log().WithError(err).Error("error force-closing connection")
-	}
-}

cmd/config_logging.go

@@ -1,251 +0,0 @@
-package cmd
-
-import (
-	"crypto/tls"
-	"crypto/x509"
-	"github.com/mattn/go-isatty"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/logger"
-	"io/ioutil"
-	"os"
-	"time"
-)
-
-type LoggingConfig struct {
-	Outlets *logger.Outlets
-}
-
-type MetadataFlags int64
-
-const (
-	MetadataTime MetadataFlags = 1 << iota
-	MetadataLevel
-
-	MetadataNone MetadataFlags = 0
-	MetadataAll  MetadataFlags = ^0
-)
-
-func parseLogging(i interface{}) (c *LoggingConfig, err error) {
-
-	c = &LoggingConfig{}
-	c.Outlets = logger.NewOutlets()
-
-	var asList []interface{}
-	if err = mapstructure.Decode(i, &asList); err != nil {
-		return nil, errors.Wrap(err, "mapstructure error")
-	}
-	if len(asList) == 0 {
-		// Default config
-		out := WriterOutlet{&HumanFormatter{}, os.Stdout}
-		c.Outlets.Add(out, logger.Warn)
-		return
-	}
-
-	var syslogOutlets, stdoutOutlets int
-	for lei, le := range asList {
-
-		outlet, minLevel, err := parseOutlet(le)
-		if err != nil {
-			return nil, errors.Wrapf(err, "cannot parse outlet #%d", lei)
-		}
-		var _ logger.Outlet = WriterOutlet{}
-		var _ logger.Outlet = &SyslogOutlet{}
-		switch outlet.(type) {
-		case *SyslogOutlet:
-			syslogOutlets++
-		case WriterOutlet:
-			stdoutOutlets++
-		}
-
-		c.Outlets.Add(outlet, minLevel)
-
-	}
-
-	if syslogOutlets > 1 {
-		return nil, errors.Errorf("can only define one 'syslog' outlet")
-	}
-	if stdoutOutlets > 1 {
-		return nil, errors.Errorf("can only define one 'stdout' outlet")
-	}
-
-	return c, nil
-
-}
-
-func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
-	var is string
-	switch j := i.(type) {
-	case string:
-		is = j
-	default:
-		return nil, errors.Errorf("invalid log format: wrong type: %T", i)
-	}
-
-	switch is {
-	case "human":
-		return &HumanFormatter{}, nil
-	case "logfmt":
-		return &LogfmtFormatter{}, nil
-	case "json":
-		return &JSONFormatter{}, nil
-	default:
-		return nil, errors.Errorf("invalid log format: '%s'", is)
-	}
-
-}
-
-func parseOutlet(i interface{}) (o logger.Outlet, level logger.Level, err error) {
-
-	var in struct {
-		Outlet string
-		Level  string
-		Format string
-	}
-	if err = mapstructure.Decode(i, &in); err != nil {
-		err = errors.Wrap(err, "mapstructure error")
-		return
-	}
-	if in.Outlet == "" || in.Level == "" || in.Format == "" {
-		err = errors.Errorf("must specify 'outlet', 'level' and 'format' field")
-		return
-	}
-
-	minLevel, err := logger.ParseLevel(in.Level)
-	if err != nil {
-		err = errors.Wrap(err, "cannot parse 'level' field")
-		return
-	}
-	formatter, err := parseLogFormat(in.Format)
-	if err != nil {
-		err = errors.Wrap(err, "cannot parse")
-		return
-	}
-
-	switch in.Outlet {
-	case "stdout":
-		o, err = parseStdoutOutlet(i, formatter)
-	case "tcp":
-		o, err = parseTCPOutlet(i, formatter)
-	case "syslog":
-		o, err = parseSyslogOutlet(i, formatter)
-	default:
-		err = errors.Errorf("unknown outlet type '%s'", in.Outlet)
-	}
-	return o, minLevel, err
-
-}
-
-func parseStdoutOutlet(i interface{}, formatter EntryFormatter) (WriterOutlet, error) {
-
-	var in struct {
-		Time bool
-	}
-	if err := mapstructure.Decode(i, &in); err != nil {
-		return WriterOutlet{}, errors.Wrap(err, "invalid structure for stdout outlet")
-	}
-
-	flags := MetadataAll
-	writer := os.Stdout
-	if !isatty.IsTerminal(writer.Fd()) && !in.Time {
-		flags &= ^MetadataTime
-	}
-
-	formatter.SetMetadataFlags(flags)
-	return WriterOutlet{
-		formatter,
-		os.Stdout,
-	}, nil
-}
-
-func parseTCPOutlet(i interface{}, formatter EntryFormatter) (out *TCPOutlet, err error) {
-
-	var in struct {
-		Net           string
-		Address       string
-		RetryInterval string `mapstructure:"retry_interval"`
-		TLS           *struct {
-			CA   string
-			Cert string
-			Key  string
-		}
-	}
-	if err = mapstructure.Decode(i, &in); err != nil {
-		return nil, errors.Wrap(err, "mapstructure error")
-	}
-
-	retryInterval, err := time.ParseDuration(in.RetryInterval)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
-	}
-
-	if len(in.Net) == 0 {
-		return nil, errors.New("field 'net' must not be empty")
-	}
-	if len(in.Address) == 0 {
-		return nil, errors.New("field 'address' must not be empty")
-	}
-
-	var tlsConfig *tls.Config
-	if in.TLS != nil {
-
-		cert, err := tls.LoadX509KeyPair(in.TLS.Cert, in.TLS.Key)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot load client cert")
-		}
-
-		var rootCAs *x509.CertPool
-		if in.TLS.CA == "" {
-			if rootCAs, err = x509.SystemCertPool(); err != nil {
-				return nil, errors.Wrap(err, "cannot open system cert pool")
-			}
-		} else {
-			rootCAs = x509.NewCertPool()
-			rootCAPEM, err := ioutil.ReadFile(in.TLS.CA)
-			if err != nil {
-				return nil, errors.Wrap(err, "cannot load CA cert")
-			}
-			if !rootCAs.AppendCertsFromPEM(rootCAPEM) {
-				return nil, errors.New("cannot parse CA cert")
-			}
-		}
-		if err != nil && in.TLS.CA == "" {
-			return nil, errors.Wrap(err, "cannot load root ca pool")
-		}
-
-		tlsConfig = &tls.Config{
-			Certificates: []tls.Certificate{cert},
-			RootCAs:      rootCAs,
-		}
-
-		tlsConfig.BuildNameToCertificate()
-	}
-
-	formatter.SetMetadataFlags(MetadataAll)
-	return NewTCPOutlet(formatter, in.Net, in.Address, tlsConfig, retryInterval), nil
-
-}
-
-func parseSyslogOutlet(i interface{}, formatter EntryFormatter) (out *SyslogOutlet, err error) {
-
-	var in struct {
-		RetryInterval string `mapstructure:"retry_interval"`
-	}
-	if err = mapstructure.Decode(i, &in); err != nil {
-		return nil, errors.Wrap(err, "mapstructure error")
-	}
-
-	out = &SyslogOutlet{}
-	out.Formatter = formatter
-	out.Formatter.SetMetadataFlags(MetadataNone)
-
-	out.RetryInterval = 0 // default to 0 as we assume local syslog will just work
-	if in.RetryInterval != "" {
-		out.RetryInterval, err = time.ParseDuration(in.RetryInterval)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
-		}
-	}
-
-	return
-}

cmd/config_parse.go

@@ -1,294 +0,0 @@
-package cmd
-
-import (
-	"io/ioutil"
-
-	"fmt"
-	yaml "github.com/go-yaml/yaml"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"os"
-	"regexp"
-	"strconv"
-	"time"
-)
-
-var ConfigFileDefaultLocations []string = []string{
-	"/etc/zrepl/zrepl.yml",
-	"/usr/local/etc/zrepl/zrepl.yml",
-}
-
-const (
-	JobNameControl string = "control"
-)
-
-var ReservedJobNames []string = []string{
-	JobNameControl,
-}
-
-type ConfigParsingContext struct {
-	Global *Global
-}
-
-func ParseConfig(path string) (config *Config, err error) {
-
-	if path == "" {
-		// Try default locations
-		for _, l := range ConfigFileDefaultLocations {
-			stat, err := os.Stat(l)
-			if err != nil {
-				continue
-			}
-			if !stat.Mode().IsRegular() {
-				err = errors.Errorf("file at default location is not a regular file: %s", l)
-				continue
-			}
-			path = l
-			break
-		}
-	}
-
-	var i interface{}
-
-	var bytes []byte
-
-	if bytes, err = ioutil.ReadFile(path); err != nil {
-		err = errors.WithStack(err)
-		return
-	}
-
-	if err = yaml.Unmarshal(bytes, &i); err != nil {
-		err = errors.WithStack(err)
-		return
-	}
-
-	return parseConfig(i)
-}
-
-func parseConfig(i interface{}) (c *Config, err error) {
-
-	var asMap struct {
-		Global map[string]interface{}
-		Jobs   []map[string]interface{}
-	}
-	if err := mapstructure.Decode(i, &asMap); err != nil {
-		return nil, errors.Wrap(err, "config root must be a dict")
-	}
-
-	c = &Config{}
-
-	// Parse global with defaults
-	c.Global.Serve.Stdinserver.SockDir = "/var/run/zrepl/stdinserver"
-	c.Global.Control.Sockpath = "/var/run/zrepl/control"
-
-	err = mapstructure.Decode(asMap.Global, &c.Global)
-	if err != nil {
-		err = errors.Wrap(err, "mapstructure error on 'global' section: %s")
-		return
-	}
-
-	if c.Global.logging, err = parseLogging(asMap.Global["logging"]); err != nil {
-		return nil, errors.Wrap(err, "cannot parse logging section")
-	}
-
-	cpc := ConfigParsingContext{&c.Global}
-	jpc := JobParsingContext{cpc}
-
-	// Jobs
-	c.Jobs = make(map[string]Job, len(asMap.Jobs))
-	for i := range asMap.Jobs {
-		job, err := parseJob(jpc, asMap.Jobs[i])
-		if err != nil {
-			// Try to find its name
-			namei, ok := asMap.Jobs[i]["name"]
-			if !ok {
-				namei = fmt.Sprintf("<no name, entry #%d in list>", i)
-			}
-			err = errors.Wrapf(err, "cannot parse job '%v'", namei)
-			return nil, err
-		}
-		jn := job.JobName()
-		if _, ok := c.Jobs[jn]; ok {
-			err = errors.Errorf("duplicate job name: %s", jn)
-			return nil, err
-		}
-		c.Jobs[job.JobName()] = job
-	}
-
-	cj, err := NewControlJob(JobNameControl, jpc.Global.Control.Sockpath)
-	if err != nil {
-		err = errors.Wrap(err, "cannot create control job")
-		return
-	}
-	c.Jobs[JobNameControl] = cj
-
-	return c, nil
-
-}
-
-func extractStringField(i map[string]interface{}, key string, notempty bool) (field string, err error) {
-	vi, ok := i[key]
-	if !ok {
-		err = errors.Errorf("must have field '%s'", key)
-		return "", err
-	}
-	field, ok = vi.(string)
-	if !ok {
-		err = errors.Errorf("'%s' field must have type string", key)
-		return "", err
-	}
-	if notempty && len(field) <= 0 {
-		err = errors.Errorf("'%s' field must not be empty", key)
-		return "", err
-	}
-	return
-}
-
-type JobParsingContext struct {
-	ConfigParsingContext
-}
-
-func parseJob(c JobParsingContext, i map[string]interface{}) (j Job, err error) {
-
-	name, err := extractStringField(i, "name", true)
-	if err != nil {
-		return
-	}
-
-	for _, r := range ReservedJobNames {
-		if name == r {
-			err = errors.Errorf("job name '%s' is reserved", name)
-			return
-		}
-	}
-
-	jobtype, err := extractStringField(i, "type", true)
-	if err != nil {
-		return
-	}
-
-	switch jobtype {
-	case "pull":
-		return parsePullJob(c, name, i)
-	case "source":
-		return parseSourceJob(c, name, i)
-	case "local":
-		return parseLocalJob(c, name, i)
-	default:
-		return nil, errors.Errorf("unknown job type '%s'", jobtype)
-	}
-
-}
-
-func parseConnect(i map[string]interface{}) (c RWCConnecter, err error) {
-
-	t, err := extractStringField(i, "type", true)
-	if err != nil {
-		return nil, err
-	}
-
-	switch t {
-	case "ssh+stdinserver":
-		return parseSSHStdinserverConnecter(i)
-	default:
-		return nil, errors.Errorf("unknown connection type '%s'", t)
-	}
-
-}
-
-func parseInitialReplPolicy(v interface{}, defaultPolicy InitialReplPolicy) (p InitialReplPolicy, err error) {
-	s, ok := v.(string)
-	if !ok {
-		goto err
-	}
-
-	switch {
-	case s == "":
-		p = defaultPolicy
-	case s == "most_recent":
-		p = InitialReplPolicyMostRecent
-	case s == "all":
-		p = InitialReplPolicyAll
-	default:
-		goto err
-	}
-
-	return
-
-err:
-	err = errors.New(fmt.Sprintf("expected InitialReplPolicy, got %#v", v))
-	return
-}
-
-func parsePrunePolicy(v map[string]interface{}, willSeeBookmarks bool) (p PrunePolicy, err error) {
-
-	policyName, err := extractStringField(v, "policy", true)
-	if err != nil {
-		return
-	}
-
-	switch policyName {
-	case "grid":
-		return parseGridPrunePolicy(v, willSeeBookmarks)
-	case "noprune":
-		return NoPrunePolicy{}, nil
-	default:
-		err = errors.Errorf("unknown policy '%s'", policyName)
-		return
-	}
-}
-
-func parseAuthenticatedChannelListenerFactory(c JobParsingContext, v map[string]interface{}) (p AuthenticatedChannelListenerFactory, err error) {
-
-	t, err := extractStringField(v, "type", true)
-	if err != nil {
-		return nil, err
-	}
-
-	switch t {
-	case "stdinserver":
-		return parseStdinserverListenerFactory(c, v)
-	default:
-		err = errors.Errorf("unknown type '%s'", t)
-		return
-	}
-
-}
-
-var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
-
-func parsePostitiveDuration(e string) (d time.Duration, err error) {
-	comps := durationStringRegex.FindStringSubmatch(e)
-	if len(comps) != 3 {
-		err = fmt.Errorf("does not match regex: %s %#v", e, comps)
-		return
-	}
-
-	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
-	if err != nil {
-		return 0, err
-	}
-	if durationFactor <= 0 {
-		return 0, errors.New("duration must be positive integer")
-	}
-
-	var durationUnit time.Duration
-	switch comps[2] {
-	case "s":
-		durationUnit = time.Second
-	case "m":
-		durationUnit = time.Minute
-	case "h":
-		durationUnit = time.Hour
-	case "d":
-		durationUnit = 24 * time.Hour
-	case "w":
-		durationUnit = 24 * 7 * time.Hour
-	default:
-		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
-		return
-	}
-
-	d = time.Duration(durationFactor) * durationUnit
-	return
-}

cmd/config_prune_grid.go

@@ -1,245 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/util"
-	"github.com/zrepl/zrepl/zfs"
-	"math"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-	"time"
-)
-
-type GridPrunePolicy struct {
-	RetentionGrid *util.RetentionGrid
-	MaxBookmarks  int
-}
-
-const GridPrunePolicyMaxBookmarksKeepAll = -1
-
-type retentionGridAdaptor struct {
-	zfs.FilesystemVersion
-}
-
-func (a retentionGridAdaptor) Date() time.Time {
-	return a.Creation
-}
-
-func (a retentionGridAdaptor) LessThan(b util.RetentionGridEntry) bool {
-	return a.CreateTXG < b.(retentionGridAdaptor).CreateTXG
-}
-
-// Prune filters snapshots with the retention grid.
-// Bookmarks are deleted such that KeepBookmarks are kept in the end.
-// The oldest bookmarks are removed first.
-func (p *GridPrunePolicy) Prune(_ *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error) {
-	skeep, sremove := p.pruneSnapshots(versions)
-	keep, remove = p.pruneBookmarks(skeep)
-	remove = append(remove, sremove...)
-	return keep, remove, nil
-}
-
-func (p *GridPrunePolicy) pruneSnapshots(versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion) {
-
-	// Build adaptors for retention grid
-	keep = []zfs.FilesystemVersion{}
-	adaptors := make([]util.RetentionGridEntry, 0)
-	for fsv := range versions {
-		if versions[fsv].Type != zfs.Snapshot {
-			keep = append(keep, versions[fsv])
-			continue
-		}
-		adaptors = append(adaptors, retentionGridAdaptor{versions[fsv]})
-	}
-
-	sort.SliceStable(adaptors, func(i, j int) bool {
-		return adaptors[i].LessThan(adaptors[j])
-	})
-	now := adaptors[len(adaptors)-1].Date()
-
-	// Evaluate retention grid
-	keepa, removea := p.RetentionGrid.FitEntries(now, adaptors)
-
-	// Revert adaptors
-	for i := range keepa {
-		keep = append(keep, keepa[i].(retentionGridAdaptor).FilesystemVersion)
-	}
-	remove = make([]zfs.FilesystemVersion, len(removea))
-	for i := range removea {
-		remove[i] = removea[i].(retentionGridAdaptor).FilesystemVersion
-	}
-	return
-
-}
-
-func (p *GridPrunePolicy) pruneBookmarks(versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion) {
-
-	if p.MaxBookmarks == GridPrunePolicyMaxBookmarksKeepAll {
-		return versions, []zfs.FilesystemVersion{}
-	}
-
-	keep = []zfs.FilesystemVersion{}
-	bookmarks := make([]zfs.FilesystemVersion, 0)
-	for fsv := range versions {
-		if versions[fsv].Type != zfs.Bookmark {
-			keep = append(keep, versions[fsv])
-			continue
-		}
-		bookmarks = append(bookmarks, versions[fsv])
-	}
-
-	if len(bookmarks) == 0 {
-		return keep, []zfs.FilesystemVersion{}
-	}
-	if len(bookmarks) < p.MaxBookmarks {
-		keep = append(keep, bookmarks...)
-		return keep, []zfs.FilesystemVersion{}
-	}
-
-	// NOTE: sorting descending by descending by createtxg <=> sorting ascending wrt creation time
-	sort.SliceStable(bookmarks, func(i, j int) bool {
-		return (bookmarks[i].CreateTXG > bookmarks[j].CreateTXG)
-	})
-
-	keep = append(keep, bookmarks[:p.MaxBookmarks]...)
-	remove = bookmarks[p.MaxBookmarks:]
-
-	return keep, remove
-}
-
-func parseGridPrunePolicy(e map[string]interface{}, willSeeBookmarks bool) (p *GridPrunePolicy, err error) {
-
-	const KeepBookmarksAllString = "all"
-	var i struct {
-		Grid          string
-		KeepBookmarks string `mapstructure:"keep_bookmarks"`
-	}
-
-	dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{Result: &i, WeaklyTypedInput: true})
-	if err != nil {
-		err = errors.Wrap(err, "mapstructure error")
-		return
-	}
-	if err = dec.Decode(e); err != nil {
-		err = errors.Wrapf(err, "mapstructure error")
-		return
-	}
-
-	// Parse grid
-	intervals, err := parseRetentionGridIntervalsString(i.Grid)
-	if err != nil {
-		err = fmt.Errorf("cannot parse retention grid: %s", err)
-		return
-	}
-	// Assert intervals are of increasing length (not necessarily required, but indicates config mistake)
-	lastDuration := time.Duration(0)
-	for i := range intervals {
-
-		if intervals[i].Length < lastDuration {
-			// If all intervals before were keep=all, this is ok
-			allPrevKeepCountAll := true
-			for j := i - 1; allPrevKeepCountAll && j >= 0; j-- {
-				allPrevKeepCountAll = intervals[j].KeepCount == util.RetentionGridKeepCountAll
-			}
-			if allPrevKeepCountAll {
-				goto isMonotonicIncrease
-			}
-			err = errors.New("retention grid interval length must be monotonically increasing")
-			return
-		}
-	isMonotonicIncrease:
-		lastDuration = intervals[i].Length
-
-	}
-
-	// Parse KeepBookmarks
-	keepBookmarks := 0
-	if i.KeepBookmarks == KeepBookmarksAllString || (i.KeepBookmarks == "" && !willSeeBookmarks) {
-		keepBookmarks = GridPrunePolicyMaxBookmarksKeepAll
-	} else {
-		i, err := strconv.ParseInt(i.KeepBookmarks, 10, 32)
-		if err != nil || i <= 0 || i > math.MaxInt32 {
-			return nil, errors.Errorf("keep_bookmarks must be positive integer or 'all'")
-		}
-		keepBookmarks = int(i)
-	}
-	return &GridPrunePolicy{
-		util.NewRetentionGrid(intervals),
-		keepBookmarks,
-	}, nil
-}
-
-var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
-
-func parseRetentionGridIntervalString(e string) (intervals []util.RetentionInterval, err error) {
-
-	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
-	if comps == nil {
-		err = fmt.Errorf("retention string does not match expected format")
-		return
-	}
-
-	times, err := strconv.Atoi(comps[1])
-	if err != nil {
-		return nil, err
-	} else if times <= 0 {
-		return nil, fmt.Errorf("contains factor <= 0")
-	}
-
-	duration, err := parsePostitiveDuration(comps[2])
-	if err != nil {
-		return nil, err
-	}
-
-	keepCount := 1
-	if comps[3] != "" {
-		// Decompose key=value, comma separated
-		// For now, only keep_count is supported
-		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
-		res := re.FindStringSubmatch(comps[4])
-		if res == nil || len(res) != 2 {
-			err = fmt.Errorf("interval parameter contains unknown parameters")
-			return
-		}
-		if res[1] == "all" {
-			keepCount = util.RetentionGridKeepCountAll
-		} else {
-			keepCount, err = strconv.Atoi(res[1])
-			if err != nil {
-				err = fmt.Errorf("cannot parse keep_count value")
-				return
-			}
-		}
-	}
-
-	intervals = make([]util.RetentionInterval, times)
-	for i := range intervals {
-		intervals[i] = util.RetentionInterval{
-			Length:    duration,
-			KeepCount: keepCount,
-		}
-	}
-
-	return
-
-}
-
-func parseRetentionGridIntervalsString(s string) (intervals []util.RetentionInterval, err error) {
-
-	ges := strings.Split(s, "|")
-	intervals = make([]util.RetentionInterval, 0, 7*len(ges))
-
-	for intervalIdx, e := range ges {
-		parsed, err := parseRetentionGridIntervalString(e)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
-		}
-		intervals = append(intervals, parsed...)
-	}
-
-	return
-}

cmd/config_prune_none.go

@@ -1,11 +0,0 @@
-package cmd
-
-import "github.com/zrepl/zrepl/zfs"
-
-type NoPrunePolicy struct{}
-
-func (p NoPrunePolicy) Prune(fs *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error) {
-	keep = versions
-	remove = []zfs.FilesystemVersion{}
-	return
-}

cmd/config_serve_stdinserver.go

@@ -1,56 +0,0 @@
-package cmd
-
-import (
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/problame/go-netssh"
-	"io"
-	"path"
-)
-
-type StdinserverListenerFactory struct {
-	ClientIdentity string `mapstructure:"client_identity"`
-	sockpath       string
-}
-
-func parseStdinserverListenerFactory(c JobParsingContext, i map[string]interface{}) (f *StdinserverListenerFactory, err error) {
-
-	f = &StdinserverListenerFactory{}
-
-	if err = mapstructure.Decode(i, f); err != nil {
-		return nil, errors.Wrap(err, "mapstructure error")
-	}
-	if !(len(f.ClientIdentity) > 0) {
-		err = errors.Errorf("must specify 'client_identity'")
-		return
-	}
-
-	f.sockpath = path.Join(c.Global.Serve.Stdinserver.SockDir, f.ClientIdentity)
-
-	return
-}
-
-func (f *StdinserverListenerFactory) Listen() (al AuthenticatedChannelListener, err error) {
-
-	if err = PreparePrivateSockpath(f.sockpath); err != nil {
-		return nil, err
-	}
-
-	l, err := netssh.Listen(f.sockpath)
-	if err != nil {
-		return nil, err
-	}
-	return StdinserverListener{l}, nil
-}
-
-type StdinserverListener struct {
-	l *netssh.Listener
-}
-
-func (l StdinserverListener) Accept() (ch io.ReadWriteCloser, err error) {
-	return l.l.Accept()
-}
-
-func (l StdinserverListener) Close() (err error) {
-	return l.l.Close()
-}

cmd/config_test.go

@@ -1,222 +0,0 @@
-package cmd
-
-import (
-	"testing"
-	"time"
-
-	"github.com/kr/pretty"
-	"github.com/stretchr/testify/assert"
-	"github.com/zrepl/zrepl/util"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
-
-	paths := []string{
-		"./sampleconf/localbackup/host1.yml",
-		"./sampleconf/pullbackup/backuphost.yml",
-		"./sampleconf/pullbackup/productionhost.yml",
-		"./sampleconf/random/debugging.yml",
-		"./sampleconf/random/logging.yml",
-	}
-
-	for _, p := range paths {
-
-		c, err := ParseConfig(p)
-		if err != nil {
-			t.Errorf("error parsing %s:\n%+v", p, err)
-		}
-
-		t.Logf("file: %s", p)
-		t.Log(pretty.Sprint(c))
-
-	}
-
-}
-
-func TestParseRetentionGridStringParsing(t *testing.T) {
-
-	intervals, err := parseRetentionGridIntervalsString("2x10m(keep=2) | 1x1h | 3x1w")
-
-	assert.Nil(t, err)
-	assert.Len(t, intervals, 6)
-	proto := util.RetentionInterval{
-		KeepCount: 2,
-		Length:    10 * time.Minute,
-	}
-	assert.EqualValues(t, proto, intervals[0])
-	assert.EqualValues(t, proto, intervals[1])
-
-	proto.KeepCount = 1
-	proto.Length = 1 * time.Hour
-	assert.EqualValues(t, proto, intervals[2])
-
-	proto.Length = 7 * 24 * time.Hour
-	assert.EqualValues(t, proto, intervals[3])
-	assert.EqualValues(t, proto, intervals[4])
-	assert.EqualValues(t, proto, intervals[5])
-
-	intervals, err = parseRetentionGridIntervalsString("|")
-	assert.Error(t, err)
-	intervals, err = parseRetentionGridIntervalsString("2x10m")
-	assert.NoError(t, err)
-
-	intervals, err = parseRetentionGridIntervalsString("1x10m(keep=all)")
-	assert.NoError(t, err)
-	assert.Len(t, intervals, 1)
-	assert.EqualValues(t, util.RetentionGridKeepCountAll, intervals[0].KeepCount)
-
-}
-
-func TestDatasetMapFilter(t *testing.T) {
-
-	expectMapping := func(m map[string]string, from, to string) {
-		dmf, err := parseDatasetMapFilter(m, false)
-		if err != nil {
-			t.Logf("expect test map to be valid: %s", err)
-			t.FailNow()
-		}
-		fromPath, err := zfs.NewDatasetPath(from)
-		if err != nil {
-			t.Logf("expect test from path to be valid: %s", err)
-			t.FailNow()
-		}
-
-		res, err := dmf.Map(fromPath)
-		if to == "" {
-			assert.Nil(t, res)
-			assert.Nil(t, err)
-			t.Logf("%s => NOT MAPPED", fromPath.ToString())
-			return
-		}
-
-		assert.Nil(t, err)
-		toPath, err := zfs.NewDatasetPath(to)
-		if err != nil {
-			t.Logf("expect test to path to be valid: %s", err)
-			t.FailNow()
-		}
-		assert.True(t, res.Equal(toPath))
-	}
-
-	expectFilter := func(m map[string]string, path string, pass bool) {
-		dmf, err := parseDatasetMapFilter(m, true)
-		if err != nil {
-			t.Logf("expect test filter to be valid: %s", err)
-			t.FailNow()
-		}
-		p, err := zfs.NewDatasetPath(path)
-		if err != nil {
-			t.Logf("expect test path to be valid: %s", err)
-			t.FailNow()
-		}
-		res, err := dmf.Filter(p)
-		assert.Nil(t, err)
-		assert.Equal(t, pass, res)
-	}
-
-	map1 := map[string]string{
-		"a/b/c<":     "root1",
-		"a/b<":       "root2",
-		"<":          "root3/b/c",
-		"b":          "!",
-		"a/b/c/d/e<": "!",
-		"q<":         "root4/1/2",
-	}
-
-	expectMapping(map1, "a/b/c", "root1")
-	expectMapping(map1, "a/b/c/d", "root1/d")
-	expectMapping(map1, "a/b/c/d/e", "")
-	expectMapping(map1, "a/b/e", "root2/e")
-	expectMapping(map1, "a/b", "root2")
-	expectMapping(map1, "x", "root3/b/c/x")
-	expectMapping(map1, "x/y", "root3/b/c/x/y")
-	expectMapping(map1, "q", "root4/1/2")
-	expectMapping(map1, "b", "")
-	expectMapping(map1, "q/r", "root4/1/2/r")
-
-	filter1 := map[string]string{
-		"<":    "!",
-		"a<":   "ok",
-		"a/b<": "!",
-	}
-
-	expectFilter(filter1, "b", false)
-	expectFilter(filter1, "a", true)
-	expectFilter(filter1, "a/d", true)
-	expectFilter(filter1, "a/b", false)
-	expectFilter(filter1, "a/b/c", false)
-
-	filter2 := map[string]string{}
-	expectFilter(filter2, "foo", false) // default to omit
-
-}
-
-func TestDatasetMapFilter_AsFilter(t *testing.T) {
-
-	mapspec := map[string]string{
-		"a/b/c<":     "root1",
-		"a/b<":       "root2",
-		"<":          "root3/b/c",
-		"b":          "!",
-		"a/b/c/d/e<": "!",
-		"q<":         "root4/1/2",
-	}
-
-	m, err := parseDatasetMapFilter(mapspec, false)
-	assert.Nil(t, err)
-
-	f := m.AsFilter()
-
-	t.Logf("Mapping:\n%s\nFilter:\n%s", pretty.Sprint(m), pretty.Sprint(f))
-
-	tf := func(f zfs.DatasetFilter, path string, pass bool) {
-		p, err := zfs.NewDatasetPath(path)
-		assert.Nil(t, err)
-		r, err := f.Filter(p)
-		assert.Nil(t, err)
-		assert.Equal(t, pass, r)
-	}
-
-	tf(f, "a/b/c", true)
-	tf(f, "a/b", true)
-	tf(f, "b", false)
-	tf(f, "a/b/c/d/e", false)
-	tf(f, "a/b/c/d/e/f", false)
-	tf(f, "a", true)
-
-}
-
-func TestDatasetMapFilter_InvertedFilter(t *testing.T) {
-	mapspec := map[string]string{
-		"a/b":      "1/2",
-		"a/b/c<":   "3",
-		"a/b/c/d<": "1/2/a",
-		"a/b/d":    "!",
-	}
-
-	m, err := parseDatasetMapFilter(mapspec, false)
-	assert.Nil(t, err)
-
-	inv, err := m.InvertedFilter()
-	assert.Nil(t, err)
-
-	t.Log(pretty.Sprint(inv))
-
-	expectMapping := func(m *DatasetMapFilter, ps string, expRes bool) {
-		p, err := zfs.NewDatasetPath(ps)
-		assert.Nil(t, err)
-		r, err := m.Filter(p)
-		assert.Nil(t, err)
-		assert.Equal(t, expRes, r)
-	}
-
-	expectMapping(inv, "4", false)
-	expectMapping(inv, "3", true)
-	expectMapping(inv, "3/x", true)
-	expectMapping(inv, "1", false)
-	expectMapping(inv, "1/2", true)
-	expectMapping(inv, "1/2/3", false)
-	expectMapping(inv, "1/2/a/b", true)
-
-}

cmd/control.go

@@ -1,305 +0,0 @@
-package cmd
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"github.com/dustin/go-humanize"
-	"github.com/pkg/errors"
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/logger"
-	"io"
-	golog "log"
-	"net"
-	"net/http"
-	"os"
-	"sort"
-	"strings"
-	"time"
-)
-
-var controlCmd = &cobra.Command{
-	Use:   "control",
-	Short: "control zrepl daemon",
-}
-
-var pprofCmd = &cobra.Command{
-	Use:   "pprof off | [on TCP_LISTEN_ADDRESS]",
-	Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
-	Run:   doControlPProf,
-	PreRunE: func(cmd *cobra.Command, args []string) error {
-		if cmd.Flags().NArg() < 1 {
-			goto enargs
-		}
-		switch cmd.Flags().Arg(0) {
-		case "on":
-			pprofCmdArgs.msg.Run = true
-			if cmd.Flags().NArg() != 2 {
-				return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
-			}
-			pprofCmdArgs.msg.HttpListenAddress = cmd.Flags().Arg(1)
-		case "off":
-			if cmd.Flags().NArg() != 1 {
-				goto enargs
-			}
-			pprofCmdArgs.msg.Run = false
-		}
-		return nil
-	enargs:
-		return errors.New("invalid number of positional arguments")
-
-	},
-}
-var pprofCmdArgs struct {
-	msg PprofServerControlMsg
-}
-
-var controlVersionCmd = &cobra.Command{
-	Use:   "version",
-	Short: "print version of running zrepl daemon",
-	Run:   doControLVersionCmd,
-}
-
-var controlStatusCmdArgs struct {
-	format      string
-	level       logger.Level
-	onlyShowJob string
-}
-
-var controlStatusCmd = &cobra.Command{
-	Use:   "status [JOB_NAME]",
-	Short: "get current status",
-	Run:   doControlStatusCmd,
-}
-
-func init() {
-	RootCmd.AddCommand(controlCmd)
-	controlCmd.AddCommand(pprofCmd)
-	controlCmd.AddCommand(controlVersionCmd)
-	controlCmd.AddCommand(controlStatusCmd)
-	controlStatusCmd.Flags().StringVar(&controlStatusCmdArgs.format, "format", "human", "output format (human|raw)")
-	controlStatusCmdArgs.level = logger.Warn
-	controlStatusCmd.Flags().Var(&controlStatusCmdArgs.level, "level", "minimum log level to show")
-}
-
-func controlHttpClient() (client http.Client, err error) {
-
-	conf, err := ParseConfig(rootArgs.configFile)
-	if err != nil {
-		return http.Client{}, err
-	}
-
-	return http.Client{
-		Transport: &http.Transport{
-			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
-				return net.Dial("unix", conf.Global.Control.Sockpath)
-			},
-		},
-	}, nil
-}
-
-func doControlPProf(cmd *cobra.Command, args []string) {
-
-	log := golog.New(os.Stderr, "", 0)
-
-	die := func() {
-		log.Printf("exiting after error")
-		os.Exit(1)
-	}
-
-	log.Printf("connecting to zrepl daemon")
-	httpc, err := controlHttpClient()
-	if err != nil {
-		log.Printf("error parsing config: %s", err)
-		die()
-	}
-
-	var buf bytes.Buffer
-	if err := json.NewEncoder(&buf).Encode(&pprofCmdArgs.msg); err != nil {
-		log.Printf("error marshaling request: %s", err)
-		die()
-	}
-	_, err = httpc.Post("http://unix"+ControlJobEndpointPProf, "application/json", &buf)
-	if err != nil {
-		log.Printf("error: %s", err)
-		die()
-	}
-
-	log.Printf("finished")
-}
-
-func doControLVersionCmd(cmd *cobra.Command, args []string) {
-
-	log := golog.New(os.Stderr, "", 0)
-
-	die := func() {
-		log.Printf("exiting after error")
-		os.Exit(1)
-	}
-
-	httpc, err := controlHttpClient()
-	if err != nil {
-		log.Printf("could not connect to daemon: %s", err)
-		die()
-	}
-
-	resp, err := httpc.Get("http://unix" + ControlJobEndpointVersion)
-	if err != nil {
-		log.Printf("error: %s", err)
-		die()
-	} else if resp.StatusCode != http.StatusOK {
-		var msg bytes.Buffer
-		io.CopyN(&msg, resp.Body, 4096)
-		log.Printf("error: %s", msg.String())
-		die()
-	}
-
-	var info ZreplVersionInformation
-	err = json.NewDecoder(resp.Body).Decode(&info)
-	if err != nil {
-		log.Printf("error unmarshaling response: %s", err)
-		die()
-	}
-
-	fmt.Println(info.String())
-
-}
-
-func doControlStatusCmd(cmd *cobra.Command, args []string) {
-
-	log := golog.New(os.Stderr, "", 0)
-
-	die := func() {
-		log.Print("exiting after error")
-		os.Exit(1)
-	}
-
-	if len(args) == 1 {
-		controlStatusCmdArgs.onlyShowJob = args[0]
-	} else if len(args) > 1 {
-		log.Print("can only specify one job as positional argument")
-		cmd.Usage()
-		die()
-	}
-
-	httpc, err := controlHttpClient()
-	if err != nil {
-		log.Printf("could not connect to daemon: %s", err)
-		die()
-	}
-
-	resp, err := httpc.Get("http://unix" + ControlJobEndpointStatus)
-	if err != nil {
-		log.Printf("error: %s", err)
-		die()
-	} else if resp.StatusCode != http.StatusOK {
-		var msg bytes.Buffer
-		io.CopyN(&msg, resp.Body, 4096)
-		log.Printf("error: %s", msg.String())
-		die()
-	}
-
-	var status DaemonStatus
-	err = json.NewDecoder(resp.Body).Decode(&status)
-	if err != nil {
-		log.Printf("error unmarshaling response: %s", err)
-		die()
-	}
-
-	switch controlStatusCmdArgs.format {
-	case "raw":
-		enc := json.NewEncoder(os.Stdout)
-		enc.SetIndent("", "  ")
-		if err := enc.Encode(status); err != nil {
-			log.Panic(err)
-		}
-	case "human":
-
-		formatter := HumanFormatter{}
-		formatter.SetMetadataFlags(MetadataAll)
-		formatter.SetIgnoreFields([]string{
-			logJobField,
-		})
-		jobNames := make([]string, 0, len(status.Jobs))
-		for name, _ := range status.Jobs {
-			jobNames = append(jobNames, name)
-		}
-		sort.Slice(jobNames, func(i, j int) bool {
-			return strings.Compare(jobNames[i], jobNames[j]) == -1
-		})
-		now := time.Now()
-		for _, name := range jobNames {
-
-			if controlStatusCmdArgs.onlyShowJob != "" && name != controlStatusCmdArgs.onlyShowJob {
-				continue
-			}
-
-			job := status.Jobs[name]
-			jobLogEntries := make([]logger.Entry, 0)
-			informAboutError := false
-
-			fmt.Printf("Job '%s':\n", name)
-			for _, task := range job.Tasks {
-
-				var header bytes.Buffer
-				fmt.Fprintf(&header, "  Task '%s': ", task.Name)
-				if !task.Idle {
-					fmt.Fprint(&header, strings.Join(task.ActivityStack, "."))
-				} else {
-					fmt.Fprint(&header, "<idle>")
-				}
-				fmt.Fprint(&header, " ")
-				const TASK_STALLED_HOLDOFF_DURATION = 10 * time.Second
-				sinceLastUpdate := now.Sub(task.LastUpdate)
-				if !task.Idle || task.ProgressRx != 0 || task.ProgressTx != 0 {
-					fmt.Fprintf(&header, "(%s / %s , Rx/Tx",
-						humanize.Bytes(uint64(task.ProgressRx)),
-						humanize.Bytes(uint64(task.ProgressTx)))
-					if task.Idle {
-						fmt.Fprint(&header, ", values from last run")
-					}
-					fmt.Fprint(&header, ")")
-				}
-				fmt.Fprint(&header, "\n")
-				if !task.Idle && !task.LastUpdate.IsZero() && sinceLastUpdate >= TASK_STALLED_HOLDOFF_DURATION {
-					informAboutError = true
-					fmt.Fprintf(&header, "    WARNING: last update %s ago at %s)",
-						sinceLastUpdate.String(),
-						task.LastUpdate.Format(HumanFormatterDateFormat))
-					fmt.Fprint(&header, "\n")
-				}
-				io.Copy(os.Stdout, &header)
-
-				jobLogEntries = append(jobLogEntries, task.LogEntries...)
-				informAboutError = informAboutError || task.MaxLogLevel >= logger.Warn
-			}
-
-			sort.Slice(jobLogEntries, func(i, j int) bool {
-				return jobLogEntries[i].Time.Before(jobLogEntries[j].Time)
-			})
-			if informAboutError {
-				fmt.Println("    WARNING: Some tasks encountered problems since the last time they left idle state:")
-				fmt.Println("             check the logs below or your log file for more information.")
-				fmt.Println("             Use the --level flag if you need debug information.")
-				fmt.Println()
-			}
-			for _, e := range jobLogEntries {
-				if e.Level < controlStatusCmdArgs.level {
-					continue
-				}
-				formatted, err := formatter.Format(&e)
-				if err != nil {
-					panic(err)
-				}
-				fmt.Printf("      %s\n", string(formatted))
-			}
-			fmt.Println()
-
-		}
-	default:
-		log.Printf("invalid output format '%s'", controlStatusCmdArgs.format)
-		die()
-	}
-
-}

cmd/daemon.go

@@ -1,394 +0,0 @@
-package cmd
-
-import (
-	"container/list"
-	"context"
-	"fmt"
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/logger"
-	"io"
-	"os"
-	"os/signal"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-)
-
-// daemonCmd represents the daemon command
-var daemonCmd = &cobra.Command{
-	Use:   "daemon",
-	Short: "start daemon",
-	Run:   doDaemon,
-}
-
-func init() {
-	RootCmd.AddCommand(daemonCmd)
-}
-
-type Job interface {
-	JobName() string
-	JobStart(ctxt context.Context)
-	JobStatus(ctxt context.Context) (*JobStatus, error)
-}
-
-func doDaemon(cmd *cobra.Command, args []string) {
-
-	conf, err := ParseConfig(rootArgs.configFile)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "error parsing config: %s\n", err)
-		os.Exit(1)
-	}
-
-	log := logger.NewLogger(conf.Global.logging.Outlets, 1*time.Second)
-
-	log.Info(NewZreplVersionInformation().String())
-	log.Debug("starting daemon")
-	ctx := context.WithValue(context.Background(), contextKeyLog, log)
-	ctx = context.WithValue(ctx, contextKeyLog, log)
-
-	d := NewDaemon(conf)
-	d.Loop(ctx)
-
-}
-
-type contextKey string
-
-const (
-	contextKeyLog    contextKey = contextKey("log")
-	contextKeyDaemon contextKey = contextKey("daemon")
-)
-
-type Daemon struct {
-	conf      *Config
-	startedAt time.Time
-}
-
-func NewDaemon(initialConf *Config) *Daemon {
-	return &Daemon{conf: initialConf}
-}
-
-func (d *Daemon) Loop(ctx context.Context) {
-
-	d.startedAt = time.Now()
-
-	log := ctx.Value(contextKeyLog).(Logger)
-
-	ctx, cancel := context.WithCancel(ctx)
-	ctx = context.WithValue(ctx, contextKeyDaemon, d)
-
-	sigChan := make(chan os.Signal, 1)
-	finishs := make(chan Job)
-
-	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
-
-	log.Info("starting jobs from config")
-	i := 0
-	for _, job := range d.conf.Jobs {
-		logger := log.WithField(logJobField, job.JobName())
-		logger.Info("starting")
-		i++
-		jobCtx := context.WithValue(ctx, contextKeyLog, logger)
-		go func(j Job) {
-			j.JobStart(jobCtx)
-			finishs <- j
-		}(job)
-	}
-
-	finishCount := 0
-outer:
-	for {
-		select {
-		case <-finishs:
-			finishCount++
-			if finishCount == len(d.conf.Jobs) {
-				log.Info("all jobs finished")
-				break outer
-			}
-
-		case sig := <-sigChan:
-			log.WithField("signal", sig).Info("received signal")
-			log.Info("cancelling all jobs")
-			cancel()
-		}
-	}
-
-	signal.Stop(sigChan)
-	cancel() // make go vet happy
-
-	log.Info("exiting")
-
-}
-
-// Representation of a Job's status that is composed of Tasks
-type JobStatus struct {
-	// Statuses of all tasks of this job
-	Tasks []*TaskStatus
-	// Error != "" if JobStatus() returned an error
-	JobStatusError string
-}
-
-// Representation of a Daemon's status that is composed of Jobs
-type DaemonStatus struct {
-	StartedAt time.Time
-	Jobs      map[string]*JobStatus
-}
-
-func (d *Daemon) Status() (s *DaemonStatus) {
-
-	s = &DaemonStatus{}
-	s.StartedAt = d.startedAt
-
-	s.Jobs = make(map[string]*JobStatus, len(d.conf.Jobs))
-
-	for name, j := range d.conf.Jobs {
-		status, err := j.JobStatus(context.TODO())
-		if err != nil {
-			s.Jobs[name] = &JobStatus{nil, err.Error()}
-			continue
-		}
-		s.Jobs[name] = status
-	}
-
-	return
-}
-
-// Representation of a Task's status
-type TaskStatus struct {
-	Name string
-	// Whether the task is idle.
-	Idle bool
-	// The stack of activities the task is currently executing.
-	// The first element is the root activity and equal to Name.
-	ActivityStack []string
-	// Number of bytes received by the task since it last left idle state.
-	ProgressRx int64
-	// Number of bytes sent by the task since it last left idle state.
-	ProgressTx int64
-	// Log entries emitted by the task since it last left idle state.
-	// Only contains the log entries emitted through the task's logger
-	// (provided by Task.Log()).
-	LogEntries []logger.Entry
-	// The maximum log level of LogEntries.
-	// Only valid if len(LogEntries) > 0.
-	MaxLogLevel logger.Level
-	// Last time something about the Task changed
-	LastUpdate time.Time
-}
-
-// An instance of Task tracks  a single thread of activity that is part of a Job.
-type Task struct {
-	// Stack of activities the task is currently in
-	// Members are instances of taskActivity
-	activities *list.List
-	// Last time activities was changed (not the activities inside, the list)
-	activitiesLastUpdate time.Time
-	// Protects Task members from modification
-	rwl sync.RWMutex
-}
-
-// Structure that describes the progress a Task has made
-type taskProgress struct {
-	rx         int64
-	tx         int64
-	lastUpdate time.Time
-	logEntries []logger.Entry
-	mtx        sync.RWMutex
-}
-
-func newTaskProgress() (p *taskProgress) {
-	return &taskProgress{
-		logEntries: make([]logger.Entry, 0),
-	}
-}
-
-func (p *taskProgress) UpdateIO(drx, dtx int64) {
-	p.mtx.Lock()
-	defer p.mtx.Unlock()
-	p.rx += drx
-	p.tx += dtx
-	p.lastUpdate = time.Now()
-}
-
-func (p *taskProgress) UpdateLogEntry(entry logger.Entry) {
-	p.mtx.Lock()
-	defer p.mtx.Unlock()
-	// FIXME: ensure maximum size (issue #48)
-	p.logEntries = append(p.logEntries, entry)
-	p.lastUpdate = time.Now()
-}
-
-func (p *taskProgress) DeepCopy() (out taskProgress) {
-	p.mtx.RLock()
-	defer p.mtx.RUnlock()
-	out.rx, out.tx = p.rx, p.tx
-	out.lastUpdate = p.lastUpdate
-	out.logEntries = make([]logger.Entry, len(p.logEntries))
-	for i := range p.logEntries {
-		out.logEntries[i] = p.logEntries[i]
-	}
-	return
-}
-
-// returns a copy of this taskProgress, the mutex carries no semantic value
-func (p *taskProgress) Read() (out taskProgress) {
-	p.mtx.RLock()
-	defer p.mtx.RUnlock()
-	return p.DeepCopy()
-}
-
-// Element of a Task's activity stack
-type taskActivity struct {
-	name   string
-	idle   bool
-	logger *logger.Logger
-	// The progress of the task that is updated by UpdateIO() and UpdateLogEntry()
-	//
-	// Progress happens on a task-level and is thus global to the task.
-	// That's why progress is just a pointer to the current taskProgress:
-	// we reset progress when leaving the idle root activity
-	progress *taskProgress
-}
-
-func NewTask(name string, lg *logger.Logger) *Task {
-	t := &Task{
-		activities: list.New(),
-	}
-	rootLogger := lg.ReplaceField(logTaskField, name).
-		WithOutlet(t, logger.Debug)
-	rootAct := &taskActivity{name, true, rootLogger, newTaskProgress()}
-	t.activities.PushFront(rootAct)
-	return t
-}
-
-// callers must hold t.rwl
-func (t *Task) cur() *taskActivity {
-	return t.activities.Front().Value.(*taskActivity)
-}
-
-// buildActivityStack returns the stack of activity names
-// t.rwl must be held, but the slice can be returned since strings are immutable
-func (t *Task) buildActivityStack() []string {
-	comps := make([]string, 0, t.activities.Len())
-	for e := t.activities.Back(); e != nil; e = e.Prev() {
-		act := e.Value.(*taskActivity)
-		comps = append(comps, act.name)
-	}
-	return comps
-}
-
-// Start a sub-activity.
-// Must always be matched with a call to t.Finish()
-// --- consider using defer for this purpose.
-func (t *Task) Enter(activity string) {
-	t.rwl.Lock()
-	defer t.rwl.Unlock()
-
-	prev := t.cur()
-	if prev.idle {
-		// reset progress when leaving idle task
-		// we leave the old progress dangling to have the user not worry about
-		prev.progress = newTaskProgress()
-	}
-	act := &taskActivity{activity, false, nil, prev.progress}
-	t.activities.PushFront(act)
-	stack := t.buildActivityStack()
-	activityField := strings.Join(stack, ".")
-	act.logger = prev.logger.ReplaceField(logTaskField, activityField)
-
-	t.activitiesLastUpdate = time.Now()
-}
-
-func (t *Task) UpdateProgress(dtx, drx int64) {
-	t.rwl.RLock()
-	p := t.cur().progress // protected by own rwlock
-	t.rwl.RUnlock()
-	p.UpdateIO(dtx, drx)
-}
-
-// Returns a wrapper io.Reader that updates this task's _current_ progress value.
-// Progress updates after this task resets its progress value are discarded.
-func (t *Task) ProgressUpdater(r io.Reader) *IOProgressUpdater {
-	t.rwl.RLock()
-	defer t.rwl.RUnlock()
-	return &IOProgressUpdater{r, t.cur().progress}
-}
-
-func (t *Task) Status() *TaskStatus {
-	t.rwl.RLock()
-	defer t.rwl.RUnlock()
-	// NOTE
-	// do not return any state in TaskStatus that is protected by t.rwl
-
-	cur := t.cur()
-	stack := t.buildActivityStack()
-	prog := cur.progress.Read()
-
-	var maxLevel logger.Level
-	for _, entry := range prog.logEntries {
-		if maxLevel < entry.Level {
-			maxLevel = entry.Level
-		}
-	}
-
-	lastUpdate := prog.lastUpdate
-	if lastUpdate.Before(t.activitiesLastUpdate) {
-		lastUpdate = t.activitiesLastUpdate
-	}
-
-	s := &TaskStatus{
-		Name:          stack[0],
-		ActivityStack: stack,
-		Idle:          cur.idle,
-		ProgressRx:    prog.rx,
-		ProgressTx:    prog.tx,
-		LogEntries:    prog.logEntries,
-		MaxLogLevel:   maxLevel,
-		LastUpdate:    lastUpdate,
-	}
-
-	return s
-}
-
-// Finish a sub-activity.
-// Corresponds to a preceding call to t.Enter()
-func (t *Task) Finish() {
-	t.rwl.Lock()
-	defer t.rwl.Unlock()
-	top := t.activities.Front()
-	if top.Next() == nil {
-		return // cannot remove root activity
-	}
-	t.activities.Remove(top)
-	t.activitiesLastUpdate = time.Now()
-
-}
-
-// Returns a logger derived from the logger passed to the constructor function.
-// The logger's task field contains the current activity stack joined by '.'.
-func (t *Task) Log() *logger.Logger {
-	t.rwl.RLock()
-	defer t.rwl.RUnlock()
-	// FIXME should influence TaskStatus's LastUpdate field
-	return t.cur().logger
-}
-
-// implement logger.Outlet interface
-func (t *Task) WriteEntry(entry logger.Entry) error {
-	t.rwl.RLock()
-	defer t.rwl.RUnlock()
-	t.cur().progress.UpdateLogEntry(entry)
-	return nil
-}
-
-type IOProgressUpdater struct {
-	r io.Reader
-	p *taskProgress
-}
-
-func (u *IOProgressUpdater) Read(p []byte) (n int, err error) {
-	n, err = u.r.Read(p)
-	u.p.UpdateIO(int64(n), 0)
-	return
-
-}

cmd/handler.go

@@ -1,181 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"io"
-
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type DatasetMapping interface {
-	Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error)
-}
-
-type FilesystemRequest struct {
-	Roots []string // may be nil, indicating interest in all filesystems
-}
-
-type FilesystemVersionsRequest struct {
-	Filesystem *zfs.DatasetPath
-}
-
-type InitialTransferRequest struct {
-	Filesystem        *zfs.DatasetPath
-	FilesystemVersion zfs.FilesystemVersion
-}
-
-type IncrementalTransferRequest struct {
-	Filesystem *zfs.DatasetPath
-	From       zfs.FilesystemVersion
-	To         zfs.FilesystemVersion
-}
-
-type Handler struct {
-	logger Logger
-	dsf    zfs.DatasetFilter
-	fsvf   zfs.FilesystemVersionFilter
-}
-
-func NewHandler(logger Logger, dsfilter zfs.DatasetFilter, snapfilter zfs.FilesystemVersionFilter) (h Handler) {
-	return Handler{logger, dsfilter, snapfilter}
-}
-
-func registerEndpoints(server rpc.RPCServer, handler Handler) (err error) {
-	err = server.RegisterEndpoint("FilesystemRequest", handler.HandleFilesystemRequest)
-	if err != nil {
-		panic(err)
-	}
-	err = server.RegisterEndpoint("FilesystemVersionsRequest", handler.HandleFilesystemVersionsRequest)
-	if err != nil {
-		panic(err)
-	}
-	err = server.RegisterEndpoint("InitialTransferRequest", handler.HandleInitialTransferRequest)
-	if err != nil {
-		panic(err)
-	}
-	err = server.RegisterEndpoint("IncrementalTransferRequest", handler.HandleIncrementalTransferRequest)
-	if err != nil {
-		panic(err)
-	}
-	return nil
-}
-
-func (h Handler) HandleFilesystemRequest(r *FilesystemRequest, roots *[]*zfs.DatasetPath) (err error) {
-
-	log := h.logger.WithField("endpoint", "FilesystemRequest")
-
-	log.WithField("request", r).Debug("request")
-	log.WithField("dataset_filter", h.dsf).Debug("dsf")
-
-	allowed, err := zfs.ZFSListMapping(h.dsf)
-	if err != nil {
-		log.WithError(err).Error("error listing filesystems")
-		return
-	}
-
-	log.WithField("response", allowed).Debug("response")
-	*roots = allowed
-	return
-}
-
-func (h Handler) HandleFilesystemVersionsRequest(r *FilesystemVersionsRequest, versions *[]zfs.FilesystemVersion) (err error) {
-
-	log := h.logger.WithField("endpoint", "FilesystemVersionsRequest")
-
-	log.WithField("request", r).Debug("request")
-
-	// allowed to request that?
-	if h.pullACLCheck(r.Filesystem, nil); err != nil {
-		log.WithError(err).Warn("pull ACL check failed")
-		return
-	}
-
-	// find our versions
-	vs, err := zfs.ZFSListFilesystemVersions(r.Filesystem, h.fsvf)
-	if err != nil {
-		log.WithError(err).Error("cannot list filesystem versions")
-		return
-	}
-
-	log.WithField("response", vs).Debug("response")
-
-	*versions = vs
-	return
-
-}
-
-func (h Handler) HandleInitialTransferRequest(r *InitialTransferRequest, stream *io.Reader) (err error) {
-
-	log := h.logger.WithField("endpoint", "InitialTransferRequest")
-
-	log.WithField("request", r).Debug("request")
-	if err = h.pullACLCheck(r.Filesystem, &r.FilesystemVersion); err != nil {
-		log.WithError(err).Warn("pull ACL check failed")
-		return
-	}
-
-	log.Debug("invoking zfs send")
-
-	s, err := zfs.ZFSSend(r.Filesystem, &r.FilesystemVersion, nil)
-	if err != nil {
-		log.WithError(err).Error("cannot send filesystem")
-	}
-	*stream = s
-
-	return
-
-}
-
-func (h Handler) HandleIncrementalTransferRequest(r *IncrementalTransferRequest, stream *io.Reader) (err error) {
-
-	log := h.logger.WithField("endpoint", "IncrementalTransferRequest")
-	log.WithField("request", r).Debug("request")
-	if err = h.pullACLCheck(r.Filesystem, &r.From); err != nil {
-		log.WithError(err).Warn("pull ACL check failed")
-		return
-	}
-	if err = h.pullACLCheck(r.Filesystem, &r.To); err != nil {
-		log.WithError(err).Warn("pull ACL check failed")
-		return
-	}
-
-	log.Debug("invoking zfs send")
-
-	s, err := zfs.ZFSSend(r.Filesystem, &r.From, &r.To)
-	if err != nil {
-		log.WithError(err).Error("cannot send filesystem")
-	}
-
-	*stream = s
-	return
-
-}
-
-func (h Handler) pullACLCheck(p *zfs.DatasetPath, v *zfs.FilesystemVersion) (err error) {
-	var fsAllowed, vAllowed bool
-	fsAllowed, err = h.dsf.Filter(p)
-	if err != nil {
-		err = fmt.Errorf("error evaluating ACL: %s", err)
-		return
-	}
-	if !fsAllowed {
-		err = fmt.Errorf("ACL prohibits access to %s", p.ToString())
-		return
-	}
-	if v == nil {
-		return
-	}
-
-	vAllowed, err = h.fsvf.Filter(*v)
-	if err != nil {
-		err = errors.Wrap(err, "error evaluating version filter")
-		return
-	}
-	if !vAllowed {
-		err = fmt.Errorf("ACL prohibits access to %s", v.ToAbsPath(p))
-		return
-	}
-	return
-}

cmd/main.go

@@ -1,47 +0,0 @@
-// zrepl replicates ZFS filesystems & volumes between pools
-//
-// Code Organization
-//
-// The cmd package uses github.com/spf13/cobra for its CLI.
-//
-// It combines the other packages in the zrepl project to implement zrepl functionality.
-//
-// Each subcommand's code is in the corresponding *.go file.
-// All other *.go files contain code shared by the subcommands.
-package cmd
-
-import (
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/logger"
-)
-
-//
-//type Logger interface {
-//	Printf(format string, v ...interface{})
-//}
-
-var (
-	zreplVersion string // set by build infrastructure
-)
-
-type Logger = *logger.Logger
-
-var RootCmd = &cobra.Command{
-	Use:   "zrepl",
-	Short: "ZFS dataset replication",
-	Long: `Replicate ZFS filesystems & volumes between pools:
-
-  - push & pull mode
-  - automatic snapshot creation & pruning
-  - local / over the network
-  - ACLs instead of blank SSH access`,
-}
-
-var rootArgs struct {
-	configFile string
-}
-
-func init() {
-	//cobra.OnInitialize(initConfig)
-	RootCmd.PersistentFlags().StringVar(&rootArgs.configFile, "config", "", "config file path")
-}

cmd/prune.go

@@ -1,138 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"github.com/zrepl/zrepl/zfs"
-	"time"
-)
-
-type Pruner struct {
-	task           *Task
-	Now            time.Time
-	DryRun         bool
-	DatasetFilter  zfs.DatasetFilter
-	SnapshotPrefix string
-	PrunePolicy    PrunePolicy
-}
-
-type PruneResult struct {
-	Filesystem *zfs.DatasetPath
-	All        []zfs.FilesystemVersion
-	Keep       []zfs.FilesystemVersion
-	Remove     []zfs.FilesystemVersion
-}
-
-func (p *Pruner) filterFilesystems() (filesystems []*zfs.DatasetPath, stop bool) {
-	p.task.Enter("filter_fs")
-	defer p.task.Finish()
-	filesystems, err := zfs.ZFSListMapping(p.DatasetFilter)
-	if err != nil {
-		p.task.Log().WithError(err).Error("error applying filesystem filter")
-		return nil, true
-	}
-	if len(filesystems) <= 0 {
-		p.task.Log().Info("no filesystems matching filter")
-		return nil, true
-	}
-	return filesystems, false
-}
-
-func (p *Pruner) filterVersions(fs *zfs.DatasetPath) (fsversions []zfs.FilesystemVersion, stop bool) {
-	p.task.Enter("filter_versions")
-	defer p.task.Finish()
-	log := p.task.Log().WithField(logFSField, fs.ToString())
-
-	filter := NewPrefixFilter(p.SnapshotPrefix)
-	fsversions, err := zfs.ZFSListFilesystemVersions(fs, filter)
-	if err != nil {
-		log.WithError(err).Error("error listing filesytem versions")
-		return nil, true
-	}
-	if len(fsversions) == 0 {
-		log.WithField("prefix", p.SnapshotPrefix).Info("no filesystem versions matching prefix")
-		return nil, true
-	}
-	return fsversions, false
-}
-
-func (p *Pruner) pruneFilesystem(fs *zfs.DatasetPath) (r PruneResult, valid bool) {
-	p.task.Enter("prune_fs")
-	defer p.task.Finish()
-	log := p.task.Log().WithField(logFSField, fs.ToString())
-
-	fsversions, stop := p.filterVersions(fs)
-	if stop {
-		return
-	}
-
-	p.task.Enter("prune_policy")
-	keep, remove, err := p.PrunePolicy.Prune(fs, fsversions)
-	p.task.Finish()
-	if err != nil {
-		log.WithError(err).Error("error evaluating prune policy")
-		return
-	}
-
-	log.WithField("fsversions", fsversions).
-		WithField("keep", keep).
-		WithField("remove", remove).
-		Debug("prune policy debug dump")
-
-	r = PruneResult{fs, fsversions, keep, remove}
-
-	makeFields := func(v zfs.FilesystemVersion) (fields map[string]interface{}) {
-		fields = make(map[string]interface{})
-		fields["version"] = v.ToAbsPath(fs)
-		timeSince := v.Creation.Sub(p.Now)
-		fields["age_ns"] = timeSince
-		const day time.Duration = 24 * time.Hour
-		days := timeSince / day
-		remainder := timeSince % day
-		fields["age_str"] = fmt.Sprintf("%dd%s", days, remainder)
-		return
-	}
-
-	for _, v := range remove {
-		fields := makeFields(v)
-		log.WithFields(fields).Info("destroying version")
-		// echo what we'll do and exec zfs destroy if not dry run
-		// TODO special handling for EBUSY (zfs hold)
-		// TODO error handling for clones? just echo to cli, skip over, and exit with non-zero status code (we're idempotent)
-		if !p.DryRun {
-			p.task.Enter("destroy")
-			err := zfs.ZFSDestroyFilesystemVersion(fs, v)
-			p.task.Finish()
-			if err != nil {
-				log.WithFields(fields).WithError(err).Error("error destroying version")
-			}
-		}
-	}
-	return r, true
-}
-
-func (p *Pruner) Run(ctx context.Context) (r []PruneResult, err error) {
-	p.task.Enter("run")
-	defer p.task.Finish()
-
-	if p.DryRun {
-		p.task.Log().Info("doing dry run")
-	}
-
-	filesystems, stop := p.filterFilesystems()
-	if stop {
-		return
-	}
-
-	r = make([]PruneResult, 0, len(filesystems))
-
-	for _, fs := range filesystems {
-		res, ok := p.pruneFilesystem(fs)
-		if ok {
-			r = append(r, res)
-		}
-	}
-
-	return
-
-}

cmd/replication.go

@@ -1,323 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"io"
-
-	"bytes"
-	"encoding/json"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type localPullACL struct{}
-
-func (a localPullACL) Filter(p *zfs.DatasetPath) (pass bool, err error) {
-	return true, nil
-}
-
-const DEFAULT_INITIAL_REPL_POLICY = InitialReplPolicyMostRecent
-
-type InitialReplPolicy string
-
-const (
-	InitialReplPolicyMostRecent InitialReplPolicy = "most_recent"
-	InitialReplPolicyAll        InitialReplPolicy = "all"
-)
-
-type Puller struct {
-	task              *Task
-	Remote            rpc.RPCClient
-	Mapping           DatasetMapping
-	InitialReplPolicy InitialReplPolicy
-}
-
-type remoteLocalMapping struct {
-	Remote *zfs.DatasetPath
-	Local  *zfs.DatasetPath
-}
-
-func (p *Puller) getRemoteFilesystems() (rfs []*zfs.DatasetPath, ok bool) {
-	p.task.Enter("fetch_remote_fs_list")
-	defer p.task.Finish()
-
-	fsr := FilesystemRequest{}
-	if err := p.Remote.Call("FilesystemRequest", &fsr, &rfs); err != nil {
-		p.task.Log().WithError(err).Error("cannot fetch remote filesystem list")
-		return nil, false
-	}
-	return rfs, true
-}
-
-func (p *Puller) buildReplMapping(remoteFilesystems []*zfs.DatasetPath) (replMapping map[string]remoteLocalMapping, ok bool) {
-	p.task.Enter("build_repl_mapping")
-	defer p.task.Finish()
-
-	replMapping = make(map[string]remoteLocalMapping, len(remoteFilesystems))
-	for fs := range remoteFilesystems {
-		var err error
-		var localFs *zfs.DatasetPath
-		localFs, err = p.Mapping.Map(remoteFilesystems[fs])
-		if err != nil {
-			err := fmt.Errorf("error mapping %s: %s", remoteFilesystems[fs], err)
-			p.task.Log().WithError(err).WithField(logMapFromField, remoteFilesystems[fs]).Error("cannot map")
-			return nil, false
-		}
-		if localFs == nil {
-			continue
-		}
-		p.task.Log().WithField(logMapFromField, remoteFilesystems[fs].ToString()).
-			WithField(logMapToField, localFs.ToString()).Debug("mapping")
-		m := remoteLocalMapping{remoteFilesystems[fs], localFs}
-		replMapping[m.Local.ToString()] = m
-	}
-	return replMapping, true
-}
-
-// returns true if the receiving filesystem (local side) exists and can have child filesystems
-func (p *Puller) replFilesystem(m remoteLocalMapping, localFilesystemState map[string]zfs.FilesystemState) (localExists bool) {
-
-	p.task.Enter("repl_fs")
-	defer p.task.Finish()
-	var err error
-	remote := p.Remote
-
-	log := p.task.Log().
-		WithField(logMapFromField, m.Remote.ToString()).
-		WithField(logMapToField, m.Local.ToString())
-
-	log.Debug("examining local filesystem state")
-	localState, localExists := localFilesystemState[m.Local.ToString()]
-	var versions []zfs.FilesystemVersion
-	switch {
-	case !localExists:
-		log.Info("local filesystem does not exist")
-	case localState.Placeholder:
-		log.Info("local filesystem is marked as placeholder")
-	default:
-		log.Debug("local filesystem exists")
-		log.Debug("requesting local filesystem versions")
-		if versions, err = zfs.ZFSListFilesystemVersions(m.Local, nil); err != nil {
-			log.WithError(err).Error("cannot get local filesystem versions")
-			return false
-		}
-	}
-
-	log.Info("requesting remote filesystem versions")
-	r := FilesystemVersionsRequest{
-		Filesystem: m.Remote,
-	}
-	var theirVersions []zfs.FilesystemVersion
-	if err = remote.Call("FilesystemVersionsRequest", &r, &theirVersions); err != nil {
-		log.WithError(err).Error("cannot get remote filesystem versions")
-		log.Warn("stopping replication for all filesystems mapped as children of receiving filesystem")
-		return false
-	}
-
-	log.Debug("computing diff between remote and local filesystem versions")
-	diff := zfs.MakeFilesystemDiff(versions, theirVersions)
-	log.WithField("diff", diff).Debug("diff between local and remote filesystem")
-
-	if localState.Placeholder && diff.Conflict != zfs.ConflictAllRight {
-		panic("internal inconsistency: local placeholder implies ConflictAllRight")
-	}
-
-	switch diff.Conflict {
-	case zfs.ConflictAllRight:
-
-		log.WithField("replication_policy", p.InitialReplPolicy).Info("performing initial sync, following policy")
-
-		if p.InitialReplPolicy != InitialReplPolicyMostRecent {
-			panic(fmt.Sprintf("policy '%s' not implemented", p.InitialReplPolicy))
-		}
-
-		snapsOnly := make([]zfs.FilesystemVersion, 0, len(diff.MRCAPathRight))
-		for s := range diff.MRCAPathRight {
-			if diff.MRCAPathRight[s].Type == zfs.Snapshot {
-				snapsOnly = append(snapsOnly, diff.MRCAPathRight[s])
-			}
-		}
-
-		if len(snapsOnly) < 1 {
-			log.Warn("cannot perform initial sync: no remote snapshots")
-			return false
-		}
-
-		r := InitialTransferRequest{
-			Filesystem:        m.Remote,
-			FilesystemVersion: snapsOnly[len(snapsOnly)-1],
-		}
-
-		log.WithField("version", r.FilesystemVersion).Debug("requesting snapshot stream")
-
-		var stream io.Reader
-
-		if err = remote.Call("InitialTransferRequest", &r, &stream); err != nil {
-			log.WithError(err).Error("cannot request initial transfer")
-			return false
-		}
-		log.Debug("received initial transfer request response")
-
-		log.Debug("invoke zfs receive")
-		recvArgs := []string{"-u"}
-		if localState.Placeholder {
-			log.Info("receive with forced rollback to replace placeholder filesystem")
-			recvArgs = append(recvArgs, "-F")
-		}
-		progressStream := p.task.ProgressUpdater(stream)
-		if err = zfs.ZFSRecv(m.Local, progressStream, recvArgs...); err != nil {
-			log.WithError(err).Error("cannot receive stream")
-			return false
-		}
-		log.Info("finished receiving stream") // TODO rx delta
-
-		// TODO unify with recv path of ConflictIncremental
-		log.Debug("configuring properties of received filesystem")
-		props := zfs.NewZFSProperties()
-		props.Set("readonly", "on")
-		if err = zfs.ZFSSet(m.Local, props); err != nil {
-			log.WithError(err).Error("cannot set readonly property")
-		}
-
-		log.Info("finished initial transfer")
-		return true
-
-	case zfs.ConflictIncremental:
-
-		if len(diff.IncrementalPath) < 2 {
-			log.Info("remote and local are in sync")
-			return true
-		}
-
-		log.Info("following incremental path from diff")
-		for i := 0; i < len(diff.IncrementalPath)-1; i++ {
-
-			from, to := diff.IncrementalPath[i], diff.IncrementalPath[i+1]
-
-			log, _ := log.WithField(logIncFromField, from.Name).WithField(logIncToField, to.Name), 0
-
-			log.Debug("requesting incremental snapshot stream")
-			r := IncrementalTransferRequest{
-				Filesystem: m.Remote,
-				From:       from,
-				To:         to,
-			}
-			var stream io.Reader
-			if err = remote.Call("IncrementalTransferRequest", &r, &stream); err != nil {
-				log.WithError(err).Error("cannot request incremental snapshot stream")
-				return false
-			}
-
-			log.Debug("invoking zfs receive")
-			progressStream := p.task.ProgressUpdater(stream)
-			// TODO protect against malicious incremental stream
-			if err = zfs.ZFSRecv(m.Local, progressStream); err != nil {
-				log.WithError(err).Error("cannot receive stream")
-				return false
-			}
-			log.Info("finished incremental transfer") // TODO increment rx
-
-		}
-		log.Info("finished following incremental path") // TODO path rx
-		return true
-
-	case zfs.ConflictNoCommonAncestor:
-		fallthrough
-	case zfs.ConflictDiverged:
-
-		var jsonDiff bytes.Buffer
-		if err := json.NewEncoder(&jsonDiff).Encode(diff); err != nil {
-			log.WithError(err).Error("cannot JSON-encode diff")
-			return false
-		}
-
-		var problem, resolution string
-
-		switch diff.Conflict {
-		case zfs.ConflictNoCommonAncestor:
-			problem = "remote and local filesystem have snapshots, but no common one"
-			resolution = "perform manual establish a common snapshot history"
-		case zfs.ConflictDiverged:
-			problem = "remote and local filesystem share a history but have diverged"
-			resolution = "perform manual replication or delete snapshots on the receiving" +
-				"side  to establish an incremental replication parse"
-		}
-
-		log.WithField("diff", jsonDiff.String()).
-			WithField("problem", problem).
-			WithField("resolution", resolution).
-			Error("manual conflict resolution required")
-
-		return false
-
-	}
-
-	panic("should not be reached")
-}
-
-func (p *Puller) Pull() {
-	p.task.Enter("run")
-	defer p.task.Finish()
-
-	p.task.Log().Info("request remote filesystem list")
-	remoteFilesystems, ok := p.getRemoteFilesystems()
-	if !ok {
-		return
-	}
-
-	p.task.Log().Debug("map remote filesystems to local paths and determine order for per-filesystem sync")
-	replMapping, ok := p.buildReplMapping(remoteFilesystems)
-	if !ok {
-
-	}
-
-	p.task.Log().Debug("build cache for already present local filesystem state")
-	p.task.Enter("cache_local_fs_state")
-	localFilesystemState, err := zfs.ZFSListFilesystemState()
-	p.task.Finish()
-	if err != nil {
-		p.task.Log().WithError(err).Error("cannot request local filesystem state")
-		return
-	}
-
-	localTraversal := zfs.NewDatasetPathForest()
-	for _, m := range replMapping {
-		localTraversal.Add(m.Local)
-	}
-
-	p.task.Log().Info("start per-filesystem sync")
-	localTraversal.WalkTopDown(func(v zfs.DatasetPathVisit) bool {
-
-		p.task.Enter("tree_walk")
-		defer p.task.Finish()
-
-		log := p.task.Log().WithField(logFSField, v.Path.ToString())
-
-		if v.FilledIn {
-			if _, exists := localFilesystemState[v.Path.ToString()]; exists {
-				// No need to verify if this is a placeholder or not. It is sufficient
-				// to know we can add child filesystems to it
-				return true
-			}
-			log.Debug("create placeholder filesystem")
-			p.task.Enter("create_placeholder")
-			err = zfs.ZFSCreatePlaceholderFilesystem(v.Path)
-			p.task.Finish()
-			if err != nil {
-				log.Error("cannot create placeholder filesystem")
-				return false
-			}
-			return true
-		}
-
-		m, ok := replMapping[v.Path.ToString()]
-		if !ok {
-			panic("internal inconsistency: replMapping should contain mapping for any path that was not filled in by WalkTopDown()")
-		}
-
-		return p.replFilesystem(m, localFilesystemState)
-	})
-
-	return
-
-}

cmd/sampleconf/localbackup/host1.yml

@@ -1,29 +0,0 @@
-jobs:
-- name: mirror_local
-  type: local
-
-  # snapshot the filesystems matched by the left-hand-side of the mapping
-  # every 10m with zrepl_ as prefix
-  mapping: {
-    "zroot/var/db<":    "storage/backups/local/zroot/var/db",
-    "zroot/usr/home<":  "storage/backups/local/zroot/usr/home",
-    "zroot/usr/home/paranoid":  "!", #don't backup paranoid user
-    "zroot/poudriere/ports<": "!", #don't backup the ports trees
-  }
-  snapshot_prefix: zrepl_
-  interval: 10m
-  initial_repl_policy: most_recent
-
-  # keep one hour of 10m interval snapshots of filesystems matched by
-  # the left-hand-side of the mapping
-  prune_lhs:
-    policy: grid
-    grid: 1x1h(keep=all)
-    keep_bookmarks: all
-
-  # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
-  prune_rhs:
-    policy: grid
-    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-
-

cmd/sampleconf/pullbackup/backuphost.yml

@@ -1,27 +0,0 @@
-jobs:
-- name: fullbackup_prod1
-  type: pull
-  # connect to remote using ssh / stdinserver command
-  connect:
-    type: ssh+stdinserver
-    host: prod1.example.com
-    user: root
-    port: 22
-    identity_file: /root/.ssh/id_ed25519
-
-  # pull (=ask for new snapshots) every 10m, prune afterwards
-  # this will leave us at most 10m behind production
-  interval: 10m
-
-  # pull all offered filesystems to storage/backups/zrepl/pull/prod1.example.com
-  mapping: {
-    "<":"storage/backups/zrepl/pull/prod1.example.com"
-  }
-  initial_repl_policy: most_recent
-
-  # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
-  snapshot_prefix: zrepl_
-  prune:
-    policy: grid
-    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-

cmd/sampleconf/pullbackup/productionhost.yml

@@ -1,47 +0,0 @@
-global:
-  serve:
-    stdinserver:
-      # Directory where AF_UNIX sockets for stdinserver command are placed.
-      #
-      # `zrepl stdinserver CLIENT_IDENTITY`
-      #   * connects to the socket in $sockdir/CLIENT_IDENTITY
-      #   * sends its stdin / stdout file descriptors to the `zrepl daemon` process (see cmsg(3))
-      #   * does nothing more
-      #
-      # This enables a setup where `zrepl daemon` is not directly exposed to the internet
-      # but instead all traffic is tunnelled through SSH.
-      # The server with the source job has an authorized_keys file entry for the public key
-      # used by the corresponding pull job
-      #
-      #   command="/mnt/zrepl stdinserver CLIENT_IDENTITY" ssh-ed25519 AAAAC3NzaC1E... zrepl@pullingserver
-      #
-      # Below is the default value.
-      sockdir: /var/run/zrepl/stdinserver
-
-jobs:
-
-- name: fullbackup_prod1
-  # expect remote to connect via ssh+stdinserver with fullbackup_prod1 as client_identity
-  type: source
-  serve:
-    type: stdinserver # see global.serve.stdinserver for explanation
-    client_identity: fullbackup_prod1
-
-  # snapshot these filesystems every 10m with zrepl_ as prefix
-  filesystems: {
-    "zroot/var/db<": "ok",
-    "zroot/usr/home<": "ok",
-    "zroot/var/tmp": "!", #don't backup /tmp
-  }
-  snapshot_prefix: zrepl_
-  interval: 10m
-
-
-  # keep 1 hour of snapshots (6 at 10m interval)
-  # and  one day of bookmarks in case pull doesn't work (link down, etc)
-  # => keep_bookmarks = 24h / interval = 24h / 10m = 144
-  prune:
-    policy: grid
-    grid: 1x1h(keep=all)
-    keep_bookmarks: 144
-

cmd/sampleconf/pushbackup/backuphost.yml

@@ -1,20 +0,0 @@
-jobs:
-- name: fullbackup_prod1
-
-  # expect remote to connect via ssh+stdinserver with fullbackup_prod1 as client_identity
-  type: push-sink
-  serve:
-    type: stdinserver
-    client_identity: fullbackup_prod1
-
-  # map all pushed datasets to storage/backups/zrepl/sink/prod1.example.com
-  mapping: {
-    "<":"storage/backups/zrepl/sink/prod1.example.com"
-  }
-
-  # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
-  prune:
-    policy: grid
-    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-
-

cmd/sampleconf/pushbackup/productionhost.yml

@@ -1,26 +0,0 @@
-jobs:
-- name: fullbackup_prod1
-
-  # connect to remote using ssh / stdinserver command
-  type: push
-  connect:
-      type: ssh+stdinserver
-      host: prod1.example.com
-      user: root
-      port: 22
-      identity_file: /root/.ssh/id_ed25519
-
-  # snapshot these datsets every 10m with zrepl_ as prefix
-  filesystems: {
-    "zroot/var/db<": "ok",
-    "zroot/usr/home<": "!",
-  }
-  snapshot_prefix: zrepl_
-  interval: 10m
-
-  # keep a one day window 10m interval snapshots in case push doesn't work (link down, etc)
-  # (we cannot keep more than one day because this host will run out of disk space)
-  prune:
-    policy: grid
-    grid: 1x1d(keep=all)
-

cmd/sampleconf/random/debugging.yml

@@ -1,33 +0,0 @@
-global:
-  serve:
-    stdinserver:
-      sockdir: /var/run/zrepl/stdinserver
-
-jobs:
-
-- name: debian2_pull
-  # JOB DEBUGGING OPTIONS
-  # should be equal for all job types, but each job implements the debugging itself
-  # => consult job documentation for supported options
-  debug:
-    conn: # debug the io.ReadWriteCloser connection
-      read_dump: /tmp/connlog_read   # dump results of Read() invocations to this file
-      write_dump: /tmp/connlog_write # dump results of Write() invocations to this file
-    rpc: # debug the RPC protocol implementation
-      log: true # log output from rpc layer to the job log
-
-  # ... just to make the unit tests pass.
-  # check other examples, e.g. localbackup or pullbackup for what the sutff below means
-  type: source
-  serve:
-    type: stdinserver
-    client_identity: debian2
-  filesystems: {
-    "pool1/db<": ok
-  }
-  snapshot_prefix: zrepl_
-  interval: 1s
-  prune:
-    policy: grid
-    grid: 1x10s(keep=all)
-    keep_bookmarks: all

cmd/sampleconf/random/logging.yml

@@ -1,24 +0,0 @@
-global:
-  logging:
-
-    - outlet: stdout
-      level: warn
-      format: human
-
-    - outlet: tcp
-      level: debug
-      format: json
-      net: tcp
-      address: 127.0.0.1:8080
-      retry_interval: 1s
-      tls: # if not specified, use plain TCP
-        ca: sampleconf/random/logging/logserver.crt
-        cert: sampleconf/random/logging/client.crt
-        key: sampleconf/random/logging/client.key
-
-    - outlet: syslog
-      level: debug
-      format: logfmt
-
-
-jobs: []

cmd/sampleconf/random/logging/client.crt

@@ -1,19 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDIzCCAgsCAQEwDQYJKoZIhvcNAQELBQAwWTELMAkGA1UEBhMCQVUxEzARBgNV
-BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0
-ZDESMBAGA1UEAwwJbG9nc2VydmVyMB4XDTE3MDkyNDEyMzAzNloXDTE3MTAyNDEy
-MzAzNlowVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNV
-BAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xpZW50MIIB
-IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/xJTUlqApeJGzRD+w2J8sZS
-Bo+s+04T987L/M6gaCo8aDSTEb/ZH3XSoU5JEmO6kPpwNNapOsaEhTCjndZQdm5F
-uqiUtAg1uW0HCkBEIDkGr9bFHDKzpewGmmMgfQ2+hfiBR/4ZCrc/vd9P0W9BiWQS
-Dtc7p22XraWPVL8HlSz5K/Ih+V6i8O+kBltZkusiJh2bWPoRp/netiTZuc6du+Wp
-kpWp1OBaTU4GXIAlLj5afF14BBphRQK983Yhaz53BkA7OQ76XxowynMjmuLQVGmK
-f1R9zEJuohTX9XIr1tp/ueRHcS4Awk6LcNZUMCV6270FNSIw2f4hbOZvep+t2wID
-AQABMA0GCSqGSIb3DQEBCwUAA4IBAQACK3OeNzScpiNwz/jpg/usQzvXbZ/wDvml
-YLjtzn/A65ox8a8BhxvH1ydyoCM2YAGYX7+y7qXJnMgRO/v8565CQIVcznHhg9ST
-3828/WqZ3bXf2DV5GxKKQf7hPmBnyVUUhn/Ny91MECED27lZucWiX/bczN8ffDeh
-M3+ngezcJxsOBd4x0gLrqIJCoaFRSeepOaFEW6GHQ8loxE9GmA7FQd2phIpJHFSd
-Z7nQl7X5C1iN2OboEApJHwtmNVC45UlOpg53vo2sDTLhSfdogstiWi8x1HmvhIGM
-j3XHs0Illvo9OwVrmgUph8zQ7pvr/AFrTOIbhgzl/9uVUk5ApwFM
------END CERTIFICATE-----

cmd/sampleconf/random/logging/client.csr

@@ -1,16 +0,0 @@
------BEGIN CERTIFICATE REQUEST-----
-MIICmzCCAYMCAQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx
-ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xp
-ZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/xJTUlqApeJGzRD
-+w2J8sZSBo+s+04T987L/M6gaCo8aDSTEb/ZH3XSoU5JEmO6kPpwNNapOsaEhTCj
-ndZQdm5FuqiUtAg1uW0HCkBEIDkGr9bFHDKzpewGmmMgfQ2+hfiBR/4ZCrc/vd9P
-0W9BiWQSDtc7p22XraWPVL8HlSz5K/Ih+V6i8O+kBltZkusiJh2bWPoRp/netiTZ
-uc6du+WpkpWp1OBaTU4GXIAlLj5afF14BBphRQK983Yhaz53BkA7OQ76XxowynMj
-muLQVGmKf1R9zEJuohTX9XIr1tp/ueRHcS4Awk6LcNZUMCV6270FNSIw2f4hbOZv
-ep+t2wIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAKnlr0Qs5KYF85u2YA7DJ5pL
-HwAx+qNoNbox5CS1aynrDBpDTWLaErviUJ+4WxRlRyTMEscMOIOKajbYhqqFmtGZ
-mu3SshZnFihErw8TOQMyU1LGGG+l6r+6ve5TciwJRLla2Y75z7izr6cyvQNRWdLr
-PvxL1/Yqr8LKha12+7o28R4SLf6/GY0GcedqoebRmtuwA/jES0PuGauEUD5lH4cj
-Me8sqRrB+IMHQ5j8hlJX4DbA8UQRUBL64sHkQzeQfWu+qkWmS5I19CFfLNrcH+OV
-yhyjGfN0q0jHyHdpckBhgzS7IIdo6P66AIlm4qpHM7Scra3JaGM7oaZPamJ6f8U=
------END CERTIFICATE REQUEST-----

cmd/sampleconf/random/logging/client.key

@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC3/ElNSWoCl4kb
-NEP7DYnyxlIGj6z7ThP3zsv8zqBoKjxoNJMRv9kfddKhTkkSY7qQ+nA01qk6xoSF
-MKOd1lB2bkW6qJS0CDW5bQcKQEQgOQav1sUcMrOl7AaaYyB9Db6F+IFH/hkKtz+9
-30/Rb0GJZBIO1zunbZetpY9UvweVLPkr8iH5XqLw76QGW1mS6yImHZtY+hGn+d62
-JNm5zp275amSlanU4FpNTgZcgCUuPlp8XXgEGmFFAr3zdiFrPncGQDs5DvpfGjDK
-cyOa4tBUaYp/VH3MQm6iFNf1civW2n+55EdxLgDCTotw1lQwJXrbvQU1IjDZ/iFs
-5m96n63bAgMBAAECggEAF4om0sWe06ARwbJJNFjCGpa3LfG5/xk5Qs5pmPnS2iD1
-Q5veaTnzjKvlfA/pF3o9B4mTS59fXY7Cq8vSU0J1XwGy2DPzeqlGPmgtq2kXjkvd
-iCfhZj8ybvsoyR3/rSBSDRADcnOXPqC9fgyRSMmESBDOoql1D3HdIzF4ii46ySIU
-/XQvExS6NWifbP+Ue6DETV8NhreO5PqjeXLITQhhndtc8MDL/8eCNOyN8XjYIWKX
-smlBYtRQYOOY9BHOQgUn6yvPHrtKJNKci+qcQNvWir66mBhY1o40MH5wTIV+8yP2
-Vbm/VzoNKIYgeROsilBW7QTwGvkDn3R11zeTqfUNSQKBgQD0eFzhJAEZi4uBw6Tg
-NKmBC5Y1IHPOsb5gKPNz9Z9j4qYRDySgYl6ISk+2EdhgUCo1NmTk8EIPQjIerUVf
-S+EogFnpsj8U9LR3OM79DaGkNULxrHqhd209/g8DtVgk7yjkxL4vmVOv8qpHMp/7
-eWsylN7AOxj2RB/eXYQBPrw+jQKBgQDAqae9HasLmvpJ9ktTv30yZSKXC+LP4A0D
-RBBmx410VpPd4CvcpCJxXmjer6B7+9L1xHYP2pvsnMBid5i0knuvyK28dYy7fldl
-CzWvb+lqNA5YYPFXQED4oEdihlQczoI1Bm06SFizeAKD1Q9e2c+lgbR/51j8xuXi
-twvhMj/YBwKBgQCZw97/iQrcC2Zq7yiUEOuQjD4lGk1c83U/vGIsTJC9XcCAOFsc
-OeMlrD/oz96d7a4unBDn4qpaOJOXsfpRT0PGmrxy/jcpMiUUW/ntNpa11v5NTeQw
-DRL8DAFbnsNbL8Yz5f+Nps35fBNYBuKTZLJlNTfKByHTO9QjpAQ0WEZEvQKBgQCi
-Ovm83EuYVSKmvxcE6Tyx/8lVqTOO2Vn7wweQlD4/lVujvE0S2L8L+XSS9w5K+GzW
-eFz10p3zarbw80YJ30L5bSEmjVE43BUZR4woMzM4M6dUsiTm1HshIE2b4ALZ0uZ/
-Ye794ceXL9nmSrVLqFsaQZLNFPCwwYb4FiyRry9lZwKBgAO9VbWcN8SEeBDKo3z8
-yRbRTc6sI+AdKY44Dfx0tqOPmTjO3mE4X1GU4sbfD2Bvg3DdjwTuxxC/jHaKu0GG
-dTM0CbrZGbDAj7E87SOcN/PWEeBckSvuQq5H3DQfwIpTmlS1l5oZn9CxRGbLqC2G
-ifnel8XWUG0ROybsr1tk4mzW
------END PRIVATE KEY-----

cmd/sampleconf/random/logging/logserver.crt

@@ -1,21 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDiDCCAnCgAwIBAgIJALhp/WvTQeg/MA0GCSqGSIb3DQEBCwUAMFkxCzAJBgNV
-BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
-aWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMMCWxvZ3NlcnZlcjAeFw0xNzA5MjQxMjI3
-MDRaFw0yNzA5MjIxMjI3MDRaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21l
-LVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNV
-BAMMCWxvZ3NlcnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKs3
-TLYfXhV3hap71tOkhPQlM+m0EKRAo8Nua50Cci5UhDo4JkVpyYok1h+NFkqmjU2b
-IiIuGvsZZPOWYjbWWnSJE4+n5pBFBzcfNQ4d8xVxjANImFn6Tcehhj0WkbDIv/Ge
-364XUgywS7u3EGQj/FO7vZ8KHlUxBHNuPIOPHftwIVRyleh5K32UyBaSpSmnqGos
-rvI1byMuznavcZpOs4vlebZ+Jy6a20iKf9fj/0f0t0O+F5x3JIk07D3zSywhJ4RM
-M0mGIUmYXbh2SMh+f61KDZLDANpz/pMAPbUJe0mxEtBf0tnwK1gEqc3SLwA0EwiM
-8Hnn2iaH5Ln20UE3LOkCAwEAAaNTMFEwHQYDVR0OBBYEFDXoDcwx9SngzZcRYCeP
-BplBecfiMB8GA1UdIwQYMBaAFDXoDcwx9SngzZcRYCePBplBecfiMA8GA1UdEwEB
-/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBADyNvs4AA91x3gurQb1pcPVhK6nR
-mkYSTN1AsDKSRi/X2iCUmR7G7FlF7XW8mntTpHvVzcs+gr94WckH5wqEOA5iZnaw
-PXUWexmdXUge4hmC2q6kBQ5e2ykhSJMRVZXvOLZOZV9qitceamHESV1cKZSNMvZM
-aCSVA1RK61/nUzs04pVp5PFPv9gFxJp9ki39FYFdsgZmM5RZ5I/FqxxvTJzu4RnH
-VPjsMopzARYwJw6dV2bKdFSYOE8B/Vs3Yv0GxjrABw2ko4PkBPTjLIz22x6+Hd9r
-K9BQi4pVmQfvppF5+SORSftlHSS+N47b0DD1rW1f5R6QGi71dFuJGikOwvY=
------END CERTIFICATE-----

cmd/sampleconf/random/logging/logserver.key

@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCrN0y2H14Vd4Wq
-e9bTpIT0JTPptBCkQKPDbmudAnIuVIQ6OCZFacmKJNYfjRZKpo1NmyIiLhr7GWTz
-lmI21lp0iROPp+aQRQc3HzUOHfMVcYwDSJhZ+k3HoYY9FpGwyL/xnt+uF1IMsEu7
-txBkI/xTu72fCh5VMQRzbjyDjx37cCFUcpXoeSt9lMgWkqUpp6hqLK7yNW8jLs52
-r3GaTrOL5Xm2ficumttIin/X4/9H9LdDvhecdySJNOw980ssISeETDNJhiFJmF24
-dkjIfn+tSg2SwwDac/6TAD21CXtJsRLQX9LZ8CtYBKnN0i8ANBMIjPB559omh+S5
-9tFBNyzpAgMBAAECggEBAIY8ZwJq+WKvQLb3POjWFf8so9TY/ispGrwAeJKy9j5o
-uPrERw0o8YBDfTVjclS43BQ6Srqtly3DLSjlgL8ps+WmCxYYN2ZpGE0ZRIl65bis
-O2/fnML+wbiAZTTD2xnVatfPDeP6GLQmDFpyHoHEzPIBQZvNXRbBxZGSnhMvQ/x7
-FhqSBQG4kf3b1XDCENIbFEVOBOCg7WtMiIgjEGS7QnW3I65/Zt+Ts1LXRZbz+6na
-Gmi0PGHA/oLUh1NRzsF4zuZn6fFzja5zw4mkt+JvCWEoxg1QhRAxRp6QQwmZ6MIc
-1rw1D4Z+c5UEKyqHeIwZj4M6UNPhCfTXVm47c9eSiGECgYEA4U8pB+7eRo2fqX0C
-nWsWMcmsULJvwplQnUSFenUayPn3E8ammS/ZBHksoKhj82vwIdDbtS1hQZn8Bzsi
-atc8au0wz0YRDcVDzHX4HknXVQayHtP/FTPeSr5hwpoY8vhEbySuxBTBkXCrp4dx
-u5ErfOiYEP3Q1ZvPRywelrATu20CgYEAwonV5dgOcen/4oAirlnvufc2NfqhAQwJ
-FJ/JSVMAcXxPYu3sZMv0dGWrX8mLc+P1+XMCuV/7eBM/vU2LbDzmpeUV8sJfB2jw
-wyKqKXZwBgeq60btriA4f+0ElwRGgU2KSiniUuuTX2JmyftFQx4cVAQRCFk27NY0
-09psSsYyre0CgYBo6unabdtH029EB5iOIW3GZXk+Yrk0TxyA/4WAjsOYTv5FUT4H
-G4bdVGf5sDBLDDpYJOAKsEUXvVLlMx5FzlCuIiGWg7QxS2jU7yJJSG1jhKixPlsM
-Toj3GUyAyC1SB1Ymw1g2qsuwpFzquGG3zFQJ6G3Xi7oRnmqZY+wik3+8yQKBgB11
-SdKYOPe++2SNCrNkIw0CBk9+OEs0S1u4Jn7X9sU4kbzlUlqhF89YZe8HUfqmlmTD
-qbHwet/f6lL8HxSw1Cxi2EP+cu1oUqz53tKQgL4pAxTFlNA9SND2Ty+fEh4aY8p/
-NSphSduzxuTnC8HyGVAPnZSqDcsnVLCP7r4T7TCxAoGAbJygkkk/gZ9pT4fZoIaq
-8CMR8FTfxtkwCuZsWccSMUOWtx9nqet3gbCpKHfyoYZiKB4ke+lnUz4uFS16Y3hG
-kN0hFfvfoNa8eB2Ox7vs60cMMfWJac0H7KSaDDy+EvbhE2KtQADT0eWxMyhzGR8p
-5CbIivB0QCjeQIA8dOQpE8E=
------END PRIVATE KEY-----

cmd/test.go

@@ -1,215 +0,0 @@
-package cmd
-
-import (
-	"os"
-
-	"bytes"
-	"context"
-	"fmt"
-	"sort"
-	"strings"
-
-	"github.com/kr/pretty"
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/zfs"
-	"time"
-)
-
-var testCmd = &cobra.Command{
-	Use:              "test",
-	Short:            "test configuration",
-	PersistentPreRun: testCmdGlobalInit,
-}
-
-var testCmdGlobal struct {
-	log  Logger
-	conf *Config
-}
-
-var testConfigSyntaxCmd = &cobra.Command{
-	Use:   "config",
-	Short: "parse config file and dump parsed datastructure",
-	Run:   doTestConfig,
-}
-
-var testDatasetMapFilter = &cobra.Command{
-	Use:     "pattern jobname test/zfs/dataset/path",
-	Short:   "test dataset mapping / filter specified in config",
-	Example: ` zrepl test pattern my_pull_job tank/tmp`,
-	Run:     doTestDatasetMapFilter,
-}
-
-var testPrunePolicyArgs struct {
-	side        PrunePolicySide
-	showKept    bool
-	showRemoved bool
-}
-
-var testPrunePolicyCmd = &cobra.Command{
-	Use:   "prune jobname",
-	Short: "do a dry-run of the pruning part of a job",
-	Run:   doTestPrunePolicy,
-}
-
-func init() {
-	RootCmd.AddCommand(testCmd)
-	testCmd.AddCommand(testConfigSyntaxCmd)
-	testCmd.AddCommand(testDatasetMapFilter)
-
-	testPrunePolicyCmd.Flags().VarP(&testPrunePolicyArgs.side, "side", "s", "prune_lhs (left) or prune_rhs (right)")
-	testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showKept, "kept", false, "show kept snapshots")
-	testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showRemoved, "removed", true, "show removed snapshots")
-	testCmd.AddCommand(testPrunePolicyCmd)
-}
-
-func testCmdGlobalInit(cmd *cobra.Command, args []string) {
-
-	out := logger.NewOutlets()
-	out.Add(WriterOutlet{&NoFormatter{}, os.Stdout}, logger.Info)
-	log := logger.NewLogger(out, 1*time.Second)
-	testCmdGlobal.log = log
-
-	var err error
-	if testCmdGlobal.conf, err = ParseConfig(rootArgs.configFile); err != nil {
-		testCmdGlobal.log.Printf("error parsing config file: %s", err)
-		os.Exit(1)
-	}
-
-}
-
-func doTestConfig(cmd *cobra.Command, args []string) {
-
-	log, conf := testCmdGlobal.log, testCmdGlobal.conf
-
-	log.Printf("config ok")
-	log.Printf("%# v", pretty.Formatter(conf))
-	return
-}
-
-func doTestDatasetMapFilter(cmd *cobra.Command, args []string) {
-
-	log, conf := testCmdGlobal.log, testCmdGlobal.conf
-
-	if len(args) != 2 {
-		log.Printf("specify job name as first postitional argument, test input as second")
-		log.Printf(cmd.UsageString())
-		os.Exit(1)
-	}
-	n, i := args[0], args[1]
-
-	jobi, err := conf.LookupJob(n)
-	if err != nil {
-		log.Printf("%s", err)
-		os.Exit(1)
-	}
-
-	var mf *DatasetMapFilter
-	switch j := jobi.(type) {
-	case *PullJob:
-		mf = j.Mapping
-	case *SourceJob:
-		mf = j.Filesystems
-	case *LocalJob:
-		mf = j.Mapping
-	default:
-		panic("incomplete implementation")
-	}
-
-	ip, err := zfs.NewDatasetPath(i)
-	if err != nil {
-		log.Printf("cannot parse test input as ZFS dataset path: %s", err)
-		os.Exit(1)
-	}
-
-	if mf.filterMode {
-		pass, err := mf.Filter(ip)
-		if err != nil {
-			log.Printf("error evaluating filter: %s", err)
-			os.Exit(1)
-		}
-		log.Printf("filter result: %v", pass)
-	} else {
-		res, err := mf.Map(ip)
-		if err != nil {
-			log.Printf("error evaluating mapping: %s", err)
-			os.Exit(1)
-		}
-		toStr := "NO MAPPING"
-		if res != nil {
-			toStr = res.ToString()
-		}
-		log.Printf("%s => %s", ip.ToString(), toStr)
-
-	}
-
-}
-
-func doTestPrunePolicy(cmd *cobra.Command, args []string) {
-
-	log, conf := testCmdGlobal.log, testCmdGlobal.conf
-
-	if cmd.Flags().NArg() != 1 {
-		log.Printf("specify job name as first positional argument")
-		log.Printf(cmd.UsageString())
-		os.Exit(1)
-	}
-
-	jobname := cmd.Flags().Arg(0)
-	jobi, err := conf.LookupJob(jobname)
-	if err != nil {
-		log.Printf("%s", err)
-		os.Exit(1)
-	}
-
-	jobp, ok := jobi.(PruningJob)
-	if !ok {
-		log.Printf("job doesn't do any prunes")
-		os.Exit(0)
-	}
-
-	log.Printf("job dump:\n%s", pretty.Sprint(jobp))
-
-	task := NewTask("", log)
-	pruner, err := jobp.Pruner(task, testPrunePolicyArgs.side, true)
-	if err != nil {
-		log.Printf("cannot create test pruner: %s", err)
-		os.Exit(1)
-	}
-
-	log.Printf("start pruning")
-
-	ctx := context.WithValue(context.Background(), contextKeyLog, log)
-	result, err := pruner.Run(ctx)
-	if err != nil {
-		log.Printf("error running pruner: %s", err)
-		os.Exit(1)
-	}
-
-	sort.Slice(result, func(i, j int) bool {
-		return strings.Compare(result[i].Filesystem.ToString(), result[j].Filesystem.ToString()) == -1
-	})
-
-	var b bytes.Buffer
-	for _, r := range result {
-		fmt.Fprintf(&b, "%s\n", r.Filesystem.ToString())
-
-		if testPrunePolicyArgs.showKept {
-			fmt.Fprintf(&b, "\tkept:\n")
-			for _, v := range r.Keep {
-				fmt.Fprintf(&b, "\t- %s\n", v.Name)
-			}
-		}
-
-		if testPrunePolicyArgs.showRemoved {
-			fmt.Fprintf(&b, "\tremoved:\n")
-			for _, v := range r.Remove {
-				fmt.Fprintf(&b, "\t- %s\n", v.Name)
-			}
-		}
-
-	}
-
-	log.Printf("pruning result:\n%s", b.String())
-
-}

cmd/version.go

@@ -1,42 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"github.com/spf13/cobra"
-	"runtime"
-)
-
-var versionCmd = &cobra.Command{
-	Use:   "version",
-	Short: "print version of zrepl binary (for running daemon 'zrepl control version' command)",
-	Run:   doVersion,
-}
-
-func init() {
-	RootCmd.AddCommand(versionCmd)
-}
-
-func doVersion(cmd *cobra.Command, args []string) {
-	fmt.Println(NewZreplVersionInformation().String())
-}
-
-type ZreplVersionInformation struct {
-	Version         string
-	RuntimeGOOS     string
-	RuntimeGOARCH   string
-	RUNTIMECompiler string
-}
-
-func NewZreplVersionInformation() *ZreplVersionInformation {
-	return &ZreplVersionInformation{
-		Version:         zreplVersion,
-		RuntimeGOOS:     runtime.GOOS,
-		RuntimeGOARCH:   runtime.GOARCH,
-		RUNTIMECompiler: runtime.Compiler,
-	}
-}
-
-func (i *ZreplVersionInformation) String() string {
-	return fmt.Sprintf("zrepl version=%s GOOS=%s GOARCH=%s Compiler=%s",
-		i.Version, i.RuntimeGOOS, i.RuntimeGOARCH, i.RUNTIMECompiler)
-}

config/config.go

@@ -0,0 +1,755 @@
+package config
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log/syslog"
+	"os"
+	"reflect"
+	"regexp"
+	"strconv"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/robfig/cron/v3"
+	"github.com/zrepl/yaml-config"
+
+	"github.com/zrepl/zrepl/util/datasizeunit"
+	zfsprop "github.com/zrepl/zrepl/zfs/property"
+)
+
+type ParseFlags uint
+
+const (
+	ParseFlagsNone        ParseFlags = 0
+	ParseFlagsNoCertCheck ParseFlags = 1 << iota
+)
+
+type Config struct {
+	Jobs   []JobEnum `yaml:"jobs"`
+	Global *Global   `yaml:"global,optional,fromdefaults"`
+}
+
+func (c *Config) Job(name string) (*JobEnum, error) {
+	for _, j := range c.Jobs {
+		if j.Name() == name {
+			return &j, nil
+		}
+	}
+	return nil, fmt.Errorf("job %q not defined in config", name)
+}
+
+type JobEnum struct {
+	Ret interface{}
+}
+
+func (j JobEnum) Name() string {
+	var name string
+	switch v := j.Ret.(type) {
+	case *SnapJob:
+		name = v.Name
+	case *PushJob:
+		name = v.Name
+	case *SinkJob:
+		name = v.Name
+	case *PullJob:
+		name = v.Name
+	case *SourceJob:
+		name = v.Name
+	default:
+		panic(fmt.Sprintf("unknown job type %T", v))
+	}
+	return name
+}
+
+type ActiveJob struct {
+	Type               string                `yaml:"type"`
+	Name               string                `yaml:"name"`
+	Connect            ConnectEnum           `yaml:"connect"`
+	Pruning            PruningSenderReceiver `yaml:"pruning"`
+	Debug              JobDebugSettings      `yaml:"debug,optional"`
+	Replication        *Replication          `yaml:"replication,optional,fromdefaults"`
+	ConflictResolution *ConflictResolution   `yaml:"conflict_resolution,optional,fromdefaults"`
+}
+
+type ConflictResolution struct {
+	InitialReplication string `yaml:"initial_replication,optional,default=most_recent"`
+}
+
+type PassiveJob struct {
+	Type  string           `yaml:"type"`
+	Name  string           `yaml:"name"`
+	Serve ServeEnum        `yaml:"serve"`
+	Debug JobDebugSettings `yaml:"debug,optional"`
+}
+
+type SnapJob struct {
+	Type         string            `yaml:"type"`
+	Name         string            `yaml:"name"`
+	Pruning      PruningLocal      `yaml:"pruning"`
+	Debug        JobDebugSettings  `yaml:"debug,optional"`
+	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
+	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+}
+
+type SendOptions struct {
+	Encrypted        bool `yaml:"encrypted,optional,default=false"`
+	Raw              bool `yaml:"raw,optional,default=false"`
+	SendProperties   bool `yaml:"send_properties,optional,default=false"`
+	BackupProperties bool `yaml:"backup_properties,optional,default=false"`
+	LargeBlocks      bool `yaml:"large_blocks,optional,default=false"`
+	Compressed       bool `yaml:"compressed,optional,default=false"`
+	EmbeddedData     bool `yaml:"embedded_data,optional,default=false"`
+	Saved            bool `yaml:"saved,optional,default=false"`
+
+	BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
+}
+
+type RecvOptions struct {
+	// Note: we cannot enforce encrypted recv as the ZFS cli doesn't provide a mechanism for it
+	// Encrypted bool `yaml:"may_encrypted"`
+	// Future:
+	// Reencrypt bool `yaml:"reencrypt"`
+
+	Properties *PropertyRecvOptions `yaml:"properties,fromdefaults"`
+
+	BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
+
+	Placeholder *PlaceholderRecvOptions `yaml:"placeholder,fromdefaults"`
+}
+
+var _ yaml.Unmarshaler = &datasizeunit.Bits{}
+
+type BandwidthLimit struct {
+	Max            datasizeunit.Bits `yaml:"max,default=-1 B"`
+	BucketCapacity datasizeunit.Bits `yaml:"bucket_capacity,default=128 KiB"`
+}
+
+type Replication struct {
+	Protection  *ReplicationOptionsProtection  `yaml:"protection,optional,fromdefaults"`
+	Concurrency *ReplicationOptionsConcurrency `yaml:"concurrency,optional,fromdefaults"`
+}
+
+type ReplicationOptionsProtection struct {
+	Initial     string `yaml:"initial,optional,default=guarantee_resumability"`
+	Incremental string `yaml:"incremental,optional,default=guarantee_resumability"`
+}
+
+type ReplicationOptionsConcurrency struct {
+	Steps         int `yaml:"steps,optional,default=1"`
+	SizeEstimates int `yaml:"size_estimates,optional,default=4"`
+}
+
+type PropertyRecvOptions struct {
+	Inherit  []zfsprop.Property          `yaml:"inherit,optional"`
+	Override map[zfsprop.Property]string `yaml:"override,optional"`
+}
+
+type PlaceholderRecvOptions struct {
+	Encryption string `yaml:"encryption,default=unspecified"`
+}
+
+type PushJob struct {
+	ActiveJob    `yaml:",inline"`
+	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
+	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+	Send         *SendOptions      `yaml:"send,fromdefaults,optional"`
+}
+
+func (j *PushJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
+func (j *PushJob) GetSendOptions() *SendOptions      { return j.Send }
+
+type PullJob struct {
+	ActiveJob `yaml:",inline"`
+	RootFS    string                   `yaml:"root_fs"`
+	Interval  PositiveDurationOrManual `yaml:"interval"`
+	Recv      *RecvOptions             `yaml:"recv,fromdefaults,optional"`
+}
+
+func (j *PullJob) GetRootFS() string             { return j.RootFS }
+func (j *PullJob) GetAppendClientIdentity() bool { return false }
+func (j *PullJob) GetRecvOptions() *RecvOptions  { return j.Recv }
+
+type PositiveDurationOrManual struct {
+	Interval time.Duration
+	Manual   bool
+}
+
+var _ yaml.Unmarshaler = (*PositiveDurationOrManual)(nil)
+
+func (i *PositiveDurationOrManual) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	var s string
+	if err := u(&s, true); err != nil {
+		return err
+	}
+	switch s {
+	case "manual":
+		i.Manual = true
+		i.Interval = 0
+	case "":
+		return fmt.Errorf("value must not be empty")
+	default:
+		i.Manual = false
+		i.Interval, err = time.ParseDuration(s)
+		if err != nil {
+			return err
+		}
+		if i.Interval <= 0 {
+			return fmt.Errorf("value must be a positive duration, got %q", s)
+		}
+	}
+	return nil
+}
+
+type SinkJob struct {
+	PassiveJob `yaml:",inline"`
+	RootFS     string       `yaml:"root_fs"`
+	Recv       *RecvOptions `yaml:"recv,optional,fromdefaults"`
+}
+
+func (j *SinkJob) GetRootFS() string             { return j.RootFS }
+func (j *SinkJob) GetAppendClientIdentity() bool { return true }
+func (j *SinkJob) GetRecvOptions() *RecvOptions  { return j.Recv }
+
+type SourceJob struct {
+	PassiveJob   `yaml:",inline"`
+	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
+	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+	Send         *SendOptions      `yaml:"send,optional,fromdefaults"`
+}
+
+func (j *SourceJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
+func (j *SourceJob) GetSendOptions() *SendOptions      { return j.Send }
+
+type FilesystemsFilter map[string]bool
+
+type SnapshottingEnum struct {
+	Ret interface{}
+}
+
+type SnapshottingPeriodic struct {
+	Type     string        `yaml:"type"`
+	Prefix   string        `yaml:"prefix"`
+	Interval time.Duration `yaml:"interval,positive"`
+	Hooks    HookList      `yaml:"hooks,optional"`
+}
+
+type CronSpec struct {
+	Schedule cron.Schedule
+}
+
+var _ yaml.Unmarshaler = &CronSpec{}
+
+func (s *CronSpec) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
+	var specString string
+	if err := unmarshal(&specString, false); err != nil {
+		return err
+	}
+
+	// Use standard cron format.
+	// Disable the various "descriptors" (@daily, etc)
+	// They are just aliases to "top of hour", "midnight", etc.
+	parser := cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.SecondOptional)
+
+	sched, err := parser.Parse(specString)
+	if err != nil {
+		return errors.Wrap(err, "cron syntax invalid")
+	}
+	s.Schedule = sched
+	return nil
+}
+
+type SnapshottingCron struct {
+	Type   string   `yaml:"type"`
+	Prefix string   `yaml:"prefix"`
+	Cron   CronSpec `yaml:"cron"`
+	Hooks  HookList `yaml:"hooks,optional"`
+}
+
+type SnapshottingManual struct {
+	Type string `yaml:"type"`
+}
+
+type PruningSenderReceiver struct {
+	KeepSender   []PruningEnum `yaml:"keep_sender"`
+	KeepReceiver []PruningEnum `yaml:"keep_receiver"`
+}
+
+type PruningLocal struct {
+	Keep []PruningEnum `yaml:"keep"`
+}
+
+type LoggingOutletEnumList []LoggingOutletEnum
+
+func (l *LoggingOutletEnumList) SetDefault() {
+	def := `
+type: "stdout"
+time: true
+level: "warn"
+format: "human"
+`
+	s := &StdoutLoggingOutlet{}
+	err := yaml.UnmarshalStrict([]byte(def), &s)
+	if err != nil {
+		panic(err)
+	}
+	*l = []LoggingOutletEnum{LoggingOutletEnum{Ret: s}}
+}
+
+var _ yaml.Defaulter = &LoggingOutletEnumList{}
+
+type Global struct {
+	Logging    *LoggingOutletEnumList `yaml:"logging,optional,fromdefaults"`
+	Monitoring []MonitoringEnum       `yaml:"monitoring,optional"`
+	Control    *GlobalControl         `yaml:"control,optional,fromdefaults"`
+	Serve      *GlobalServe           `yaml:"serve,optional,fromdefaults"`
+}
+
+func Default(i interface{}) {
+	v := reflect.ValueOf(i)
+	if v.Kind() != reflect.Ptr {
+		panic(v)
+	}
+	y := `{}`
+	err := yaml.Unmarshal([]byte(y), v.Interface())
+	if err != nil {
+		panic(err)
+	}
+}
+
+type ConnectEnum struct {
+	Ret interface{}
+}
+
+type ConnectCommon struct {
+	Type string `yaml:"type"`
+}
+
+type TCPConnect struct {
+	ConnectCommon `yaml:",inline"`
+	Address       string        `yaml:"address,hostport"`
+	DialTimeout   time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
+}
+
+type TLSConnect struct {
+	ConnectCommon `yaml:",inline"`
+	Address       string        `yaml:"address,hostport"`
+	Ca            string        `yaml:"ca"`
+	Cert          string        `yaml:"cert"`
+	Key           string        `yaml:"key"`
+	ServerCN      string        `yaml:"server_cn"`
+	DialTimeout   time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
+}
+
+type SSHStdinserverConnect struct {
+	ConnectCommon        `yaml:",inline"`
+	Host                 string        `yaml:"host"`
+	User                 string        `yaml:"user"`
+	Port                 uint16        `yaml:"port"`
+	IdentityFile         string        `yaml:"identity_file"`
+	TransportOpenCommand []string      `yaml:"transport_open_command,optional"` //TODO unused
+	SSHCommand           string        `yaml:"ssh_command,optional"`            //TODO unused
+	Options              []string      `yaml:"options,optional"`
+	DialTimeout          time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
+}
+
+type LocalConnect struct {
+	ConnectCommon  `yaml:",inline"`
+	ListenerName   string        `yaml:"listener_name"`
+	ClientIdentity string        `yaml:"client_identity"`
+	DialTimeout    time.Duration `yaml:"dial_timeout,zeropositive,default=2s"`
+}
+
+type ServeEnum struct {
+	Ret interface{}
+}
+
+type ServeCommon struct {
+	Type string `yaml:"type"`
+}
+
+type TCPServe struct {
+	ServeCommon    `yaml:",inline"`
+	Listen         string            `yaml:"listen,hostport"`
+	ListenFreeBind bool              `yaml:"listen_freebind,default=false"`
+	Clients        map[string]string `yaml:"clients"`
+}
+
+type TLSServe struct {
+	ServeCommon      `yaml:",inline"`
+	Listen           string        `yaml:"listen,hostport"`
+	ListenFreeBind   bool          `yaml:"listen_freebind,default=false"`
+	Ca               string        `yaml:"ca"`
+	Cert             string        `yaml:"cert"`
+	Key              string        `yaml:"key"`
+	ClientCNs        []string      `yaml:"client_cns"`
+	HandshakeTimeout time.Duration `yaml:"handshake_timeout,zeropositive,default=10s"`
+}
+
+type StdinserverServer struct {
+	ServeCommon      `yaml:",inline"`
+	ClientIdentities []string `yaml:"client_identities"`
+}
+
+type LocalServe struct {
+	ServeCommon  `yaml:",inline"`
+	ListenerName string `yaml:"listener_name"`
+}
+
+type PruningEnum struct {
+	Ret interface{}
+}
+
+type PruneKeepNotReplicated struct {
+	Type                 string `yaml:"type"`
+	KeepSnapshotAtCursor bool   `yaml:"keep_snapshot_at_cursor,optional,default=true"`
+}
+
+type PruneKeepLastN struct {
+	Type  string `yaml:"type"`
+	Count int    `yaml:"count"`
+	Regex string `yaml:"regex,optional"`
+}
+
+type PruneKeepRegex struct { // FIXME rename to KeepRegex
+	Type   string `yaml:"type"`
+	Regex  string `yaml:"regex"`
+	Negate bool   `yaml:"negate,optional,default=false"`
+}
+
+type LoggingOutletEnum struct {
+	Ret interface{}
+}
+
+type LoggingOutletCommon struct {
+	Type   string `yaml:"type"`
+	Level  string `yaml:"level"`
+	Format string `yaml:"format"`
+}
+
+type StdoutLoggingOutlet struct {
+	LoggingOutletCommon `yaml:",inline"`
+	Time                bool `yaml:"time,default=true"`
+	Color               bool `yaml:"color,default=true"`
+}
+
+type SyslogLoggingOutlet struct {
+	LoggingOutletCommon `yaml:",inline"`
+	Facility            *SyslogFacility `yaml:"facility,optional,fromdefaults"`
+	RetryInterval       time.Duration   `yaml:"retry_interval,positive,default=10s"`
+}
+
+type TCPLoggingOutlet struct {
+	LoggingOutletCommon `yaml:",inline"`
+	Address             string               `yaml:"address,hostport"`
+	Net                 string               `yaml:"net,default=tcp"`
+	RetryInterval       time.Duration        `yaml:"retry_interval,positive,default=10s"`
+	TLS                 *TCPLoggingOutletTLS `yaml:"tls,optional"`
+}
+
+type TCPLoggingOutletTLS struct {
+	CA   string `yaml:"ca"`
+	Cert string `yaml:"cert"`
+	Key  string `yaml:"key"`
+}
+
+type MonitoringEnum struct {
+	Ret interface{}
+}
+
+type PrometheusMonitoring struct {
+	Type           string `yaml:"type"`
+	Listen         string `yaml:"listen,hostport"`
+	ListenFreeBind bool   `yaml:"listen_freebind,default=false"`
+}
+
+type SyslogFacility syslog.Priority
+
+func (f *SyslogFacility) SetDefault() {
+	*f = SyslogFacility(syslog.LOG_LOCAL0)
+}
+
+var _ yaml.Defaulter = (*SyslogFacility)(nil)
+
+type GlobalControl struct {
+	SockPath string `yaml:"sockpath,default=/var/run/zrepl/control"`
+}
+
+type GlobalServe struct {
+	StdinServer *GlobalStdinServer `yaml:"stdinserver,optional,fromdefaults"`
+}
+
+type GlobalStdinServer struct {
+	SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
+}
+
+type JobDebugSettings struct {
+	Conn *struct {
+		ReadDump  string `yaml:"read_dump"`
+		WriteDump string `yaml:"write_dump"`
+	} `yaml:"conn,optional"`
+	RPCLog bool `yaml:"rpc_log,optional,default=false"`
+}
+
+type HookList []HookEnum
+
+type HookEnum struct {
+	Ret interface{}
+}
+
+type HookCommand struct {
+	Path               string            `yaml:"path"`
+	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
+	Filesystems        FilesystemsFilter `yaml:"filesystems,optional,default={'<': true}"`
+	HookSettingsCommon `yaml:",inline"`
+}
+
+type HookPostgresCheckpoint struct {
+	HookSettingsCommon `yaml:",inline"`
+	DSN                string            `yaml:"dsn"`
+	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
+	Filesystems        FilesystemsFilter `yaml:"filesystems"` // required, user should not CHECKPOINT for every FS
+}
+
+type HookMySQLLockTables struct {
+	HookSettingsCommon `yaml:",inline"`
+	DSN                string            `yaml:"dsn"`
+	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
+	Filesystems        FilesystemsFilter `yaml:"filesystems"`
+}
+
+type HookSettingsCommon struct {
+	Type       string `yaml:"type"`
+	ErrIsFatal bool   `yaml:"err_is_fatal,optional,default=false"`
+}
+
+func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) {
+	var in struct {
+		Type string
+	}
+	if err := u(&in, true); err != nil {
+		return nil, err
+	}
+	if in.Type == "" {
+		return nil, &yaml.TypeError{Errors: []string{"must specify type"}}
+	}
+
+	v, ok := types[in.Type]
+	if !ok {
+		return nil, &yaml.TypeError{Errors: []string{fmt.Sprintf("invalid type name %q", in.Type)}}
+	}
+	if err := u(v, false); err != nil {
+		return nil, err
+	}
+	return v, nil
+}
+
+func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"snap":   &SnapJob{},
+		"push":   &PushJob{},
+		"sink":   &SinkJob{},
+		"pull":   &PullJob{},
+		"source": &SourceJob{},
+	})
+	return
+}
+
+func (t *ConnectEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"tcp":             &TCPConnect{},
+		"tls":             &TLSConnect{},
+		"ssh+stdinserver": &SSHStdinserverConnect{},
+		"local":           &LocalConnect{},
+	})
+	return
+}
+
+func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"tcp":         &TCPServe{},
+		"tls":         &TLSServe{},
+		"stdinserver": &StdinserverServer{},
+		"local":       &LocalServe{},
+	})
+	return
+}
+
+func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"not_replicated": &PruneKeepNotReplicated{},
+		"last_n":         &PruneKeepLastN{},
+		"grid":           &PruneGrid{},
+		"regex":          &PruneKeepRegex{},
+	})
+	return
+}
+
+func (t *SnapshottingEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"periodic": &SnapshottingPeriodic{},
+		"manual":   &SnapshottingManual{},
+		"cron":     &SnapshottingCron{},
+	})
+	return
+}
+
+func (t *LoggingOutletEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"stdout": &StdoutLoggingOutlet{},
+		"syslog": &SyslogLoggingOutlet{},
+		"tcp":    &TCPLoggingOutlet{},
+	})
+	return
+}
+
+func (t *MonitoringEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"prometheus": &PrometheusMonitoring{},
+	})
+	return
+}
+
+func (t *SyslogFacility) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	var s string
+	if err := u(&s, true); err != nil {
+		return err
+	}
+	var level syslog.Priority
+	switch s {
+	case "kern":
+		level = syslog.LOG_KERN
+	case "user":
+		level = syslog.LOG_USER
+	case "mail":
+		level = syslog.LOG_MAIL
+	case "daemon":
+		level = syslog.LOG_DAEMON
+	case "auth":
+		level = syslog.LOG_AUTH
+	case "syslog":
+		level = syslog.LOG_SYSLOG
+	case "lpr":
+		level = syslog.LOG_LPR
+	case "news":
+		level = syslog.LOG_NEWS
+	case "uucp":
+		level = syslog.LOG_UUCP
+	case "cron":
+		level = syslog.LOG_CRON
+	case "authpriv":
+		level = syslog.LOG_AUTHPRIV
+	case "ftp":
+		level = syslog.LOG_FTP
+	case "local0":
+		level = syslog.LOG_LOCAL0
+	case "local1":
+		level = syslog.LOG_LOCAL1
+	case "local2":
+		level = syslog.LOG_LOCAL2
+	case "local3":
+		level = syslog.LOG_LOCAL3
+	case "local4":
+		level = syslog.LOG_LOCAL4
+	case "local5":
+		level = syslog.LOG_LOCAL5
+	case "local6":
+		level = syslog.LOG_LOCAL6
+	case "local7":
+		level = syslog.LOG_LOCAL7
+	default:
+		return fmt.Errorf("invalid syslog level: %q", s)
+	}
+	*t = SyslogFacility(level)
+	return nil
+}
+
+func (t *HookEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"command":             &HookCommand{},
+		"postgres-checkpoint": &HookPostgresCheckpoint{},
+		"mysql-lock-tables":   &HookMySQLLockTables{},
+	})
+	return
+}
+
+var ConfigFileDefaultLocations = []string{
+	"/etc/zrepl/zrepl.yml",
+	"/usr/local/etc/zrepl/zrepl.yml",
+}
+
+func ParseConfig(path string) (i *Config, err error) {
+
+	if path == "" {
+		// Try default locations
+		for _, l := range ConfigFileDefaultLocations {
+			stat, statErr := os.Stat(l)
+			if statErr != nil {
+				continue
+			}
+			if !stat.Mode().IsRegular() {
+				err = errors.Errorf("file at default location is not a regular file: %s", l)
+				return
+			}
+			path = l
+			break
+		}
+	}
+
+	var bytes []byte
+
+	if bytes, err = ioutil.ReadFile(path); err != nil {
+		return
+	}
+
+	return ParseConfigBytes(bytes)
+}
+
+func ParseConfigBytes(bytes []byte) (*Config, error) {
+	var c *Config
+	if err := yaml.UnmarshalStrict(bytes, &c); err != nil {
+		return nil, err
+	}
+	if c == nil {
+		return nil, fmt.Errorf("config is empty or only consists of comments")
+	}
+	return c, nil
+}
+
+var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
+
+func parsePositiveDuration(e string) (d time.Duration, err error) {
+	comps := durationStringRegex.FindStringSubmatch(e)
+	if len(comps) != 3 {
+		err = fmt.Errorf("does not match regex: %s %#v", e, comps)
+		return
+	}
+
+	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+	if durationFactor <= 0 {
+		return 0, errors.New("duration must be positive integer")
+	}
+
+	var durationUnit time.Duration
+	switch comps[2] {
+	case "s":
+		durationUnit = time.Second
+	case "m":
+		durationUnit = time.Minute
+	case "h":
+		durationUnit = time.Hour
+	case "d":
+		durationUnit = 24 * time.Hour
+	case "w":
+		durationUnit = 24 * 7 * time.Hour
+	default:
+		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
+		return
+	}
+
+	d = time.Duration(durationFactor) * durationUnit
+	return
+}

config/config_dep_test.go

@@ -0,0 +1,90 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/zrepl/yaml-config"
+)
+
+type A struct {
+	B  *B     `yaml:"b,optional,fromdefaults"`
+	A1 string `yaml:"a1,optional"`
+}
+
+type B struct {
+	C *C     `yaml:"c,optional,fromdefaults"`
+	D string `yaml:"d,default=ddd"`
+	E string `yaml:"e,optional"`
+}
+
+type C struct {
+	Q string `yaml:"q,optional"`
+	R string `yaml:"r,default=r"`
+}
+
+func TestDepFromDefaults(t *testing.T) {
+
+	type testcase struct {
+		name   string
+		yaml   string
+		expect *A
+	}
+
+	tcs := []testcase{
+		{
+			name: "empty",
+			yaml: `{}`,
+			expect: &A{
+				B: &B{
+					C: &C{
+						R: "r",
+					},
+					D: "ddd",
+				},
+			},
+		},
+		{
+			name: "a1 set",
+			yaml: `{"a1":"blah"}`,
+			expect: &A{
+				A1: "blah",
+				B: &B{
+					C: &C{
+						R: "r",
+					},
+					D: "ddd",
+				},
+			},
+		},
+		{
+			name: "D set",
+			yaml: `
+b:
+  d: 4d
+`,
+			expect: &A{
+				B: &B{
+					D: "4d",
+					C: &C{
+						R: "r",
+					},
+				},
+			},
+		},
+	}
+
+	for tci := range tcs {
+		t.Run(fmt.Sprintf("%d-%s", tci, tcs[tci].name), func(t *testing.T) {
+			tc := tcs[tci]
+
+			var a A
+			err := yaml.UnmarshalStrict([]byte(tc.yaml), &a)
+			require.NoError(t, err)
+
+			require.Equal(t, tc.expect, &a)
+		})
+	}
+
+}

config/config_global_test.go

@@ -0,0 +1,115 @@
+package config
+
+import (
+	"fmt"
+	"log/syslog"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/zrepl/yaml-config"
+)
+
+func testValidGlobalSection(t *testing.T, s string) *Config {
+	jobdef := `
+jobs:
+- name: dummyjob
+  type: sink
+  serve:
+    type: tcp
+    listen: ":2342"
+    clients: {
+      "10.0.0.1":"foo"
+    }
+  root_fs: zroot/foo
+`
+	_, err := ParseConfigBytes([]byte(jobdef))
+	require.NoError(t, err)
+	return testValidConfig(t, s+jobdef)
+}
+
+func TestOutletTypes(t *testing.T) {
+	conf := testValidGlobalSection(t, `
+global:
+  logging:
+  - type: stdout
+    level: debug
+    format: human
+  - type: syslog
+    level: info
+    retry_interval: 20s
+    format: human
+  - type: tcp
+    level: debug
+    format: json
+    address: logserver.example.com:1234
+  - type: tcp
+    level: debug
+    format: json
+    address: encryptedlogserver.example.com:1234
+    retry_interval: 20s 
+    tls:
+      ca: /etc/zrepl/log/ca.crt
+      cert: /etc/zrepl/log/key.pem
+      key: /etc/zrepl/log/cert.pem
+`)
+	assert.Equal(t, 4, len(*conf.Global.Logging))
+	assert.NotNil(t, (*conf.Global.Logging)[3].Ret.(*TCPLoggingOutlet).TLS)
+}
+
+func TestDefaultLoggingOutlet(t *testing.T) {
+	conf := testValidGlobalSection(t, "")
+	assert.Equal(t, 1, len(*conf.Global.Logging))
+	o := (*conf.Global.Logging)[0].Ret.(*StdoutLoggingOutlet)
+	assert.Equal(t, "warn", o.Level)
+	assert.Equal(t, "human", o.Format)
+}
+
+func TestPrometheusMonitoring(t *testing.T) {
+	conf := testValidGlobalSection(t, `
+global:
+  monitoring:
+    - type: prometheus
+      listen: ':9811'
+`)
+	assert.Equal(t, ":9811", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
+}
+
+func TestSyslogLoggingOutletFacility(t *testing.T) {
+	type SyslogFacilityPriority struct {
+		Facility string
+		Priority syslog.Priority
+	}
+	syslogFacilitiesPriorities := []SyslogFacilityPriority{
+		{"", syslog.LOG_LOCAL0}, // default
+		{"kern", syslog.LOG_KERN}, {"daemon", syslog.LOG_DAEMON}, {"auth", syslog.LOG_AUTH},
+		{"syslog", syslog.LOG_SYSLOG}, {"lpr", syslog.LOG_LPR}, {"news", syslog.LOG_NEWS},
+		{"uucp", syslog.LOG_UUCP}, {"cron", syslog.LOG_CRON}, {"authpriv", syslog.LOG_AUTHPRIV},
+		{"ftp", syslog.LOG_FTP}, {"local0", syslog.LOG_LOCAL0}, {"local1", syslog.LOG_LOCAL1},
+		{"local2", syslog.LOG_LOCAL2}, {"local3", syslog.LOG_LOCAL3}, {"local4", syslog.LOG_LOCAL4},
+		{"local5", syslog.LOG_LOCAL5}, {"local6", syslog.LOG_LOCAL6}, {"local7", syslog.LOG_LOCAL7},
+	}
+
+	for _, sFP := range syslogFacilitiesPriorities {
+		logcfg := fmt.Sprintf(`
+global:
+  logging:
+  - type: syslog
+    level: info
+    format: human
+    facility: %s
+`, sFP.Facility)
+		conf := testValidGlobalSection(t, logcfg)
+		assert.Equal(t, 1, len(*conf.Global.Logging))
+		assert.True(t, SyslogFacility(sFP.Priority) == *(*conf.Global.Logging)[0].Ret.(*SyslogLoggingOutlet).Facility)
+	}
+}
+
+func TestLoggingOutletEnumList_SetDefaults(t *testing.T) {
+	e := &LoggingOutletEnumList{}
+	var i yaml.Defaulter = e
+	require.NotPanics(t, func() {
+		i.SetDefault()
+		assert.Equal(t, "warn", (*e)[0].Ret.(*StdoutLoggingOutlet).Level)
+	})
+}

config/config_jobid_test.go

@@ -0,0 +1 @@
+package config

config/config_minimal_test.go

@@ -0,0 +1,40 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfigEmptyFails(t *testing.T) {
+	conf, err := testConfig(t, "\n")
+	assert.Nil(t, conf)
+	assert.Error(t, err)
+}
+
+func TestJobsOnlyWorks(t *testing.T) {
+	testValidConfig(t, `
+jobs:
+- name: push
+  type: push
+  # snapshot the filesystems matched by the left-hand-side of the mapping
+  # every 10m with zrepl_ as prefix
+  connect:
+    type: tcp
+    address: localhost:2342
+  filesystems: {
+    "pool1/var/db<": true,
+    "pool1/usr/home<": true,
+    "pool1/usr/home/paranoid": false, #don't backup paranoid user
+    "pool1/poudriere/ports<": false #don't backup the ports trees
+  }
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    keep_receiver:
+    - type: last_n 
+      count: 1
+`)
+}

config/config_positiveintervalormanual_test.go

@@ -0,0 +1,41 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/zrepl/yaml-config"
+)
+
+func TestPositiveDurationOrManual(t *testing.T) {
+	cases := []struct {
+		Comment, Input string
+		Result         *PositiveDurationOrManual
+	}{
+		{"empty is error", "", nil},
+		{"negative is error", "-1s", nil},
+		{"zero seconds is error", "0s", nil},
+		{"zero is error", "0", nil},
+		{"non-manual is error", "something", nil},
+		{"positive seconds works", "1s", &PositiveDurationOrManual{Manual: false, Interval: 1 * time.Second}},
+		{"manual works", "manual", &PositiveDurationOrManual{Manual: true, Interval: 0}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.Comment, func(t *testing.T) {
+			var out struct {
+				FieldName PositiveDurationOrManual `yaml:"fieldname"`
+			}
+			input := fmt.Sprintf("\nfieldname: %s\n", tc.Input)
+			err := yaml.UnmarshalStrict([]byte(input), &out)
+			if tc.Result == nil {
+				assert.Error(t, err)
+				t.Logf("%#v", out)
+			} else {
+				assert.Equal(t, *tc.Result, out.FieldName)
+			}
+		})
+	}
+
+}

config/config_recv_test.go

@@ -0,0 +1,134 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	zfsprop "github.com/zrepl/zrepl/zfs/property"
+)
+
+func TestRecvOptions(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: pull
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  root_fs: "zreplplatformtest"
+  %s
+  interval: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+
+`
+
+	recv_properties_empty := `
+  recv:
+    properties:
+`
+
+	recv_inherit_empty := `
+  recv:
+    properties:
+      inherit:
+`
+
+	recv_inherit := `
+  recv:
+    properties:
+      inherit:
+        - testprop
+`
+
+	recv_override_empty := `
+  recv:
+    properties:
+      override:
+`
+
+	recv_override := `
+  recv:
+    properties:
+      override:
+        testprop2: "test123"
+`
+
+	recv_override_and_inherit := `
+  recv:
+    properties:
+      inherit:
+        - testprop
+      override:
+        testprop2: "test123"
+`
+
+	recv_empty := `
+  recv: {}
+`
+
+	recv_not_specified := `
+`
+
+	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
+
+	t.Run("recv_inherit_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_inherit_empty))
+		assert.NotNil(t, c)
+	})
+
+	t.Run("recv_inherit", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_inherit))
+		inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
+		assert.NotEmpty(t, inherit)
+		assert.Contains(t, inherit, zfsprop.Property("testprop"))
+	})
+
+	t.Run("recv_override_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_override_empty))
+		assert.NotNil(t, c)
+	})
+
+	t.Run("recv_override", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_override))
+		override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
+		require.Len(t, override, 1)
+		require.Equal(t, "test123", override["testprop2"])
+
+	})
+
+	t.Run("recv_override_and_inherit", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_override_and_inherit))
+		inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
+		override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
+		assert.NotEmpty(t, inherit)
+		assert.Contains(t, inherit, zfsprop.Property("testprop"))
+		assert.NotEmpty(t, override)
+		assert.Equal(t, "test123", override["testprop2"])
+	})
+
+	t.Run("recv_properties_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_properties_empty))
+		assert.NotNil(t, c)
+	})
+
+	t.Run("recv_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_empty))
+		assert.NotNil(t, c)
+	})
+
+	t.Run("send_not_specified", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_not_specified))
+		assert.NotNil(t, c)
+	})
+
+}

config/config_send_test.go

@@ -0,0 +1,146 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSendOptions(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  %s
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+	encrypted_false := `
+  send:
+    encrypted: false
+`
+
+	encrypted_true := `
+  send:
+    encrypted: true
+`
+
+	raw_true := `
+  send:
+    raw: true
+
+`
+
+	raw_false := `
+  send:
+    raw: false
+
+`
+
+	raw_and_encrypted := `
+  send:
+    encrypted: true
+    raw: true
+`
+
+	properties_and_encrypted := `
+  send:
+    encrypted: true
+    send_properties: true
+`
+
+	properties_true := `
+  send:
+    send_properties: true
+`
+
+	properties_false := `
+  send:
+    send_properties: false
+`
+
+	send_empty := `
+  send: {}
+`
+
+	send_not_specified := `
+`
+
+	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
+
+	t.Run("encrypted_false", func(t *testing.T) {
+		c := testValidConfig(t, fill(encrypted_false))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, false, encrypted)
+	})
+	t.Run("encrypted_true", func(t *testing.T) {
+		c := testValidConfig(t, fill(encrypted_true))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, true, encrypted)
+	})
+
+	t.Run("send_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(send_empty))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, false, encrypted)
+	})
+
+	t.Run("properties_and_encrypted", func(t *testing.T) {
+		c := testValidConfig(t, fill(properties_and_encrypted))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
+		assert.Equal(t, true, encrypted)
+		assert.Equal(t, true, properties)
+	})
+
+	t.Run("properties_false", func(t *testing.T) {
+		c := testValidConfig(t, fill(properties_false))
+		properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
+		assert.Equal(t, false, properties)
+	})
+
+	t.Run("properties_true", func(t *testing.T) {
+		c := testValidConfig(t, fill(properties_true))
+		properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
+		assert.Equal(t, true, properties)
+	})
+
+	t.Run("raw_true", func(t *testing.T) {
+		c := testValidConfig(t, fill(raw_true))
+		raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
+		assert.Equal(t, true, raw)
+	})
+
+	t.Run("raw_false", func(t *testing.T) {
+		c := testValidConfig(t, fill(raw_false))
+		raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
+		assert.Equal(t, false, raw)
+	})
+
+	t.Run("raw_and_encrypted", func(t *testing.T) {
+		c := testValidConfig(t, fill(raw_and_encrypted))
+		raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, true, raw)
+		assert.Equal(t, true, encrypted)
+	})
+
+	t.Run("send_not_specified", func(t *testing.T) {
+		c := testValidConfig(t, fill(send_not_specified))
+		assert.NotNil(t, c)
+	})
+
+}

config/config_snapshotting_test.go

@@ -0,0 +1,90 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSnapshotting(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  %s
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+	manual := `
+  snapshotting:
+    type: manual
+`
+	periodic := `
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 10m
+`
+
+	hooks := `
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 10m
+    hooks:
+    - type: command
+      path: /tmp/path/to/command
+    - type: command
+      path: /tmp/path/to/command
+      filesystems: { "zroot<": true, "<": false }
+    - type: postgres-checkpoint
+      dsn: "host=localhost port=5432 user=postgres sslmode=disable"
+      filesystems: {
+          "tank/postgres/data11": true
+      }
+    - type: mysql-lock-tables
+      dsn: "root@tcp(localhost)/"
+      filesystems: {
+        "tank/mysql": true
+      }
+`
+
+	fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
+	var c *Config
+
+	t.Run("manual", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(manual))
+		snm := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingManual)
+		assert.Equal(t, "manual", snm.Type)
+	})
+
+	t.Run("periodic", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(periodic))
+		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
+		assert.Equal(t, "periodic", snp.Type)
+		assert.Equal(t, 10*time.Minute, snp.Interval)
+		assert.Equal(t, "zrepl_", snp.Prefix)
+	})
+
+	t.Run("hooks", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(hooks))
+		hs := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic).Hooks
+		assert.Equal(t, hs[0].Ret.(*HookCommand).Filesystems["<"], true)
+		assert.Equal(t, hs[1].Ret.(*HookCommand).Filesystems["zroot<"], true)
+		assert.Equal(t, hs[2].Ret.(*HookPostgresCheckpoint).Filesystems["tank/postgres/data11"], true)
+		assert.Equal(t, hs[3].Ret.(*HookMySQLLockTables).Filesystems["tank/mysql"], true)
+	})
+
+}

config/config_test.go

@@ -0,0 +1,140 @@
+package config
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+	"text/template"
+
+	"github.com/kr/pretty"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/zrepl/yaml-config"
+)
+
+func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
+	paths, err := filepath.Glob("./samples/*")
+	if err != nil {
+		t.Errorf("glob failed: %+v", err)
+	}
+
+	for _, p := range paths {
+
+		if path.Ext(p) != ".yml" {
+			t.Logf("skipping file %s", p)
+			continue
+		}
+
+		t.Run(p, func(t *testing.T) {
+			c, err := ParseConfig(p)
+			if err != nil {
+				t.Errorf("error parsing %s:\n%+v", p, err)
+			}
+
+			t.Logf("file: %s", p)
+			t.Log(pretty.Sprint(c))
+		})
+
+	}
+
+}
+
+// template must be a template/text template with a single '{{ . }}' as placeholder for val
+//
+//nolint:deadcode,unused
+func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
+	tmp, err := template.New("master").Parse(tmpl)
+	if err != nil {
+		panic(err)
+	}
+	var buf bytes.Buffer
+	err = tmp.Execute(&buf, val)
+	if err != nil {
+		panic(err)
+	}
+	return testValidConfig(t, buf.String())
+}
+
+func testValidConfig(t *testing.T, input string) *Config {
+	t.Helper()
+	conf, err := testConfig(t, input)
+	require.NoError(t, err)
+	require.NotNil(t, conf)
+	return conf
+}
+
+func testConfig(t *testing.T, input string) (*Config, error) {
+	t.Helper()
+	return ParseConfigBytes([]byte(input))
+}
+
+func trimSpaceEachLineAndPad(s, pad string) string {
+	var out strings.Builder
+	scan := bufio.NewScanner(strings.NewReader(s))
+	for scan.Scan() {
+		fmt.Fprintf(&out, "%s%s\n", pad, bytes.TrimSpace(scan.Bytes()))
+	}
+	return out.String()
+}
+
+func TestTrimSpaceEachLineAndPad(t *testing.T) {
+	foo := `
+	foo
+	bar baz 
+	`
+	assert.Equal(t, "  \n  foo\n  bar baz\n  \n", trimSpaceEachLineAndPad(foo, "  "))
+}
+
+func TestCronSpec(t *testing.T) {
+
+	expectAccept := []string{
+		`"* * * * *"`,
+		`"0-10 * * * *"`,
+		`"* 0-5,8,12 * * *"`,
+	}
+
+	expectFail := []string{
+		`* * * *`,
+		``,
+		`23`,
+		`"@reboot"`,
+		`"@every 1h30m"`,
+		`"@daily"`,
+		`* * * * * *`,
+	}
+
+	for _, input := range expectAccept {
+		t.Run(input, func(t *testing.T) {
+			s := fmt.Sprintf("spec: %s\n", input)
+			var v struct {
+				Spec CronSpec
+			}
+			v.Spec.Schedule = nil
+			t.Logf("input:\n%s", s)
+			err := yaml.UnmarshalStrict([]byte(s), &v)
+			t.Logf("error: %T %s", err, err)
+			require.NoError(t, err)
+			require.NotNil(t, v.Spec.Schedule)
+		})
+	}
+
+	for _, input := range expectFail {
+		t.Run(input, func(t *testing.T) {
+			s := fmt.Sprintf("spec: %s\n", input)
+			var v struct {
+				Spec CronSpec
+			}
+			v.Spec.Schedule = nil
+			t.Logf("input: %q", s)
+			err := yaml.UnmarshalStrict([]byte(s), &v)
+			t.Logf("error: %T %s", err, err)
+			require.Error(t, err)
+			require.Nil(t, v.Spec.Schedule)
+		})
+	}
+
+}

config/config_transport_test.go

@@ -0,0 +1,99 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestTransportConnect(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+%s
+  filesystems: {"<": true}
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+
+	mconf := func(s string) string { return fmt.Sprintf(tmpl, s) }
+
+	type test struct {
+		Name        string
+		ExpectError bool
+		Connect     string
+	}
+
+	testTable := []test{
+		{
+			Name:        "tcp_with_address_and_port",
+			ExpectError: false,
+			Connect: `
+			type: tcp
+			address: 10.0.0.23:42
+			`,
+		},
+		{
+			Name:        "tls_with_host_and_port",
+			ExpectError: false,
+			Connect: `
+			type: tls
+			address: "server1.foo.bar:8888"
+			ca:   /etc/zrepl/ca.crt
+			cert: /etc/zrepl/backupserver.fullchain
+			key:  /etc/zrepl/backupserver.key
+			server_cn: "server1"
+			`,
+		},
+		{
+			Name:        "tcp_without_port",
+			ExpectError: true,
+			Connect: `
+			type: tcp
+			address: 10.0.0.23
+			`,
+		},
+		{
+			Name:        "tls_without_port",
+			ExpectError: true,
+			Connect: `
+			type: tls
+			address: 10.0.0.23
+			ca:   /etc/zrepl/ca.crt
+			cert: /etc/zrepl/backupserver.fullchain
+			key:  /etc/zrepl/backupserver.key
+			server_cn: "server1"
+			`,
+		},
+	}
+
+	for _, tc := range testTable {
+		t.Run(tc.Name, func(t *testing.T) {
+			require.NotEmpty(t, tc.Connect)
+			connect := trimSpaceEachLineAndPad(tc.Connect, "    ")
+			conf := mconf(connect)
+			config, err := testConfig(t, conf)
+			if tc.ExpectError && err == nil {
+				t.Errorf("expected test failure, but got valid config %v", config)
+				return
+			}
+			if !tc.ExpectError && err != nil {
+				t.Errorf("not expecint test failure but got error: %s", err)
+				return
+			}
+			t.Logf("error=%v", err)
+			t.Logf("config=%v", config)
+		})
+	}
+
+}

config/retentiongrid.go

@@ -0,0 +1,119 @@
+package config
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type RetentionIntervalList []RetentionInterval
+
+type PruneGrid struct {
+	Type  string                `yaml:"type"`
+	Grid  RetentionIntervalList `yaml:"grid"`
+	Regex string                `yaml:"regex"`
+}
+
+type RetentionInterval struct {
+	length    time.Duration
+	keepCount int
+}
+
+func (i *RetentionInterval) Length() time.Duration {
+	return i.length
+}
+
+func (i *RetentionInterval) KeepCount() int {
+	return i.keepCount
+}
+
+const RetentionGridKeepCountAll int = -1
+
+func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	var in string
+	if err := u(&in, true); err != nil {
+		return err
+	}
+
+	intervals, err := ParseRetentionIntervalSpec(in)
+	if err != nil {
+		return err
+	}
+
+	*t = intervals
+
+	return nil
+}
+
+var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
+
+func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {
+
+	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
+	if comps == nil {
+		err = fmt.Errorf("retention string does not match expected format")
+		return
+	}
+
+	times, err := strconv.Atoi(comps[1])
+	if err != nil {
+		return nil, err
+	} else if times <= 0 {
+		return nil, fmt.Errorf("contains factor <= 0")
+	}
+
+	duration, err := parsePositiveDuration(comps[2])
+	if err != nil {
+		return nil, err
+	}
+
+	keepCount := 1
+	if comps[3] != "" {
+		// Decompose key=value, comma separated
+		// For now, only keep_count is supported
+		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
+		res := re.FindStringSubmatch(comps[4])
+		if res == nil || len(res) != 2 {
+			err = fmt.Errorf("interval parameter contains unknown parameters")
+			return
+		}
+		if res[1] == "all" {
+			keepCount = RetentionGridKeepCountAll
+		} else {
+			keepCount, err = strconv.Atoi(res[1])
+			if err != nil {
+				err = fmt.Errorf("cannot parse keep_count value")
+				return
+			}
+		}
+	}
+
+	intervals = make([]RetentionInterval, times)
+	for i := range intervals {
+		intervals[i] = RetentionInterval{
+			length:    duration,
+			keepCount: keepCount,
+		}
+	}
+
+	return
+
+}
+
+func ParseRetentionIntervalSpec(s string) (intervals []RetentionInterval, err error) {
+
+	ges := strings.Split(s, "|")
+	intervals = make([]RetentionInterval, 0, 7*len(ges))
+
+	for intervalIdx, e := range ges {
+		parsed, err := parseRetentionGridIntervalString(e)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
+		}
+		intervals = append(intervals, parsed...)
+	}
+
+	return
+}

config/samples/bandwidth_limit.yml

@@ -0,0 +1,41 @@
+
+jobs:
+  - type: sink
+    name: "limited_sink"
+    root_fs: "fs0"
+    recv:
+      bandwidth_limit:
+        max: 12345 B
+    serve:
+      type: local
+      listener_name: localsink
+
+  - type: push
+    name: "limited_push"
+    connect:
+      type: local
+      listener_name: localsink
+      client_identity: local_backup
+    filesystems: {
+      "root<": true,
+    }
+    send:
+      bandwidth_limit:
+        max: 54321 B
+        bucket_capacity: 1024 B
+    snapshotting:
+      type: manual
+    pruning:
+      keep_sender:
+        - type: last_n
+          count: 1
+      keep_receiver:
+        - type: last_n
+          count: 1
+
+  - type: sink
+    name: "nolimit_sink"
+    root_fs: "fs1"
+    serve:
+      type: local
+      listener_name: localsink

config/samples/hooks/template.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -e
+
+[ "$ZREPL_DRYRUN" = "true" ] && DRYRUN="echo DRYRUN: "
+
+pre_snapshot() {
+    $DRYRUN date
+}
+
+post_snapshot() {
+    $DRYRUN date
+}
+
+case "$ZREPL_HOOKTYPE" in
+    pre_snapshot|post_snapshot)
+        "$ZREPL_HOOKTYPE"
+        ;;
+    *)
+        printf 'Unrecognized hook type: %s\n' "$ZREPL_HOOKTYPE"
+        exit 255
+        ;;
+esac

config/samples/local.yml

@@ -0,0 +1,31 @@
+
+jobs:
+  - type: sink
+    name: "local_sink"
+    root_fs: "storage/zrepl/sink"
+    serve:
+      type: local
+      listener_name: localsink
+
+  - type: push
+    name: "backup_system"
+    connect:
+      type: local
+      listener_name: localsink
+      client_identity: local_backup
+    filesystems: {
+      "system<": true,
+    }
+    snapshotting:
+      type: periodic
+      interval: 10m
+      prefix: zrepl_
+    pruning:
+      keep_sender:
+      - type: not_replicated
+      - type: last_n
+        count: 10
+      keep_receiver:
+      - type: grid
+        grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+        regex: "zrepl_.*"

config/samples/pull.yml

@@ -0,0 +1,24 @@
+jobs:
+- name: pull_servers
+  type: pull
+  connect:
+    type: tls
+    address: "server1.foo.bar:8888"
+    ca: "/certs/ca.crt"
+    cert: "/certs/cert.crt"
+    key: "/certs/key.pem"
+    server_cn: "server1"
+  root_fs: "pool2/backup_servers"
+  interval: 10m
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    - type: last_n
+      count: 10
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 14x1d
+      regex: "zrepl_.*"
+    keep_receiver:
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+      regex: "zrepl_.*"

config/samples/pull_ssh.yml

@@ -0,0 +1,28 @@
+jobs:
+
+- name: pull_servers
+  type: pull
+  connect:
+    type: ssh+stdinserver
+    host: app-srv.example.com
+    user: root
+    port: 22
+    identity_file: /etc/zrepl/ssh/identity
+    options: # optional, default [], `-o` arguments passed to ssh
+    - "Compression=yes"
+  root_fs: "pool2/backup_servers"
+  interval: 10m
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    - type: last_n
+      count: 10
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 14x1d
+      regex: "^zrepl_.*"
+    keep_receiver:
+    - type: regex
+      regex: keep_
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+      regex: "^zrepl_.*"

config/samples/push.yml

@@ -0,0 +1,26 @@
+jobs:
+  - type: push
+    name: "push"
+    filesystems: {
+      "<": true,
+      "tmp": false
+    }
+    connect:
+      type: tcp
+      address: "backup-server.foo.bar:8888"
+    snapshotting:
+      type: manual
+    send:
+      encrypted: false
+    pruning:
+      keep_sender:
+        - type: not_replicated
+        - type: last_n
+          count: 10
+        - type: grid
+          grid: 1x1h(keep=all) | 24x1h | 14x1d
+          regex: "^zrepl_.*"
+      keep_receiver:
+        - type: grid
+          grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+          regex: "^zrepl_.*"

config/samples/quickstart_backup_to_external_disk.yml

@@ -0,0 +1,88 @@
+# This config serves as an example for a local zrepl installation that
+# backups the entire zpool `system` to `backuppool/zrepl/sink`
+#
+# The requirements covered by this setup are described in the zrepl documentation's
+# quick start section which inlines this example.
+#
+# CUSTOMIZATIONS YOU WILL LIKELY WANT TO APPLY:
+# - adjust the name of the production pool `system` in the `filesystems` filter of jobs `snapjob` and `push_to_drive`
+# - adjust the name of the backup pool `backuppool` in the `backuppool_sink` job
+# - adjust the occurences of `myhostname` to the name of the system you are backing up (cannot be easily changed once you start replicating)
+# - make sure the `zrepl_` prefix is not being used by any other zfs tools you might have installed (it likely isn't)
+
+jobs:
+
+# this job takes care of snapshot creation + pruning
+- name: snapjob
+  type: snap
+  filesystems: {
+      "system<": true,
+  }
+  # create snapshots with prefix `zrepl_` every 15 minutes
+  snapshotting:
+    type: periodic
+    interval: 15m
+    prefix: zrepl_
+  pruning:
+    keep:
+    # fade-out scheme for snapshots starting with `zrepl_`
+    # - keep all created in the last hour
+    # - then destroy snapshots such that we keep 24 each 1 hour apart
+    # - then destroy snapshots such that we keep 14 each 1 day apart
+    # - then destroy all older snapshots
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 14x1d
+      regex: "^zrepl_.*"
+    # keep all snapshots that don't have the `zrepl_` prefix
+    - type: regex
+      negate: true
+      regex: "^zrepl_.*"
+
+# This job pushes to the local sink defined in job `backuppool_sink`.
+# We trigger replication manually from the command line / udev rules using
+#  `zrepl signal wakeup push_to_drive`
+- type: push
+  name: push_to_drive
+  connect:
+    type: local
+    listener_name: backuppool_sink
+    client_identity: myhostname
+  filesystems: {
+      "system<": true
+  }
+  send:
+    encrypted: true
+  replication:
+    protection:
+      initial: guarantee_resumability
+      # Downgrade protection to guarantee_incremental which uses zfs bookmarks instead of zfs holds.
+      # Thus, when we yank out the backup drive during replication
+      # - we might not be able to resume the interrupted replication step because the partially received `to` snapshot of a `from`->`to` step may be pruned any time
+      # - but in exchange we get back the disk space allocated by `to` when we prune it
+      # - and because we still have the bookmarks created by `guarantee_incremental`, we can still do incremental replication of `from`->`to2` in the future
+      incremental: guarantee_incremental
+  snapshotting:
+    type: manual
+  pruning:
+    # no-op prune rule on sender (keep all snapshots), job `snapshot` takes care of this
+    keep_sender:
+    - type: regex
+      regex: ".*"
+    # retain 
+    keep_receiver:
+    # longer retention on the backup drive, we have more space there
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 360x1d
+      regex: "^zrepl_.*"
+    # retain all non-zrepl snapshots on the backup drive
+    - type: regex
+      negate: true
+      regex: "^zrepl_.*"
+
+# This job receives from job `push_to_drive` into `backuppool/zrepl/sink/myhostname`
+- type: sink
+  name: backuppool_sink
+  root_fs: "backuppool/zrepl/sink"
+  serve:
+    type: local
+    listener_name: backuppool_sink

config/samples/quickstart_continuous_server_backup_receiver.yml

@@ -0,0 +1,12 @@
+jobs:
+- name: sink
+  type: sink
+  serve:
+      type: tls
+      listen: ":8888"
+      ca: "/etc/zrepl/prod.crt"
+      cert: "/etc/zrepl/backups.crt"
+      key: "/etc/zrepl/backups.key"
+      client_cns:
+        - "prod"
+  root_fs: "storage/zrepl/sink"

config/samples/quickstart_continuous_server_backup_sender.yml

@@ -0,0 +1,28 @@
+jobs:
+- name: prod_to_backups
+  type: push
+  connect:
+    type: tls
+    address: "backups.example.com:8888"
+    ca: /etc/zrepl/backups.crt
+    cert: /etc/zrepl/prod.crt
+    key:  /etc/zrepl/prod.key
+    server_cn: "backups"
+  filesystems: {
+    "zroot<": true,
+    "zroot/var/tmp<": false,
+    "zroot/usr/home/paranoid": false
+  }
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 10m
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 30x1d | 6x30d
+      regex: "^zrepl_"

config/samples/quickstart_fan_out_replication_source.yml

@@ -0,0 +1,59 @@
+jobs:
+# Separate job for snapshots and pruning
+- name: snapshots
+  type: snap
+  filesystems:
+    'tank<': true # all filesystems
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 10m
+  pruning:
+    keep:
+      # Keep non-zrepl snapshots
+      - type: regex
+        negate: true
+        regex: '^zrepl_'
+      # Time-based snapshot retention
+      - type: grid
+        grid: 1x1h(keep=all) | 24x1h | 30x1d | 12x30d
+        regex: '^zrepl_'
+
+# Source job for target B
+- name: target_b
+  type: source
+  serve:
+    type: tls
+    listen: :8888
+    ca: /etc/zrepl/b.example.com.crt
+    cert: /etc/zrepl/a.example.com.crt
+    key: /etc/zrepl/a.example.com.key
+    client_cns:
+      - b.example.com
+  filesystems:
+    'tank<': true # all filesystems
+  # Snapshots are handled by the separate snap job
+  snapshotting:
+    type: manual
+
+# Source job for target C
+- name: target_c
+  type: source
+  serve:
+    type: tls
+    listen: :8889
+    ca: /etc/zrepl/c.example.com.crt
+    cert: /etc/zrepl/a.example.com.crt
+    key: /etc/zrepl/a.example.com.key
+    client_cns:
+      - c.example.com
+  filesystems:
+    'tank<': true # all filesystems
+  # Snapshots are handled by the separate snap job
+  snapshotting:
+    type: manual
+
+# Source jobs for remaining targets. Each one should listen on a different port
+# and reference the correct certificate and client CN.
+# - name: target_c
+#   ...

config/samples/quickstart_fan_out_replication_target.yml

@@ -0,0 +1,30 @@
+jobs:
+# Pull from source server A
+- name: source_a
+  type: pull
+  connect:
+    type: tls
+    # Use the correct port for this specific client (eg. B is 8888, C is 8889, etc.)
+    address: a.example.com:8888
+    ca: /etc/zrepl/a.example.com.crt
+    # Use the correct key pair for this specific client
+    cert: /etc/zrepl/b.example.com.crt
+    key: /etc/zrepl/b.example.com.key
+    server_cn: a.example.com
+  root_fs: pool0/backup
+  interval: 10m
+  pruning:
+    keep_sender:
+      # Source does the pruning in its snap job
+      - type: regex
+        regex: '.*'
+    # Receiver-side pruning can be configured as desired on each target server
+    keep_receiver:
+      # Keep non-zrepl snapshots
+      - type: regex
+        negate: true
+        regex: '^zrepl_'
+      # Time-based snapshot retention
+      - type: grid
+        grid: 1x1h(keep=all) | 24x1h | 30x1d | 12x30d
+        regex: '^zrepl_'

config/samples/sink.yml

@@ -0,0 +1,13 @@
+jobs:
+  - type: sink
+    name: "laptop_sink"
+    root_fs: "pool2/backup_laptops"
+    serve:
+      type: tls
+      listen: "192.168.122.189:8888"
+      ca: "ca.pem"
+      cert: "cert.pem"
+      key: "key.pem"
+      client_cns:
+        - "laptop1"
+        - "homeserver"