build: tooling to use CircleCI artifacts for releasing

Also, include RPMs and DEBs in the sha256sum.txt

.circleci/config.yml

@@ -0,0 +1,297 @@
+version: 2.1
+orbs:
+  # NB: 1.7.2 is not the Go version, but the Orb version
+  # https://circleci.com/developer/orbs/orb/circleci/go#usage-go-modules-cache
+  go: circleci/go@1.7.2
+commands:
+  setup-home-local-bin:
+    steps:
+      - run:
+          shell: /bin/bash -euo pipefail
+          command: |
+            mkdir -p "$HOME/.local/bin"
+            line='export PATH="$HOME/.local/bin:$PATH"'
+            if grep "$line" $BASH_ENV >/dev/null; then
+              echo "$line" >> $BASH_ENV
+            fi
+
+  invoke-lazy-sh:
+    parameters:
+      subcommand:
+        type: string
+    steps:
+      - run:
+          environment:
+            TERM: xterm
+          command: ./lazy.sh <<parameters.subcommand>>
+
+  apt-update-and-install-common-deps:
+    steps:
+      - run: sudo apt-get update
+      - run: sudo apt-get install -y gawk make
+      # CircleCI doesn't update its cimg/go images.
+      # So, need to update manually to get up-to-date trust chains.
+      # The need for this was required for cimg/go:1.12, but let's future proof this here and now.
+      - run: sudo apt-get install -y git ca-certificates
+
+  install-godep:
+    steps:
+      - apt-update-and-install-common-deps
+      - invoke-lazy-sh:
+          subcommand: godep
+
+  install-docdep:
+    steps:
+      - apt-update-and-install-common-deps
+      - run: sudo apt install python3 python3-pip libgirepository1.0-dev
+      - invoke-lazy-sh:
+          subcommand: docdep
+
+  docs-publish-sh:
+    parameters:
+      push:
+        type: boolean
+    steps:
+      - checkout
+      - run:
+          command: |
+            git config --global user.email "zreplbot@cschwarz.com"
+            git config --global user.name "zrepl-github-io-ci"
+
+      # https://circleci.com/docs/2.0/add-ssh-key/#adding-multiple-keys-with-blank-hostnames
+      - run: ssh-add -D
+      # the default circleci ssh config only additional ssh keys for Host !github.com
+      - run:
+          command: |
+            cat > ~/.ssh/config \<<EOF
+            Host *
+              IdentityFile /home/circleci/.ssh/id_rsa_458e62c517f6c480e40452126ce47421
+            EOF
+      - add_ssh_keys:
+          fingerprints:
+            # deploy key for zrepl.github.io
+            - "45:8e:62:c5:17:f6:c4:80:e4:04:52:12:6c:e4:74:21"
+
+      # caller must install-docdep
+      - when:
+          condition: << parameters.push >>
+          steps:
+            - run: bash -x docs/publish.sh -c -a -P
+      - when:
+          condition:
+            not: << parameters.push >>
+          steps:
+            - run: bash -x docs/publish.sh -c -a
+
+parameters:
+  do_ci:
+    type: boolean
+    default: true
+
+  do_release:
+    type: boolean
+    default: false
+
+  release_docker_baseimage_tag:
+    type: string
+    default: "1.21"
+
+workflows:
+  version: 2
+
+  ci:
+    when: << pipeline.parameters.do_ci >>
+    jobs:
+      - quickcheck-docs
+      - quickcheck-go: &quickcheck-go-smoketest
+          name: quickcheck-go-amd64-linux-1.21
+          goversion: &latest-go-release "1.21"
+          goos: linux
+          goarch: amd64
+      - test-go-on-latest-go-release:
+          goversion: *latest-go-release
+      - quickcheck-go:
+          requires:
+            - quickcheck-go-amd64-linux-1.21 #quickcheck-go-smoketest.name
+          matrix: &quickcheck-go-matrix
+            alias: quickcheck-go-matrix
+            parameters:
+              goversion: [*latest-go-release, "1.20"]
+              goos: ["linux", "freebsd"]
+              goarch: ["amd64", "arm64"]
+            exclude:
+              # don't re-do quickcheck-go-smoketest
+              - goversion: *latest-go-release
+                goos: linux
+                goarch: amd64
+      - platformtest:
+          matrix:
+            parameters:
+              goversion: [*latest-go-release]
+              goos: ["linux"]
+              goarch: ["amd64"]
+          requires:
+            - quickcheck-go-<< matrix.goarch >>-<< matrix.goos >>-<< matrix.goversion >>
+
+  release:
+    when: << pipeline.parameters.do_release >>
+    jobs:
+      - release-build
+      - release-deb:
+          requires:
+            - release-build
+      - release-rpm:
+          requires:
+            - release-build
+      - release-upload:
+          requires:
+            - release-build
+            - release-deb
+            - release-rpm
+
+  publish-zrepl.github.io:
+    jobs:
+      - publish-zrepl-github-io:
+          filters:
+            branches:
+              only:
+                - stable
+
+jobs:
+  quickcheck-docs:
+    docker:
+      - image: cimg/base:2023.09
+    steps:
+      - checkout
+      - install-docdep
+      # do the current docs build
+      - run: make docs
+      # does the publish.sh script still work?
+      - docs-publish-sh:
+          push: false
+
+  quickcheck-go:
+    parameters:
+      goversion:
+        type: string
+      goos:
+        type: string
+      goarch:
+        type: string
+    docker:
+      - image: cimg/go:<<parameters.goversion>>
+    environment:
+      GOOS: <<parameters.goos>>
+      GOARCH: <<parameters.goarch>>
+
+    steps:
+      - checkout
+
+      - go/load-cache:
+          key: quickcheck-<<parameters.goversion>>
+      - install-godep
+      - run: go mod download
+      - run: cd build && go mod download
+      - go/save-cache:
+          key: quickcheck-<<parameters.goversion>>
+
+      - run: make formatcheck
+      - run: make generate-platform-test-list
+      - run: make zrepl-bin test-platform-bin
+      - run: make vet
+      - run: make lint
+
+      - run: rm -f artifacts/generate-platform-test-list
+      - store_artifacts:
+          path: artifacts
+      - persist_to_workspace:
+          root: .
+          paths: [.]
+
+  platformtest:
+    parameters:
+      goversion:
+        type: string
+      goos:
+        type: string
+      goarch:
+        type: string
+    machine:
+      image: ubuntu-2204:current
+    resource_class: medium
+    environment:
+      GOOS: <<parameters.goos>>
+      GOARCH: <<parameters.goarch>>
+    steps:
+      - attach_workspace:
+          at: .
+      - run: sudo apt-get update
+      - run: sudo apt-get install -y zfsutils-linux
+      - run: sudo zfs version
+      - run: sudo make test-platform GOOS="$GOOS" GOARCH="$GOARCH"
+
+  test-go-on-latest-go-release:
+    parameters:
+      goversion:
+        type: string
+    docker:
+      - image: cimg/go:<<parameters.goversion>>
+    steps:
+      - checkout
+      - go/load-cache:
+          key: make-test-go
+      - run: make test-go
+      - go/save-cache:
+          key: make-test-go
+
+  release-build:
+    machine:
+      image: ubuntu-2004:202201-02
+    steps:
+      - checkout
+      - run: make release-docker RELEASE_DOCKER_BASEIMAGE_TAG=<<pipeline.parameters.release_docker_baseimage_tag>>
+      - persist_to_workspace:
+          root: .
+          paths: [.]
+  release-deb:
+    machine:
+      image: ubuntu-2004:202201-02
+    steps:
+      - attach_workspace:
+          at: .
+      - run: make debs-docker
+      - persist_to_workspace:
+          root: .
+          paths:
+            - "artifacts/*.deb"
+
+  release-rpm:
+    machine:
+      image: ubuntu-2004:202201-02
+    steps:
+      - attach_workspace:
+          at: .
+      - run: make rpms-docker
+      - persist_to_workspace:
+          root: .
+          paths:
+            - "artifacts/*.rpm"
+
+  release-upload:
+    docker:
+      - image: cimg/base:2020.08
+    steps:
+      - attach_workspace:
+          at: .
+      - run: make wrapup-and-checksum
+      - store_artifacts:
+          path: artifacts
+
+  publish-zrepl-github-io:
+    docker:
+      - image: cimg/base:2023.09
+    steps:
+      - checkout
+      - install-docdep
+      - docs-publish-sh:
+          push: true

.circleci/download_artifacts.py

@@ -0,0 +1,84 @@
+import argparse
+from pathlib import Path
+import re
+
+import requests
+
+import time
+
+import os
+import argparse
+from pathlib import Path
+
+
+circle_token = os.environ.get('CIRCLE_TOKEN')
+if not circle_token:
+    raise ValueError('CIRCLE_TOKEN environment variable not set')
+
+parser = argparse.ArgumentParser(description='Download artifacts from CircleCI')
+parser.add_argument('build_num', type=str, help='Build number')
+parser.add_argument('dst', type=Path, help='Destination directory')
+parser.add_argument('--prefix', type=str, default='', help='Filter for prefix')
+parser.add_argument('--match', type=str, default='.*', help='Only include paths matching the given regex')
+
+args = parser.parse_args()
+
+res = requests.get(
+    f"https://circleci.com/api/v1.1/project/github/zrepl/zrepl/{args.build_num}/artifacts",
+    headers={
+        "Circle-Token": circle_token,
+    },
+)
+res.raise_for_status()
+
+# https://circleci.com/docs/api/v1/index.html#artifacts-of-a-job
+# [ {
+#   "path" : "raw-test-output/go-test-report.xml",
+#   "pretty_path" : "raw-test-output/go-test-report.xml",
+#   "node_index" : 0,
+#   "url" : "https://24-88881093-gh.circle-artifacts.com/0/raw-test-output/go-test-report.xml"
+# }, {
+#   "path" : "raw-test-output/go-test.out",
+#   "pretty_path" : "raw-test-output/go-test.out",
+#   "node_index" : 0,
+#   "url" : "https://24-88881093-gh.circle-artifacts.com/0/raw-test-output/go-test.out"
+# } ]
+res = res.json()
+
+for artifact in res:
+    if not artifact["pretty_path"].startswith(args.prefix):
+        continue
+    if not re.match(args.match, artifact["pretty_path"]):
+        continue
+    stripped = artifact["pretty_path"][len(args.prefix):]
+    print(f"Downloading {artifact['pretty_path']} to {args.dst / stripped}")
+    artifact_rel = Path(stripped)
+    artifact_dst = args.dst / artifact_rel
+    artifact_dst.parent.mkdir(parents=True, exist_ok=True)
+
+    res = requests.get(
+        artifact["url"],
+        headers={
+            "Circle-Token": circle_token,
+        },
+        stream=True,
+    )
+    res.raise_for_status()
+
+    total_size = int(res.headers.get("Content-Length", 0))
+    block_size = 128 * 1024
+    with open(artifact_dst, "wb") as f:
+        progress = 0
+        start_time = time.time()
+        for chunk in res.iter_content(chunk_size=block_size):
+            f.write(chunk)
+            progress += len(chunk)
+            percent = progress / total_size * 100
+            elapsed_time = time.time() - start_time
+            if elapsed_time >= 5:
+                print(f"Downloaded {progress}/{total_size} bytes ({percent:.2f}%)", end="\r")
+                start_time = time.time()
+        print(f"Downloaded {progress}/{total_size} bytes ({percent:.2f}%)")
+    print("Download complete!")
+
+print("All files downloaded")

.circleci/trigger_debian_binary_packaging_workflow.bash

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+COMMIT="$1"
+GO_VERSION="$2"
+
+curl -v -X POST https://api.github.com/repos/zrepl/debian-binary-packaging/dispatches \
+    -H 'Accept: application/vnd.github.v3+json' \
+    -H "Authorization: token $GITHUB_ACCESS_TOKEN" \
+    --data '{"event_type": "push", "client_payload": { "zrepl_main_repo_commit": "'"$COMMIT"'", "go_version": "'"$GO_VERSION"'" }}'

.github/FUNDING.yml

@@ -0,0 +1,5 @@
+github: problame
+patreon: zrepl
+liberapay: zrepl
+custom: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96
+ko_fi: problame

.gitignore

@@ -1,3 +1,13 @@
+# Build
+artifacts/
+
+# Golang
+vendor/
+
+# IDEs
+.idea/
+.vscode/
+
 # Compiled Object files, Static and Dynamic libs (Shared Objects)
 *.o
 *.a

.gitmodules

@@ -1,4 +0,0 @@
-[submodule "docs/themes/docdock"]
-	path = docs/themes/docdock
-	url = git@github.com:zrepl/hugo-theme-docdock.git
-

.golangci.yml

@@ -0,0 +1,16 @@
+linters:
+  enable:
+  - goimports
+
+issues:
+  exclude-rules:
+    - path: _test\.go
+      linters:
+        - errcheck
+    # Disable staticcheck 'Empty body in an if or else branch' as it's useful
+    # to put a comment into an empty else-clause that explains why whatever
+    # is done in the if-caluse is not necessary if the condition is false.
+    - linters:
+        - staticcheck
+      text: "SA9003:"
+

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2017 Christian Schwarz
+Copyright (c) 2017-2018 Christian Schwarz
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -0,0 +1,349 @@
+.PHONY: generate build test vet cover release docs docs-clean clean format lint platformtest
+.PHONY: release bins-all release-noarch
+.DEFAULT_GOAL := zrepl-bin
+
+ARTIFACTDIR := artifacts
+
+ifdef ZREPL_VERSION
+    _ZREPL_VERSION := $(ZREPL_VERSION)
+endif
+ifndef _ZREPL_VERSION
+    _ZREPL_VERSION := $(shell git describe --always --dirty 2>/dev/null || echo "ZREPL_BUILD_INVALID_VERSION" )
+    ifeq ($(_ZREPL_VERSION),ZREPL_BUILD_INVALID_VERSION) # can't use .SHELLSTATUS because Debian Stretch is still on gmake 4.1
+        $(error cannot infer variable ZREPL_VERSION using git and variable is not overriden by make invocation)
+    endif
+endif
+
+GO := go
+GOOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOOS"')
+GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"')
+GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"')
+GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"')
+GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
+GO_ENV_VARS := GO111MODULE=on
+GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
+GO_MOD_READONLY := -mod=readonly
+GO_EXTRA_BUILDFLAGS :=
+GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
+GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
+GOLANGCI_LINT := golangci-lint
+GOCOVMERGE := gocovmerge
+RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.21
+RELEASE_DOCKER_BASEIMAGE ?= golang:$(RELEASE_DOCKER_BASEIMAGE_TAG)
+
+ifneq ($(GOARM),)
+	ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM)
+else
+	ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)
+endif
+
+.PHONY: printvars
+printvars:
+	@echo GOOS=$(GOOS)
+	@echo GOARCH=$(GOARCH)
+	@echo GOARM=$(GOARM)
+
+
+##################### PRODUCING A RELEASE #############
+.PHONY: release wrapup-and-checksum check-git-clean sign clean
+
+release: clean
+	# no cross-platform support for target test
+	$(MAKE) test-go
+	$(MAKE) bins-all
+	$(MAKE) noarch
+
+release-docker: $(ARTIFACTDIR)
+	sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > artifacts/release-docker.Dockerfile
+	docker build -t zrepl_release --pull -f artifacts/release-docker.Dockerfile .
+	docker run --rm -i -v $(CURDIR):/src -u $$(id -u):$$(id -g) \
+		zrepl_release \
+		make release GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+
+debs-docker:
+	$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=deb
+rpms-docker:
+	$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=rpm
+_debs_or_rpms_docker: # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
+	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=amd64
+	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=arm64
+	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=arm GOARM=7
+	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=386
+
+rpm: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
+	$(eval _ZREPL_RPM_VERSION := $(subst -,.,$(_ZREPL_VERSION)))
+	$(eval _ZREPL_RPM_TOPDIR_ABS := $(CURDIR)/$(ARTIFACTDIR)/rpmbuild)
+	rm -rf "$(_ZREPL_RPM_TOPDIR_ABS)"
+	mkdir "$(_ZREPL_RPM_TOPDIR_ABS)"
+	mkdir -p "$(_ZREPL_RPM_TOPDIR_ABS)"/{SPECS,RPMS,BUILD,BUILDROOT}
+	sed "s/^Version:.*/Version:          $(_ZREPL_RPM_VERSION)/g" \
+	    packaging/rpm/zrepl.spec >  $(_ZREPL_RPM_TOPDIR_ABS)/SPECS/zrepl.spec
+
+	# see /usr/lib/rpm/platform
+ifeq ($(GOARCH),amd64)
+	$(eval _ZREPL_RPMBUILD_TARGET := x86_64)
+else ifeq ($(GOARCH), 386)
+	$(eval _ZREPL_RPMBUILD_TARGET := i386)
+else ifeq ($(GOARCH), arm64)
+	$(eval _ZREPL_RPMBUILD_TARGET := aarch64)
+else ifeq ($(GOARCH), arm)
+	$(eval _ZREPL_RPMBUILD_TARGET := armv7hl)
+else
+	$(eval _ZREPL_RPMBUILD_TARGET := $(GOARCH))
+endif
+	rpmbuild \
+		--build-in-place \
+		--define "_sourcedir $(CURDIR)" \
+		--define "_topdir $(_ZREPL_RPM_TOPDIR_ABS)" \
+		--define "_zrepl_binary_filename zrepl-$(ZREPL_TARGET_TUPLE)" \
+		--target $(_ZREPL_RPMBUILD_TARGET) \
+		-bb "$(_ZREPL_RPM_TOPDIR_ABS)"/SPECS/zrepl.spec
+	cp "$(_ZREPL_RPM_TOPDIR_ABS)"/RPMS/$(_ZREPL_RPMBUILD_TARGET)/zrepl-$(_ZREPL_RPM_VERSION)*.rpm $(ARTIFACTDIR)/
+
+rpm-docker:
+	docker build -t zrepl_rpm_pkg --pull -f packaging/rpm/Dockerfile .
+	docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
+		zrepl_rpm_pkg \
+		make rpm GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+
+deb: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
+
+	cp packaging/deb/debian/changelog.template packaging/deb/debian/changelog
+	sed -i 's/DATE_DASH_R_OUTPUT/$(shell date -R)/' packaging/deb/debian/changelog
+	VERSION="$(subst -,.,$(_ZREPL_VERSION))"; \
+		export VERSION="$${VERSION#v}"; \
+		sed -i 's/VERSION/'"$$VERSION"'/' packaging/deb/debian/changelog
+
+ifeq ($(GOARCH), arm)
+	$(eval  DEB_HOST_ARCH := armhf)
+else ifeq ($(GOARCH), 386)
+	$(eval DEB_HOST_ARCH := i386)
+else
+	$(eval DEB_HOST_ARCH := $(GOARCH))
+endif
+
+	export ZREPL_DPKG_ZREPL_BINARY_FILENAME=zrepl-$(ZREPL_TARGET_TUPLE); \
+		dpkg-buildpackage -b --no-sign --host-arch $(DEB_HOST_ARCH)
+	cp ../*.deb artifacts/
+
+deb-docker:
+	docker build -t zrepl_debian_pkg --pull -f packaging/deb/Dockerfile .
+	# Use a small open file limit to make fakeroot work. If we don't
+	# specify it, docker daemon will use its file limit. I don't know
+	# what changed (Docker, its systemd service, its Go version). But I
+	# observed fakeroot iterating close(i) up to i > 1000000, which costs
+	# a good amount of CPU time and makes the build slow.
+	docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
+		--ulimit nofile=1024:1024 \
+		zrepl_debian_pkg \
+		make deb GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+
+# expects `release`, `deb` & `rpm` targets to have run before
+NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
+wrapup-and-checksum:
+	rm -f $(NOARCH_TARBALL)
+	tar --mtime='1970-01-01' --sort=name \
+		--transform 's/$(ARTIFACTDIR)/zrepl-$(_ZREPL_VERSION)-noarch/' \
+		--transform 's#dist#zrepl-$(_ZREPL_VERSION)-noarch/dist#' \
+		--transform 's#config/samples#zrepl-$(_ZREPL_VERSION)-noarch/config#' \
+		-acf $(NOARCH_TARBALL) \
+		$(ARTIFACTDIR)/docs/html \
+		$(ARTIFACTDIR)/bash_completion \
+		$(ARTIFACTDIR)/_zrepl.zsh_completion \
+		$(ARTIFACTDIR)/go_env.txt \
+		dist \
+		config/samples
+	rm -rf "$(ARTIFACTDIR)/release"
+	mkdir -p "$(ARTIFACTDIR)/release"
+	cp -l $(ARTIFACTDIR)/zrepl-* \
+		$(ARTIFACTDIR)/platformtest-* \
+		"$(ARTIFACTDIR)/release"
+	cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
+
+check-git-clean:
+	@# note that we use ZREPL_VERSION and not _ZREPL_VERSION because we want to detect the override
+	@if git describe --always --dirty 2>/dev/null | grep dirty >/dev/null; then \
+        echo '[INFO] either git reports checkout is dirty or git is not installed or this is not a git checkout'; \
+		if [ "$(ZREPL_VERSION)" = "" ]; then \
+			echo '[WARN] git checkout is dirty and make variable ZREPL_VERSION was not used to override'; \
+			git status; \
+			echo "git diff:";  \
+			git diff | cat; \
+			exit 1; \
+		fi; \
+	fi;
+
+tag-release:
+	test -n "$(ZREPL_TAG_VERSION)" || exit 1
+	git tag -u '328A6627FA98061D!' -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)"
+
+sign:
+	gpg -u '328A6627FA98061D!' \
+		--armor \
+		--detach-sign $(ARTIFACTDIR)/release/sha512sum.txt
+
+clean: docs-clean
+	rm -rf "$(ARTIFACTDIR)"
+
+download-circleci-release:
+	rm -rf "$(ARTIFACTDIR)"
+	mkdir -p "$(ARTIFACTDIR)/release"
+	python3 .circleci/download_artifacts.py --prefix 'artifacts/release/' "$(BUILD_NUM)" "$(ARTIFACTDIR)/release"
+
+##################### BINARIES #####################
+.PHONY: bins-all lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin generate-platform-test-list
+
+BINS_ALL_TARGETS := zrepl-bin test-platform-bin vet lint
+GO_SUPPORTS_ILLUMOS := $(shell $(GO) version | gawk -F '.' '/^go version /{split($$0, comps, " "); split(comps[3], v, "."); if (v[1] == "go1" && v[2] >= 13) { print "illumos"; } else { print "noillumos"; }}')
+bins-all:
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=amd64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=386
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=arm	  GOARM=7
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=arm64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=amd64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm     GOARM=7
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=386
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=darwin    GOARCH=amd64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=solaris   GOARCH=amd64
+ifeq ($(GO_SUPPORTS_ILLUMOS), illumos)
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=illumos   GOARCH=amd64
+else ifeq ($(GO_SUPPORTS_ILLUMOS), noillumos)
+	@echo "SKIPPING ILLUMOS BUILD BECAUSE GO VERSION DOESN'T SUPPORT IT"
+else
+	@echo "CANNOT DETERMINE WHETHER GO VERSION SUPPORTS GOOS=illumos"; exit 1
+endif
+
+lint:
+	$(GO_ENV_VARS) $(GOLANGCI_LINT) run ./...
+
+vet:
+	$(GO_ENV_VARS) $(GO) vet $(GO_BUILDFLAGS) ./...
+
+test-go: $(ARTIFACTDIR)
+	rm -f "$(ARTIFACTDIR)/gotest.cover"
+ifeq ($(COVER),1)
+	$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
+		-coverpkg github.com/zrepl/zrepl/... \
+		-covermode atomic \
+		-coverprofile "$(ARTIFACTDIR)/gotest.cover" \
+		./...
+else
+	$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
+		./...
+endif
+
+zrepl-bin:
+	$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(ZREPL_TARGET_TUPLE)"
+
+generate-platform-test-list:
+	$(GO_BUILD) -o $(ARTIFACTDIR)/generate-platform-test-list ./platformtest/tests/gen
+
+COVER_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-cover-$(ZREPL_TARGET_TUPLE)
+cover-platform-bin:
+	$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
+		-c -o "$(COVER_PLATFORM_BIN_PATH)" \
+		-covermode=atomic -cover -coverpkg github.com/zrepl/zrepl/... \
+		./platformtest/harness
+cover-platform:
+	# do not track dependency on cover-platform-bin to allow build of binary outside of test VM
+	export _TEST_PLATFORM_CMD="$(COVER_PLATFORM_BIN_PATH) \
+		-test.coverprofile \"$(ARTIFACTDIR)/platformtest.cover\" \
+		-test.v \
+		__DEVEL--i-heard-you-like-tests"; \
+		$(MAKE) _test-or-cover-platform-impl
+
+TEST_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)
+test-platform-bin:
+	$(GO_BUILD) -o "$(TEST_PLATFORM_BIN_PATH)" ./platformtest/harness
+test-platform:
+	export _TEST_PLATFORM_CMD="\"$(TEST_PLATFORM_BIN_PATH)\""; \
+		$(MAKE) _test-or-cover-platform-impl
+
+ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
+ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
+ZREPL_PLATFORMTEST_MOUNTPOINT := /tmp/zreplplatformtest.pool
+ZREPL_PLATFORMTEST_ZFS_LOG := /tmp/zreplplatformtest.zfs.log
+# ZREPL_PLATFORMTEST_STOP_AND_KEEP := -failure.stop-and-keep-pool
+ZREPL_PLATFORMTEST_ARGS :=
+_test-or-cover-platform-impl: $(ARTIFACTDIR)
+ifndef _TEST_PLATFORM_CMD
+	$(error _TEST_PLATFORM_CMD is undefined, caller 'cover-platform' or 'test-platform' should have defined it)
+endif
+	rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
+	rm -f "$(ARTIFACTDIR)/platformtest.cover"
+	platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
+		$(_TEST_PLATFORM_CMD) \
+		-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
+		-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
+		-mountpoint "$(ZREPL_PLATFORMTEST_MOUNTPOINT)" \
+		$(ZREPL_PLATFORMTEST_STOP_AND_KEEP) \
+		$(ZREPL_PLATFORMTEST_ARGS)
+
+cover-merge: $(ARTIFACTDIR)
+	$(GOCOVMERGE) $(ARTIFACTDIR)/platformtest.cover $(ARTIFACTDIR)/gotest.cover > $(ARTIFACTDIR)/merged.cover
+cover-html: cover-merge
+	$(GO) tool cover -html "$(ARTIFACTDIR)/merged.cover" -o "$(ARTIFACTDIR)/merged.cover.html"
+
+cover-full:
+	test "$$(id -u)" = "0" || echo "MUST RUN AS ROOT" 1>&2
+	$(MAKE) test-go COVER=1
+	$(MAKE) cover-platform-bin
+	$(MAKE) cover-platform
+	$(MAKE) cover-html
+
+##################### DEV TARGETS #####################
+# not part of the build, must do that manually
+.PHONY: generate formatcheck format
+
+generate: generate-platform-test-list
+	protoc -I=replication/logic/pdu --go_out=replication/logic/pdu --go-grpc_out=replication/logic/pdu replication/logic/pdu/pdu.proto
+	protoc -I=rpc/grpcclientidentity/example --go_out=rpc/grpcclientidentity/example/pdu --go-grpc_out=rpc/grpcclientidentity/example/pdu rpc/grpcclientidentity/example/grpcauth.proto
+	$(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./...
+
+GOIMPORTS := goimports -srcdir . -local 'github.com/zrepl/zrepl'
+FINDSRCFILES := find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go'
+
+formatcheck:
+	@# goimports doesn't have a knob to exit with non-zero status code if formatting is needed
+	@# see https://go-review.googlesource.com/c/tools/+/237378
+	@ affectedfiles=$$($(GOIMPORTS) -l $(shell $(FINDSRCFILES)) | tee /dev/stderr | wc -l); test "$$affectedfiles" = 0
+
+format:
+	@ $(GOIMPORTS) -w -d $(shell  $(FINDSRCFILES))
+
+##################### NOARCH #####################
+.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
+
+
+$(ARTIFACTDIR):
+	mkdir -p "$@"
+$(ARTIFACTDIR)/docs: $(ARTIFACTDIR)
+	mkdir -p "$@"
+
+noarch: $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs
+	# pass
+
+$(ARTIFACTDIR)/bash_completion:
+	$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
+	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion bash "$@"
+
+$(ARTIFACTDIR)/_zrepl.zsh_completion:
+	$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
+	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion zsh "$@"
+
+$(ARTIFACTDIR)/go_env.txt:
+	$(GO_ENV_VARS) $(GO) env > $@
+	$(GO) version >> $@
+
+docs: $(ARTIFACTDIR)/docs
+	# https://www.sphinx-doc.org/en/master/man/sphinx-build.html
+	$(MAKE) -C docs \
+		html \
+		BUILDDIR=../artifacts/docs \
+		SPHINXOPTS="-W --keep-going -n"
+
+docs-clean:
+	$(MAKE) -C docs \
+		clean \
+		BUILDDIR=../artifacts/docs

README.md

@@ -1,2 +1,135 @@
+[![GitHub license](https://img.shields.io/github/license/zrepl/zrepl.svg)](https://github.com/zrepl/zrepl/blob/master/LICENSE)
+[![Language: Go](https://img.shields.io/badge/lang-Go-6ad7e5.svg)](https://golang.org/)
+[![User Docs](https://img.shields.io/badge/docs-web-blue.svg)](https://zrepl.github.io)
+[![Support me on Patreon](https://img.shields.io/badge/dynamic/json?color=yellow&label=Patreon&query=data.attributes.patron_count&url=https%3A%2F%2Fwww.patreon.com%2Fapi%2Fcampaigns%2F3095079)](https://patreon.com/zrepl)
+[![Donate via GitHub Sponsors](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&style=flat&color=yellow)](https://github.com/sponsors/problame)
+[![Donate via Liberapay](https://img.shields.io/liberapay/patrons/zrepl.svg?logo=liberapay)](https://liberapay.com/zrepl/donate)
+[![Donate via PayPal](https://img.shields.io/badge/donate-paypal-yellow.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
+[![Twitter](https://img.shields.io/twitter/url/https/github.com/zrepl/zrepl.svg?style=social)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl)
+[![Chat](https://img.shields.io/badge/chat-matrix-blue.svg)](https://matrix.to/#/#zrepl:matrix.org)
+
 # zrepl
-ZFS dataset replication. Not a shell script.
+zrepl is a one-stop ZFS backup & replication solution.
+
+## User Documentation
+
+**User Documentation** can be found at [zrepl.github.io](https://zrepl.github.io).
+
+## Bug Reports
+
+1. If the issue is reproducible, enable debug logging, reproduce and capture the log.
+2. Open an issue on GitHub, with logs pasted as GitHub gists / inline.
+
+## Feature Requests
+
+1. Does your feature request require default values / some kind of configuration?
+   If so, think of an expressive configuration example.
+2. Think of at least one use case that generalizes from your concrete application.
+3. Open an issue on GitHub with example conf & use case attached.
+4. **Optional**: [Post a bounty](https://www.bountysource.com/teams/zrepl) on the issue, or [contact Christian Schwarz](https://cschwarz.com) for contract work.
+
+The above does not apply if you already implemented everything.
+Check out the *Coding Workflow* section below for details.
+
+## Building, Releasing, Downstream-Packaging
+
+This section provides an overview of the zrepl build & release process.
+Check out `docs/installation/compile-from-source.rst` for build-from-source instructions.
+
+### Overview
+
+zrepl is written in [Go](https://golang.org) and uses [Go modules](https://github.com/golang/go/wiki/Modules) to manage dependencies.
+The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
+
+Install **build dependencies** using  `./lazy.sh devsetup`.
+`lazy.sh` uses `python3-pip` to fetch the build dependencies for the docs - you might want to use a [venv](https://docs.python.org/3/library/venv.html).
+If you just want to install the Go dependencies, run `./lazy.sh godep`.
+
+The **test suite** is split into pure **Go tests** (`make test-go`) and **platform tests** that interact with ZFS and thus generally **require root privileges** (`sudo make test-platform`).
+Platform tests run on their own pool with the name `zreplplatformtest`, which is created using the file vdev in `/tmp`.
+
+For a full **code coverage** profile, run `make test-go COVER=1 && sudo make test-platform && make cover-merge`.
+An HTML report can be generated using `make cover-html`.
+
+**Code generation** is triggered by `make generate`. Generated code is committed to the source tree.
+
+### Build & Release Process
+
+**The `Makefile` is catering to the needs of developers & CI, not distro packagers**.
+It provides phony targets for
+* local development (building, running tests, etc)
+* building a release in Docker (used by the CI & release management)
+* building .deb and .rpm packages out of the release artifacts.
+
+**Build tooling & dependencies** are documented as code in `lazy.sh`.
+Go dependencies are then fetched by the go command and pip dependencies are pinned through a `requirements.txt`.
+
+**We use CircleCI for continuous integration**.
+There are two workflows:
+
+* `ci` runs for every commit / branch / tag pushed to GitHub.
+  It is supposed to run very fast (<5min and provides quick feedback to developers).
+  It runs formatting checks, lints and tests on the most important OSes / architectures.
+  Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
+
+* `release` runs
+  * on manual triggers through the CircleCI API (in order to produce a release)
+  * periodically on `master`
+  Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
+
+**Releases** are issued via Git tags + GitHub Releases feature.
+The procedure to issue a release is as follows:
+* Issue the source release:
+  * Git tag the release on the `master` branch.
+  * Push the tag.
+  * Run `./docs/publish.sh` to re-build & push zrepl.github.io.
+* Issue the official binary release:
+  * Run the `release` pipeline (triggered via CircleCI API)
+  * Download the artifacts to the release manager's machine.
+  * Create a GitHub release, edit the changelog, upload all the release artifacts, including .rpm and .deb files.
+  * Issue the GitHub release.
+  * Add the .rpm and .deb files to the official zrepl repos, publish those.
+
+**Official binary releases are not re-built when Go receives an update. If the Go update is critical to zrepl (e.g. a Go security update that affects zrepl), we'd issue a new source release**.
+The rationale for this is that whereas distros provide a mechanism for this (`$zrepl_source_release-$distro_package_revision`), GitHub Releases doesn't which means we'd need to update the existing GitHub release's assets, which nobody would notice (no RSS feed updates, etc.).
+Downstream packagers can read the changelog to determine whether they want to push that minor release into their distro or simply skip it.
+
+### Additional Notes to Distro Package Maintainers
+
+* Run the platform tests (Docs -> Usage -> Platform Tests) **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
+* Ship a default config that adheres to your distro's `hier` and logging system.
+* Ship a service manager file and _please_ try to upstream it to this repository.
+  * `dist/systemd` contains a Systemd unit template.
+* Ship other material provided in `./dist`, e.g. in `/usr/share/zrepl/`.
+* Have a look at the `Makefile`'s `ZREPL_VERSION` variable and how it passed to Go's `ldFlags`.
+  This is how `zrepl version` knows what version number to show.
+  Your build system should set the `ldFlags` flags appropriately and add a prefix or suffix that indicates that the given zrepl binary is a distro build, not an official one.
+* Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
+
+
+## Contributing Code
+
+* Open an issue when starting to hack on a new feature
+* Commits should reference the issue they are related to
+* Docs improvements not documenting new features do not require an issue.
+
+### Breaking Changes
+
+Backward-incompatible changes must be documented in the git commit message and are listed in `docs/changelog.rst`.
+
+### Glossary & Naming Inconsistencies
+
+In ZFS, *dataset* refers to the objects *filesystem*, *ZVOL* and *snapshot*. <br />
+However, we need a word for *filesystem* & *ZVOL* but not a snapshot, bookmark, etc.
+
+Toward the user, the following terminology is used:
+
+* **filesystem**: a ZFS filesystem or a ZVOL
+* **filesystem version**: a ZFS snapshot or a bookmark
+
+Sadly, the zrepl implementation is inconsistent in its use of these words:
+variables and types are often named *dataset* when they in fact refer to a *filesystem*.
+
+There will not be a big refactoring (an attempt was made, but it's destroying too much history without much gain).
+
+However, new contributions & patches should fix naming without further notice in the commit message.

build.Dockerfile

@@ -0,0 +1,36 @@
+FROM !SUBSTITUTED_BY_MAKEFILE
+
+RUN apt-get update && apt-get install -y \
+    python3-pip \
+    python3-venv \
+    unzip \
+    gawk
+
+ADD build.installprotoc.bash ./
+RUN bash build.installprotoc.bash
+
+# setup venv
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+ADD lazy.sh /tmp/lazy.sh
+ADD docs/requirements.txt /tmp/requirements.txt
+ENV ZREPL_LAZY_DOCS_REQPATH=/tmp/requirements.txt
+RUN /tmp/lazy.sh docdep
+
+# prepare volume mount of git checkout to /zrepl
+RUN mkdir -p /src/github.com/zrepl/zrepl
+RUN mkdir -p /.cache && chmod -R 0777 /.cache
+
+# $GOPATH is /go
+# Go 1.12 doesn't use modules within GOPATH, but 1.13 and later do
+# => store source outside of GOPATH
+WORKDIR /src
+
+# Install build tools (e.g. protobuf generator, stringer) into $GOPATH/bin
+ADD build/ /tmp/build
+RUN /tmp/lazy.sh godep
+
+RUN chmod -R 0777 /go
+

build.installprotoc.bash

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+
+MACH=$(uname -m)
+MACH="${MACH/aarch64/aarch_64}"
+
+VERSION=3.6.1
+FILENAME=protoc-"$VERSION"-linux-"$MACH".zip
+
+if [ -e "$FILENAME" ]; then
+    echo "$FILENAME" already exists 1>&2
+    exit 1
+fi
+
+wget https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
+
+stat "$FILENAME"
+
+sha256sum -c --ignore-missing <<EOF
+6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807  protoc-3.6.1-linux-x86_64.zip
+af8e5aaaf39ddec62ec8dd2be1b8d9602c6da66564883a16393ade5f71170922  protoc-3.6.1-linux-aarch_64.zip
+EOF
+
+unzip -d /usr "$FILENAME"

build/go.mod

@@ -0,0 +1,40 @@
+module github.com/zrepl/zrepl/build
+
+go 1.12
+
+require (
+	github.com/alvaroloes/enumer v1.1.1
+	github.com/breml/bidichk v0.2.6 // indirect
+	github.com/breml/errchkjson v0.3.5 // indirect
+	github.com/chavacava/garif v0.1.0 // indirect
+	github.com/daixiang0/gci v0.11.1 // indirect
+	github.com/golangci/golangci-lint v1.54.2
+	github.com/golangci/revgrep v0.5.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/jgautheron/goconst v1.6.0 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/mgechev/revive v1.3.3 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/polyfloyd/go-errorlint v1.4.5 // indirect
+	github.com/prometheus/client_golang v1.16.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/ryanrolds/sqlclosecheck v0.5.1 // indirect
+	github.com/spf13/viper v1.16.0 // indirect
+	github.com/stretchr/objx v0.5.1 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tetafro/godot v1.4.15 // indirect
+	github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad
+	github.com/xen0n/gosmopolitan v1.2.2 // indirect
+	gitlab.com/bosi/decorder v0.4.1 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.25.0 // indirect
+	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/exp/typeparams v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/tools v0.13.0
+	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0
+	google.golang.org/protobuf v1.31.0
+	mvdan.cc/unparam v0.0.0-20230815095028-f7c6fb1088f0 // indirect
+)

build/tools.go

@@ -0,0 +1,15 @@
+//go:build tools
+// +build tools
+
+package main
+
+// the lines are parsed by lazy.sh, do not edit
+import (
+	_ "github.com/alvaroloes/enumer"
+	_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
+	_ "github.com/wadey/gocovmerge"
+	_ "golang.org/x/tools/cmd/goimports"
+	_ "golang.org/x/tools/cmd/stringer"
+	_ "google.golang.org/grpc/cmd/protoc-gen-go-grpc"
+	_ "google.golang.org/protobuf/cmd/protoc-gen-go"
+)

cli/cli.go

@@ -0,0 +1,155 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+
+	"github.com/zrepl/zrepl/config"
+)
+
+var rootArgs struct {
+	configPath string
+}
+
+var rootCmd = &cobra.Command{
+	Use:   "zrepl",
+	Short: "One-stop ZFS replication solution",
+}
+
+func init() {
+	rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
+}
+
+var genCompletionCmd = &cobra.Command{
+	Use:   "gencompletion",
+	Short: "generate shell auto-completions",
+}
+
+type completionCmdInfo struct {
+	genFunc func(outpath string) error
+	help    string
+}
+
+var completionCmdMap = map[string]completionCmdInfo{
+	"zsh": {
+		rootCmd.GenZshCompletionFile,
+		"  save to file `_zrepl` in your zsh's $fpath",
+	},
+	"bash": {
+		rootCmd.GenBashCompletionFile,
+		"  save to a path and source that path in your .bashrc",
+	},
+}
+
+func init() {
+	for sh, info := range completionCmdMap {
+		sh, info := sh, info
+		genCompletionCmd.AddCommand(&cobra.Command{
+			Use:     fmt.Sprintf("%s path/to/out/file", sh),
+			Short:   fmt.Sprintf("generate %s completions", sh),
+			Example: info.help,
+			Run: func(cmd *cobra.Command, args []string) {
+				if len(args) != 1 {
+					fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
+					err := cmd.Usage()
+					if err != nil {
+						panic(err)
+					}
+					os.Exit(1)
+				}
+				if err := info.genFunc(args[0]); err != nil {
+					fmt.Fprintf(os.Stderr, "error generating %s completion: %s", sh, err)
+					os.Exit(1)
+				}
+			},
+		})
+	}
+	rootCmd.AddCommand(genCompletionCmd)
+}
+
+type Subcommand struct {
+	Use              string
+	Short            string
+	Example          string
+	NoRequireConfig  bool
+	Run              func(ctx context.Context, subcommand *Subcommand, args []string) error
+	SetupFlags       func(f *pflag.FlagSet)
+	SetupSubcommands func() []*Subcommand
+
+	config    *config.Config
+	configErr error
+}
+
+func (s *Subcommand) ConfigParsingError() error {
+	return s.configErr
+}
+
+func (s *Subcommand) Config() *config.Config {
+	if !s.NoRequireConfig && s.config == nil {
+		panic("command that requires config is running and has no config set")
+	}
+	return s.config
+}
+
+func (s *Subcommand) run(cmd *cobra.Command, args []string) {
+	s.tryParseConfig()
+	ctx := context.Background()
+	endTask := trace.WithTaskFromStackUpdateCtx(&ctx)
+	defer endTask()
+	err := s.Run(ctx, s, args)
+	endTask()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+		os.Exit(1)
+	}
+}
+
+func (s *Subcommand) tryParseConfig() {
+	config, err := config.ParseConfig(rootArgs.configPath)
+	s.configErr = err
+	if err != nil {
+		if s.NoRequireConfig {
+			// doesn't matter
+			return
+		} else {
+			fmt.Fprintf(os.Stderr, "could not parse config: %s\n", err)
+			os.Exit(1)
+		}
+	}
+	s.config = config
+}
+
+func AddSubcommand(s *Subcommand) {
+	addSubcommandToCobraCmd(rootCmd, s)
+}
+
+func addSubcommandToCobraCmd(c *cobra.Command, s *Subcommand) {
+	cmd := cobra.Command{
+		Use:     s.Use,
+		Short:   s.Short,
+		Example: s.Example,
+	}
+	if s.SetupSubcommands == nil {
+		cmd.Run = s.run
+	} else {
+		for _, sub := range s.SetupSubcommands() {
+			addSubcommandToCobraCmd(&cmd, sub)
+		}
+	}
+	if s.SetupFlags != nil {
+		s.SetupFlags(cmd.Flags())
+	}
+	c.AddCommand(&cmd)
+}
+
+func Run() {
+	if err := rootCmd.Execute(); err != nil {
+		os.Exit(1)
+	}
+}

client/configcheck.go

@@ -0,0 +1,131 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/kr/pretty"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+	"github.com/zrepl/yaml-config"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/logger"
+)
+
+var configcheckArgs struct {
+	format        string
+	what          string
+	skipCertCheck bool
+}
+
+var ConfigcheckCmd = &cli.Subcommand{
+	Use:   "configcheck",
+	Short: "check if config can be parsed without errors",
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]")
+		f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]")
+		f.BoolVar(&configcheckArgs.skipCertCheck, "skip-cert-check", false, "skip checking cert files")
+	},
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		formatMap := map[string]func(interface{}){
+			"": func(i interface{}) {},
+			"pretty": func(i interface{}) {
+				if _, err := pretty.Println(i); err != nil {
+					panic(err)
+				}
+			},
+			"json": func(i interface{}) {
+				if err := json.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
+					panic(err)
+				}
+			},
+			"yaml": func(i interface{}) {
+				if err := yaml.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
+					panic(err)
+				}
+			},
+		}
+
+		formatter, ok := formatMap[configcheckArgs.format]
+		if !ok {
+			return fmt.Errorf("unsupported --format %q", configcheckArgs.format)
+		}
+
+		var hadErr bool
+
+		parseFlags := config.ParseFlagsNone
+
+		if configcheckArgs.skipCertCheck {
+			parseFlags |= config.ParseFlagsNoCertCheck
+		}
+
+		// further: try to build jobs
+		confJobs, err := job.JobsFromConfig(subcommand.Config(), parseFlags)
+
+		if err != nil {
+			err := errors.Wrap(err, "cannot build jobs from config")
+			if configcheckArgs.what == "jobs" {
+				return err
+			} else {
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				confJobs = nil
+				hadErr = true
+			}
+		}
+
+		// further: try to build logging outlets
+		outlets, err := logging.OutletsFromConfig(*subcommand.Config().Global.Logging)
+		if err != nil {
+			err := errors.Wrap(err, "cannot build logging from config")
+			if configcheckArgs.what == "logging" {
+				return err
+			} else {
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				outlets = nil
+				hadErr = true
+			}
+		}
+
+		whatMap := map[string]func(){
+			"all": func() {
+				o := struct {
+					config  *config.Config
+					jobs    []job.Job
+					logging *logger.Outlets
+				}{
+					subcommand.Config(),
+					confJobs,
+					outlets,
+				}
+				formatter(o)
+			},
+			"config": func() {
+				formatter(subcommand.Config())
+			},
+			"jobs": func() {
+				formatter(confJobs)
+			},
+			"logging": func() {
+				formatter(outlets)
+			},
+		}
+
+		wf, ok := whatMap[configcheckArgs.what]
+		if !ok {
+			return fmt.Errorf("unsupported --format %q", configcheckArgs.what)
+		}
+		wf()
+
+		if hadErr {
+			return fmt.Errorf("config parsing failed")
+		} else {
+			return nil
+		}
+	},
+}

client/jsonclient.go

@@ -0,0 +1,49 @@
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net"
+	"net/http"
+
+	"github.com/pkg/errors"
+)
+
+func controlHttpClient(sockpath string) (client http.Client, err error) {
+	return http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", sockpath)
+			},
+		},
+	}, nil
+}
+
+func jsonRequestResponse(c http.Client, endpoint string, req interface{}, res interface{}) error {
+	var buf bytes.Buffer
+	encodeErr := json.NewEncoder(&buf).Encode(req)
+	if encodeErr != nil {
+		return encodeErr
+	}
+
+	resp, err := c.Post("http://unix"+endpoint, "application/json", &buf)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		var msg bytes.Buffer
+		_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
+		return errors.Errorf("%s", msg.String())
+	}
+
+	decodeError := json.NewDecoder(resp.Body).Decode(&res)
+	if decodeError != nil {
+		return decodeError
+	}
+
+	return nil
+}

client/migrate.go

@@ -0,0 +1,268 @@
+package client
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/fatih/color"
+	"github.com/kr/pretty"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+)
+
+var (
+	MigrateCmd = &cli.Subcommand{
+		Use:   "migrate",
+		Short: "perform migration of the on-disk / zfs properties",
+		SetupSubcommands: func() []*cli.Subcommand {
+			return migrations
+		},
+	}
+)
+
+var migrations = []*cli.Subcommand{
+	&cli.Subcommand{
+		Use: "0.0.X:0.1:placeholder",
+		Run: doMigratePlaceholder0_1,
+		SetupFlags: func(f *pflag.FlagSet) {
+			f.BoolVar(&migratePlaceholder0_1Args.dryRun, "dry-run", false, "dry run")
+		},
+	},
+	&cli.Subcommand{
+		Use: "replication-cursor:v1-v2",
+		Run: doMigrateReplicationCursor,
+		SetupFlags: func(f *pflag.FlagSet) {
+			f.BoolVar(&migrateReplicationCursorArgs.dryRun, "dry-run", false, "dry run")
+		},
+	},
+}
+
+var migratePlaceholder0_1Args struct {
+	dryRun bool
+}
+
+func doMigratePlaceholder0_1(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	if len(args) != 0 {
+		return fmt.Errorf("migration does not take arguments, got %v", args)
+	}
+
+	cfg := sc.Config()
+
+	allFSS, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
+	if err != nil {
+		return errors.Wrap(err, "cannot list filesystems")
+	}
+
+	type workItem struct {
+		jobName string
+		rootFS  *zfs.DatasetPath
+		fss     []*zfs.DatasetPath
+	}
+	var wis []workItem
+	for i, j := range cfg.Jobs {
+		var rfsS string
+		switch job := j.Ret.(type) {
+		case *config.SinkJob:
+			rfsS = job.RootFS
+		case *config.PullJob:
+			rfsS = job.RootFS
+		default:
+			fmt.Printf("ignoring job %q (%d/%d, type %T)\n", j.Name(), i, len(cfg.Jobs), j.Ret)
+			continue
+		}
+		rfs, err := zfs.NewDatasetPath(rfsS)
+		if err != nil {
+			return errors.Wrapf(err, "root fs for job %q is not a valid dataset path", j.Name())
+		}
+		var fss []*zfs.DatasetPath
+		for _, fs := range allFSS {
+			if fs.HasPrefix(rfs) {
+				fss = append(fss, fs)
+			}
+		}
+		wis = append(wis, workItem{j.Name(), rfs, fss})
+	}
+
+	for _, wi := range wis {
+		fmt.Printf("job %q => migrate filesystems below root_fs %q\n", wi.jobName, wi.rootFS.ToString())
+		if len(wi.fss) == 0 {
+			fmt.Printf("\tno filesystems\n")
+			continue
+		}
+		for _, fs := range wi.fss {
+			fmt.Printf("\t%q ... ", fs.ToString())
+			r, err := zfs.ZFSMigrateHashBasedPlaceholderToCurrent(ctx, fs, migratePlaceholder0_1Args.dryRun)
+			if err != nil {
+				fmt.Printf("error: %s\n", err)
+			} else if !r.NeedsModification {
+				fmt.Printf("unchanged (placeholder=%v)\n", r.OriginalState.IsPlaceholder)
+			} else {
+				fmt.Printf("migrate (placeholder=%v) (old value = %q)\n",
+					r.OriginalState.IsPlaceholder, r.OriginalState.RawLocalPropertyValue)
+			}
+		}
+	}
+
+	return nil
+}
+
+var migrateReplicationCursorArgs struct {
+	dryRun bool
+}
+
+var bold = color.New(color.Bold)
+var succ = color.New(color.FgGreen)
+var fail = color.New(color.FgRed)
+
+var migrateReplicationCursorSkipSentinel = fmt.Errorf("skipping this filesystem")
+
+func doMigrateReplicationCursor(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	if len(args) != 0 {
+		return fmt.Errorf("migration does not take arguments, got %v", args)
+	}
+
+	cfg := sc.Config()
+	jobs, err := job.JobsFromConfig(cfg, config.ParseFlagsNone)
+	if err != nil {
+		fmt.Printf("cannot parse config:\n%s\n\n", err)
+		fmt.Printf("NOTE: this migration was released together with a change in job name requirements.\n")
+		return fmt.Errorf("exiting migration after error")
+	}
+
+	v1cursorJobs := make([]job.Job, 0, len(cfg.Jobs))
+	for i, j := range cfg.Jobs {
+		if jobs[i].Name() != j.Name() {
+			panic("implementation error")
+		}
+		switch j.Ret.(type) {
+		case *config.PushJob:
+			v1cursorJobs = append(v1cursorJobs, jobs[i])
+		case *config.SourceJob:
+			v1cursorJobs = append(v1cursorJobs, jobs[i])
+		default:
+			fmt.Printf("ignoring job %q (%d/%d, type %T), not supposed to create v1 replication cursors\n", j.Name(), i, len(cfg.Jobs), j.Ret)
+			continue
+		}
+	}
+
+	// scan all filesystems for v1 replication cursors
+
+	fss, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
+	if err != nil {
+		return errors.Wrap(err, "list filesystems")
+	}
+
+	var hadError bool
+	for _, fs := range fss {
+
+		bold.Printf("INSPECT FILESYSTEM %q\n", fs.ToString())
+
+		err := doMigrateReplicationCursorFS(ctx, v1cursorJobs, fs)
+		if err == migrateReplicationCursorSkipSentinel {
+			bold.Printf("FILESYSTEM SKIPPED\n")
+		} else if err != nil {
+			hadError = true
+			fail.Printf("MIGRATION FAILED: %s\n", err)
+		} else {
+			succ.Printf("FILESYSTEM %q COMPLETE\n", fs.ToString())
+		}
+	}
+
+	if hadError {
+		fail.Printf("\n\none or more filesystems could not be migrated, please inspect output and or re-run migration")
+		return errors.Errorf("")
+	}
+	return nil
+}
+
+func doMigrateReplicationCursorFS(ctx context.Context, v1CursorJobs []job.Job, fs *zfs.DatasetPath) error {
+
+	var owningJob job.Job = nil
+	for _, job := range v1CursorJobs {
+		conf := job.SenderConfig()
+		if conf == nil {
+			continue
+		}
+		pass, err := conf.FSF.Filter(fs)
+		if err != nil {
+			return errors.Wrapf(err, "filesystem filter error in job %q for fs %q", job.Name(), fs.ToString())
+		}
+		if !pass {
+			continue
+		}
+		if owningJob != nil {
+			return errors.Errorf("jobs %q and %q both match %q\ncannot attribute replication cursor to either one", owningJob.Name(), job.Name(), fs)
+		}
+		owningJob = job
+	}
+	if owningJob == nil {
+		fmt.Printf("no job's Filesystems filter matches\n")
+		return migrateReplicationCursorSkipSentinel
+	}
+	fmt.Printf("identified owning job %q\n", owningJob.Name())
+
+	bookmarks, err := zfs.ZFSListFilesystemVersions(ctx, fs, zfs.ListFilesystemVersionsOptions{
+		Types: zfs.Bookmarks,
+	})
+	if err != nil {
+		return errors.Wrapf(err, "list filesystem versions of %q", fs.ToString())
+	}
+
+	var oldCursor *zfs.FilesystemVersion
+	for i, fsv := range bookmarks {
+		_, _, err := endpoint.ParseReplicationCursorBookmarkName(fsv.ToAbsPath(fs))
+		if err != endpoint.ErrV1ReplicationCursor {
+			continue
+		}
+
+		if oldCursor != nil {
+			fmt.Printf("unexpected v1 replication cursor candidate: %q", fsv.ToAbsPath(fs))
+			return errors.Wrap(err, "multiple filesystem versions identified as v1 replication cursors")
+		}
+
+		oldCursor = &bookmarks[i]
+
+	}
+
+	if oldCursor == nil {
+		bold.Printf("no v1 replication cursor found for filesystem %q\n", fs.ToString())
+		return migrateReplicationCursorSkipSentinel
+	}
+
+	fmt.Printf("found v1 replication cursor:\n%s\n", pretty.Sprint(oldCursor))
+
+	mostRecentNew, err := endpoint.GetMostRecentReplicationCursorOfJob(ctx, fs.ToString(), owningJob.SenderConfig().JobID)
+	if err != nil {
+		return errors.Wrapf(err, "get most recent v2 replication cursor")
+	}
+
+	if mostRecentNew == nil {
+		return errors.Errorf("no v2 replication cursor found for job %q on filesystem %q", owningJob.SenderConfig().JobID, fs.ToString())
+	}
+
+	fmt.Printf("most recent v2 replication cursor:\n%#v", oldCursor)
+
+	if !(mostRecentNew.CreateTXG >= oldCursor.CreateTXG) {
+		return errors.Errorf("v1 replication cursor createtxg is higher than v2 cursor's, skipping this filesystem")
+	}
+
+	fmt.Printf("determined that v2 cursor is bookmark of same or newer version than v1 cursor\n")
+	fmt.Printf("destroying v1 cursor %q\n", oldCursor.ToAbsPath(fs))
+
+	if migrateReplicationCursorArgs.dryRun {
+		succ.Printf("DRY RUN\n")
+	} else {
+		if err := zfs.ZFSDestroyFilesystemVersion(ctx, fs, oldCursor); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

client/migrate_test.go

@@ -0,0 +1,16 @@
+package client
+
+import "testing"
+
+func TestMigrationsUnambiguousNames(t *testing.T) {
+	names := make(map[string]bool)
+	for _, mig := range migrations {
+		if _, ok := names[mig.Use]; ok {
+			t.Errorf("duplicate migration name %q", mig.Use)
+			t.FailNow()
+			return
+		} else {
+			names[mig.Use] = true
+		}
+	}
+}

client/pprof.go

@@ -0,0 +1,10 @@
+package client
+
+import "github.com/zrepl/zrepl/cli"
+
+var PprofCmd = &cli.Subcommand{
+	Use: "pprof",
+	SetupSubcommands: func() []*cli.Subcommand {
+		return []*cli.Subcommand{PprofListenCmd, pprofActivityTraceCmd}
+	},
+}

client/pprof_activity_trace.go

@@ -0,0 +1,44 @@
+package client
+
+import (
+	"context"
+	"io"
+	"log"
+	"os"
+
+	"golang.org/x/net/websocket"
+
+	"github.com/zrepl/zrepl/cli"
+)
+
+var pprofActivityTraceCmd = &cli.Subcommand{
+	Use:   "activity-trace ZREPL_PPROF_HOST:ZREPL_PPROF_PORT",
+	Short: "attach to zrepl daemon with activated pprof listener and dump an activity-trace to stdout",
+	Run:   runPProfActivityTrace,
+}
+
+func runPProfActivityTrace(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+	log := log.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Printf("exiting after error")
+		os.Exit(1)
+	}
+
+	if len(args) != 1 {
+		log.Printf("exactly one positional argument is required")
+		die()
+	}
+
+	url := "ws://" + args[0] + "/debug/zrepl/activity-trace" // FIXME dont' repeat that
+
+	log.Printf("attaching to activity trace stream %s", url)
+	ws, err := websocket.Dial(url, "", url)
+	if err != nil {
+		log.Printf("error: %s", err)
+		die()
+	}
+
+	_, err = io.Copy(os.Stdout, ws)
+	return err
+}

client/pprof_listen.go

@@ -0,0 +1,68 @@
+package client
+
+import (
+	"context"
+	"errors"
+	"log"
+	"os"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+)
+
+var pprofListenCmd struct {
+	daemon.PprofServerControlMsg
+}
+
+var PprofListenCmd = &cli.Subcommand{
+	Use:   "listen off | [on TCP_LISTEN_ADDRESS]",
+	Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		if len(args) < 1 {
+			goto enargs
+		}
+		switch args[0] {
+		case "on":
+			pprofListenCmd.Run = true
+			if len(args) != 2 {
+				return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
+			}
+			pprofListenCmd.HttpListenAddress = args[1]
+		case "off":
+			if len(args) != 1 {
+				goto enargs
+			}
+			pprofListenCmd.Run = false
+		}
+
+		RunPProf(subcommand.Config())
+		return nil
+	enargs:
+		return errors.New("invalid number of positional arguments")
+
+	},
+}
+
+func RunPProf(conf *config.Config) {
+	log := log.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Printf("exiting after error")
+		os.Exit(1)
+	}
+
+	log.Printf("connecting to zrepl daemon")
+
+	httpc, err := controlHttpClient(conf.Global.Control.SockPath)
+	if err != nil {
+		log.Printf("error creating http client: %s", err)
+		die()
+	}
+	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofListenCmd.PprofServerControlMsg, struct{}{})
+	if err != nil {
+		log.Printf("error sending control message: %s", err)
+		die()
+	}
+	log.Printf("finished")
+}

client/signal.go

@@ -0,0 +1,42 @@
+package client
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+)
+
+var SignalCmd = &cli.Subcommand{
+	Use:   "signal [wakeup|reset] JOB",
+	Short: "wake up a job from wait state or abort its current invocation",
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		return runSignalCmd(subcommand.Config(), args)
+	},
+}
+
+func runSignalCmd(config *config.Config, args []string) error {
+	if len(args) != 2 {
+		return errors.Errorf("Expected 2 arguments: [wakeup|reset] JOB")
+	}
+
+	httpc, err := controlHttpClient(config.Global.Control.SockPath)
+	if err != nil {
+		return err
+	}
+
+	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointSignal,
+		struct {
+			Name string
+			Op   string
+		}{
+			Name: args[1],
+			Op:   args[0],
+		},
+		struct{}{},
+	)
+	return err
+}

client/status/client/client.go

@@ -0,0 +1,111 @@
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net"
+	"net/http"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/daemon"
+)
+
+type Client struct {
+	h *http.Client
+}
+
+func New(network, addr string) (*Client, error) {
+	httpc, err := makeControlHttpClient(func(_ context.Context) (net.Conn, error) { return net.Dial(network, addr) })
+	if err != nil {
+		return nil, err
+	}
+	return &Client{httpc}, nil
+}
+
+func (c *Client) Status() (s daemon.Status, _ error) {
+	err := jsonRequestResponse(c.h, daemon.ControlJobEndpointStatus,
+		struct{}{},
+		&s,
+	)
+	return s, err
+}
+
+func (c *Client) StatusRaw() ([]byte, error) {
+	var r json.RawMessage
+	err := jsonRequestResponse(c.h, daemon.ControlJobEndpointStatus, struct{}{}, &r)
+	if err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+
+func (c *Client) signal(job, sig string) error {
+	return jsonRequestResponse(c.h, daemon.ControlJobEndpointSignal,
+		struct {
+			Name string
+			Op   string
+		}{
+			Name: job,
+			Op:   sig,
+		},
+		struct{}{},
+	)
+}
+
+func (c *Client) SignalWakeup(job string) error {
+	return c.signal(job, "wakeup")
+}
+
+func (c *Client) SignalReset(job string) error {
+	return c.signal(job, "reset")
+}
+
+func makeControlHttpClient(dialfunc func(context.Context) (net.Conn, error)) (client *http.Client, err error) {
+	return &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+				return dialfunc(ctx)
+			},
+		},
+	}, nil
+}
+
+func jsonRequestResponse(c *http.Client, endpoint string, req interface{}, res interface{}) error {
+	var buf bytes.Buffer
+	encodeErr := json.NewEncoder(&buf).Encode(req)
+	if encodeErr != nil {
+		return encodeErr
+	}
+
+	hreq, err := http.NewRequest("POST", "http://unix"+endpoint, &buf)
+	if err != nil {
+		return err
+	}
+	hreq.Header.Set("Content-Type", "application/json")
+	// Prevent EOF errors when client request frequency and server keepalive close are at the same time.
+	// Found this by watching http.Server.ConnState changes, then found
+	// https://stackoverflow.com/questions/17714494/golang-http-request-results-in-eof-errors-when-making-multiple-requests-successi
+	// Note: The issue seems even more prounounced with local TCP sockets than unix domain sockets. So, I used that for debugging.
+	hreq.Close = true
+	resp, err := c.Do(hreq)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		var msg bytes.Buffer
+		_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
+		return errors.Errorf("%s", msg.String())
+	}
+
+	decodeError := json.NewDecoder(resp.Body).Decode(&res)
+	if decodeError != nil {
+		return decodeError
+	}
+
+	return nil
+}

client/status/status.go

@@ -0,0 +1,97 @@
+package status
+
+import (
+	"context"
+	"os"
+	"time"
+
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/client/status/client"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+	"github.com/zrepl/zrepl/util/choices"
+)
+
+type Client interface {
+	Status() (daemon.Status, error)
+	StatusRaw() ([]byte, error)
+	SignalWakeup(job string) error
+	SignalReset(job string) error
+}
+
+type statusFlags struct {
+	Mode  choices.Choices
+	Job   string
+	Delay time.Duration
+}
+
+var statusv2Flags statusFlags
+
+type statusv2Mode int
+
+const (
+	StatusV2ModeInteractive statusv2Mode = 1 + iota
+	StatusV2ModeDump
+	StatusV2ModeRaw
+	StatusV2ModeLegacy
+)
+
+var Subcommand = &cli.Subcommand{
+	Use:   "status",
+	Short: "retrieve & display daemon status information",
+	SetupFlags: func(f *pflag.FlagSet) {
+		statusv2Flags.Mode.Init(
+			"interactive", StatusV2ModeInteractive,
+			"dump", StatusV2ModeDump,
+			"raw", StatusV2ModeRaw,
+			"legacy", StatusV2ModeLegacy,
+		)
+		statusv2Flags.Mode.SetTypeString("mode")
+		statusv2Flags.Mode.SetDefaultValue(StatusV2ModeInteractive)
+		f.Var(&statusv2Flags.Mode, "mode", statusv2Flags.Mode.Usage())
+		f.StringVar(&statusv2Flags.Job, "job", "", "only show specified job (works in \"dump\" and \"interactive\" mode)")
+		f.DurationVarP(&statusv2Flags.Delay, "delay", "d", 1*time.Second, "use -d 3s for 3 seconds delay (minimum delay is 1s)")
+	},
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		return runStatusV2Command(ctx, subcommand.Config(), args)
+	},
+}
+
+func runStatusV2Command(ctx context.Context, config *config.Config, args []string) error {
+
+	c, err := client.New("unix", config.Global.Control.SockPath)
+	if err != nil {
+		return errors.Wrapf(err, "connect to daemon socket at %q", config.Global.Control.SockPath)
+	}
+
+	mode := statusv2Flags.Mode.Value().(statusv2Mode)
+
+	if !isatty.IsTerminal(os.Stdout.Fd()) && mode != StatusV2ModeDump && mode != StatusV2ModeRaw {
+		dumpmode, err := statusv2Flags.Mode.InputForChoice(StatusV2ModeDump)
+		if err != nil {
+			panic(err)
+		}
+		rawmode, err := statusv2Flags.Mode.InputForChoice(StatusV2ModeRaw)
+		if err != nil {
+			panic(err)
+		}
+		return errors.Errorf("error: stdout is not a tty, please use --mode %s or --mode %s", dumpmode, rawmode)
+	}
+
+	switch mode {
+	case StatusV2ModeInteractive:
+		return interactive(c, statusv2Flags)
+	case StatusV2ModeDump:
+		return dump(c, statusv2Flags.Job)
+	case StatusV2ModeRaw:
+		return raw(c)
+	case StatusV2ModeLegacy:
+		return legacy(c, statusv2Flags)
+	default:
+		panic("unreachable")
+	}
+}

client/status/status_dump.go

@@ -0,0 +1,70 @@
+package status
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/gdamore/tcell"
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/client/status/viewmodel"
+)
+
+func dump(c Client, job string) error {
+	s, err := c.Status()
+	if err != nil {
+		return err
+	}
+
+	if job != "" {
+		if _, ok := s.Jobs[job]; !ok {
+			return errors.Errorf("job %q not found", job)
+		}
+	}
+
+	width := (1 << 31) - 1
+	wrap := false
+	hline := strings.Repeat("-", 80)
+	if isatty.IsTerminal(os.Stdout.Fd()) {
+		wrap = true
+		screen, err := tcell.NewScreen()
+		if err != nil {
+			return errors.Wrap(err, "get terminal dimensions")
+		}
+		if err := screen.Init(); err != nil {
+			return errors.Wrap(err, "init screen")
+		}
+		width, _ = screen.Size()
+		screen.Fini()
+		hline = strings.Repeat("-", width)
+	}
+
+	m := viewmodel.New()
+	params := viewmodel.Params{
+		Report:                  s.Jobs,
+		ReportFetchError:        nil,
+		SelectedJob:             nil,
+		FSFilter:                func(s string) bool { return true },
+		DetailViewWidth:         width,
+		DetailViewWrap:          wrap,
+		ShortKeybindingOverview: "",
+	}
+	m.Update(params)
+	for _, j := range m.Jobs() {
+		if job != "" && j.Name() != job {
+			continue
+		}
+		params.SelectedJob = j
+		m.Update(params)
+		fmt.Println(m.SelectedJob().FullDescription())
+		if job != "" {
+			return nil
+		} else {
+			fmt.Println(hline)
+		}
+	}
+
+	return nil
+}

client/status/status_interactive.go

@@ -0,0 +1,347 @@
+package status
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gdamore/tcell/v2"
+	tview "gitlab.com/tslocum/cview"
+
+	"github.com/zrepl/zrepl/client/status/viewmodel"
+)
+
+func interactive(c Client, flag statusFlags) error {
+
+	// Set this so we don't overwrite the default terminal colors
+	// See https://github.com/rivo/tview/blob/master/styles.go
+	tview.Styles.PrimitiveBackgroundColor = tcell.ColorDefault
+	tview.Styles.ContrastBackgroundColor = tcell.ColorDefault
+	tview.Styles.PrimaryTextColor = tcell.ColorDefault
+	tview.Styles.BorderColor = tcell.ColorDefault
+	app := tview.NewApplication()
+
+	jobDetailSplit := tview.NewFlex()
+	jobMenu := tview.NewTreeView()
+	jobMenuRoot := tview.NewTreeNode("jobs")
+	jobMenuRoot.SetSelectable(true)
+	jobMenu.SetRoot(jobMenuRoot)
+	jobMenu.SetCurrentNode(jobMenuRoot)
+	jobMenu.SetSelectedTextColor(tcell.ColorGreen)
+	jobTextDetail := tview.NewTextView()
+	jobTextDetail.SetWrap(false)
+
+	jobMenu.SetBorder(true)
+	jobTextDetail.SetBorder(true)
+
+	toolbarSplit := tview.NewFlex()
+	toolbarSplit.SetDirection(tview.FlexRow)
+	inputBarContainer := tview.NewFlex()
+	fsFilterInput := tview.NewInputField()
+	fsFilterInput.SetBorder(false)
+	fsFilterInput.SetFieldBackgroundColor(tcell.ColorDefault)
+	inputBarLabel := tview.NewTextView()
+	inputBarLabel.SetText("[::b]FILTER ")
+	inputBarLabel.SetDynamicColors(true)
+	inputBarContainer.AddItem(inputBarLabel, 7, 1, false)
+	inputBarContainer.AddItem(fsFilterInput, 0, 10, false)
+	toolbarSplit.AddItem(inputBarContainer, 1, 0, false)
+	toolbarSplit.AddItem(jobDetailSplit, 0, 10, false)
+
+	bottombar := tview.NewFlex()
+	bottombar.SetDirection(tview.FlexColumn)
+	bottombarDateView := tview.NewTextView()
+	bottombar.AddItem(bottombarDateView, len(time.Now().String()), 0, false)
+	bottomBarStatus := tview.NewTextView()
+	bottomBarStatus.SetDynamicColors(true)
+	bottomBarStatus.SetTextAlign(tview.AlignRight)
+	bottombar.AddItem(bottomBarStatus, 0, 10, false)
+	toolbarSplit.AddItem(bottombar, 1, 0, false)
+
+	tabbableWithJobMenu := []tview.Primitive{jobMenu, jobTextDetail, fsFilterInput}
+	tabbableWithoutJobMenu := []tview.Primitive{jobTextDetail, fsFilterInput}
+	var tabbable []tview.Primitive
+	tabbableActiveIndex := 0
+	tabbableRedraw := func() {
+		if len(tabbable) == 0 {
+			app.SetFocus(nil)
+			return
+		}
+		if tabbableActiveIndex >= len(tabbable) {
+			app.SetFocus(tabbable[0])
+			return
+		}
+		app.SetFocus(tabbable[tabbableActiveIndex])
+	}
+	tabbableCycle := func() {
+		if len(tabbable) == 0 {
+			return
+		}
+		tabbableActiveIndex = (tabbableActiveIndex + 1) % len(tabbable)
+		app.SetFocus(tabbable[tabbableActiveIndex])
+		tabbableRedraw()
+	}
+
+	jobMenuVisisble := false
+	reconfigureJobDetailSplit := func(setJobMenuVisible bool) {
+		if jobMenuVisisble == setJobMenuVisible {
+			return
+		}
+		jobMenuVisisble = setJobMenuVisible
+		if setJobMenuVisible {
+			jobDetailSplit.RemoveItem(jobTextDetail)
+			jobDetailSplit.AddItem(jobMenu, 0, 1, true)
+			jobDetailSplit.AddItem(jobTextDetail, 0, 5, false)
+			tabbable = tabbableWithJobMenu
+		} else {
+			jobDetailSplit.RemoveItem(jobMenu)
+			tabbable = tabbableWithoutJobMenu
+		}
+		tabbableRedraw()
+	}
+
+	showModal := func(m *tview.Modal, modalDoneFunc func(idx int, label string)) {
+		preModalFocus := app.GetFocus()
+		m.SetDoneFunc(func(idx int, label string) {
+			if modalDoneFunc != nil {
+				modalDoneFunc(idx, label)
+			}
+			app.SetRoot(toolbarSplit, true)
+			app.SetFocus(preModalFocus)
+			app.Draw()
+		})
+		app.SetRoot(m, true)
+		app.Draw()
+	}
+
+	app.SetRoot(toolbarSplit, true)
+	// initial focus
+	tabbableActiveIndex = len(tabbable)
+	tabbableCycle()
+	reconfigureJobDetailSplit(true)
+
+	m := viewmodel.New()
+	params := &viewmodel.Params{
+		Report:                  nil,
+		SelectedJob:             nil,
+		FSFilter:                func(_ string) bool { return true },
+		DetailViewWidth:         100,
+		DetailViewWrap:          false,
+		ShortKeybindingOverview: "[::b]Q[::-] quit  [::b]<TAB>[::-] switch panes [::b]W[::-] wrap lines  [::b]Shift+M[::-] toggle navbar  [::b]Shift+S[::-] signal job [::b]</>[::-] filter filesystems",
+	}
+	paramsMtx := &sync.Mutex{}
+	var redraw func()
+	viewmodelupdate := func(cb func(*viewmodel.Params)) {
+		paramsMtx.Lock()
+		defer paramsMtx.Unlock()
+		cb(params)
+		m.Update(*params)
+	}
+	redraw = func() {
+		jobs := m.Jobs()
+		if flag.Job != "" {
+			job_found := false
+			for _, job := range jobs {
+				if strings.Compare(flag.Job, job.Name()) == 0 {
+					jobs = []*viewmodel.Job{job}
+					job_found = true
+					break
+				}
+			}
+			if !job_found {
+				jobs = nil
+			}
+		}
+		redrawJobsList := false
+		var selectedJobN *tview.TreeNode
+		if len(jobMenuRoot.GetChildren()) == len(jobs) {
+			for i, jobN := range jobMenuRoot.GetChildren() {
+				if jobN.GetReference().(*viewmodel.Job) != jobs[i] {
+					redrawJobsList = true
+					break
+				}
+				if jobN.GetReference().(*viewmodel.Job) == m.SelectedJob() {
+					selectedJobN = jobN
+				}
+			}
+		} else {
+			redrawJobsList = true
+		}
+		if redrawJobsList {
+			selectedJobN = nil
+			children := make([]*tview.TreeNode, len(jobs))
+			for i := range jobs {
+				jobN := tview.NewTreeNode(jobs[i].JobTreeTitle())
+				jobN.SetReference(jobs[i])
+				jobN.SetSelectable(true)
+				children[i] = jobN
+				jobN.SetSelectedFunc(func() {
+					viewmodelupdate(func(p *viewmodel.Params) {
+						p.SelectedJob = jobN.GetReference().(*viewmodel.Job)
+					})
+				})
+				if jobs[i] == m.SelectedJob() {
+					selectedJobN = jobN
+				}
+			}
+			jobMenuRoot.SetChildren(children)
+		}
+
+		if selectedJobN != nil && jobMenu.GetCurrentNode() != selectedJobN {
+			jobMenu.SetCurrentNode(selectedJobN)
+		} else if selectedJobN == nil {
+			// select something, otherwise selection breaks (likely bug in tview)
+			jobMenu.SetCurrentNode(jobMenuRoot)
+		}
+
+		if selJ := m.SelectedJob(); selJ != nil {
+			jobTextDetail.SetText(selJ.FullDescription())
+		} else {
+			jobTextDetail.SetText("please select a job")
+		}
+
+		bottombardatestring := m.DateString()
+		bottombarDateView.SetText(bottombardatestring)
+		bottombar.ResizeItem(bottombarDateView, len(bottombardatestring), 0)
+
+		bottomBarStatus.SetText(m.BottomBarStatus())
+
+		app.Draw()
+
+	}
+
+	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				app.Suspend(func() {
+					panic(err)
+				})
+			}
+		}()
+		for {
+			st, err := c.Status()
+			viewmodelupdate(func(p *viewmodel.Params) {
+				p.Report = st.Jobs
+				p.ReportFetchError = err
+			})
+			app.QueueUpdateDraw(redraw)
+
+			time.Sleep(flag.Delay)
+		}
+	}()
+
+	jobMenu.SetChangedFunc(func(jobN *tview.TreeNode) {
+		viewmodelupdate(func(p *viewmodel.Params) {
+			p.SelectedJob, _ = jobN.GetReference().(*viewmodel.Job)
+		})
+		redraw()
+		jobTextDetail.ScrollToBeginning()
+	})
+	jobMenu.SetSelectedFunc(func(jobN *tview.TreeNode) {
+		app.SetFocus(jobTextDetail)
+	})
+
+	app.SetBeforeDrawFunc(func(screen tcell.Screen) bool {
+		viewmodelupdate(func(p *viewmodel.Params) {
+			_, _, p.DetailViewWidth, _ = jobTextDetail.GetInnerRect()
+		})
+		return false
+	})
+
+	app.SetInputCapture(func(e *tcell.EventKey) *tcell.EventKey {
+		if e.Key() == tcell.KeyTab {
+			tabbableCycle()
+			return nil
+		}
+
+		if e.Key() == tcell.KeyRune && app.GetFocus() == fsFilterInput {
+			return e
+		}
+
+		if e.Key() == tcell.KeyRune && e.Rune() == '/' {
+			if app.GetFocus() != fsFilterInput {
+				app.SetFocus(fsFilterInput)
+			}
+			return e
+		}
+
+		if e.Key() == tcell.KeyRune && e.Rune() == 'M' {
+			reconfigureJobDetailSplit(!jobMenuVisisble)
+			return nil
+		}
+
+		if e.Key() == tcell.KeyRune && e.Rune() == 'q' {
+			app.Stop()
+		}
+
+		if e.Key() == tcell.KeyRune && e.Rune() == 'S' {
+			job, ok := jobMenu.GetCurrentNode().GetReference().(*viewmodel.Job)
+			if !ok {
+				return nil
+			}
+			signals := []string{"wakeup", "reset"}
+			clientFuncs := []func(job string) error{c.SignalWakeup, c.SignalReset}
+			sigMod := tview.NewModal()
+			sigMod.SetBackgroundColor(tcell.ColorDefault)
+			sigMod.SetBorder(true)
+			sigMod.GetForm().SetButtonTextColorFocused(tcell.ColorGreen)
+			sigMod.AddButtons(signals)
+			sigMod.SetText(fmt.Sprintf("Send a signal to job %q", job.Name()))
+			showModal(sigMod, func(idx int, _ string) {
+				go func() {
+					if idx == -1 {
+						return
+					}
+					err := clientFuncs[idx](job.Name())
+					if err != nil {
+						app.QueueUpdate(func() {
+							me := tview.NewModal()
+							me.SetText(fmt.Sprintf("signal error: %s", err))
+							me.AddButtons([]string{"Close"})
+							showModal(me, nil)
+						})
+					}
+				}()
+			})
+		}
+
+		return e
+	})
+
+	fsFilterInput.SetChangedFunc(func(searchterm string) {
+		viewmodelupdate(func(p *viewmodel.Params) {
+			p.FSFilter = func(fs string) bool {
+				r, err := regexp.Compile(searchterm)
+				if err != nil {
+					return true
+				}
+				return r.MatchString(fs)
+			}
+		})
+		redraw()
+		jobTextDetail.ScrollToBeginning()
+	})
+	fsFilterInput.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+		if event.Key() == tcell.KeyEnter {
+			app.SetFocus(jobTextDetail)
+			return nil
+		}
+		return event
+	})
+
+	jobTextDetail.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+		if event.Key() == tcell.KeyRune && event.Rune() == 'w' {
+			// toggle wrapping
+			viewmodelupdate(func(p *viewmodel.Params) {
+				p.DetailViewWrap = !p.DetailViewWrap
+			})
+			redraw()
+			return nil
+		}
+		return event
+	})
+
+	return app.Run()
+}

client/status/status_legacy.go

@@ -0,0 +1,128 @@
+package status
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gdamore/tcell/v2"
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+	tview "gitlab.com/tslocum/cview"
+
+	"github.com/zrepl/zrepl/client/status/viewmodel"
+)
+
+func legacy(c Client, flag statusFlags) error {
+
+	// Set this so we don't overwrite the default terminal colors
+	// See https://github.com/rivo/tview/blob/master/styles.go
+	tview.Styles.PrimitiveBackgroundColor = tcell.ColorDefault
+	tview.Styles.ContrastBackgroundColor = tcell.ColorDefault
+	tview.Styles.PrimaryTextColor = tcell.ColorDefault
+	tview.Styles.BorderColor = tcell.ColorDefault
+	app := tview.NewApplication()
+
+	textView := tview.NewTextView()
+	textView.SetWrap(true)
+	textView.SetScrollable(true) // so that it allows us to set scroll position
+	textView.SetScrollBarVisibility(tview.ScrollBarNever)
+
+	app.SetRoot(textView, true)
+
+	width := (1 << 31) - 1
+	wrap := false
+	if isatty.IsTerminal(os.Stdout.Fd()) {
+		wrap = true
+		screen, err := tcell.NewScreen()
+		if err != nil {
+			return errors.Wrap(err, "get terminal dimensions")
+		}
+		if err := screen.Init(); err != nil {
+			return errors.Wrap(err, "init screen")
+		}
+		width, _ = screen.Size()
+		screen.Fini()
+	}
+
+	paramsMtx := &sync.Mutex{}
+	params := viewmodel.Params{
+		Report:                  nil,
+		ReportFetchError:        nil,
+		SelectedJob:             nil,
+		FSFilter:                func(s string) bool { return true },
+		DetailViewWidth:         width,
+		DetailViewWrap:          wrap,
+		ShortKeybindingOverview: "",
+	}
+
+	redraw := func() {
+		textView.Clear()
+
+		paramsMtx.Lock()
+		defer paramsMtx.Unlock()
+
+		if params.ReportFetchError != nil {
+			fmt.Fprintln(textView, params.ReportFetchError.Error())
+		} else if params.Report != nil {
+			m := viewmodel.New()
+			m.Update(params)
+			for _, j := range m.Jobs() {
+				if flag.Job != "" && j.Name() != flag.Job {
+					continue
+				}
+				params.SelectedJob = j
+				m.Update(params)
+				fmt.Fprintln(textView, m.SelectedJob().FullDescription())
+				if flag.Job != "" {
+					break
+				} else {
+					hline := strings.Repeat("-", params.DetailViewWidth)
+					fmt.Fprintln(textView, hline)
+				}
+			}
+		} else {
+			fmt.Fprintln(textView, "waiting for request results")
+		}
+		textView.ScrollToBeginning()
+	}
+
+	app.SetBeforeDrawFunc(func(screen tcell.Screen) bool {
+		// sync resizes to `params`
+		paramsMtx.Lock()
+		_, _, newWidth, _ := textView.GetInnerRect()
+		if newWidth != params.DetailViewWidth {
+			params.DetailViewWidth = newWidth
+			app.QueueUpdateDraw(redraw)
+		}
+		paramsMtx.Unlock()
+
+		textView.ScrollToBeginning() // has the effect of inhibiting user scrolls
+		return false
+	})
+
+	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				app.Suspend(func() {
+					panic(err)
+				})
+			}
+		}()
+		for {
+			st, err := c.Status()
+			paramsMtx.Lock()
+			params.Report = st.Jobs
+			params.ReportFetchError = err
+			paramsMtx.Unlock()
+
+			app.QueueUpdateDraw(redraw)
+
+			time.Sleep(flag.Delay)
+		}
+	}()
+
+	return app.Run()
+}

client/status/status_raw.go

@@ -0,0 +1,18 @@
+package status
+
+import (
+	"bytes"
+	"io"
+	"os"
+)
+
+func raw(c Client) error {
+	b, err := c.StatusRaw()
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(os.Stdout, bytes.NewReader(b)); err != nil {
+		return err
+	}
+	return nil
+}

client/status/viewmodel/bytecountbinary.go

@@ -0,0 +1,26 @@
+package viewmodel
+
+import (
+	"fmt"
+	"math"
+)
+
+func ByteCountBinaryUint(b uint64) string {
+	if b > math.MaxInt64 {
+		panic(b)
+	}
+	return ByteCountBinary(int64(b))
+}
+
+func ByteCountBinary(b int64) string {
+	const unit = 1024
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := unit, 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
+}

client/status/viewmodel/bytesprogresshistory.go

@@ -0,0 +1,49 @@
+package viewmodel
+
+import "time"
+
+type byteProgressMeasurement struct {
+	time time.Time
+	val  uint64
+}
+
+type bytesProgressHistory struct {
+	last        *byteProgressMeasurement // pointer as poor man's optional
+	changeCount int
+	lastChange  time.Time
+}
+
+func (p *bytesProgressHistory) Update(currentVal uint64) (bytesPerSecondAvg int64, changeCount int) {
+
+	if p.last == nil {
+		p.last = &byteProgressMeasurement{
+			time: time.Now(),
+			val:  currentVal,
+		}
+		return 0, 0
+	}
+
+	if p.last.val != currentVal {
+		p.changeCount++
+		p.lastChange = time.Now()
+	}
+
+	if time.Since(p.lastChange) > 3*time.Second {
+		p.last = nil
+		return 0, 0
+	}
+
+	var deltaV int64
+	if currentVal >= p.last.val {
+		deltaV = int64(currentVal - p.last.val)
+	} else {
+		deltaV = -int64(p.last.val - currentVal)
+	}
+	deltaT := time.Since(p.last.time)
+	rate := float64(deltaV) / deltaT.Seconds()
+
+	p.last.time = time.Now()
+	p.last.val = currentVal
+
+	return int64(rate), p.changeCount
+}

client/status/viewmodel/render.go

@@ -0,0 +1,660 @@
+package viewmodel
+
+import (
+	"fmt"
+	"math"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/go-playground/validator/v10"
+	yaml "github.com/zrepl/yaml-config"
+
+	"github.com/zrepl/zrepl/client/status/viewmodel/stringbuilder"
+	"github.com/zrepl/zrepl/daemon"
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/daemon/pruner"
+	"github.com/zrepl/zrepl/daemon/snapper"
+	"github.com/zrepl/zrepl/replication/report"
+)
+
+type M struct {
+	jobs            map[string]*Job
+	jobsList        []*Job
+	selectedJob     *Job
+	dateString      string
+	bottomBarStatus string
+}
+
+type Job struct {
+	// long-lived
+	name         string
+	byteProgress *bytesProgressHistory
+
+	lastStatus      *job.Status
+	fulldescription string
+}
+
+func New() *M {
+	return &M{
+		jobs:        make(map[string]*Job),
+		jobsList:    make([]*Job, 0),
+		selectedJob: nil,
+	}
+}
+
+type FilterFunc func(string) bool
+
+type Params struct {
+	Report                  map[string]*job.Status
+	ReportFetchError        error
+	SelectedJob             *Job
+	FSFilter                FilterFunc `validate:"required"`
+	DetailViewWidth         int        `validate:"gte=1"`
+	DetailViewWrap          bool
+	ShortKeybindingOverview string
+}
+
+var validate = validator.New()
+
+func (m *M) Update(p Params) {
+
+	if err := validate.Struct(p); err != nil {
+		panic(err)
+	}
+
+	if p.ReportFetchError != nil {
+		m.bottomBarStatus = fmt.Sprintf("[red::]status fetch: %s", p.ReportFetchError)
+	} else {
+		m.bottomBarStatus = p.ShortKeybindingOverview
+		for jobname, st := range p.Report {
+			// TODO handle job renames & deletions
+			j, ok := m.jobs[jobname]
+			if !ok {
+				j = &Job{
+					name:         jobname,
+					byteProgress: &bytesProgressHistory{},
+				}
+				m.jobs[jobname] = j
+				m.jobsList = append(m.jobsList, j)
+			}
+			j.lastStatus = st
+		}
+	}
+
+	// filter out internal jobs
+	var jobsList []*Job
+	for _, j := range m.jobsList {
+		if daemon.IsInternalJobName(j.name) {
+			continue
+		}
+		jobsList = append(jobsList, j)
+	}
+	m.jobsList = jobsList
+
+	// determinism!
+	sort.Slice(m.jobsList, func(i, j int) bool {
+		return strings.Compare(m.jobsList[i].name, m.jobsList[j].name) < 0
+	})
+
+	// try to not lose the selected job
+	m.selectedJob = nil
+	for _, j := range m.jobsList {
+		j.updateFullDescription(p)
+		if j == p.SelectedJob {
+			m.selectedJob = j
+		}
+	}
+
+	m.dateString = time.Now().Format(time.RFC3339)
+
+}
+
+func (m *M) BottomBarStatus() string { return m.bottomBarStatus }
+
+func (m *M) Jobs() []*Job { return m.jobsList }
+
+// may be nil
+func (m *M) SelectedJob() *Job { return m.selectedJob }
+
+func (m *M) DateString() string { return m.dateString }
+
+func (j *Job) updateFullDescription(p Params) {
+	width := p.DetailViewWidth
+	if !p.DetailViewWrap {
+		width = 10000000 // FIXME
+	}
+	b := stringbuilder.New(stringbuilder.Config{
+		IndentMultiplier: 3,
+		Width:            width,
+	})
+	drawJob(b, j.name, j.lastStatus, j.byteProgress, p.FSFilter)
+	j.fulldescription = b.String()
+}
+
+func (j *Job) JobTreeTitle() string {
+	return j.name
+}
+
+func (j *Job) FullDescription() string {
+	return j.fulldescription
+}
+
+func (j *Job) Name() string {
+	return j.name
+}
+
+func drawJob(t *stringbuilder.B, name string, v *job.Status, history *bytesProgressHistory, fsfilter FilterFunc) {
+
+	t.Printf("Job: %s\n", name)
+	t.Printf("Type: %s\n\n", v.Type)
+
+	if v.Type == job.TypePush || v.Type == job.TypePull {
+		activeStatus, ok := v.JobSpecific.(*job.ActiveSideStatus)
+		if !ok || activeStatus == nil {
+			t.Printf("ActiveSideStatus is null")
+			t.Newline()
+			return
+		}
+
+		t.Printf("Replication:")
+		t.AddIndentAndNewline(1)
+		renderReplicationReport(t, activeStatus.Replication, history, fsfilter)
+		t.AddIndentAndNewline(-1)
+
+		t.Printf("Pruning Sender:")
+		t.AddIndentAndNewline(1)
+		renderPrunerReport(t, activeStatus.PruningSender, fsfilter)
+		t.AddIndentAndNewline(-1)
+
+		t.Printf("Pruning Receiver:")
+		t.AddIndentAndNewline(1)
+		renderPrunerReport(t, activeStatus.PruningReceiver, fsfilter)
+		t.AddIndentAndNewline(-1)
+
+		if v.Type == job.TypePush {
+			t.Printf("Snapshotting:")
+			t.AddIndentAndNewline(1)
+			renderSnapperReport(t, activeStatus.Snapshotting, fsfilter)
+			t.AddIndentAndNewline(-1)
+		}
+
+	} else if v.Type == job.TypeSnap {
+		snapStatus, ok := v.JobSpecific.(*job.SnapJobStatus)
+		if !ok || snapStatus == nil {
+			t.Printf("SnapJobStatus is null")
+			t.Newline()
+			return
+		}
+		t.Printf("Pruning snapshots:")
+		t.AddIndentAndNewline(1)
+		renderPrunerReport(t, snapStatus.Pruning, fsfilter)
+		t.AddIndentAndNewline(-1)
+		t.Printf("Snapshotting:")
+		t.AddIndentAndNewline(1)
+		renderSnapperReport(t, snapStatus.Snapshotting, fsfilter)
+		t.AddIndentAndNewline(-1)
+	} else if v.Type == job.TypeSource {
+
+		st := v.JobSpecific.(*job.PassiveStatus)
+		t.Printf("Snapshotting:\n")
+		t.AddIndent(1)
+		renderSnapperReport(t, st.Snapper, fsfilter)
+		t.AddIndentAndNewline(-1)
+
+	} else {
+		t.Printf("No status representation for job type '%s', dumping as YAML", v.Type)
+		t.Newline()
+		asYaml, err := yaml.Marshal(v.JobSpecific)
+		if err != nil {
+			t.Printf("Error marshaling status to YAML: %s", err)
+			t.Newline()
+			return
+		}
+		t.Write(string(asYaml))
+		t.Newline()
+	}
+}
+
+func printFilesystemStatus(t *stringbuilder.B, rep *report.FilesystemReport, maxFS int) {
+
+	expected, replicated, containsInvalidSizeEstimates := rep.BytesSum()
+	sizeEstimationImpreciseNotice := ""
+	if containsInvalidSizeEstimates {
+		sizeEstimationImpreciseNotice = " (some steps lack size estimation)"
+	}
+	if rep.CurrentStep < len(rep.Steps) && rep.Steps[rep.CurrentStep].Info.BytesExpected == 0 {
+		sizeEstimationImpreciseNotice = " (step lacks size estimation)"
+	}
+
+	userVisisbleCurrentStep, userVisibleTotalSteps := rep.CurrentStep, len(rep.Steps)
+	// `.CurrentStep` is == len(rep.Steps) if all steps are done.
+	// Until then, it's an index into .Steps that starts at 0.
+	// For the user, we want it to start at 1.
+	if rep.CurrentStep >= len(rep.Steps) {
+		// rep.CurrentStep is what we want to show.
+		// We check for >= and not == for robustness.
+	} else {
+		// We're not done yet, so, make step count start at 1
+		// (The `.State` is included in the output, indicating we're not done yet)
+		userVisisbleCurrentStep = rep.CurrentStep + 1
+	}
+	status := fmt.Sprintf("%s (step %d/%d, %s/%s)%s",
+		strings.ToUpper(string(rep.State)),
+		userVisisbleCurrentStep, userVisibleTotalSteps,
+		ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected),
+		sizeEstimationImpreciseNotice,
+	)
+
+	activeIndicator := " "
+	if rep.BlockedOn == report.FsBlockedOnNothing &&
+		(rep.State == report.FilesystemPlanning || rep.State == report.FilesystemStepping) {
+		activeIndicator = "*"
+	}
+	t.AddIndent(1)
+	t.Printf("%s %s %s ",
+		activeIndicator,
+		stringbuilder.RightPad(rep.Info.Name, maxFS, " "),
+		status)
+
+	next := ""
+	if err := rep.Error(); err != nil {
+		next = err.Err
+	} else if rep.State != report.FilesystemDone {
+		if nextStep := rep.NextStep(); nextStep != nil {
+			if nextStep.IsIncremental() {
+				next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
+			} else {
+				next = fmt.Sprintf("next: full send %s", nextStep.Info.To)
+			}
+			attribs := []string{}
+
+			if nextStep.Info.Resumed {
+				attribs = append(attribs, "resumed")
+			}
+
+			if len(attribs) > 0 {
+				next += fmt.Sprintf(" (%s)", strings.Join(attribs, ", "))
+			}
+		} else {
+			next = "" // individual FSes may still be in planning state
+		}
+
+	}
+	t.Printf("%s", next)
+
+	t.AddIndent(-1)
+	t.Newline()
+}
+
+func renderReplicationReport(t *stringbuilder.B, rep *report.Report, history *bytesProgressHistory, fsfilter FilterFunc) {
+	if rep == nil {
+		t.Printf("...\n")
+		return
+	}
+
+	if rep.WaitReconnectError != nil {
+		t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: %s", rep.WaitReconnectError)
+		t.Newline()
+	}
+	if !rep.WaitReconnectSince.IsZero() {
+		delta := time.Until(rep.WaitReconnectUntil).Round(time.Second)
+		if rep.WaitReconnectUntil.IsZero() || delta > 0 {
+			var until string
+			if rep.WaitReconnectUntil.IsZero() {
+				until = "waiting indefinitely"
+			} else {
+				until = fmt.Sprintf("hard fail in %s @ %s", delta, rep.WaitReconnectUntil)
+			}
+			t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnecting with exponential backoff (since %s) (%s)",
+				rep.WaitReconnectSince, until)
+		} else {
+			t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnects reached hard-fail timeout @ %s", rep.WaitReconnectUntil)
+		}
+		t.Newline()
+	}
+
+	// TODO visualize more than the latest attempt by folding all attempts into one
+	if len(rep.Attempts) == 0 {
+		t.Printf("no attempts made yet")
+		return
+	} else {
+		t.Printf("Attempt #%d", len(rep.Attempts))
+		if len(rep.Attempts) > 1 {
+			t.Printf(". Previous attempts failed with the following statuses:")
+			t.AddIndentAndNewline(1)
+			for i, a := range rep.Attempts[:len(rep.Attempts)-1] {
+				t.PrintfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)\n", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt))
+			}
+			t.AddIndentAndNewline(-1)
+		} else {
+			t.Newline()
+		}
+	}
+
+	latest := rep.Attempts[len(rep.Attempts)-1]
+	sort.Slice(latest.Filesystems, func(i, j int) bool {
+		return latest.Filesystems[i].Info.Name < latest.Filesystems[j].Info.Name
+	})
+
+	// apply filter
+	filtered := make([]*report.FilesystemReport, 0, len(latest.Filesystems))
+	for _, fs := range latest.Filesystems {
+		if !fsfilter(fs.Info.Name) {
+			continue
+		}
+		filtered = append(filtered, fs)
+	}
+	latest.Filesystems = filtered
+
+	t.Printf("Status: %s", latest.State)
+	t.Newline()
+	if !latest.FinishAt.IsZero() {
+		t.Printf("Last Run: %s (lasted %s)\n", latest.FinishAt.Round(time.Second), latest.FinishAt.Sub(latest.StartAt).Round(time.Second))
+	} else {
+		t.Printf("Started: %s (lasting %s)\n", latest.StartAt.Round(time.Second), time.Since(latest.StartAt).Round(time.Second))
+	}
+
+	if latest.State == report.AttemptPlanningError {
+		t.Printf("Problem: ")
+		t.PrintfDrawIndentedAndWrappedIfMultiline("%s", latest.PlanError)
+		t.Newline()
+	} else if latest.State == report.AttemptFanOutError {
+		t.Printf("Problem: one or more of the filesystems encountered errors")
+		t.Newline()
+	}
+
+	if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
+		// Draw global progress bar
+		// Progress: [---------------]
+		expected, replicated, containsInvalidSizeEstimates := latest.BytesSum()
+		rate, changeCount := history.Update(replicated)
+		eta := time.Duration(0)
+		if rate > 0 {
+			eta = time.Duration((float64(expected)-float64(replicated))/float64(rate)) * time.Second
+		}
+
+		if !latest.State.IsTerminal() {
+			t.Write("Progress: ")
+			t.DrawBar(50, replicated, expected, changeCount)
+			t.Write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected), ByteCountBinary(rate)))
+			if eta != 0 {
+				t.Write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
+			}
+			t.Newline()
+		}
+		if containsInvalidSizeEstimates {
+			t.Write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
+			t.Newline()
+		}
+
+		if len(latest.Filesystems) == 0 {
+			t.Write("NOTE: no filesystems were considered for replication!")
+			t.Newline()
+		}
+
+		var maxFSLen int
+		for _, fs := range latest.Filesystems {
+			if len(fs.Info.Name) > maxFSLen {
+				maxFSLen = len(fs.Info.Name)
+			}
+		}
+		for _, fs := range latest.Filesystems {
+			printFilesystemStatus(t, fs, maxFSLen)
+		}
+
+	}
+}
+
+func humanizeDuration(duration time.Duration) string {
+	days := int64(duration.Hours() / 24)
+	hours := int64(math.Mod(duration.Hours(), 24))
+	minutes := int64(math.Mod(duration.Minutes(), 60))
+	seconds := int64(math.Mod(duration.Seconds(), 60))
+
+	var parts []string
+
+	force := false
+	chunks := []int64{days, hours, minutes, seconds}
+	for i, chunk := range chunks {
+		if force || chunk > 0 {
+			padding := 0
+			if force {
+				padding = 2
+			}
+			parts = append(parts, fmt.Sprintf("%*d%c", padding, chunk, "dhms"[i]))
+			force = true
+		}
+	}
+
+	return strings.Join(parts, " ")
+}
+
+func renderPrunerReport(t *stringbuilder.B, r *pruner.Report, fsfilter FilterFunc) {
+	if r == nil {
+		t.Printf("...\n")
+		return
+	}
+
+	state, err := pruner.StateString(r.State)
+	if err != nil {
+		t.Printf("Status: %q (parse error: %q)\n", r.State, err)
+		return
+	}
+
+	t.Printf("Status: %s", state)
+	t.Newline()
+
+	if r.Error != "" {
+		t.Printf("Error: %s\n", r.Error)
+	}
+
+	type commonFS struct {
+		*pruner.FSReport
+		completed bool
+	}
+	all := make([]commonFS, 0, len(r.Pending)+len(r.Completed))
+	for i := range r.Pending {
+		all = append(all, commonFS{&r.Pending[i], false})
+	}
+	for i := range r.Completed {
+		all = append(all, commonFS{&r.Completed[i], true})
+	}
+
+	// filter all
+	filtered := make([]commonFS, 0, len(all))
+	for _, fs := range all {
+		if fsfilter(fs.FSReport.Filesystem) {
+			filtered = append(filtered, fs)
+		}
+	}
+	all = filtered
+
+	switch state {
+	case pruner.Plan:
+		fallthrough
+	case pruner.PlanErr:
+		return
+	}
+
+	if len(all) == 0 {
+		t.Printf("nothing to do\n")
+		return
+	}
+
+	var totalDestroyCount, completedDestroyCount int
+	var maxFSname int
+	for _, fs := range all {
+		totalDestroyCount += len(fs.DestroyList)
+		if fs.completed {
+			completedDestroyCount += len(fs.DestroyList)
+		}
+		if maxFSname < len(fs.Filesystem) {
+			maxFSname = len(fs.Filesystem)
+		}
+	}
+
+	// global progress bar
+	if !state.IsTerminal() {
+		progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
+		t.Write("Progress: ")
+		t.Write("[")
+		t.Write(stringbuilder.Times("=", progress))
+		t.Write(">")
+		t.Write(stringbuilder.Times("-", 80-progress))
+		t.Write("]")
+		t.Printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
+		t.Newline()
+	}
+
+	sort.SliceStable(all, func(i, j int) bool {
+		return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
+	})
+
+	// Draw a table-like representation of 'all'
+	for _, fs := range all {
+		t.Write(stringbuilder.RightPad(fs.Filesystem, maxFSname, " "))
+		t.Write(" ")
+		if !fs.SkipReason.NotSkipped() {
+			t.Printf("skipped: %s\n", fs.SkipReason)
+			continue
+		}
+		if fs.LastError != "" {
+			if strings.ContainsAny(fs.LastError, "\r\n") {
+				t.Printf("ERROR:")
+				t.PrintfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError)
+			} else {
+				t.PrintfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError)
+			}
+			t.Newline()
+			continue
+		}
+
+		pruneRuleActionStr := fmt.Sprintf("(destroy %d of %d snapshots)",
+			len(fs.DestroyList), len(fs.SnapshotList))
+
+		if fs.completed {
+			t.Printf("Completed  %s\n", pruneRuleActionStr)
+			continue
+		}
+
+		t.Write("Pending    ") // whitespace is padding 10
+		if len(fs.DestroyList) == 1 {
+			t.Write(fs.DestroyList[0].Name)
+		} else {
+			t.Write(pruneRuleActionStr)
+		}
+		t.Newline()
+	}
+
+}
+
+func renderSnapperReport(t *stringbuilder.B, r *snapper.Report, fsfilter FilterFunc) {
+	if r == nil {
+		t.Printf("<no snapshotting report available>\n")
+		return
+	}
+	t.Printf("Type: %s\n", r.Type)
+	if r.Periodic != nil {
+		renderSnapperReportPeriodic(t, r.Periodic, fsfilter)
+	} else if r.Cron != nil {
+		renderSnapperReportCron(t, r.Cron, fsfilter)
+	} else {
+		t.Printf("<no details available>")
+	}
+}
+
+func renderSnapperReportPeriodic(t *stringbuilder.B, r *snapper.PeriodicReport, fsfilter FilterFunc) {
+	t.Printf("Status: %s", r.State)
+	t.Newline()
+
+	if r.Error != "" {
+		t.Printf("Error: %s\n", r.Error)
+	}
+	if !r.SleepUntil.IsZero() {
+		t.Printf("Sleep until: %s\n", r.SleepUntil)
+	}
+
+	renderSnapperPlanReportFilesystem(t, r.Progress, fsfilter)
+}
+
+func renderSnapperReportCron(t *stringbuilder.B, r *snapper.CronReport, fsfilter FilterFunc) {
+	t.Printf("State: %s\n", r.State)
+
+	now := time.Now()
+	if r.WakeupTime.After(now) {
+		t.Printf("Sleep until: %s (%s remaining)\n", r.WakeupTime, r.WakeupTime.Sub(now).Round(time.Second))
+	} else {
+		t.Printf("Started: %s (lasting %s)\n", r.WakeupTime, now.Sub(r.WakeupTime).Round(time.Second))
+	}
+
+	renderSnapperPlanReportFilesystem(t, r.Progress, fsfilter)
+}
+
+func renderSnapperPlanReportFilesystem(t *stringbuilder.B, fss []*snapper.ReportFilesystem, fsfilter FilterFunc) {
+	sort.Slice(fss, func(i, j int) bool {
+		return strings.Compare(fss[i].Path, fss[j].Path) == -1
+	})
+
+	dur := func(d time.Duration) string {
+		return d.Round(100 * time.Millisecond).String()
+	}
+
+	type row struct {
+		path, state, duration, remainder, hookReport string
+	}
+	var widths struct {
+		path, state, duration int
+	}
+	rows := make([]*row, 0, len(fss))
+	for _, fs := range fss {
+		if !fsfilter(fs.Path) {
+			continue
+		}
+		r := &row{
+			path:  fs.Path,
+			state: fs.State.String(),
+		}
+		if fs.HooksHadError {
+			r.hookReport = fs.Hooks // FIXME render here, not in daemon
+		}
+		switch fs.State {
+		case snapper.SnapPending:
+			r.duration = "..."
+			r.remainder = ""
+		case snapper.SnapStarted:
+			r.duration = dur(time.Since(fs.StartAt))
+			r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
+		case snapper.SnapDone:
+			fallthrough
+		case snapper.SnapError:
+			r.duration = dur(fs.DoneAt.Sub(fs.StartAt))
+			r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
+		}
+		rows = append(rows, r)
+		if len(r.path) > widths.path {
+			widths.path = len(r.path)
+		}
+		if len(r.state) > widths.state {
+			widths.state = len(r.state)
+		}
+		if len(r.duration) > widths.duration {
+			widths.duration = len(r.duration)
+		}
+	}
+
+	for _, r := range rows {
+		path := stringbuilder.RightPad(r.path, widths.path, " ")
+		state := stringbuilder.RightPad(r.state, widths.state, " ")
+		duration := stringbuilder.RightPad(r.duration, widths.duration, " ")
+		t.Printf("%s %s %s", path, state, duration)
+		t.PrintfDrawIndentedAndWrappedIfMultiline(" %s", r.remainder)
+		if r.hookReport != "" {
+			t.AddIndent(1)
+			t.Newline()
+			t.Printf("%s", r.hookReport)
+			t.AddIndent(-1)
+		}
+		t.Newline()
+	}
+}

client/status/viewmodel/stringbuilder/stringbuilder.go

@@ -0,0 +1,119 @@
+package stringbuilder
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/go-playground/validator/v10"
+)
+
+type B struct {
+	// const
+	indentMultiplier int
+
+	// mut
+	sb     *strings.Builder
+	indent int
+	width  int
+	x, y   int
+}
+
+type Config struct {
+	IndentMultiplier int `validate:"gte=1"`
+	Width            int `validate:"gte=1"`
+}
+
+var validate = validator.New()
+
+func New(config Config) *B {
+
+	if err := validate.Struct(config); err != nil {
+		panic(err)
+	}
+
+	return &B{sb: &strings.Builder{}, width: config.Width, indentMultiplier: config.IndentMultiplier}
+}
+
+func (b *B) String() string { return b.sb.String() }
+
+func (w *B) Newline() {
+	w.Write("\n")
+}
+
+func (w *B) PrintfDrawIndentedAndWrappedIfMultiline(format string, args ...interface{}) {
+	whole := fmt.Sprintf(format, args...)
+	if strings.ContainsAny(whole, "\n\r") {
+		w.AddIndent(1)
+		defer w.AddIndent(-1)
+	}
+	w.Write(whole)
+}
+
+func (w *B) Printf(format string, args ...interface{}) {
+	whole := fmt.Sprintf(format, args...)
+	w.Write(whole)
+}
+
+func (t *B) AddIndent(delta int) {
+	t.indent += delta * t.indentMultiplier
+}
+
+func (t *B) AddIndentAndNewline(delta int) {
+	t.indent += delta * t.indentMultiplier
+	t.Write("\n")
+}
+
+func (w *B) Write(s string) {
+	for _, c := range s {
+		if c == '\n' {
+			fmt.Fprint(w.sb, "\n")
+			w.x = 0
+			fmt.Fprint(w.sb, Times(" ", w.indent-w.x))
+			w.x = w.indent
+			w.y++
+			continue
+		}
+		if w.x >= w.width {
+			fmt.Fprint(w.sb, "\n")
+			w.x = 0
+			fmt.Fprint(w.sb, Times(" ", w.indent-w.x))
+			w.x = w.indent
+		}
+		fmt.Fprintf(w.sb, "%c", c)
+		w.x++
+	}
+}
+
+func Times(str string, n int) (out string) {
+	for i := 0; i < n; i++ {
+		out += str
+	}
+	return
+}
+
+func RightPad(str string, length int, pad string) string {
+	if len(str) > length {
+		return str[:length]
+	}
+	return str + strings.Repeat(pad, length-len(str))
+}
+
+// changeCount = 0 indicates stall / no progress
+func (w *B) DrawBar(length int, bytes, totalBytes uint64, changeCount int) {
+	const arrowPositions = `>\|/`
+	var completedLength int
+	if totalBytes > 0 {
+		completedLength = int(uint64(length) * bytes / totalBytes)
+		if completedLength > length {
+			completedLength = length
+		}
+	} else if totalBytes == bytes {
+		completedLength = length
+	}
+
+	w.Write("[")
+	w.Write(Times("=", completedLength))
+	w.Write(string(arrowPositions[changeCount%len(arrowPositions)]))
+	w.Write(Times("-", length-completedLength))
+	w.Write("]")
+}

client/stdinserver.go

@@ -0,0 +1,51 @@
+package client
+
+import (
+	"os"
+
+	"github.com/problame/go-netssh"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+
+	"context"
+	"errors"
+	"log"
+	"path"
+)
+
+var StdinserverCmd = &cli.Subcommand{
+	Use:   "stdinserver CLIENT_IDENTITY",
+	Short: "stdinserver transport mode (started from authorized_keys file as forced command)",
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		return runStdinserver(subcommand.Config(), args)
+	},
+}
+
+func runStdinserver(config *config.Config, args []string) error {
+
+	// NOTE: the netssh proxying protocol requires exiting with non-zero status if anything goes wrong
+	defer os.Exit(1)
+
+	log := log.New(os.Stderr, "", log.LUTC|log.Ldate|log.Ltime)
+
+	if len(args) != 1 || args[0] == "" {
+		err := errors.New("must specify client_identity as positional argument")
+		return err
+	}
+
+	identity := args[0]
+	unixaddr := path.Join(config.Global.Serve.StdinServer.SockDir, identity)
+
+	log.Printf("proxying client identity '%s' to zrepl daemon '%s'", identity, unixaddr)
+
+	ctx := netssh.ContextWithLog(context.TODO(), log)
+
+	err := netssh.Proxy(ctx, unixaddr)
+	if err == nil {
+		log.Print("proxying finished successfully, exiting with status 0")
+		os.Exit(0)
+	}
+	log.Printf("error proxying: %s", err)
+	return nil
+}

client/testcmd.go

@@ -0,0 +1,220 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var TestCmd = &cli.Subcommand{
+	Use: "test",
+	SetupSubcommands: func() []*cli.Subcommand {
+		return []*cli.Subcommand{testFilter, testPlaceholder, testDecodeResumeToken}
+	},
+}
+
+var testFilterArgs struct {
+	job   string
+	all   bool
+	input string
+}
+
+var testFilter = &cli.Subcommand{
+	Use:   "filesystems --job JOB [--all | --input INPUT]",
+	Short: "test filesystems filter specified in push or source job",
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&testFilterArgs.job, "job", "", "the name of the push or source job")
+		f.StringVar(&testFilterArgs.input, "input", "", "a filesystem name to test against the job's filters")
+		f.BoolVar(&testFilterArgs.all, "all", false, "test all local filesystems")
+	},
+	Run: runTestFilterCmd,
+}
+
+func runTestFilterCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+
+	if testFilterArgs.job == "" {
+		return fmt.Errorf("must specify --job flag")
+	}
+	if !(testFilterArgs.all != (testFilterArgs.input != "")) { // xor
+		return fmt.Errorf("must set one: --all or --input")
+	}
+
+	conf := subcommand.Config()
+
+	var confFilter config.FilesystemsFilter
+	job, err := conf.Job(testFilterArgs.job)
+	if err != nil {
+		return err
+	}
+	switch j := job.Ret.(type) {
+	case *config.SourceJob:
+		confFilter = j.Filesystems
+	case *config.PushJob:
+		confFilter = j.Filesystems
+	case *config.SnapJob:
+		confFilter = j.Filesystems
+	default:
+		return fmt.Errorf("job type %T does not have filesystems filter", j)
+	}
+
+	f, err := filters.DatasetMapFilterFromConfig(confFilter)
+	if err != nil {
+		return fmt.Errorf("filter invalid: %s", err)
+	}
+
+	var fsnames []string
+	if testFilterArgs.input != "" {
+		fsnames = []string{testFilterArgs.input}
+	} else {
+		out, err := zfs.ZFSList(ctx, []string{"name"})
+		if err != nil {
+			return fmt.Errorf("could not list ZFS filesystems: %s", err)
+		}
+		for _, row := range out {
+
+			fsnames = append(fsnames, row[0])
+		}
+	}
+
+	fspaths := make([]*zfs.DatasetPath, len(fsnames))
+	for i, fsname := range fsnames {
+		path, err := zfs.NewDatasetPath(fsname)
+		if err != nil {
+			return err
+		}
+		fspaths[i] = path
+	}
+
+	hadFilterErr := false
+	for _, in := range fspaths {
+		var res string
+		var errStr string
+		pass, err := f.Filter(in)
+		if err != nil {
+			res = "ERROR"
+			errStr = err.Error()
+			hadFilterErr = true
+		} else if pass {
+			res = "ACCEPT"
+		} else {
+			res = "REJECT"
+		}
+		fmt.Printf("%s\t%s\t%s\n", res, in.ToString(), errStr)
+	}
+
+	if hadFilterErr {
+		return fmt.Errorf("filter errors occurred")
+	}
+	return nil
+}
+
+var testPlaceholderArgs struct {
+	ds  string
+	all bool
+}
+
+var testPlaceholder = &cli.Subcommand{
+	Use:   "placeholder [--all | --dataset DATASET]",
+	Short: fmt.Sprintf("list received placeholder filesystems (zfs property %q)", zfs.PlaceholderPropertyName),
+	Example: `
+	placeholder --all
+	placeholder --dataset path/to/sink/clientident/fs`,
+	NoRequireConfig: true,
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&testPlaceholderArgs.ds, "dataset", "", "dataset path (not required to exist)")
+		f.BoolVar(&testPlaceholderArgs.all, "all", false, "list tab-separated placeholder status of all filesystems")
+	},
+	Run: runTestPlaceholder,
+}
+
+func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+
+	var checkDPs []*zfs.DatasetPath
+	var datasetWasExplicitArgument bool
+
+	// all actions first
+	if testPlaceholderArgs.all {
+		datasetWasExplicitArgument = false
+		out, err := zfs.ZFSList(ctx, []string{"name"})
+		if err != nil {
+			return errors.Wrap(err, "could not list ZFS filesystems")
+		}
+		for _, row := range out {
+			dp, err := zfs.NewDatasetPath(row[0])
+			if err != nil {
+				panic(err)
+			}
+			checkDPs = append(checkDPs, dp)
+		}
+	} else {
+		datasetWasExplicitArgument = true
+		dp, err := zfs.NewDatasetPath(testPlaceholderArgs.ds)
+		if err != nil {
+			return err
+		}
+		if dp.Empty() {
+			return fmt.Errorf("must specify --dataset DATASET or --all")
+		}
+		checkDPs = append(checkDPs, dp)
+	}
+
+	fmt.Printf("IS_PLACEHOLDER\tDATASET\tzrepl:placeholder\n")
+	for _, dp := range checkDPs {
+		ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, dp)
+		if err != nil {
+			return errors.Wrap(err, "cannot get placeholder state")
+		}
+		if !ph.FSExists {
+			if datasetWasExplicitArgument {
+				return errors.Errorf("filesystem %q does not exist", ph.FS)
+			} else {
+				// got deleted between ZFSList and ZFSGetFilesystemPlaceholderState
+				continue
+			}
+		}
+		is := "yes"
+		if !ph.IsPlaceholder {
+			is = "no"
+		}
+		fmt.Printf("%s\t%s\t%s\n", is, dp.ToString(), ph.RawLocalPropertyValue)
+	}
+	return nil
+}
+
+var testDecodeResumeTokenArgs struct {
+	token string
+}
+
+var testDecodeResumeToken = &cli.Subcommand{
+	Use:   "decoderesumetoken --token TOKEN",
+	Short: "decode resume token",
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&testDecodeResumeTokenArgs.token, "token", "", "the resume token obtained from the receive_resume_token property")
+	},
+	Run: runTestDecodeResumeTokenCmd,
+}
+
+func runTestDecodeResumeTokenCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+	if testDecodeResumeTokenArgs.token == "" {
+		return fmt.Errorf("token argument must be specified")
+	}
+	token, err := zfs.ParseResumeToken(ctx, testDecodeResumeTokenArgs.token)
+	if err != nil {
+		return err
+	}
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", " ")
+	if err := enc.Encode(&token); err != nil {
+		panic(err)
+	}
+	return nil
+}

client/version.go

@@ -0,0 +1,75 @@
+package client
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+	"github.com/zrepl/zrepl/version"
+)
+
+var versionArgs struct {
+	Show      string
+	Config    *config.Config
+	ConfigErr error
+}
+
+var VersionCmd = &cli.Subcommand{
+	Use:             "version",
+	Short:           "print version of zrepl binary and running daemon",
+	NoRequireConfig: true,
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)")
+	},
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		versionArgs.Config = subcommand.Config()
+		versionArgs.ConfigErr = subcommand.ConfigParsingError()
+		return runVersionCmd()
+	},
+}
+
+func runVersionCmd() error {
+	args := versionArgs
+
+	if args.Show != "daemon" && args.Show != "client" && args.Show != "" {
+		return fmt.Errorf("show flag must be 'client' or 'server' or be left empty")
+	}
+
+	var clientVersion, daemonVersion *version.ZreplVersionInformation
+	if args.Show == "client" || args.Show == "" {
+		clientVersion = version.NewZreplVersionInformation()
+		fmt.Printf("client: %s\n", clientVersion.String())
+	}
+	if args.Show == "daemon" || args.Show == "" {
+
+		if args.ConfigErr != nil {
+			return fmt.Errorf("config parsing error: %s", args.ConfigErr)
+		}
+
+		httpc, err := controlHttpClient(args.Config.Global.Control.SockPath)
+		if err != nil {
+			return fmt.Errorf("server: error: %s\n", err)
+		}
+
+		var info version.ZreplVersionInformation
+		err = jsonRequestResponse(httpc, daemon.ControlJobEndpointVersion, "", &info)
+		if err != nil {
+			return fmt.Errorf("server: error: %s\n", err)
+		}
+		daemonVersion = &info
+		fmt.Printf("server: %s\n", daemonVersion.String())
+	}
+
+	if args.Show == "" {
+		if clientVersion.Version != daemonVersion.Version {
+			fmt.Fprintf(os.Stderr, "WARNING: client version != daemon version, restart zrepl daemon\n")
+		}
+	}
+
+	return nil
+}

client/zfsabstractions.go

@@ -0,0 +1,147 @@
+package client
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var (
+	ZFSAbstractionsCmd = &cli.Subcommand{
+		Use:   "zfs-abstraction",
+		Short: "manage abstractions that zrepl builds on top of ZFS",
+		SetupSubcommands: func() []*cli.Subcommand {
+			return []*cli.Subcommand{
+				zabsCmdList,
+				zabsCmdReleaseAll,
+				zabsCmdReleaseStale,
+				zabsCmdCreate,
+			}
+		},
+	}
+)
+
+// a common set of CLI flags that map to the fields of an
+// endpoint.ListZFSHoldsAndBookmarksQuery
+type zabsFilterFlags struct {
+	Filesystems FilesystemsFilterFlag
+	Job         JobIDFlag
+	Types       AbstractionTypesFlag
+	Concurrency int64
+}
+
+// produce a query from the CLI flags
+func (f zabsFilterFlags) Query() (endpoint.ListZFSHoldsAndBookmarksQuery, error) {
+	q := endpoint.ListZFSHoldsAndBookmarksQuery{
+		FS:          f.Filesystems.FlagValue(),
+		What:        f.Types.FlagValue(),
+		JobID:       f.Job.FlagValue(),
+		Concurrency: f.Concurrency,
+	}
+	return q, q.Validate()
+}
+
+func (f *zabsFilterFlags) registerZabsFilterFlags(s *pflag.FlagSet, verb string) {
+	// Note: the default value is defined in the .FlagValue methods
+	s.Var(&f.Filesystems, "fs", fmt.Sprintf("only %s holds on the specified filesystem [default: all filesystems] [comma-separated list of <dataset-pattern>:<ok|!> pairs]", verb))
+	s.Var(&f.Job, "job", fmt.Sprintf("only %s holds created by the specified job [default: any job]", verb))
+
+	variants := make([]string, 0, len(endpoint.AbstractionTypesAll))
+	for v := range endpoint.AbstractionTypesAll {
+		variants = append(variants, string(v))
+	}
+	sort.Strings(variants)
+	variantsJoined := strings.Join(variants, "|")
+	s.Var(&f.Types, "type", fmt.Sprintf("only %s holds of the specified type [default: all] [comma-separated list of %s]", verb, variantsJoined))
+
+	s.Int64VarP(&f.Concurrency, "concurrency", "p", 1, "number of concurrently queried filesystems")
+}
+
+type JobIDFlag struct{ J *endpoint.JobID }
+
+func (f *JobIDFlag) Set(s string) error {
+	if len(s) == 0 {
+		*f = JobIDFlag{J: nil}
+		return nil
+	}
+
+	jobID, err := endpoint.MakeJobID(s)
+	if err != nil {
+		return err
+	}
+	*f = JobIDFlag{J: &jobID}
+	return nil
+}
+func (f JobIDFlag) Type() string               { return "job-ID" }
+func (f JobIDFlag) String() string             { return fmt.Sprint(f.J) }
+func (f JobIDFlag) FlagValue() *endpoint.JobID { return f.J }
+
+type AbstractionTypesFlag map[endpoint.AbstractionType]bool
+
+func (f *AbstractionTypesFlag) Set(s string) error {
+	ats, err := endpoint.AbstractionTypeSetFromStrings(strings.Split(s, ","))
+	if err != nil {
+		return err
+	}
+	*f = AbstractionTypesFlag(ats)
+	return nil
+}
+func (f AbstractionTypesFlag) Type() string { return "abstraction-type" }
+func (f AbstractionTypesFlag) String() string {
+	return endpoint.AbstractionTypeSet(f).String()
+}
+func (f AbstractionTypesFlag) FlagValue() map[endpoint.AbstractionType]bool {
+	if len(f) > 0 {
+		return f
+	}
+	return endpoint.AbstractionTypesAll
+}
+
+type FilesystemsFilterFlag struct {
+	F endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter
+}
+
+func (flag *FilesystemsFilterFlag) Set(s string) error {
+	mappings := strings.Split(s, ",")
+	if len(mappings) == 1 && !strings.Contains(mappings[0], ":") {
+		flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &mappings[0],
+		}
+		return nil
+	}
+
+	f := filters.NewDatasetMapFilter(len(mappings), true)
+	for _, m := range mappings {
+		thisMappingErr := fmt.Errorf("expecting comma-separated list of <dataset-pattern>:<ok|!> pairs, got %q", m)
+		lhsrhs := strings.SplitN(m, ":", 2)
+		if len(lhsrhs) != 2 {
+			return thisMappingErr
+		}
+		err := f.Add(lhsrhs[0], lhsrhs[1])
+		if err != nil {
+			return fmt.Errorf("%s: %s", thisMappingErr, err)
+		}
+	}
+	flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+		Filter: f,
+	}
+	return nil
+}
+func (flag FilesystemsFilterFlag) Type() string { return "filesystem filter spec" }
+func (flag FilesystemsFilterFlag) String() string {
+	return fmt.Sprintf("%v", flag.F)
+}
+func (flag FilesystemsFilterFlag) FlagValue() endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter {
+	var z FilesystemsFilterFlag
+	if flag == z {
+		return endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{Filter: zfs.NoFilter()}
+	}
+	return flag.F
+}

client/zfsabstractions_create.go

@@ -0,0 +1,14 @@
+package client
+
+import "github.com/zrepl/zrepl/cli"
+
+var zabsCmdCreate = &cli.Subcommand{
+	Use:             "create",
+	NoRequireConfig: true,
+	Short:           `create zrepl ZFS abstractions (mostly useful for debugging & development, users should not need to use this command)`,
+	SetupSubcommands: func() []*cli.Subcommand {
+		return []*cli.Subcommand{
+			zabsCmdCreateStepHold,
+		}
+	},
+}

client/zfsabstractions_create_step_hold.go

@@ -0,0 +1,58 @@
+package client
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var zabsCreateStepHoldFlags struct {
+	target string
+	jobid  JobIDFlag
+}
+
+var zabsCmdCreateStepHold = &cli.Subcommand{
+	Use:             "step",
+	Run:             doZabsCreateStep,
+	NoRequireConfig: true,
+	Short:           `create a step hold or bookmark`,
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVarP(&zabsCreateStepHoldFlags.target, "target", "t", "", "snapshot to be held / bookmark to be held")
+		f.VarP(&zabsCreateStepHoldFlags.jobid, "jobid", "j", "jobid for which the hold is installed")
+	},
+}
+
+func doZabsCreateStep(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	if len(args) > 0 {
+		return errors.New("subcommand takes no arguments")
+	}
+
+	f := &zabsCreateStepHoldFlags
+
+	fs, _, _, err := zfs.DecomposeVersionString(f.target)
+	if err != nil {
+		return errors.Wrapf(err, "%q invalid target", f.target)
+	}
+
+	if f.jobid.FlagValue() == nil {
+		return errors.Errorf("jobid must be set")
+	}
+
+	v, err := zfs.ZFSGetFilesystemVersion(ctx, f.target)
+	if err != nil {
+		return errors.Wrapf(err, "get info about target %q", f.target)
+	}
+
+	step, err := endpoint.HoldStep(ctx, fs, v, *f.jobid.FlagValue())
+	if err != nil {
+		return errors.Wrap(err, "create step hold")
+	}
+	fmt.Println(step.String())
+	return nil
+}

client/zfsabstractions_list.go

@@ -0,0 +1,100 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/util/chainlock"
+)
+
+var zabsListFlags struct {
+	Filter zabsFilterFlags
+	Json   bool
+}
+
+var zabsCmdList = &cli.Subcommand{
+	Use:             "list",
+	Short:           `list zrepl ZFS abstractions`,
+	Run:             doZabsList,
+	NoRequireConfig: true,
+	SetupFlags: func(f *pflag.FlagSet) {
+		zabsListFlags.Filter.registerZabsFilterFlags(f, "list")
+		f.BoolVar(&zabsListFlags.Json, "json", false, "emit JSON")
+	},
+}
+
+func doZabsList(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsListFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	abstractions, errors, drainDone, err := endpoint.ListAbstractionsStreamed(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+	defer drainDone()
+
+	var line chainlock.L
+	var wg sync.WaitGroup
+	defer wg.Wait()
+
+	// print results
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		enc := json.NewEncoder(os.Stdout)
+		for a := range abstractions {
+			func() {
+				defer line.Lock().Unlock()
+				if zabsListFlags.Json {
+					enc.SetIndent("", "  ")
+					if err := enc.Encode(a); err != nil {
+						panic(err)
+					}
+					fmt.Println()
+				} else {
+					fmt.Println(a)
+				}
+			}()
+		}
+	}()
+
+	// print errors to stderr
+	errorColor := color.New(color.FgRed)
+	var errorsSlice []endpoint.ListAbstractionsError
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for err := range errors {
+			func() {
+				defer line.Lock().Unlock()
+				errorsSlice = append(errorsSlice, err)
+				errorColor.Fprintf(os.Stderr, "%s\n", err)
+			}()
+		}
+	}()
+	wg.Wait()
+	if len(errorsSlice) > 0 {
+		errorColor.Add(color.Bold).Fprintf(os.Stderr, "there were errors in listing the abstractions")
+		return fmt.Errorf("")
+	} else {
+		return nil
+	}
+
+}

client/zfsabstractions_release.go

@@ -0,0 +1,143 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+)
+
+// shared between release-all and release-step
+var zabsReleaseFlags struct {
+	Filter zabsFilterFlags
+	Json   bool
+	DryRun bool
+}
+
+func registerZabsReleaseFlags(s *pflag.FlagSet) {
+	zabsReleaseFlags.Filter.registerZabsFilterFlags(s, "release")
+	s.BoolVar(&zabsReleaseFlags.Json, "json", false, "emit json instead of pretty-printed")
+	s.BoolVar(&zabsReleaseFlags.DryRun, "dry-run", false, "do a dry-run")
+}
+
+var zabsCmdReleaseAll = &cli.Subcommand{
+	Use:             "release-all",
+	Run:             doZabsReleaseAll,
+	NoRequireConfig: true,
+	Short:           `(DANGEROUS) release ALL zrepl ZFS abstractions (mostly useful for uninstalling zrepl completely or for "de-zrepl-ing" a filesystem)`,
+	SetupFlags:      registerZabsReleaseFlags,
+}
+
+var zabsCmdReleaseStale = &cli.Subcommand{
+	Use:             "release-stale",
+	Run:             doZabsReleaseStale,
+	NoRequireConfig: true,
+	Short:           `release stale zrepl ZFS abstractions (useful if zrepl has a bug and does not do it by itself)`,
+	SetupFlags:      registerZabsReleaseFlags,
+}
+
+func doZabsReleaseAll(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsReleaseFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	abstractions, listErrors, err := endpoint.ListAbstractions(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+	if len(listErrors) > 0 {
+		color.New(color.FgRed).Fprintf(os.Stderr, "there were errors in listing the abstractions:\n%s\n", listErrors)
+		// proceed anyways with rest of abstractions
+	}
+
+	return doZabsRelease_Common(ctx, abstractions)
+}
+
+func doZabsReleaseStale(ctx context.Context, sc *cli.Subcommand, args []string) error {
+
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsReleaseFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	stalenessInfo, err := endpoint.ListStale(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+
+	return doZabsRelease_Common(ctx, stalenessInfo.Stale)
+}
+
+func doZabsRelease_Common(ctx context.Context, destroy []endpoint.Abstraction) error {
+
+	if zabsReleaseFlags.DryRun {
+		if zabsReleaseFlags.Json {
+			m, err := json.MarshalIndent(destroy, "", "  ")
+			if err != nil {
+				panic(err)
+			}
+			if _, err := os.Stdout.Write(m); err != nil {
+				panic(err)
+			}
+			fmt.Println()
+		} else {
+			for _, a := range destroy {
+				fmt.Printf("would destroy %s\n", a)
+			}
+		}
+		return nil
+	}
+
+	outcome := endpoint.BatchDestroy(ctx, destroy)
+	hadErr := false
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	colorErr := color.New(color.FgRed)
+	printfSuccess := color.New(color.FgGreen).FprintfFunc()
+	printfSection := color.New(color.Bold).FprintfFunc()
+
+	for res := range outcome {
+		hadErr = hadErr || res.DestroyErr != nil
+		if zabsReleaseFlags.Json {
+			err := enc.Encode(res)
+			if err != nil {
+				colorErr.Fprintf(os.Stderr, "cannot marshal there were errors in destroying the abstractions")
+			}
+		} else {
+			printfSection(os.Stdout, "destroy %s ...", res.Abstraction)
+			if res.DestroyErr != nil {
+				colorErr.Fprintf(os.Stdout, " failed:\n%s\n", res.DestroyErr)
+			} else {
+				printfSuccess(os.Stdout, " OK\n")
+			}
+		}
+	}
+
+	if hadErr {
+		colorErr.Add(color.Bold).Fprintf(os.Stderr, "there were errors in destroying the abstractions")
+		return fmt.Errorf("")
+	} else {
+		return nil
+	}
+}

cmd/autosnap.go

@@ -1,91 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/jobrun"
-	"github.com/zrepl/zrepl/zfs"
-	"os"
-	"sync"
-	"time"
-)
-
-var AutosnapCmd = &cobra.Command{
-	Use:   "autosnap",
-	Short: "perform automatic snapshotting",
-	Run:   cmdAutosnap,
-}
-
-func init() {
-	RootCmd.AddCommand(AutosnapCmd)
-}
-
-func cmdAutosnap(cmd *cobra.Command, args []string) {
-
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		runner.Start()
-	}()
-
-	if len(args) < 1 {
-		log.Printf("must specify exactly one job as positional argument")
-		os.Exit(1)
-	}
-
-	snap, ok := conf.Autosnaps[args[0]]
-	if !ok {
-		log.Printf("could not find autosnap job: %s", args[0])
-		os.Exit(1)
-	}
-
-	job := jobrun.Job{
-		Name:           snap.JobName,
-		RepeatStrategy: snap.Interval,
-		RunFunc: func(log jobrun.Logger) error {
-			log.Printf("doing autosnap: %v", snap)
-			ctx := AutosnapContext{snap}
-			return doAutosnap(ctx, log)
-		},
-	}
-	runner.AddJob(job)
-
-	wg.Wait()
-
-}
-
-type AutosnapContext struct {
-	Autosnap *Autosnap
-}
-
-func doAutosnap(ctx AutosnapContext, log Logger) (err error) {
-
-	snap := ctx.Autosnap
-
-	filesystems, err := zfs.ZFSListMapping(snap.DatasetFilter)
-	if err != nil {
-		return fmt.Errorf("cannot filter datasets: %s", err)
-	}
-
-	suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
-	snapname := fmt.Sprintf("%s%s", snap.Prefix, suffix)
-
-	hadError := false
-
-	for _, fs := range filesystems { // optimization: use recursive snapshots / channel programs here
-		log.Printf("snapshotting filesystem %s@%s", fs, snapname)
-		err := zfs.ZFSSnapshot(fs, snapname, false)
-		if err != nil {
-			log.Printf("error snapshotting %s: %s", fs, err)
-			hadError = true
-		}
-	}
-
-	if hadError {
-		err = fmt.Errorf("errors occurred during autosnap, check logs for details")
-	}
-
-	return
-
-}

cmd/config.go

@@ -1,730 +0,0 @@
-package cmd
-
-import (
-	"errors"
-	"fmt"
-	"github.com/jinzhu/copier"
-	"github.com/mitchellh/mapstructure"
-	"github.com/zrepl/zrepl/jobrun"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/sshbytestream"
-	. "github.com/zrepl/zrepl/util"
-	"github.com/zrepl/zrepl/zfs"
-	yaml "gopkg.in/yaml.v2"
-	"io"
-	"io/ioutil"
-	"regexp"
-	"strconv"
-	"strings"
-	"time"
-)
-
-var (
-	JobSectionPush     string = "push"
-	JobSectionPull     string = "pull"
-	JobSectionPrune    string = "prune"
-	JobSectionAutosnap string = "autosnap"
-)
-
-type Remote struct {
-	Name      string
-	Transport Transport
-}
-
-type Transport interface {
-	Connect(rpcLog Logger) (rpc.RPCRequester, error)
-}
-type LocalTransport struct {
-	Handler rpc.RPCHandler
-}
-type SSHTransport struct {
-	Host                 string
-	User                 string
-	Port                 uint16
-	IdentityFile         string   `mapstructure:"identity_file"`
-	TransportOpenCommand []string `mapstructure:"transport_open_command"`
-	SSHCommand           string   `mapstructure:"ssh_command"`
-	Options              []string
-	ConnLogReadFile      string `mapstructure:"connlog_read_file"`
-	ConnLogWriteFile     string `mapstructure:"connlog_write_file"`
-}
-
-type Push struct {
-	JobName           string // for use with jobrun package
-	To                *Remote
-	Filter            zfs.DatasetFilter
-	InitialReplPolicy rpc.InitialReplPolicy
-	RepeatStrategy    jobrun.RepeatStrategy
-}
-type Pull struct {
-	JobName           string // for use with jobrun package
-	From              *Remote
-	Mapping           DatasetMapFilter
-	InitialReplPolicy rpc.InitialReplPolicy
-	RepeatStrategy    jobrun.RepeatStrategy
-}
-
-type Prune struct {
-	JobName         string // for use with jobrun package
-	DatasetFilter   zfs.DatasetFilter
-	SnapshotFilter  zfs.FilesystemVersionFilter
-	RetentionPolicy *RetentionGrid // TODO abstract interface to support future policies?
-}
-
-type Autosnap struct {
-	JobName       string // for use with jobrun package
-	Prefix        string
-	Interval      jobrun.RepeatStrategy
-	DatasetFilter zfs.DatasetFilter
-}
-
-type Config struct {
-	Remotes   map[string]*Remote
-	Pushs     map[string]*Push            // job name -> job
-	Pulls     map[string]*Pull            // job name -> job
-	Sinks     map[string]DatasetMapFilter // client identity -> mapping
-	PullACLs  map[string]DatasetMapFilter // client identity -> filter
-	Prunes    map[string]*Prune           // job name -> job
-	Autosnaps map[string]*Autosnap        // job name -> job
-}
-
-func ParseConfig(path string) (config Config, err error) {
-
-	c := make(map[string]interface{}, 0)
-
-	var bytes []byte
-
-	if bytes, err = ioutil.ReadFile(path); err != nil {
-		return
-	}
-
-	if err = yaml.Unmarshal(bytes, &c); err != nil {
-		return
-	}
-
-	return parseMain(c)
-}
-
-func parseMain(root map[string]interface{}) (c Config, err error) {
-	if c.Remotes, err = parseRemotes(root["remotes"]); err != nil {
-		return
-	}
-
-	remoteLookup := func(name string) (remote *Remote, err error) {
-		remote = c.Remotes[name]
-		if remote == nil {
-			err = fmt.Errorf("remote '%s' not defined", name)
-		}
-		return
-	}
-
-	if c.Pushs, err = parsePushs(root["pushs"], remoteLookup); err != nil {
-		return
-	}
-	if c.Pulls, err = parsePulls(root["pulls"], remoteLookup); err != nil {
-		return
-	}
-	if c.Sinks, err = parseSinks(root["sinks"]); err != nil {
-		return
-	}
-	if c.PullACLs, err = parsePullACLs(root["pull_acls"]); err != nil {
-		return
-	}
-	if c.Prunes, err = parsePrunes(root["prune"]); err != nil {
-		return
-	}
-	if c.Autosnaps, err = parseAutosnaps(root["autosnap"]); err != nil {
-		return
-	}
-
-	return
-}
-
-func fullJobName(section, name string) (full string, err error) {
-	if len(name) < 1 {
-		err = fmt.Errorf("job name not set")
-		return
-	}
-	full = fmt.Sprintf("%s.%s", section, name)
-	return
-}
-
-func parseRemotes(v interface{}) (remotes map[string]*Remote, err error) {
-
-	asMap := make(map[string]struct {
-		Transport map[string]interface{}
-	}, 0)
-	if err = mapstructure.Decode(v, &asMap); err != nil {
-		return
-	}
-
-	remotes = make(map[string]*Remote, len(asMap))
-	for name, p := range asMap {
-
-		if name == rpc.LOCAL_TRANSPORT_IDENTITY {
-			err = errors.New(fmt.Sprintf("remote name '%s' reserved for local pulls", rpc.LOCAL_TRANSPORT_IDENTITY))
-			return
-		}
-
-		var transport Transport
-		if transport, err = parseTransport(p.Transport); err != nil {
-			return
-		}
-		remotes[name] = &Remote{
-			Name:      name,
-			Transport: transport,
-		}
-	}
-
-	return
-}
-
-func parseTransport(it map[string]interface{}) (t Transport, err error) {
-
-	if len(it) != 1 {
-		err = errors.New("ambiguous transport type")
-		return
-	}
-
-	for key, val := range it {
-		switch key {
-		case "ssh":
-			t := SSHTransport{}
-			if err = mapstructure.Decode(val, &t); err != nil {
-				err = errors.New(fmt.Sprintf("could not parse ssh transport: %s", err))
-				return nil, err
-			}
-			return t, nil
-		default:
-			return nil, errors.New(fmt.Sprintf("unknown transport type '%s'\n", key))
-		}
-	}
-
-	return // unreachable
-
-}
-
-type remoteLookup func(name string) (*Remote, error)
-
-func parsePushs(v interface{}, rl remoteLookup) (p map[string]*Push, err error) {
-
-	asMap := make(map[string]struct {
-		To                string
-		Filter            map[string]string
-		InitialReplPolicy string
-		Repeat            map[string]string
-	}, 0)
-
-	if err = mapstructure.Decode(v, &asMap); err != nil {
-		return
-	}
-
-	p = make(map[string]*Push, len(asMap))
-
-	for name, e := range asMap {
-
-		var toRemote *Remote
-		if toRemote, err = rl(e.To); err != nil {
-			return
-		}
-		push := &Push{
-			To: toRemote,
-		}
-
-		if push.JobName, err = fullJobName(JobSectionPush, name); err != nil {
-			return
-		}
-		if push.Filter, err = parseDatasetMapFilter(e.Filter, true); err != nil {
-			return
-		}
-
-		if push.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, rpc.DEFAULT_INITIAL_REPL_POLICY); err != nil {
-			return
-		}
-
-		if push.RepeatStrategy, err = parseRepeatStrategy(e.Repeat); err != nil {
-			return
-		}
-
-		p[name] = push
-	}
-
-	return
-}
-
-func parsePulls(v interface{}, rl remoteLookup) (p map[string]*Pull, err error) {
-
-	asMap := make(map[string]struct {
-		From              string
-		Mapping           map[string]string
-		InitialReplPolicy string
-		Repeat            map[string]string
-	}, 0)
-
-	if err = mapstructure.Decode(v, &asMap); err != nil {
-		return
-	}
-
-	p = make(map[string]*Pull, len(asMap))
-
-	for name, e := range asMap {
-
-		if len(e.From) < 1 {
-			err = fmt.Errorf("source not set ('from' attribute is empty)")
-			return
-		}
-
-		var fromRemote *Remote
-
-		if e.From == rpc.LOCAL_TRANSPORT_IDENTITY {
-			fromRemote = &Remote{
-				Name:      rpc.LOCAL_TRANSPORT_IDENTITY,
-				Transport: LocalTransport{},
-			}
-		} else {
-			if fromRemote, err = rl(e.From); err != nil {
-				return
-			}
-		}
-
-		pull := &Pull{
-			From: fromRemote,
-		}
-		if pull.JobName, err = fullJobName(JobSectionPull, name); err != nil {
-			return
-		}
-		if pull.Mapping, err = parseDatasetMapFilter(e.Mapping, false); err != nil {
-			return
-		}
-		if pull.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, rpc.DEFAULT_INITIAL_REPL_POLICY); err != nil {
-			return
-		}
-		if pull.RepeatStrategy, err = parseRepeatStrategy(e.Repeat); err != nil {
-			return
-		}
-
-		p[name] = pull
-	}
-
-	return
-}
-
-func parseInitialReplPolicy(v interface{}, defaultPolicy rpc.InitialReplPolicy) (p rpc.InitialReplPolicy, err error) {
-	s, ok := v.(string)
-	if !ok {
-		goto err
-	}
-
-	switch {
-	case s == "":
-		p = defaultPolicy
-	case s == "most_recent":
-		p = rpc.InitialReplPolicyMostRecent
-	case s == "all":
-		p = rpc.InitialReplPolicyAll
-	default:
-		goto err
-	}
-
-	return
-
-err:
-	err = errors.New(fmt.Sprintf("expected InitialReplPolicy, got %#v", v))
-	return
-}
-
-func parseRepeatStrategy(r map[string]string) (s jobrun.RepeatStrategy, err error) {
-
-	if r == nil {
-		return jobrun.NoRepeatStrategy{}, nil
-	}
-
-	if repeatStr, ok := r["interval"]; ok {
-		d, err := parseDuration(repeatStr)
-		if err != nil {
-			return nil, err
-		}
-		s = &jobrun.PeriodicRepeatStrategy{d}
-		return s, err
-	} else {
-		return nil, fmt.Errorf("attribute 'interval' not found but required in repeat specification")
-	}
-
-}
-
-func expectList(v interface{}) (asList []interface{}, err error) {
-	var ok bool
-	if asList, ok = v.([]interface{}); !ok {
-		err = errors.New("expected list")
-	}
-	return
-}
-
-func parseSinks(v interface{}) (m map[string]DatasetMapFilter, err error) {
-	var asMap map[string]map[string]interface{}
-	if err = mapstructure.Decode(v, &asMap); err != nil {
-		return
-	}
-
-	m = make(map[string]DatasetMapFilter, len(asMap))
-
-	for identity, entry := range asMap {
-		parseSink := func() (mapping DatasetMapFilter, err error) {
-			mappingMap, ok := entry["mapping"]
-			if !ok {
-				err = fmt.Errorf("no mapping specified")
-				return
-			}
-			mapping, err = parseDatasetMapFilter(mappingMap, false)
-			return
-		}
-		mapping, sinkErr := parseSink()
-		if sinkErr != nil {
-			err = fmt.Errorf("cannot parse sink for identity '%s': %s", identity, sinkErr)
-			return
-		}
-		m[identity] = mapping
-	}
-	return
-}
-
-func parsePullACLs(v interface{}) (m map[string]DatasetMapFilter, err error) {
-	var asMap map[string]map[string]interface{}
-	if err = mapstructure.Decode(v, &asMap); err != nil {
-		return
-	}
-
-	m = make(map[string]DatasetMapFilter, len(asMap))
-
-	for identity, entry := range asMap {
-		parsePullACL := func() (filter DatasetMapFilter, err error) {
-			filterMap, ok := entry["filter"]
-			if !ok {
-				err = fmt.Errorf("no filter specified")
-				return
-			}
-			filter, err = parseDatasetMapFilter(filterMap, true)
-			return
-		}
-		filter, filterErr := parsePullACL()
-		if filterErr != nil {
-			err = fmt.Errorf("cannot parse pull-ACL for identity '%s': %s", identity, filterErr)
-			return
-		}
-		m[identity] = filter
-	}
-	return
-}
-
-func parseDatasetMapFilter(mi interface{}, filterOnly bool) (f DatasetMapFilter, err error) {
-
-	var m map[string]string
-	if err = mapstructure.Decode(mi, &m); err != nil {
-		err = fmt.Errorf("maps / filters must be specified as map[string]string: %s", err)
-		return
-	}
-
-	f = NewDatasetMapFilter(len(m), filterOnly)
-	for pathPattern, mapping := range m {
-		if err = f.Add(pathPattern, mapping); err != nil {
-			err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err)
-			return
-		}
-	}
-	return
-}
-
-func (t SSHTransport) Connect(rpcLog Logger) (r rpc.RPCRequester, err error) {
-	var stream io.ReadWriteCloser
-	var rpcTransport sshbytestream.SSHTransport
-	if err = copier.Copy(&rpcTransport, t); err != nil {
-		return
-	}
-	if stream, err = sshbytestream.Outgoing(rpcTransport); err != nil {
-		return
-	}
-	stream, err = NewReadWriteCloserLogger(stream, t.ConnLogReadFile, t.ConnLogWriteFile)
-	if err != nil {
-		return
-	}
-	return rpc.ConnectByteStreamRPC(stream, rpcLog)
-}
-
-func (t LocalTransport) Connect(rpcLog Logger) (r rpc.RPCRequester, err error) {
-	if t.Handler == nil {
-		panic("local transport with uninitialized handler")
-	}
-	return rpc.ConnectLocalRPC(t.Handler), nil
-}
-
-func (t *LocalTransport) SetHandler(handler rpc.RPCHandler) {
-	t.Handler = handler
-}
-
-func parsePrunes(m interface{}) (rets map[string]*Prune, err error) {
-
-	asList := make(map[string]map[string]interface{}, 0)
-	if err = mapstructure.Decode(m, &asList); err != nil {
-		return
-	}
-
-	rets = make(map[string]*Prune, len(asList))
-
-	for name, e := range asList {
-		var prune *Prune
-		if prune, err = parsePrune(e, name); err != nil {
-			err = fmt.Errorf("cannot parse prune job %s: %s", name, err)
-			return
-		}
-		rets[name] = prune
-	}
-
-	return
-}
-
-func parsePrune(e map[string]interface{}, name string) (prune *Prune, err error) {
-
-	// Only support grid policy for now
-	policyName, ok := e["policy"]
-	if !ok || policyName != "grid" {
-		err = fmt.Errorf("prune job with unimplemented policy '%s'", policyName)
-		return
-	}
-
-	var i struct {
-		Grid           string
-		DatasetFilter  map[string]string `mapstructure:"dataset_filter"`
-		SnapshotFilter map[string]string `mapstructure:"snapshot_filter"`
-	}
-
-	if err = mapstructure.Decode(e, &i); err != nil {
-		return
-	}
-
-	prune = &Prune{}
-
-	if prune.JobName, err = fullJobName(JobSectionPrune, name); err != nil {
-		return
-	}
-
-	// Parse grid policy
-	intervals, err := parseRetentionGridIntervalsString(i.Grid)
-	if err != nil {
-		err = fmt.Errorf("cannot parse retention grid: %s", err)
-		return
-	}
-	// Assert intervals are of increasing length (not necessarily required, but indicates config mistake)
-	lastDuration := time.Duration(0)
-	for i := range intervals {
-		if intervals[i].Length < lastDuration {
-			err = fmt.Errorf("retention grid interval length must be monotonically increasing:"+
-				"interval %d is shorter than %d", i+1, i)
-			return
-		} else {
-			lastDuration = intervals[i].Length
-		}
-	}
-	prune.RetentionPolicy = NewRetentionGrid(intervals)
-
-	// Parse filters
-	if prune.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil {
-		err = fmt.Errorf("cannot parse dataset filter: %s", err)
-		return
-	}
-	if prune.SnapshotFilter, err = parseSnapshotFilter(i.SnapshotFilter); err != nil {
-		err = fmt.Errorf("cannot parse snapshot filter: %s", err)
-		return
-	}
-
-	return
-}
-
-var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
-
-func parseDuration(e string) (d time.Duration, err error) {
-	comps := durationStringRegex.FindStringSubmatch(e)
-	if len(comps) != 3 {
-		err = fmt.Errorf("does not match regex: %s %#v", e, comps)
-		return
-	}
-
-	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
-	if err != nil {
-		return
-	}
-
-	var durationUnit time.Duration
-	switch comps[2] {
-	case "s":
-		durationUnit = time.Second
-	case "m":
-		durationUnit = time.Minute
-	case "h":
-		durationUnit = time.Hour
-	case "d":
-		durationUnit = 24 * time.Hour
-	case "w":
-		durationUnit = 24 * 7 * time.Hour
-	default:
-		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
-		return
-	}
-
-	d = time.Duration(durationFactor) * durationUnit
-	return
-
-}
-
-var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
-
-func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {
-
-	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
-	if comps == nil {
-		err = fmt.Errorf("retention string does not match expected format")
-		return
-	}
-
-	times, err := strconv.Atoi(comps[1])
-	if err != nil {
-		return nil, err
-	} else if times <= 0 {
-		return nil, fmt.Errorf("contains factor <= 0")
-	}
-
-	duration, err := parseDuration(comps[2])
-	if err != nil {
-		return nil, err
-	}
-
-	keepCount := 1
-	if comps[3] != "" {
-		// Decompose key=value, comma separated
-		// For now, only keep_count is supported
-		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
-		res := re.FindStringSubmatch(comps[4])
-		if res == nil || len(res) != 2 {
-			err = fmt.Errorf("interval parameter contains unknown parameters")
-			return
-		}
-		if res[1] == "all" {
-			keepCount = RetentionGridKeepCountAll
-		} else {
-			keepCount, err = strconv.Atoi(res[1])
-			if err != nil {
-				err = fmt.Errorf("cannot parse keep_count value")
-				return
-			}
-		}
-	}
-
-	intervals = make([]RetentionInterval, times)
-	for i := range intervals {
-		intervals[i] = RetentionInterval{
-			Length:    duration,
-			KeepCount: keepCount,
-		}
-	}
-
-	return
-
-}
-
-func parseRetentionGridIntervalsString(s string) (intervals []RetentionInterval, err error) {
-
-	ges := strings.Split(s, "|")
-	intervals = make([]RetentionInterval, 0, 7*len(ges))
-
-	for intervalIdx, e := range ges {
-		parsed, err := parseRetentionGridIntervalString(e)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
-		}
-		intervals = append(intervals, parsed...)
-	}
-
-	return
-}
-
-type prefixSnapshotFilter struct {
-	prefix string
-}
-
-func (f prefixSnapshotFilter) Filter(fsv zfs.FilesystemVersion) (accept bool, err error) {
-	return fsv.Type == zfs.Snapshot && strings.HasPrefix(fsv.Name, f.prefix), nil
-}
-
-func parseSnapshotFilter(fm map[string]string) (snapFilter zfs.FilesystemVersionFilter, err error) {
-	prefix, ok := fm["prefix"]
-	if !ok {
-		err = fmt.Errorf("unsupported snapshot filter")
-		return
-	}
-	snapFilter = prefixSnapshotFilter{prefix}
-	return
-}
-
-func parseAutosnaps(m interface{}) (snaps map[string]*Autosnap, err error) {
-
-	asMap := make(map[string]interface{}, 0)
-	if err = mapstructure.Decode(m, &asMap); err != nil {
-		return
-	}
-
-	snaps = make(map[string]*Autosnap, len(asMap))
-
-	for name, e := range asMap {
-		var snap *Autosnap
-		if snap, err = parseAutosnap(e, name); err != nil {
-			err = fmt.Errorf("cannot parse autonsap job %s: %s", name, err)
-			return
-		}
-		snaps[name] = snap
-	}
-
-	return
-
-}
-
-func parseAutosnap(m interface{}, name string) (a *Autosnap, err error) {
-
-	var i struct {
-		Prefix        string
-		Interval      string
-		DatasetFilter map[string]string `mapstructure:"dataset_filter"`
-	}
-
-	if err = mapstructure.Decode(m, &i); err != nil {
-		err = fmt.Errorf("structure unfit: %s", err)
-		return
-	}
-
-	a = &Autosnap{}
-
-	if a.JobName, err = fullJobName(JobSectionAutosnap, name); err != nil {
-		return
-	}
-
-	if len(i.Prefix) < 1 {
-		err = fmt.Errorf("prefix must not be empty")
-		return
-	}
-	a.Prefix = i.Prefix
-
-	var interval time.Duration
-	if interval, err = parseDuration(i.Interval); err != nil {
-		err = fmt.Errorf("cannot parse interval: %s", err)
-		return
-	}
-	a.Interval = &jobrun.PeriodicRepeatStrategy{interval}
-
-	if len(i.DatasetFilter) == 0 {
-		err = fmt.Errorf("dataset_filter not specified")
-		return
-	}
-	if a.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil {
-		err = fmt.Errorf("cannot parse dataset filter: %s", err)
-	}
-
-	return
-}

cmd/config_mapfilter.go

@@ -1,162 +0,0 @@
-package cmd
-
-import (
-	"errors"
-	"fmt"
-	"strings"
-
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type DatasetMapFilter struct {
-	entries []datasetMapFilterEntry
-
-	// if set, only valid filter entries can be added using Add()
-	// and Map() will always return an error
-	filterOnly bool
-}
-
-type datasetMapFilterEntry struct {
-	path *zfs.DatasetPath
-	// the mapping. since this datastructure acts as both mapping and filter
-	// we have to convert it to the desired rep dynamically
-	mapping      string
-	subtreeMatch bool
-}
-
-var NoMatchError error = errors.New("no match found in mapping")
-
-func NewDatasetMapFilter(capacity int, filterOnly bool) DatasetMapFilter {
-	return DatasetMapFilter{
-		entries: make([]datasetMapFilterEntry, 0, capacity),
-	}
-}
-
-func (m *DatasetMapFilter) Add(pathPattern, mapping string) (err error) {
-
-	if m.filterOnly {
-		if _, err = parseDatasetFilterResult(mapping); err != nil {
-			return
-		}
-	}
-
-	// assert path glob adheres to spec
-	const SUBTREE_PATTERN string = "<"
-	patternCount := strings.Count(pathPattern, SUBTREE_PATTERN)
-	switch {
-	case patternCount > 1:
-	case patternCount == 1 && !strings.HasSuffix(pathPattern, SUBTREE_PATTERN):
-		err = fmt.Errorf("pattern invalid: only one '<' at end of string allowed")
-		return
-	}
-
-	pathStr := strings.TrimSuffix(pathPattern, SUBTREE_PATTERN)
-	path, err := zfs.NewDatasetPath(pathStr)
-	if err != nil {
-		return fmt.Errorf("pattern is not a dataset path: %s", err)
-	}
-
-	entry := datasetMapFilterEntry{
-		path:         path,
-		mapping:      mapping,
-		subtreeMatch: patternCount > 0,
-	}
-	m.entries = append(m.entries, entry)
-	return
-
-}
-
-// find the most specific prefix mapping we have
-//
-// longer prefix wins over shorter prefix, direct wins over glob
-func (m DatasetMapFilter) mostSpecificPrefixMapping(path *zfs.DatasetPath) (idx int, found bool) {
-	lcp, lcp_entry_idx := -1, -1
-	direct_idx := -1
-	for e := range m.entries {
-		entry := m.entries[e]
-		ep := m.entries[e].path
-		lep := ep.Length()
-
-		switch {
-		case !entry.subtreeMatch && ep.Equal(path):
-			direct_idx = e
-			continue
-		case entry.subtreeMatch && path.HasPrefix(ep) && lep > lcp:
-			lcp = lep
-			lcp_entry_idx = e
-		default:
-			continue
-		}
-	}
-
-	if lcp_entry_idx >= 0 || direct_idx >= 0 {
-		found = true
-		switch {
-		case direct_idx >= 0:
-			idx = direct_idx
-		case lcp_entry_idx >= 0:
-			idx = lcp_entry_idx
-		}
-	}
-	return
-}
-
-func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error) {
-
-	if m.filterOnly {
-		err = fmt.Errorf("using a filter for mapping simply does not work")
-		return
-	}
-
-	mi, hasMapping := m.mostSpecificPrefixMapping(source)
-	if !hasMapping {
-		err = NoMatchError
-		return
-	}
-	me := m.entries[mi]
-
-	target, err = zfs.NewDatasetPath(me.mapping)
-	if err != nil {
-		err = fmt.Errorf("mapping target is not a dataset path: %s", err)
-		return
-	}
-	if m.entries[mi].subtreeMatch {
-		// strip common prefix
-		extendComps := source.Copy()
-		if me.path.Empty() {
-			// special case: trying to map the root => strip first component
-			extendComps.TrimNPrefixComps(1)
-		} else {
-			extendComps.TrimPrefix(me.path)
-		}
-		target.Extend(extendComps)
-	}
-	return
-}
-
-func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
-	mi, hasMapping := m.mostSpecificPrefixMapping(p)
-	if !hasMapping {
-		pass = false
-		return
-	}
-	me := m.entries[mi]
-	pass, err = parseDatasetFilterResult(me.mapping)
-	return
-}
-
-// Parse a dataset filter result
-func parseDatasetFilterResult(result string) (pass bool, err error) {
-	l := strings.ToLower(result)
-	switch strings.ToLower(l) {
-	case "ok":
-		pass = true
-		return
-	case "omit":
-		return
-	default:
-		err = fmt.Errorf("'%s' is not a valid filter result", result)
-		return
-	}
-	return
-}

cmd/config_test.go

@@ -1,123 +0,0 @@
-package cmd
-
-import (
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/zrepl/zrepl/util"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-func TestSampleConfigFileIsParsedWithoutErrors(t *testing.T) {
-	_, err := ParseConfig("./sampleconf/zrepl.yml")
-	assert.Nil(t, err)
-}
-
-func TestParseRetentionGridStringParsing(t *testing.T) {
-
-	intervals, err := parseRetentionGridIntervalsString("2x10m(keep=2) | 1x1h | 3x1w")
-
-	assert.Nil(t, err)
-	assert.Len(t, intervals, 6)
-	proto := util.RetentionInterval{
-		KeepCount: 2,
-		Length:    10 * time.Minute,
-	}
-	assert.EqualValues(t, proto, intervals[0])
-	assert.EqualValues(t, proto, intervals[1])
-
-	proto.KeepCount = 1
-	proto.Length = 1 * time.Hour
-	assert.EqualValues(t, proto, intervals[2])
-
-	proto.Length = 7 * 24 * time.Hour
-	assert.EqualValues(t, proto, intervals[3])
-	assert.EqualValues(t, proto, intervals[4])
-	assert.EqualValues(t, proto, intervals[5])
-
-	intervals, err = parseRetentionGridIntervalsString("|")
-	assert.Error(t, err)
-	intervals, err = parseRetentionGridIntervalsString("2x10m")
-	assert.NoError(t, err)
-
-	intervals, err = parseRetentionGridIntervalsString("1x10m(keep=all)")
-	assert.NoError(t, err)
-	assert.Len(t, intervals, 1)
-	assert.EqualValues(t, util.RetentionGridKeepCountAll, intervals[0].KeepCount)
-
-}
-
-func TestDatasetMapFilter(t *testing.T) {
-
-	expectMapping := func(m map[string]string, from, to string) {
-		dmf, err := parseDatasetMapFilter(m, false)
-		if err != nil {
-			t.Logf("expect test map to be valid: %s", err)
-			t.FailNow()
-		}
-		fromPath, err := zfs.NewDatasetPath(from)
-		if err != nil {
-			t.Logf("expect test from path to be valid: %s", err)
-			t.FailNow()
-		}
-		toPath, err := zfs.NewDatasetPath(to)
-		if err != nil {
-			t.Logf("expect test to path to be valid: %s", err)
-			t.FailNow()
-		}
-
-		res, err := dmf.Map(fromPath)
-		t.Logf("%s => %s", fromPath.ToString(), res.ToString())
-		assert.Nil(t, err)
-		assert.True(t, res.Equal(toPath))
-	}
-
-	expectFilter := func(m map[string]string, path string, pass bool) {
-		dmf, err := parseDatasetMapFilter(m, true)
-		if err != nil {
-			t.Logf("expect test filter to be valid: %s", err)
-			t.FailNow()
-		}
-		p, err := zfs.NewDatasetPath(path)
-		if err != nil {
-			t.Logf("expect test path to be valid: %s", err)
-			t.FailNow()
-		}
-		res, err := dmf.Filter(p)
-		assert.Nil(t, err)
-		assert.Equal(t, pass, res)
-	}
-
-	map1 := map[string]string{
-		"a/b/c<": "root1",
-		"a/b<":   "root2",
-		"<":      "root3/b/c",
-		"q<":     "root4/1/2",
-	}
-
-	expectMapping(map1, "a/b/c", "root1")
-	expectMapping(map1, "a/b/c/d", "root1/d")
-	expectMapping(map1, "a/b/e", "root2/e")
-	expectMapping(map1, "a/b", "root2")
-	expectMapping(map1, "x", "root3/b/c")
-	expectMapping(map1, "x/y", "root3/b/c/y")
-	expectMapping(map1, "q", "root4/1/2")
-	expectMapping(map1, "q/r", "root4/1/2/r")
-
-	filter1 := map[string]string{
-		"<":    "omit",
-		"a<":   "ok",
-		"a/b<": "omit",
-	}
-
-	expectFilter(filter1, "b", false)
-	expectFilter(filter1, "a", true)
-	expectFilter(filter1, "a/d", true)
-	expectFilter(filter1, "a/b", false)
-	expectFilter(filter1, "a/b/c", false)
-
-	filter2 := map[string]string{}
-	expectFilter(filter2, "foo", false) // default to omit
-
-}

cmd/handler.go

@@ -1,137 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/zfs"
-	"io"
-)
-
-type DatasetMapping interface {
-	Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error)
-}
-
-type Handler struct {
-	Logger          Logger
-	PullACL         zfs.DatasetFilter
-	SinkMappingFunc func(clientIdentity string) (mapping DatasetMapping, err error)
-}
-
-func (h Handler) HandleFilesystemRequest(r rpc.FilesystemRequest) (roots []*zfs.DatasetPath, err error) {
-
-	h.Logger.Printf("handling fsr: %#v", r)
-
-	h.Logger.Printf("using PullACL: %#v", h.PullACL)
-
-	if roots, err = zfs.ZFSListMapping(h.PullACL); err != nil {
-		h.Logger.Printf("handle fsr err: %v\n", err)
-		return
-	}
-
-	h.Logger.Printf("returning: %#v", roots)
-
-	return
-}
-
-func (h Handler) HandleFilesystemVersionsRequest(r rpc.FilesystemVersionsRequest) (versions []zfs.FilesystemVersion, err error) {
-
-	h.Logger.Printf("handling filesystem versions request: %#v", r)
-
-	// allowed to request that?
-	if h.pullACLCheck(r.Filesystem); err != nil {
-		return
-	}
-
-	// find our versions
-	if versions, err = zfs.ZFSListFilesystemVersions(r.Filesystem, nil); err != nil {
-		h.Logger.Printf("our versions error: %#v\n", err)
-		return
-	}
-
-	h.Logger.Printf("our versions: %#v\n", versions)
-	return
-
-}
-
-func (h Handler) HandleInitialTransferRequest(r rpc.InitialTransferRequest) (stream io.Reader, err error) {
-
-	h.Logger.Printf("handling initial transfer request: %#v", r)
-	if err = h.pullACLCheck(r.Filesystem); err != nil {
-		return
-	}
-
-	h.Logger.Printf("invoking zfs send")
-
-	if stream, err = zfs.ZFSSend(r.Filesystem, &r.FilesystemVersion, nil); err != nil {
-		h.Logger.Printf("error sending filesystem: %#v", err)
-	}
-
-	return
-
-}
-
-func (h Handler) HandleIncrementalTransferRequest(r rpc.IncrementalTransferRequest) (stream io.Reader, err error) {
-
-	h.Logger.Printf("handling incremental transfer request: %#v", r)
-	if err = h.pullACLCheck(r.Filesystem); err != nil {
-		return
-	}
-
-	h.Logger.Printf("invoking zfs send")
-
-	if stream, err = zfs.ZFSSend(r.Filesystem, &r.From, &r.To); err != nil {
-		h.Logger.Printf("error sending filesystem: %#v", err)
-	}
-
-	return
-
-}
-
-func (h Handler) HandlePullMeRequest(r rpc.PullMeRequest, clientIdentity string, client rpc.RPCRequester) (err error) {
-
-	// Check if we have a sink for this request
-	// Use that mapping to do what happens in doPull
-
-	h.Logger.Printf("handling PullMeRequest: %#v", r)
-
-	var sinkMapping DatasetMapping
-	sinkMapping, err = h.SinkMappingFunc(clientIdentity)
-	if err != nil {
-		h.Logger.Printf("no sink mapping for client identity '%s', denying PullMeRequest", clientIdentity)
-		err = fmt.Errorf("no sink for client identity '%s'", clientIdentity)
-		return
-	}
-
-	h.Logger.Printf("doing pull...")
-
-	err = doPull(PullContext{
-		Remote:            client,
-		Log:               h.Logger,
-		Mapping:           sinkMapping,
-		InitialReplPolicy: r.InitialReplPolicy,
-	})
-	if err != nil {
-		h.Logger.Printf("PullMeRequest failed with error: %s", err)
-		return
-	}
-
-	h.Logger.Printf("finished handling PullMeRequest: %#v", r)
-
-	return
-}
-
-func (h Handler) pullACLCheck(p *zfs.DatasetPath) (err error) {
-	var allowed bool
-	allowed, err = h.PullACL.Filter(p)
-	if err != nil {
-		err = fmt.Errorf("error evaluating ACL: %s", err)
-		h.Logger.Printf(err.Error())
-		return
-	}
-	if !allowed {
-		err = fmt.Errorf("ACL prohibits access to %s", p.ToString())
-		h.Logger.Printf(err.Error())
-		return
-	}
-	return
-}

cmd/main.go

@@ -1,104 +0,0 @@
-// zrepl replicates ZFS filesystems & volumes between pools
-//
-// Code Organization
-//
-// The cmd package uses github.com/spf13/cobra for its CLI.
-//
-// It combines the other packages in the zrepl project to implement zrepl functionality.
-//
-// Each subcommand's code is in the corresponding *.go file.
-// All other *.go files contain code shared by the subcommands.
-package cmd
-
-import (
-	"fmt"
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/jobrun"
-	"golang.org/x/sys/unix"
-	"io"
-	golog "log"
-	"net/http"
-	_ "net/http/pprof"
-	"os"
-)
-
-type Logger interface {
-	Printf(format string, v ...interface{})
-}
-
-// global state / facilities
-var (
-	conf     Config
-	runner   *jobrun.JobRunner
-	logFlags int = golog.LUTC | golog.Ldate | golog.Ltime
-	logOut   io.Writer
-	log      Logger
-)
-
-var RootCmd = &cobra.Command{
-	Use:   "zrepl",
-	Short: "ZFS dataset replication",
-	Long: `Replicate ZFS filesystems & volumes between pools:
-
-  - push & pull mode
-  - automatic snapshot creation & pruning
-  - local / over the network
-  - ACLs instead of blank SSH access`,
-}
-
-var rootArgs struct {
-	configFile string
-	stderrFile string
-	httpPprof  string
-}
-
-func init() {
-	cobra.OnInitialize(initConfig)
-	RootCmd.PersistentFlags().StringVar(&rootArgs.configFile, "config", "", "config file path")
-	RootCmd.PersistentFlags().StringVar(&rootArgs.stderrFile, "stderrFile", "-", "redirect stderr to given path")
-	RootCmd.PersistentFlags().StringVar(&rootArgs.httpPprof, "debug.pprof.http", "", "run pprof http server on given port")
-}
-
-func initConfig() {
-
-	// Logging & stderr redirection
-	if rootArgs.stderrFile != "-" {
-		file, err := os.OpenFile(rootArgs.stderrFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
-		if err != nil {
-			return
-		}
-
-		if err = unix.Dup2(int(file.Fd()), int(os.Stderr.Fd())); err != nil {
-			file.WriteString(fmt.Sprintf("error redirecting stderr file %s: %s\n", rootArgs.stderrFile, err))
-			return
-		}
-		logOut = file
-	} else {
-		logOut = os.Stderr
-	}
-
-	log = golog.New(logOut, "", logFlags)
-
-	// CPU profiling
-	if rootArgs.httpPprof != "" {
-		go func() {
-			http.ListenAndServe(rootArgs.httpPprof, nil)
-		}()
-	}
-
-	// Config
-	if rootArgs.configFile == "" {
-		log.Printf("config file not set")
-		os.Exit(1)
-	}
-	var err error
-	if conf, err = ParseConfig(rootArgs.configFile); err != nil {
-		log.Printf("error parsing config: %s", err)
-		os.Exit(1)
-	}
-
-	jobrunLogger := golog.New(os.Stderr, "jobrun ", logFlags)
-	runner = jobrun.NewJobRunner(jobrunLogger)
-	return
-
-}

cmd/prune.go

@@ -1,139 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/util"
-	"github.com/zrepl/zrepl/zfs"
-	"os"
-	"sort"
-	"time"
-)
-
-var pruneArgs struct {
-	job    string
-	dryRun bool
-}
-
-var PruneCmd = &cobra.Command{
-	Use:   "prune",
-	Short: "perform pruning",
-	Run:   cmdPrune,
-}
-
-func init() {
-	PruneCmd.Flags().StringVar(&pruneArgs.job, "job", "", "job to run")
-	PruneCmd.Flags().BoolVarP(&pruneArgs.dryRun, "dryrun", "n", false, "dry run")
-	RootCmd.AddCommand(PruneCmd)
-}
-
-func cmdPrune(cmd *cobra.Command, args []string) {
-
-	if len(args) < 1 {
-		log.Printf("must specify exactly one job as positional argument")
-		os.Exit(1)
-	}
-
-	job, ok := conf.Prunes[args[0]]
-	if !ok {
-		log.Printf("could not find prune job: %s", args[0])
-		os.Exit(1)
-	}
-
-	log.Printf("Beginning prune job:\n%s", job)
-	ctx := PruneContext{job, time.Now(), pruneArgs.dryRun}
-	err := doPrune(ctx, log)
-	if err != nil {
-		log.Printf("Prune job failed with error: %s", err)
-		os.Exit(1)
-	}
-
-}
-
-type PruneContext struct {
-	Prune  *Prune
-	Now    time.Time
-	DryRun bool
-}
-
-type retentionGridAdaptor struct {
-	zfs.FilesystemVersion
-}
-
-func (a retentionGridAdaptor) Date() time.Time {
-	return a.Creation
-}
-
-func (a retentionGridAdaptor) LessThan(b util.RetentionGridEntry) bool {
-	return a.CreateTXG < b.(retentionGridAdaptor).CreateTXG
-}
-
-func doPrune(ctx PruneContext, log Logger) error {
-
-	if ctx.DryRun {
-		log.Printf("doing dry run")
-	}
-
-	prune := ctx.Prune
-
-	// ZFSListSnapsFiltered --> todo can extend fsfilter or need new? Have already something per fs
-	// Dedicated snapshot object? Adaptor object to FilesystemVersion?
-
-	filesystems, err := zfs.ZFSListMapping(prune.DatasetFilter)
-	if err != nil {
-		return fmt.Errorf("error applying filesystem filter: %s", err)
-	}
-
-	for _, fs := range filesystems {
-
-		fsversions, err := zfs.ZFSListFilesystemVersions(fs, prune.SnapshotFilter)
-		if err != nil {
-			return fmt.Errorf("error listing filesytem versions of %s: %s", fs, err)
-		}
-
-		if len(fsversions) == 0 {
-			continue
-		}
-
-		adaptors := make([]util.RetentionGridEntry, len(fsversions))
-		for fsv := range fsversions {
-			adaptors[fsv] = retentionGridAdaptor{fsversions[fsv]}
-		}
-
-		sort.SliceStable(adaptors, func(i, j int) bool {
-			return adaptors[i].LessThan(adaptors[j])
-		})
-		ctx.Now = adaptors[len(adaptors)-1].Date()
-
-		describe := func(a retentionGridAdaptor) string {
-			timeSince := a.Creation.Sub(ctx.Now)
-			const day time.Duration = 24 * time.Hour
-			days := timeSince / day
-			remainder := timeSince % day
-			return fmt.Sprintf("%s@%dd%s from now", a.ToAbsPath(fs), days, remainder)
-		}
-
-		keep, remove := prune.RetentionPolicy.FitEntries(ctx.Now, adaptors)
-		for _, a := range remove {
-			r := a.(retentionGridAdaptor)
-			log.Printf("remove %s", describe(r))
-			// do echo what we'll do and exec zfs destroy if not dry run
-			// special handling for EBUSY (zfs hold)
-			// error handling for clones? just echo to cli, skip over, and exit with non-zero status code (we're idempotent)
-			if !ctx.DryRun {
-				err := zfs.ZFSDestroy(r.ToAbsPath(fs))
-				if err != nil {
-					// handle
-					log.Printf("error: %s", err)
-				}
-			}
-		}
-		for _, a := range keep {
-			r := a.(retentionGridAdaptor)
-			log.Printf("would keep %s", describe(r))
-		}
-	}
-
-	return nil
-
-}

cmd/replication.go

@@ -1,506 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"io"
-	"os"
-	"sync"
-	"time"
-
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/jobrun"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/util"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-var runArgs struct {
-	job  string
-	once bool
-}
-
-var RunCmd = &cobra.Command{
-	Use:   "run",
-	Short: "run push & pull replication",
-	Run:   cmdRun,
-}
-
-var PushCmd = &cobra.Command{
-	Use:   "push",
-	Short: "run push job (first positional argument)",
-	Run:   cmdPush,
-}
-
-var PullCmd = &cobra.Command{
-	Use:   "pull",
-	Short: "run pull job (first positional argument)",
-	Run:   cmdPull,
-}
-
-func init() {
-	RootCmd.AddCommand(RunCmd)
-	RunCmd.Flags().BoolVar(&runArgs.once, "once", false, "run jobs only once, regardless of configured repeat behavior")
-	RunCmd.Flags().StringVar(&runArgs.job, "job", "", "run only the given job")
-
-	RootCmd.AddCommand(PushCmd)
-	RootCmd.AddCommand(PullCmd)
-}
-
-func cmdPush(cmd *cobra.Command, args []string) {
-
-	if len(args) != 1 {
-		log.Printf("must specify exactly one job as positional argument")
-		os.Exit(1)
-	}
-	job, ok := conf.Pushs[args[0]]
-	if !ok {
-		log.Printf("could not find push job %s", args[0])
-		os.Exit(1)
-	}
-	if err := jobPush(job, log); err != nil {
-		log.Printf("error doing push: %s", err)
-		os.Exit(1)
-	}
-
-}
-
-func cmdPull(cmd *cobra.Command, args []string) {
-
-	if len(args) != 1 {
-		log.Printf("must specify exactly one job as positional argument")
-		os.Exit(1)
-	}
-	job, ok := conf.Pulls[args[0]]
-	if !ok {
-		log.Printf("could not find pull job %s", args[0])
-		os.Exit(1)
-	}
-
-	if err := jobPull(job, log); err != nil {
-		log.Printf("error doing pull: %s", err)
-		os.Exit(1)
-	}
-
-}
-
-func cmdRun(cmd *cobra.Command, args []string) {
-
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		runner.Start()
-	}()
-
-	jobs := make([]jobrun.Job, len(conf.Pulls)+len(conf.Pushs))
-	i := 0
-	for _, pull := range conf.Pulls {
-		jobs[i] = jobrun.Job{
-			Name:           fmt.Sprintf("pull.%d", i),
-			RepeatStrategy: pull.RepeatStrategy,
-			RunFunc: func(log jobrun.Logger) error {
-				log.Printf("doing pull: %v", pull)
-				return jobPull(pull, log)
-			},
-		}
-		i++
-	}
-	for _, push := range conf.Pushs {
-		jobs[i] = jobrun.Job{
-			Name:           fmt.Sprintf("push.%d", i),
-			RepeatStrategy: push.RepeatStrategy,
-			RunFunc: func(log jobrun.Logger) error {
-				log.Printf("doing push: %v", push)
-				return jobPush(push, log)
-			},
-		}
-		i++
-	}
-
-	for _, j := range jobs {
-		if runArgs.once {
-			j.RepeatStrategy = jobrun.NoRepeatStrategy{}
-		}
-		if runArgs.job != "" {
-			if runArgs.job == j.Name {
-				runner.AddJob(j)
-				break
-			}
-			continue
-		}
-		runner.AddJob(j)
-	}
-
-	for {
-		select {
-		case job := <-runner.NotificationChan():
-			log.Printf("job %s reported error: %v\n", job.Name, job.LastError)
-		}
-	}
-
-	wg.Wait()
-
-}
-
-type localPullACL struct{}
-
-func (a localPullACL) Filter(p *zfs.DatasetPath) (pass bool, err error) {
-	return true, nil
-}
-
-func jobPull(pull *Pull, log jobrun.Logger) (err error) {
-
-	if lt, ok := pull.From.Transport.(LocalTransport); ok {
-
-		lt.SetHandler(Handler{
-			Logger: log,
-			// Allow access to any dataset since we control what mapping
-			// is passed to the pull routine.
-			// All local datasets will be passed to its Map() function,
-			// but only those for which a mapping exists will actually be pulled.
-			// We can pay this small performance penalty for now.
-			PullACL: localPullACL{},
-		})
-		pull.From.Transport = lt
-		log.Printf("fixing up local transport: %#v", pull.From.Transport)
-	}
-
-	var remote rpc.RPCRequester
-
-	if remote, err = pull.From.Transport.Connect(log); err != nil {
-		return
-	}
-
-	defer closeRPCWithTimeout(log, remote, time.Second*10, "")
-
-	return doPull(PullContext{remote, log, pull.Mapping, pull.InitialReplPolicy})
-}
-
-func jobPush(push *Push, log jobrun.Logger) (err error) {
-
-	if _, ok := push.To.Transport.(LocalTransport); ok {
-		panic("no support for local pushs")
-	}
-
-	var remote rpc.RPCRequester
-	if remote, err = push.To.Transport.Connect(log); err != nil {
-		return err
-	}
-
-	defer closeRPCWithTimeout(log, remote, time.Second*10, "")
-
-	log.Printf("building handler for PullMeRequest")
-	handler := Handler{
-		Logger:          log,
-		PullACL:         push.Filter,
-		SinkMappingFunc: nil, // no need for that in the handler for PullMe
-	}
-	log.Printf("handler: %#v", handler)
-
-	r := rpc.PullMeRequest{
-		InitialReplPolicy: push.InitialReplPolicy,
-	}
-	log.Printf("doing PullMeRequest: %#v", r)
-
-	if err = remote.PullMeRequest(r, handler); err != nil {
-		log.Printf("PullMeRequest failed: %s", err)
-		return
-	}
-
-	log.Printf("push job finished")
-	return
-
-}
-
-func closeRPCWithTimeout(log Logger, remote rpc.RPCRequester, timeout time.Duration, goodbye string) {
-	log.Printf("closing rpc connection")
-
-	ch := make(chan error)
-	go func() {
-		ch <- remote.CloseRequest(rpc.CloseRequest{goodbye})
-		close(ch)
-	}()
-
-	var err error
-	select {
-	case <-time.After(timeout):
-		err = fmt.Errorf("timeout exceeded (%s)", timeout)
-	case closeRequestErr := <-ch:
-		err = closeRequestErr
-	}
-
-	if err != nil {
-		log.Printf("error closing connection: %s", err)
-		err = remote.ForceClose()
-		if err != nil {
-			log.Printf("error force-closing connection: %s", err)
-		}
-	}
-	return
-}
-
-type PullContext struct {
-	Remote            rpc.RPCRequester
-	Log               Logger
-	Mapping           DatasetMapping
-	InitialReplPolicy rpc.InitialReplPolicy
-}
-
-func doPull(pull PullContext) (err error) {
-
-	remote := pull.Remote
-	log := pull.Log
-
-	log.Printf("requesting remote filesystem list")
-	fsr := rpc.FilesystemRequest{}
-	var remoteFilesystems []*zfs.DatasetPath
-	if remoteFilesystems, err = remote.FilesystemRequest(fsr); err != nil {
-		return
-	}
-
-	log.Printf("map remote filesystems to local paths and determine order for per-filesystem sync")
-	type RemoteLocalMapping struct {
-		Remote *zfs.DatasetPath
-		Local  *zfs.DatasetPath
-	}
-	replMapping := make(map[string]RemoteLocalMapping, len(remoteFilesystems))
-	localTraversal := zfs.NewDatasetPathForest()
-	for fs := range remoteFilesystems {
-		var err error
-		var localFs *zfs.DatasetPath
-		localFs, err = pull.Mapping.Map(remoteFilesystems[fs])
-		if err != nil {
-			if err != NoMatchError {
-				err := fmt.Errorf("error mapping %s: %s", remoteFilesystems[fs], err)
-				log.Printf("%s", err)
-				return err
-			}
-			continue
-		}
-		log.Printf("%s => %s", remoteFilesystems[fs].ToString(), localFs.ToString())
-		m := RemoteLocalMapping{remoteFilesystems[fs], localFs}
-		replMapping[m.Local.ToString()] = m
-		localTraversal.Add(m.Local)
-	}
-
-	log.Printf("build cache for already present local filesystem state")
-	localFilesystemState, err := zfs.ZFSListFilesystemState()
-	if err != nil {
-		log.Printf("error requesting local filesystem state: %s", err)
-		return err
-	}
-
-	log.Printf("start per-filesystem sync")
-	localTraversal.WalkTopDown(func(v zfs.DatasetPathVisit) bool {
-
-		if v.FilledIn {
-			if _, exists := localFilesystemState[v.Path.ToString()]; exists {
-				// No need to verify if this is a placeholder or not. It is sufficient
-				// to know we can add child filesystems to it
-				return true
-			}
-			log.Printf("creating placeholder filesystem %s", v.Path.ToString())
-			err = zfs.ZFSCreatePlaceholderFilesystem(v.Path)
-			if err != nil {
-				err = fmt.Errorf("aborting, cannot create placeholder filesystem %s: %s", v.Path, err)
-				return false
-			}
-			return true
-		}
-
-		m, ok := replMapping[v.Path.ToString()]
-		if !ok {
-			panic("internal inconsistency: replMapping should contain mapping for any path that was not filled in by WalkTopDown()")
-		}
-
-		log := func(format string, args ...interface{}) {
-			log.Printf("[%s => %s]: %s", m.Remote.ToString(), m.Local.ToString(), fmt.Sprintf(format, args...))
-		}
-
-		log("examing local filesystem state")
-		localState, localExists := localFilesystemState[m.Local.ToString()]
-		var versions []zfs.FilesystemVersion
-		switch {
-		case !localExists:
-			log("local filesystem does not exist")
-		case localState.Placeholder:
-			log("local filesystem is marked as placeholder")
-		default:
-			log("local filesystem exists")
-			log("requesting local filesystem versions")
-			if versions, err = zfs.ZFSListFilesystemVersions(m.Local, nil); err != nil {
-				log("cannot get local filesystem versions: %s", err)
-				return false
-			}
-		}
-
-		log("requesting remote filesystem versions")
-		var theirVersions []zfs.FilesystemVersion
-		theirVersions, err = remote.FilesystemVersionsRequest(rpc.FilesystemVersionsRequest{
-			Filesystem: m.Remote,
-		})
-		if err != nil {
-			log("error requesting remote filesystem versions: %s", err)
-			log("stopping replication for all filesystems mapped as children of %s", m.Local.ToString())
-			return false
-		}
-
-		log("computing diff between remote and local filesystem versions")
-		diff := zfs.MakeFilesystemDiff(versions, theirVersions)
-		log("%s", diff)
-
-		if localState.Placeholder && diff.Conflict != zfs.ConflictAllRight {
-			panic("internal inconsistency: local placeholder implies ConflictAllRight")
-		}
-
-		switch diff.Conflict {
-		case zfs.ConflictAllRight:
-
-			log("performing initial sync, following policy: '%s'", pull.InitialReplPolicy)
-
-			if pull.InitialReplPolicy != rpc.InitialReplPolicyMostRecent {
-				panic(fmt.Sprintf("policy '%s' not implemented", pull.InitialReplPolicy))
-			}
-
-			snapsOnly := make([]zfs.FilesystemVersion, 0, len(diff.MRCAPathRight))
-			for s := range diff.MRCAPathRight {
-				if diff.MRCAPathRight[s].Type == zfs.Snapshot {
-					snapsOnly = append(snapsOnly, diff.MRCAPathRight[s])
-				}
-			}
-
-			if len(snapsOnly) < 1 {
-				log("cannot perform initial sync: no remote snapshots. stopping...")
-				return false
-			}
-
-			r := rpc.InitialTransferRequest{
-				Filesystem:        m.Remote,
-				FilesystemVersion: snapsOnly[len(snapsOnly)-1],
-			}
-
-			log("requesting snapshot stream for %s", r.FilesystemVersion)
-
-			var stream io.Reader
-			if stream, err = remote.InitialTransferRequest(r); err != nil {
-				log("error requesting initial transfer: %s", err)
-				return false
-			}
-			log("received initial transfer request response")
-
-			log("invoking zfs receive")
-			watcher := util.IOProgressWatcher{Reader: stream}
-			watcher.KickOff(1*time.Second, func(p util.IOProgress) {
-				log("progress on receive operation: %v bytes received", p.TotalRX)
-			})
-
-			recvArgs := []string{"-u"}
-			if localState.Placeholder {
-				log("receive with forced rollback to replace placeholder filesystem")
-				recvArgs = append(recvArgs, "-F")
-			}
-
-			if err = zfs.ZFSRecv(m.Local, &watcher, recvArgs...); err != nil {
-				log("error receiving stream: %s", err)
-				return false
-			}
-			log("finished receiving stream, %v bytes total", watcher.Progress().TotalRX)
-
-			log("configuring properties of received filesystem")
-			if err = zfs.ZFSSet(m.Local, "readonly", "on"); err != nil {
-
-			}
-
-			log("finished initial transfer")
-			return true
-
-		case zfs.ConflictIncremental:
-
-			if len(diff.IncrementalPath) < 2 {
-				log("remote and local are in sync")
-				return true
-			}
-
-			log("following incremental path from diff")
-			var pathRx uint64
-
-			for i := 0; i < len(diff.IncrementalPath)-1; i++ {
-
-				from, to := diff.IncrementalPath[i], diff.IncrementalPath[i+1]
-
-				log := func(format string, args ...interface{}) {
-					log("[%v/%v][%s => %s]: %s", i+1, len(diff.IncrementalPath)-1,
-						from.Name, to.Name, fmt.Sprintf(format, args...))
-				}
-
-				log("requesting incremental snapshot stream")
-				r := rpc.IncrementalTransferRequest{
-					Filesystem: m.Remote,
-					From:       from,
-					To:         to,
-				}
-				var stream io.Reader
-				if stream, err = remote.IncrementalTransferRequest(r); err != nil {
-					log("error requesting incremental snapshot stream: %s", err)
-					return false
-				}
-
-				log("invoking zfs receive")
-				watcher := util.IOProgressWatcher{Reader: stream}
-				watcher.KickOff(1*time.Second, func(p util.IOProgress) {
-					log("progress on receive operation: %v bytes received", p.TotalRX)
-				})
-
-				if err = zfs.ZFSRecv(m.Local, &watcher); err != nil {
-					log("error receiving stream: %s", err)
-					return false
-				}
-
-				totalRx := watcher.Progress().TotalRX
-				pathRx += totalRx
-				log("finished incremental transfer, %v bytes total", totalRx)
-
-			}
-
-			log("finished following incremental path, %v bytes total", pathRx)
-			return true
-
-		case zfs.ConflictNoCommonAncestor:
-
-			log("remote and local filesystem have snapshots, but no common one")
-			log("perform manual replication to establish a common snapshot history")
-			log("remote versions:")
-			for _, v := range diff.MRCAPathRight {
-				log(" %s (GUID %v)", v, v.Guid)
-			}
-			log("local versions:")
-			for _, v := range diff.MRCAPathLeft {
-				log(" %s (GUID %v)", v, v.Guid)
-			}
-			return false
-
-		case zfs.ConflictDiverged:
-
-			log("remote and local filesystem share a history but have diverged")
-			log("perform manual replication or delete snapshots on the receiving" +
-				"side  to establish an incremental replication parse")
-			log("remote-only versions:")
-			for _, v := range diff.MRCAPathRight {
-				log(" %s (GUID %v)", v, v.Guid)
-			}
-			log("local-only versions:")
-			for _, v := range diff.MRCAPathLeft {
-				log(" %s (GUID %v)", v, v.Guid)
-			}
-			return false
-
-		}
-
-		panic("implementation error: this should not be reached")
-		return false
-
-	})
-
-	return
-
-}

cmd/sampleconf/zrepl.yml

@@ -1,109 +0,0 @@
-remotes:
-  offsite_backups:
-    transport:
-      ssh:
-        host: 192.168.122.6
-        user: root
-        port: 22
-        identity_file: /etc/zrepl/identities/offsite_backups
-
-pushs:
-
-  offsite:
-    to: offsite_backups
-    filter: {
-      # like in pull_acls
-      "tank/var/db<":   ok,
-      "tank/usr/home<": ok,
-    }
-
-pulls:
-
-  offsite:
-    from: offsite_backups
-    mapping: {
-      # like in sinks
-    }
-
-  # local replication, only allowed in pull mode
-  # the from name 'local' is reserved for this purpose
-  homemirror:
-    from: local
-    repeat:
-      interval: 15m
-    mapping: {
-      "tank/usr/home":"mirrorpool/foo/bar"
-    }
-
-sinks:
-
-  db1:
-    mapping: {
-      # direct mapping
-      "ssdpool/var/db/postgresql9.6":"zroot/backups/db1/pg_data"
-    }
-
-  mirror1:
-    mapping: {
-      # "<" subtree wildcard matches the dataset left of < and all its children
-      "tank/foo/bar<":"zroot/backups/mirror1" 
-    }
-
-  mirror2:
-    # more specific path patterns win over less specific ones
-    # direct mappings win over subtree wildcards
-    # detailed rule precedence: check unit tests & docs for exact behavior
-    # TODO subcommand to test a mapping & filter
-    mapping: {
-      "tank<":                       "zroot/backups/mirror1/tank1",
-      "tank/cdn/root<":              "storage/cdn/root",
-      "tank/legacydb":               "legacypool/backups/legacydb",
-    }
-
-
-pull_acls:
-
-  office_backup:
-    filter: {
-      # valid filter results (right hand side): ok, omit
-      # default is to omit
-      # rule precedence is same as for mappings
-      "tank<":         omit,
-      "tank/usr/home": ok,
-    }
-
-
-prune:
-
-  clean_backups:
-    policy: grid
-    grid: 6x10m | 24x1h | 7x1d | 5 x 1w | 4 x 5w
-    dataset_filter: {
-      "tank/backups/legacyscript<": omit,
-      "tank/backups<":              ok,
-    }
-    snapshot_filter: {
-      prefix: zrepl_
-    }
-
-  hfbak_prune: # cleans up after hfbak autosnap job
-    policy: grid
-    grid: 1x1m(keep=all)
-    dataset_filter: {
-      "pool1<": ok
-    }
-    snapshot_filter: {
-      prefix: zrepl_hfbak_
-    }
-
-autosnap:
-
-  hfbak:
-    prefix: zrepl_hfbak_
-    interval: 1s
-    dataset_filter: {
-      "pool1<": ok
-    }
-    # prune: hfbak_prune
-    # future versions may inline the retention policy here, but for now,
-    # pruning has to be triggered manually (it's safe to run autosnap + prune in parallel)

cmd/stdinserver.go

@@ -1,72 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/sshbytestream"
-	"io"
-	golog "log"
-	"os"
-)
-
-var StdinserverCmd = &cobra.Command{
-	Use:   "stdinserver CLIENT_IDENTITY",
-	Short: "start in stdin server mode (from authorized_keys file)",
-	Run:   cmdStdinServer,
-}
-
-func init() {
-	RootCmd.AddCommand(StdinserverCmd)
-}
-
-func cmdStdinServer(cmd *cobra.Command, args []string) {
-
-	var err error
-	defer func() {
-		if err != nil {
-			log.Printf("stdinserver exiting with error: %s", err)
-			os.Exit(1)
-		}
-	}()
-
-	if len(args) != 1 || args[0] == "" {
-		err = fmt.Errorf("must specify client identity as positional argument")
-		return
-	}
-	identity := args[0]
-
-	pullACL, ok := conf.PullACLs[identity]
-	if !ok {
-		err = fmt.Errorf("could not find PullACL for identity '%s'", identity)
-		return
-	}
-
-	var sshByteStream io.ReadWriteCloser
-	if sshByteStream, err = sshbytestream.Incoming(); err != nil {
-		return
-	}
-
-	sinkMapping := func(identity string) (m DatasetMapping, err error) {
-		sink, ok := conf.Sinks[identity]
-		if !ok {
-			return nil, fmt.Errorf("could not find sink for identity '%s'", identity)
-		}
-		return sink, nil
-	}
-
-	sinkLogger := golog.New(logOut, fmt.Sprintf("sink[%s] ", identity), logFlags)
-	handler := Handler{
-		Logger:          sinkLogger,
-		SinkMappingFunc: sinkMapping,
-		PullACL:         pullACL,
-	}
-
-	if err = rpc.ListenByteStreamRPC(sshByteStream, identity, handler, sinkLogger); err != nil {
-		log.Printf("listenbytestreamerror: %#v\n", err)
-		os.Exit(1)
-	}
-
-	return
-
-}

config/config.go

@@ -0,0 +1,703 @@
+package config
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log/syslog"
+	"os"
+	"reflect"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/robfig/cron/v3"
+	"github.com/zrepl/yaml-config"
+
+	"github.com/zrepl/zrepl/util/datasizeunit"
+	zfsprop "github.com/zrepl/zrepl/zfs/property"
+)
+
+type ParseFlags uint
+
+const (
+	ParseFlagsNone        ParseFlags = 0
+	ParseFlagsNoCertCheck ParseFlags = 1 << iota
+)
+
+type Config struct {
+	Jobs   []JobEnum `yaml:"jobs"`
+	Global *Global   `yaml:"global,optional,fromdefaults"`
+}
+
+func (c *Config) Job(name string) (*JobEnum, error) {
+	for _, j := range c.Jobs {
+		if j.Name() == name {
+			return &j, nil
+		}
+	}
+	return nil, fmt.Errorf("job %q not defined in config", name)
+}
+
+type JobEnum struct {
+	Ret interface{}
+}
+
+func (j JobEnum) Name() string {
+	var name string
+	switch v := j.Ret.(type) {
+	case *SnapJob:
+		name = v.Name
+	case *PushJob:
+		name = v.Name
+	case *SinkJob:
+		name = v.Name
+	case *PullJob:
+		name = v.Name
+	case *SourceJob:
+		name = v.Name
+	default:
+		panic(fmt.Sprintf("unknown job type %T", v))
+	}
+	return name
+}
+
+type ActiveJob struct {
+	Type               string                `yaml:"type"`
+	Name               string                `yaml:"name"`
+	Connect            ConnectEnum           `yaml:"connect"`
+	Pruning            PruningSenderReceiver `yaml:"pruning"`
+	Replication        *Replication          `yaml:"replication,optional,fromdefaults"`
+	ConflictResolution *ConflictResolution   `yaml:"conflict_resolution,optional,fromdefaults"`
+}
+
+type ConflictResolution struct {
+	InitialReplication string `yaml:"initial_replication,optional,default=most_recent"`
+}
+
+type PassiveJob struct {
+	Type  string    `yaml:"type"`
+	Name  string    `yaml:"name"`
+	Serve ServeEnum `yaml:"serve"`
+}
+
+type SnapJob struct {
+	Type         string            `yaml:"type"`
+	Name         string            `yaml:"name"`
+	Pruning      PruningLocal      `yaml:"pruning"`
+	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
+	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+}
+
+type SendOptions struct {
+	Encrypted        bool `yaml:"encrypted,optional,default=false"`
+	Raw              bool `yaml:"raw,optional,default=false"`
+	SendProperties   bool `yaml:"send_properties,optional,default=false"`
+	BackupProperties bool `yaml:"backup_properties,optional,default=false"`
+	LargeBlocks      bool `yaml:"large_blocks,optional,default=false"`
+	Compressed       bool `yaml:"compressed,optional,default=false"`
+	EmbeddedData     bool `yaml:"embedded_data,optional,default=false"`
+	Saved            bool `yaml:"saved,optional,default=false"`
+
+	BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
+}
+
+type RecvOptions struct {
+	// Note: we cannot enforce encrypted recv as the ZFS cli doesn't provide a mechanism for it
+	// Encrypted bool `yaml:"may_encrypted"`
+	// Future:
+	// Reencrypt bool `yaml:"reencrypt"`
+
+	Properties *PropertyRecvOptions `yaml:"properties,fromdefaults"`
+
+	BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
+
+	Placeholder *PlaceholderRecvOptions `yaml:"placeholder,fromdefaults"`
+}
+
+var _ yaml.Unmarshaler = &datasizeunit.Bits{}
+
+type BandwidthLimit struct {
+	Max            datasizeunit.Bits `yaml:"max,default=-1 B"`
+	BucketCapacity datasizeunit.Bits `yaml:"bucket_capacity,default=128 KiB"`
+}
+
+type Replication struct {
+	Protection  *ReplicationOptionsProtection  `yaml:"protection,optional,fromdefaults"`
+	Concurrency *ReplicationOptionsConcurrency `yaml:"concurrency,optional,fromdefaults"`
+}
+
+type ReplicationOptionsProtection struct {
+	Initial     string `yaml:"initial,optional,default=guarantee_resumability"`
+	Incremental string `yaml:"incremental,optional,default=guarantee_resumability"`
+}
+
+type ReplicationOptionsConcurrency struct {
+	Steps         int `yaml:"steps,optional,default=1"`
+	SizeEstimates int `yaml:"size_estimates,optional,default=4"`
+}
+
+type PropertyRecvOptions struct {
+	Inherit  []zfsprop.Property          `yaml:"inherit,optional"`
+	Override map[zfsprop.Property]string `yaml:"override,optional"`
+}
+
+type PlaceholderRecvOptions struct {
+	Encryption string `yaml:"encryption,default=unspecified"`
+}
+
+type PushJob struct {
+	ActiveJob    `yaml:",inline"`
+	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
+	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+	Send         *SendOptions      `yaml:"send,fromdefaults,optional"`
+}
+
+func (j *PushJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
+func (j *PushJob) GetSendOptions() *SendOptions      { return j.Send }
+
+type PullJob struct {
+	ActiveJob `yaml:",inline"`
+	RootFS    string                   `yaml:"root_fs"`
+	Interval  PositiveDurationOrManual `yaml:"interval"`
+	Recv      *RecvOptions             `yaml:"recv,fromdefaults,optional"`
+}
+
+func (j *PullJob) GetRootFS() string             { return j.RootFS }
+func (j *PullJob) GetAppendClientIdentity() bool { return false }
+func (j *PullJob) GetRecvOptions() *RecvOptions  { return j.Recv }
+
+type PositiveDurationOrManual struct {
+	Interval time.Duration
+	Manual   bool
+}
+
+var _ yaml.Unmarshaler = (*PositiveDurationOrManual)(nil)
+
+func (i *PositiveDurationOrManual) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	var s string
+	if err := u(&s, true); err != nil {
+		return err
+	}
+	switch s {
+	case "manual":
+		i.Manual = true
+		i.Interval = 0
+	case "":
+		return fmt.Errorf("value must not be empty")
+	default:
+		i.Manual = false
+		i.Interval, err = parsePositiveDuration(s)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type SinkJob struct {
+	PassiveJob `yaml:",inline"`
+	RootFS     string       `yaml:"root_fs"`
+	Recv       *RecvOptions `yaml:"recv,optional,fromdefaults"`
+}
+
+func (j *SinkJob) GetRootFS() string             { return j.RootFS }
+func (j *SinkJob) GetAppendClientIdentity() bool { return true }
+func (j *SinkJob) GetRecvOptions() *RecvOptions  { return j.Recv }
+
+type SourceJob struct {
+	PassiveJob   `yaml:",inline"`
+	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
+	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+	Send         *SendOptions      `yaml:"send,optional,fromdefaults"`
+}
+
+func (j *SourceJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
+func (j *SourceJob) GetSendOptions() *SendOptions      { return j.Send }
+
+type FilesystemsFilter map[string]bool
+
+type SnapshottingEnum struct {
+	Ret interface{}
+}
+
+type SnapshottingPeriodic struct {
+	Type            string            `yaml:"type"`
+	Prefix          string            `yaml:"prefix"`
+	Interval        *PositiveDuration `yaml:"interval"`
+	Hooks           HookList          `yaml:"hooks,optional"`
+	TimestampFormat string            `yaml:"timestamp_format,optional,default=dense"`
+}
+
+type CronSpec struct {
+	Schedule cron.Schedule
+}
+
+var _ yaml.Unmarshaler = &CronSpec{}
+
+func (s *CronSpec) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
+	var specString string
+	if err := unmarshal(&specString, false); err != nil {
+		return err
+	}
+
+	// Use standard cron format.
+	// Disable the various "descriptors" (@daily, etc)
+	// They are just aliases to "top of hour", "midnight", etc.
+	parser := cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.SecondOptional)
+
+	sched, err := parser.Parse(specString)
+	if err != nil {
+		return errors.Wrap(err, "cron syntax invalid")
+	}
+	s.Schedule = sched
+	return nil
+}
+
+type SnapshottingCron struct {
+	Type            string   `yaml:"type"`
+	Prefix          string   `yaml:"prefix"`
+	Cron            CronSpec `yaml:"cron"`
+	Hooks           HookList `yaml:"hooks,optional"`
+	TimestampFormat string   `yaml:"timestamp_format,optional,default=dense"`
+}
+
+type SnapshottingManual struct {
+	Type string `yaml:"type"`
+}
+
+type PruningSenderReceiver struct {
+	KeepSender   []PruningEnum `yaml:"keep_sender"`
+	KeepReceiver []PruningEnum `yaml:"keep_receiver"`
+}
+
+type PruningLocal struct {
+	Keep []PruningEnum `yaml:"keep"`
+}
+
+type LoggingOutletEnumList []LoggingOutletEnum
+
+func (l *LoggingOutletEnumList) SetDefault() {
+	def := `
+type: "stdout"
+time: true
+level: "warn"
+format: "human"
+`
+	s := &StdoutLoggingOutlet{}
+	err := yaml.UnmarshalStrict([]byte(def), &s)
+	if err != nil {
+		panic(err)
+	}
+	*l = []LoggingOutletEnum{LoggingOutletEnum{Ret: s}}
+}
+
+var _ yaml.Defaulter = &LoggingOutletEnumList{}
+
+type Global struct {
+	Logging    *LoggingOutletEnumList `yaml:"logging,optional,fromdefaults"`
+	Monitoring []MonitoringEnum       `yaml:"monitoring,optional"`
+	Control    *GlobalControl         `yaml:"control,optional,fromdefaults"`
+	Serve      *GlobalServe           `yaml:"serve,optional,fromdefaults"`
+}
+
+func Default(i interface{}) {
+	v := reflect.ValueOf(i)
+	if v.Kind() != reflect.Ptr {
+		panic(v)
+	}
+	y := `{}`
+	err := yaml.Unmarshal([]byte(y), v.Interface())
+	if err != nil {
+		panic(err)
+	}
+}
+
+type ConnectEnum struct {
+	Ret interface{}
+}
+
+type ConnectCommon struct {
+	Type string `yaml:"type"`
+}
+
+type TCPConnect struct {
+	ConnectCommon `yaml:",inline"`
+	Address       string        `yaml:"address,hostport"`
+	DialTimeout   time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
+}
+
+type TLSConnect struct {
+	ConnectCommon `yaml:",inline"`
+	Address       string        `yaml:"address,hostport"`
+	Ca            string        `yaml:"ca"`
+	Cert          string        `yaml:"cert"`
+	Key           string        `yaml:"key"`
+	ServerCN      string        `yaml:"server_cn"`
+	DialTimeout   time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
+}
+
+type SSHStdinserverConnect struct {
+	ConnectCommon        `yaml:",inline"`
+	Host                 string        `yaml:"host"`
+	User                 string        `yaml:"user"`
+	Port                 uint16        `yaml:"port"`
+	IdentityFile         string        `yaml:"identity_file"`
+	TransportOpenCommand []string      `yaml:"transport_open_command,optional"` //TODO unused
+	SSHCommand           string        `yaml:"ssh_command,optional"`            //TODO unused
+	Options              []string      `yaml:"options,optional"`
+	DialTimeout          time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
+}
+
+type LocalConnect struct {
+	ConnectCommon  `yaml:",inline"`
+	ListenerName   string        `yaml:"listener_name"`
+	ClientIdentity string        `yaml:"client_identity"`
+	DialTimeout    time.Duration `yaml:"dial_timeout,zeropositive,default=2s"`
+}
+
+type ServeEnum struct {
+	Ret interface{}
+}
+
+type ServeCommon struct {
+	Type string `yaml:"type"`
+}
+
+type TCPServe struct {
+	ServeCommon    `yaml:",inline"`
+	Listen         string            `yaml:"listen,hostport"`
+	ListenFreeBind bool              `yaml:"listen_freebind,default=false"`
+	Clients        map[string]string `yaml:"clients"`
+}
+
+type TLSServe struct {
+	ServeCommon      `yaml:",inline"`
+	Listen           string        `yaml:"listen,hostport"`
+	ListenFreeBind   bool          `yaml:"listen_freebind,default=false"`
+	Ca               string        `yaml:"ca"`
+	Cert             string        `yaml:"cert"`
+	Key              string        `yaml:"key"`
+	ClientCNs        []string      `yaml:"client_cns"`
+	HandshakeTimeout time.Duration `yaml:"handshake_timeout,zeropositive,default=10s"`
+}
+
+type StdinserverServer struct {
+	ServeCommon      `yaml:",inline"`
+	ClientIdentities []string `yaml:"client_identities"`
+}
+
+type LocalServe struct {
+	ServeCommon  `yaml:",inline"`
+	ListenerName string `yaml:"listener_name"`
+}
+
+type PruningEnum struct {
+	Ret interface{}
+}
+
+type PruneKeepNotReplicated struct {
+	Type                 string `yaml:"type"`
+	KeepSnapshotAtCursor bool   `yaml:"keep_snapshot_at_cursor,optional,default=true"`
+}
+
+type PruneKeepLastN struct {
+	Type  string `yaml:"type"`
+	Count int    `yaml:"count"`
+	Regex string `yaml:"regex,optional"`
+}
+
+type PruneKeepRegex struct { // FIXME rename to KeepRegex
+	Type   string `yaml:"type"`
+	Regex  string `yaml:"regex"`
+	Negate bool   `yaml:"negate,optional,default=false"`
+}
+
+type LoggingOutletEnum struct {
+	Ret interface{}
+}
+
+type LoggingOutletCommon struct {
+	Type   string `yaml:"type"`
+	Level  string `yaml:"level"`
+	Format string `yaml:"format"`
+}
+
+type StdoutLoggingOutlet struct {
+	LoggingOutletCommon `yaml:",inline"`
+	Time                bool `yaml:"time,default=true"`
+	Color               bool `yaml:"color,default=true"`
+}
+
+type SyslogLoggingOutlet struct {
+	LoggingOutletCommon `yaml:",inline"`
+	Facility            *SyslogFacility `yaml:"facility,optional,fromdefaults"`
+	RetryInterval       time.Duration   `yaml:"retry_interval,positive,default=10s"`
+}
+
+type TCPLoggingOutlet struct {
+	LoggingOutletCommon `yaml:",inline"`
+	Address             string               `yaml:"address,hostport"`
+	Net                 string               `yaml:"net,default=tcp"`
+	RetryInterval       time.Duration        `yaml:"retry_interval,positive,default=10s"`
+	TLS                 *TCPLoggingOutletTLS `yaml:"tls,optional"`
+}
+
+type TCPLoggingOutletTLS struct {
+	CA   string `yaml:"ca"`
+	Cert string `yaml:"cert"`
+	Key  string `yaml:"key"`
+}
+
+type MonitoringEnum struct {
+	Ret interface{}
+}
+
+type PrometheusMonitoring struct {
+	Type           string `yaml:"type"`
+	Listen         string `yaml:"listen,hostport"`
+	ListenFreeBind bool   `yaml:"listen_freebind,default=false"`
+}
+
+type SyslogFacility syslog.Priority
+
+func (f *SyslogFacility) SetDefault() {
+	*f = SyslogFacility(syslog.LOG_LOCAL0)
+}
+
+var _ yaml.Defaulter = (*SyslogFacility)(nil)
+
+type GlobalControl struct {
+	SockPath string `yaml:"sockpath,default=/var/run/zrepl/control"`
+}
+
+type GlobalServe struct {
+	StdinServer *GlobalStdinServer `yaml:"stdinserver,optional,fromdefaults"`
+}
+
+type GlobalStdinServer struct {
+	SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
+}
+
+type HookList []HookEnum
+
+type HookEnum struct {
+	Ret interface{}
+}
+
+type HookCommand struct {
+	Path               string            `yaml:"path"`
+	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
+	Filesystems        FilesystemsFilter `yaml:"filesystems,optional,default={'<': true}"`
+	HookSettingsCommon `yaml:",inline"`
+}
+
+type HookPostgresCheckpoint struct {
+	HookSettingsCommon `yaml:",inline"`
+	DSN                string            `yaml:"dsn"`
+	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
+	Filesystems        FilesystemsFilter `yaml:"filesystems"` // required, user should not CHECKPOINT for every FS
+}
+
+type HookMySQLLockTables struct {
+	HookSettingsCommon `yaml:",inline"`
+	DSN                string            `yaml:"dsn"`
+	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
+	Filesystems        FilesystemsFilter `yaml:"filesystems"`
+}
+
+type HookSettingsCommon struct {
+	Type       string `yaml:"type"`
+	ErrIsFatal bool   `yaml:"err_is_fatal,optional,default=false"`
+}
+
+func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) {
+	var in struct {
+		Type string
+	}
+	if err := u(&in, true); err != nil {
+		return nil, err
+	}
+	if in.Type == "" {
+		return nil, &yaml.TypeError{Errors: []string{"must specify type"}}
+	}
+
+	v, ok := types[in.Type]
+	if !ok {
+		return nil, &yaml.TypeError{Errors: []string{fmt.Sprintf("invalid type name %q", in.Type)}}
+	}
+	if err := u(v, false); err != nil {
+		return nil, err
+	}
+	return v, nil
+}
+
+func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"snap":   &SnapJob{},
+		"push":   &PushJob{},
+		"sink":   &SinkJob{},
+		"pull":   &PullJob{},
+		"source": &SourceJob{},
+	})
+	return
+}
+
+func (t *ConnectEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"tcp":             &TCPConnect{},
+		"tls":             &TLSConnect{},
+		"ssh+stdinserver": &SSHStdinserverConnect{},
+		"local":           &LocalConnect{},
+	})
+	return
+}
+
+func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"tcp":         &TCPServe{},
+		"tls":         &TLSServe{},
+		"stdinserver": &StdinserverServer{},
+		"local":       &LocalServe{},
+	})
+	return
+}
+
+func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"not_replicated": &PruneKeepNotReplicated{},
+		"last_n":         &PruneKeepLastN{},
+		"grid":           &PruneGrid{},
+		"regex":          &PruneKeepRegex{},
+	})
+	return
+}
+
+func (t *SnapshottingEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"periodic": &SnapshottingPeriodic{},
+		"manual":   &SnapshottingManual{},
+		"cron":     &SnapshottingCron{},
+	})
+	return
+}
+
+func (t *LoggingOutletEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"stdout": &StdoutLoggingOutlet{},
+		"syslog": &SyslogLoggingOutlet{},
+		"tcp":    &TCPLoggingOutlet{},
+	})
+	return
+}
+
+func (t *MonitoringEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"prometheus": &PrometheusMonitoring{},
+	})
+	return
+}
+
+func (t *SyslogFacility) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	var s string
+	if err := u(&s, true); err != nil {
+		return err
+	}
+	var level syslog.Priority
+	switch s {
+	case "kern":
+		level = syslog.LOG_KERN
+	case "user":
+		level = syslog.LOG_USER
+	case "mail":
+		level = syslog.LOG_MAIL
+	case "daemon":
+		level = syslog.LOG_DAEMON
+	case "auth":
+		level = syslog.LOG_AUTH
+	case "syslog":
+		level = syslog.LOG_SYSLOG
+	case "lpr":
+		level = syslog.LOG_LPR
+	case "news":
+		level = syslog.LOG_NEWS
+	case "uucp":
+		level = syslog.LOG_UUCP
+	case "cron":
+		level = syslog.LOG_CRON
+	case "authpriv":
+		level = syslog.LOG_AUTHPRIV
+	case "ftp":
+		level = syslog.LOG_FTP
+	case "local0":
+		level = syslog.LOG_LOCAL0
+	case "local1":
+		level = syslog.LOG_LOCAL1
+	case "local2":
+		level = syslog.LOG_LOCAL2
+	case "local3":
+		level = syslog.LOG_LOCAL3
+	case "local4":
+		level = syslog.LOG_LOCAL4
+	case "local5":
+		level = syslog.LOG_LOCAL5
+	case "local6":
+		level = syslog.LOG_LOCAL6
+	case "local7":
+		level = syslog.LOG_LOCAL7
+	default:
+		return fmt.Errorf("invalid syslog level: %q", s)
+	}
+	*t = SyslogFacility(level)
+	return nil
+}
+
+func (t *HookEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"command":             &HookCommand{},
+		"postgres-checkpoint": &HookPostgresCheckpoint{},
+		"mysql-lock-tables":   &HookMySQLLockTables{},
+	})
+	return
+}
+
+var ConfigFileDefaultLocations = []string{
+	"/etc/zrepl/zrepl.yml",
+	"/usr/local/etc/zrepl/zrepl.yml",
+}
+
+func ParseConfig(path string) (i *Config, err error) {
+
+	if path == "" {
+		// Try default locations
+		for _, l := range ConfigFileDefaultLocations {
+			stat, statErr := os.Stat(l)
+			if statErr != nil {
+				continue
+			}
+			if !stat.Mode().IsRegular() {
+				err = errors.Errorf("file at default location is not a regular file: %s", l)
+				return
+			}
+			path = l
+			break
+		}
+	}
+
+	var bytes []byte
+
+	if bytes, err = ioutil.ReadFile(path); err != nil {
+		return
+	}
+
+	return ParseConfigBytes(bytes)
+}
+
+func ParseConfigBytes(bytes []byte) (*Config, error) {
+	var c *Config
+	if err := yaml.UnmarshalStrict(bytes, &c); err != nil {
+		return nil, err
+	}
+	if c == nil {
+		return nil, fmt.Errorf("config is empty or only consists of comments")
+	}
+	return c, nil
+}

config/config_dep_test.go

@@ -0,0 +1,90 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/zrepl/yaml-config"
+)
+
+type A struct {
+	B  *B     `yaml:"b,optional,fromdefaults"`
+	A1 string `yaml:"a1,optional"`
+}
+
+type B struct {
+	C *C     `yaml:"c,optional,fromdefaults"`
+	D string `yaml:"d,default=ddd"`
+	E string `yaml:"e,optional"`
+}
+
+type C struct {
+	Q string `yaml:"q,optional"`
+	R string `yaml:"r,default=r"`
+}
+
+func TestDepFromDefaults(t *testing.T) {
+
+	type testcase struct {
+		name   string
+		yaml   string
+		expect *A
+	}
+
+	tcs := []testcase{
+		{
+			name: "empty",
+			yaml: `{}`,
+			expect: &A{
+				B: &B{
+					C: &C{
+						R: "r",
+					},
+					D: "ddd",
+				},
+			},
+		},
+		{
+			name: "a1 set",
+			yaml: `{"a1":"blah"}`,
+			expect: &A{
+				A1: "blah",
+				B: &B{
+					C: &C{
+						R: "r",
+					},
+					D: "ddd",
+				},
+			},
+		},
+		{
+			name: "D set",
+			yaml: `
+b:
+  d: 4d
+`,
+			expect: &A{
+				B: &B{
+					D: "4d",
+					C: &C{
+						R: "r",
+					},
+				},
+			},
+		},
+	}
+
+	for tci := range tcs {
+		t.Run(fmt.Sprintf("%d-%s", tci, tcs[tci].name), func(t *testing.T) {
+			tc := tcs[tci]
+
+			var a A
+			err := yaml.UnmarshalStrict([]byte(tc.yaml), &a)
+			require.NoError(t, err)
+
+			require.Equal(t, tc.expect, &a)
+		})
+	}
+
+}

config/config_duration.go

@@ -0,0 +1,106 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
+	"time"
+
+	"github.com/kr/pretty"
+	"github.com/zrepl/yaml-config"
+)
+
+type Duration struct{ d time.Duration }
+
+func (d Duration) Duration() time.Duration { return d.d }
+
+var _ yaml.Unmarshaler = &Duration{}
+
+func (d *Duration) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
+	var s string
+	err := unmarshal(&s, false)
+	if err != nil {
+		return err
+	}
+	d.d, err = parseDuration(s)
+	if err != nil {
+		d.d = 0
+		return &yaml.TypeError{Errors: []string{fmt.Sprintf("cannot parse value %q: %s", s, err)}}
+	}
+	return nil
+}
+
+type PositiveDuration struct{ d Duration }
+
+var _ yaml.Unmarshaler = &PositiveDuration{}
+
+func (d PositiveDuration) Duration() time.Duration { return d.d.Duration() }
+
+func (d *PositiveDuration) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
+	err := d.d.UnmarshalYAML(unmarshal)
+	if err != nil {
+		return err
+	}
+	if d.d.Duration() <= 0 {
+		return fmt.Errorf("duration must be positive, got %s", d.d.Duration())
+	}
+	return nil
+}
+
+func parsePositiveDuration(e string) (time.Duration, error) {
+	d, err := parseDuration(e)
+	if err != nil {
+		return d, err
+	}
+	if d <= 0 {
+		return 0, errors.New("duration must be positive integer")
+	}
+	return d, err
+}
+
+var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*([\+-]?\d+)\s*(|s|m|h|d|w)\s*$`)
+
+func parseDuration(e string) (d time.Duration, err error) {
+	comps := durationStringRegex.FindStringSubmatch(e)
+	if comps == nil {
+		err = fmt.Errorf("must match %s", durationStringRegex)
+		return
+	}
+	if len(comps) != 3 {
+		panic(pretty.Sprint(comps))
+	}
+
+	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+
+	var durationUnit time.Duration
+	switch comps[2] {
+	case "":
+		if durationFactor != 0 {
+			err = fmt.Errorf("missing time unit")
+			return
+		} else {
+			// It's the case where user specified '0'.
+			// We want to allow this, just like time.ParseDuration.
+		}
+	case "s":
+		durationUnit = time.Second
+	case "m":
+		durationUnit = time.Minute
+	case "h":
+		durationUnit = time.Hour
+	case "d":
+		durationUnit = 24 * time.Hour
+	case "w":
+		durationUnit = 24 * 7 * time.Hour
+	default:
+		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
+		return
+	}
+
+	d = time.Duration(durationFactor) * durationUnit
+	return
+}

config/config_global_test.go

@@ -0,0 +1,115 @@
+package config
+
+import (
+	"fmt"
+	"log/syslog"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/zrepl/yaml-config"
+)
+
+func testValidGlobalSection(t *testing.T, s string) *Config {
+	jobdef := `
+jobs:
+- name: dummyjob
+  type: sink
+  serve:
+    type: tcp
+    listen: ":2342"
+    clients: {
+      "10.0.0.1":"foo"
+    }
+  root_fs: zroot/foo
+`
+	_, err := ParseConfigBytes([]byte(jobdef))
+	require.NoError(t, err)
+	return testValidConfig(t, s+jobdef)
+}
+
+func TestOutletTypes(t *testing.T) {
+	conf := testValidGlobalSection(t, `
+global:
+  logging:
+  - type: stdout
+    level: debug
+    format: human
+  - type: syslog
+    level: info
+    retry_interval: 20s
+    format: human
+  - type: tcp
+    level: debug
+    format: json
+    address: logserver.example.com:1234
+  - type: tcp
+    level: debug
+    format: json
+    address: encryptedlogserver.example.com:1234
+    retry_interval: 20s 
+    tls:
+      ca: /etc/zrepl/log/ca.crt
+      cert: /etc/zrepl/log/key.pem
+      key: /etc/zrepl/log/cert.pem
+`)
+	assert.Equal(t, 4, len(*conf.Global.Logging))
+	assert.NotNil(t, (*conf.Global.Logging)[3].Ret.(*TCPLoggingOutlet).TLS)
+}
+
+func TestDefaultLoggingOutlet(t *testing.T) {
+	conf := testValidGlobalSection(t, "")
+	assert.Equal(t, 1, len(*conf.Global.Logging))
+	o := (*conf.Global.Logging)[0].Ret.(*StdoutLoggingOutlet)
+	assert.Equal(t, "warn", o.Level)
+	assert.Equal(t, "human", o.Format)
+}
+
+func TestPrometheusMonitoring(t *testing.T) {
+	conf := testValidGlobalSection(t, `
+global:
+  monitoring:
+    - type: prometheus
+      listen: ':9811'
+`)
+	assert.Equal(t, ":9811", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
+}
+
+func TestSyslogLoggingOutletFacility(t *testing.T) {
+	type SyslogFacilityPriority struct {
+		Facility string
+		Priority syslog.Priority
+	}
+	syslogFacilitiesPriorities := []SyslogFacilityPriority{
+		{"", syslog.LOG_LOCAL0}, // default
+		{"kern", syslog.LOG_KERN}, {"daemon", syslog.LOG_DAEMON}, {"auth", syslog.LOG_AUTH},
+		{"syslog", syslog.LOG_SYSLOG}, {"lpr", syslog.LOG_LPR}, {"news", syslog.LOG_NEWS},
+		{"uucp", syslog.LOG_UUCP}, {"cron", syslog.LOG_CRON}, {"authpriv", syslog.LOG_AUTHPRIV},
+		{"ftp", syslog.LOG_FTP}, {"local0", syslog.LOG_LOCAL0}, {"local1", syslog.LOG_LOCAL1},
+		{"local2", syslog.LOG_LOCAL2}, {"local3", syslog.LOG_LOCAL3}, {"local4", syslog.LOG_LOCAL4},
+		{"local5", syslog.LOG_LOCAL5}, {"local6", syslog.LOG_LOCAL6}, {"local7", syslog.LOG_LOCAL7},
+	}
+
+	for _, sFP := range syslogFacilitiesPriorities {
+		logcfg := fmt.Sprintf(`
+global:
+  logging:
+  - type: syslog
+    level: info
+    format: human
+    facility: %s
+`, sFP.Facility)
+		conf := testValidGlobalSection(t, logcfg)
+		assert.Equal(t, 1, len(*conf.Global.Logging))
+		assert.True(t, SyslogFacility(sFP.Priority) == *(*conf.Global.Logging)[0].Ret.(*SyslogLoggingOutlet).Facility)
+	}
+}
+
+func TestLoggingOutletEnumList_SetDefaults(t *testing.T) {
+	e := &LoggingOutletEnumList{}
+	var i yaml.Defaulter = e
+	require.NotPanics(t, func() {
+		i.SetDefault()
+		assert.Equal(t, "warn", (*e)[0].Ret.(*StdoutLoggingOutlet).Level)
+	})
+}

config/config_jobid_test.go

@@ -0,0 +1 @@
+package config

config/config_minimal_test.go

@@ -0,0 +1,40 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfigEmptyFails(t *testing.T) {
+	conf, err := testConfig(t, "\n")
+	assert.Nil(t, conf)
+	assert.Error(t, err)
+}
+
+func TestJobsOnlyWorks(t *testing.T) {
+	testValidConfig(t, `
+jobs:
+- name: push
+  type: push
+  # snapshot the filesystems matched by the left-hand-side of the mapping
+  # every 10m with zrepl_ as prefix
+  connect:
+    type: tcp
+    address: localhost:2342
+  filesystems: {
+    "pool1/var/db<": true,
+    "pool1/usr/home<": true,
+    "pool1/usr/home/paranoid": false, #don't backup paranoid user
+    "pool1/poudriere/ports<": false #don't backup the ports trees
+  }
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    keep_receiver:
+    - type: last_n 
+      count: 1
+`)
+}

config/config_positiveintervalormanual_test.go

@@ -0,0 +1,41 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/zrepl/yaml-config"
+)
+
+func TestPositiveDurationOrManual(t *testing.T) {
+	cases := []struct {
+		Comment, Input string
+		Result         *PositiveDurationOrManual
+	}{
+		{"empty is error", "", nil},
+		{"negative is error", "-1s", nil},
+		{"zero seconds is error", "0s", nil},
+		{"zero is error", "0", nil},
+		{"non-manual is error", "something", nil},
+		{"positive seconds works", "1s", &PositiveDurationOrManual{Manual: false, Interval: 1 * time.Second}},
+		{"manual works", "manual", &PositiveDurationOrManual{Manual: true, Interval: 0}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.Comment, func(t *testing.T) {
+			var out struct {
+				FieldName PositiveDurationOrManual `yaml:"fieldname"`
+			}
+			input := fmt.Sprintf("\nfieldname: %s\n", tc.Input)
+			err := yaml.UnmarshalStrict([]byte(input), &out)
+			if tc.Result == nil {
+				assert.Error(t, err)
+				t.Logf("%#v", out)
+			} else {
+				assert.Equal(t, *tc.Result, out.FieldName)
+			}
+		})
+	}
+
+}

config/config_recv_test.go

@@ -0,0 +1,134 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	zfsprop "github.com/zrepl/zrepl/zfs/property"
+)
+
+func TestRecvOptions(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: pull
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  root_fs: "zreplplatformtest"
+  %s
+  interval: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+
+`
+
+	recv_properties_empty := `
+  recv:
+    properties:
+`
+
+	recv_inherit_empty := `
+  recv:
+    properties:
+      inherit:
+`
+
+	recv_inherit := `
+  recv:
+    properties:
+      inherit:
+        - testprop
+`
+
+	recv_override_empty := `
+  recv:
+    properties:
+      override:
+`
+
+	recv_override := `
+  recv:
+    properties:
+      override:
+        testprop2: "test123"
+`
+
+	recv_override_and_inherit := `
+  recv:
+    properties:
+      inherit:
+        - testprop
+      override:
+        testprop2: "test123"
+`
+
+	recv_empty := `
+  recv: {}
+`
+
+	recv_not_specified := `
+`
+
+	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
+
+	t.Run("recv_inherit_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_inherit_empty))
+		assert.NotNil(t, c)
+	})
+
+	t.Run("recv_inherit", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_inherit))
+		inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
+		assert.NotEmpty(t, inherit)
+		assert.Contains(t, inherit, zfsprop.Property("testprop"))
+	})
+
+	t.Run("recv_override_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_override_empty))
+		assert.NotNil(t, c)
+	})
+
+	t.Run("recv_override", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_override))
+		override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
+		require.Len(t, override, 1)
+		require.Equal(t, "test123", override["testprop2"])
+
+	})
+
+	t.Run("recv_override_and_inherit", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_override_and_inherit))
+		inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
+		override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
+		assert.NotEmpty(t, inherit)
+		assert.Contains(t, inherit, zfsprop.Property("testprop"))
+		assert.NotEmpty(t, override)
+		assert.Equal(t, "test123", override["testprop2"])
+	})
+
+	t.Run("recv_properties_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_properties_empty))
+		assert.NotNil(t, c)
+	})
+
+	t.Run("recv_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_empty))
+		assert.NotNil(t, c)
+	})
+
+	t.Run("send_not_specified", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_not_specified))
+		assert.NotNil(t, c)
+	})
+
+}

config/config_send_test.go

@@ -0,0 +1,146 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSendOptions(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  %s
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+	encrypted_false := `
+  send:
+    encrypted: false
+`
+
+	encrypted_true := `
+  send:
+    encrypted: true
+`
+
+	raw_true := `
+  send:
+    raw: true
+
+`
+
+	raw_false := `
+  send:
+    raw: false
+
+`
+
+	raw_and_encrypted := `
+  send:
+    encrypted: true
+    raw: true
+`
+
+	properties_and_encrypted := `
+  send:
+    encrypted: true
+    send_properties: true
+`
+
+	properties_true := `
+  send:
+    send_properties: true
+`
+
+	properties_false := `
+  send:
+    send_properties: false
+`
+
+	send_empty := `
+  send: {}
+`
+
+	send_not_specified := `
+`
+
+	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
+
+	t.Run("encrypted_false", func(t *testing.T) {
+		c := testValidConfig(t, fill(encrypted_false))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, false, encrypted)
+	})
+	t.Run("encrypted_true", func(t *testing.T) {
+		c := testValidConfig(t, fill(encrypted_true))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, true, encrypted)
+	})
+
+	t.Run("send_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(send_empty))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, false, encrypted)
+	})
+
+	t.Run("properties_and_encrypted", func(t *testing.T) {
+		c := testValidConfig(t, fill(properties_and_encrypted))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
+		assert.Equal(t, true, encrypted)
+		assert.Equal(t, true, properties)
+	})
+
+	t.Run("properties_false", func(t *testing.T) {
+		c := testValidConfig(t, fill(properties_false))
+		properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
+		assert.Equal(t, false, properties)
+	})
+
+	t.Run("properties_true", func(t *testing.T) {
+		c := testValidConfig(t, fill(properties_true))
+		properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
+		assert.Equal(t, true, properties)
+	})
+
+	t.Run("raw_true", func(t *testing.T) {
+		c := testValidConfig(t, fill(raw_true))
+		raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
+		assert.Equal(t, true, raw)
+	})
+
+	t.Run("raw_false", func(t *testing.T) {
+		c := testValidConfig(t, fill(raw_false))
+		raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
+		assert.Equal(t, false, raw)
+	})
+
+	t.Run("raw_and_encrypted", func(t *testing.T) {
+		c := testValidConfig(t, fill(raw_and_encrypted))
+		raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, true, raw)
+		assert.Equal(t, true, encrypted)
+	})
+
+	t.Run("send_not_specified", func(t *testing.T) {
+		c := testValidConfig(t, fill(send_not_specified))
+		assert.NotNil(t, c)
+	})
+
+}

config/config_snapshotting_test.go

@@ -0,0 +1,176 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSnapshotting(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  %s
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+	manual := `
+  snapshotting:
+    type: manual
+`
+	periodic := `
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    timestamp_format: dense
+    interval: 10m
+`
+	cron := `
+  snapshotting:
+    type: cron
+    prefix: zrepl_
+    timestamp_format: human
+    cron: "10 * * * *"
+`
+
+	periodicDaily := `
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 1d
+`
+
+	hooks := `
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 10m
+    hooks:
+    - type: command
+      path: /tmp/path/to/command
+    - type: command
+      path: /tmp/path/to/command
+      filesystems: { "zroot<": true, "<": false }
+    - type: postgres-checkpoint
+      dsn: "host=localhost port=5432 user=postgres sslmode=disable"
+      filesystems: {
+          "tank/postgres/data11": true
+      }
+    - type: mysql-lock-tables
+      dsn: "root@tcp(localhost)/"
+      filesystems: {
+        "tank/mysql": true
+      }
+`
+
+	fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
+	var c *Config
+
+	t.Run("manual", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(manual))
+		snm := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingManual)
+		assert.Equal(t, "manual", snm.Type)
+	})
+
+	t.Run("periodic", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(periodic))
+		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
+		assert.Equal(t, "periodic", snp.Type)
+		assert.Equal(t, 10*time.Minute, snp.Interval.Duration())
+		assert.Equal(t, "zrepl_", snp.Prefix)
+	})
+
+	t.Run("periodicDaily", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(periodicDaily))
+		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
+		assert.Equal(t, "periodic", snp.Type)
+		assert.Equal(t, 24*time.Hour, snp.Interval.Duration())
+		assert.Equal(t, "zrepl_", snp.Prefix)
+		assert.Equal(t, "dense", snp.TimestampFormat)
+	})
+
+	t.Run("cron", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(cron))
+		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingCron)
+		assert.Equal(t, "cron", snp.Type)
+		assert.Equal(t, "zrepl_", snp.Prefix)
+		assert.Equal(t, "human", snp.TimestampFormat)
+	})
+
+	t.Run("hooks", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(hooks))
+		hs := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic).Hooks
+		assert.Equal(t, hs[0].Ret.(*HookCommand).Filesystems["<"], true)
+		assert.Equal(t, hs[1].Ret.(*HookCommand).Filesystems["zroot<"], true)
+		assert.Equal(t, hs[2].Ret.(*HookPostgresCheckpoint).Filesystems["tank/postgres/data11"], true)
+		assert.Equal(t, hs[3].Ret.(*HookMySQLLockTables).Filesystems["tank/mysql"], true)
+	})
+
+}
+
+func TestSnapshottingTimestampDefaults(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  %s
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+
+	periodic := `
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 10m
+`
+	cron := `
+  snapshotting:
+    type: cron
+    prefix: zrepl_
+    cron: "10 * * * *"
+`
+
+	fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
+	var c *Config
+
+	t.Run("periodic", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(periodic))
+		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
+		assert.Equal(t, "periodic", snp.Type)
+		assert.Equal(t, 10*time.Minute, snp.Interval.Duration())
+		assert.Equal(t, "zrepl_", snp.Prefix)
+		assert.Equal(t, "dense", snp.TimestampFormat) // default was set correctly
+	})
+
+	t.Run("cron", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(cron))
+		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingCron)
+		assert.Equal(t, "cron", snp.Type)
+		assert.Equal(t, "zrepl_", snp.Prefix)
+		assert.Equal(t, "dense", snp.TimestampFormat) // default was set correctly
+	})
+}

config/config_test.go

@@ -0,0 +1,140 @@
+package config
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+	"text/template"
+
+	"github.com/kr/pretty"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/zrepl/yaml-config"
+)
+
+func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
+	paths, err := filepath.Glob("./samples/*")
+	if err != nil {
+		t.Errorf("glob failed: %+v", err)
+	}
+
+	for _, p := range paths {
+
+		if path.Ext(p) != ".yml" {
+			t.Logf("skipping file %s", p)
+			continue
+		}
+
+		t.Run(p, func(t *testing.T) {
+			c, err := ParseConfig(p)
+			if err != nil {
+				t.Errorf("error parsing %s:\n%+v", p, err)
+			}
+
+			t.Logf("file: %s", p)
+			t.Log(pretty.Sprint(c))
+		})
+
+	}
+
+}
+
+// template must be a template/text template with a single '{{ . }}' as placeholder for val
+//
+//nolint:deadcode,unused
+func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
+	tmp, err := template.New("master").Parse(tmpl)
+	if err != nil {
+		panic(err)
+	}
+	var buf bytes.Buffer
+	err = tmp.Execute(&buf, val)
+	if err != nil {
+		panic(err)
+	}
+	return testValidConfig(t, buf.String())
+}
+
+func testValidConfig(t *testing.T, input string) *Config {
+	t.Helper()
+	conf, err := testConfig(t, input)
+	require.NoError(t, err)
+	require.NotNil(t, conf)
+	return conf
+}
+
+func testConfig(t *testing.T, input string) (*Config, error) {
+	t.Helper()
+	return ParseConfigBytes([]byte(input))
+}
+
+func trimSpaceEachLineAndPad(s, pad string) string {
+	var out strings.Builder
+	scan := bufio.NewScanner(strings.NewReader(s))
+	for scan.Scan() {
+		fmt.Fprintf(&out, "%s%s\n", pad, bytes.TrimSpace(scan.Bytes()))
+	}
+	return out.String()
+}
+
+func TestTrimSpaceEachLineAndPad(t *testing.T) {
+	foo := `
+	foo
+	bar baz 
+	`
+	assert.Equal(t, "  \n  foo\n  bar baz\n  \n", trimSpaceEachLineAndPad(foo, "  "))
+}
+
+func TestCronSpec(t *testing.T) {
+
+	expectAccept := []string{
+		`"* * * * *"`,
+		`"0-10 * * * *"`,
+		`"* 0-5,8,12 * * *"`,
+	}
+
+	expectFail := []string{
+		`* * * *`,
+		``,
+		`23`,
+		`"@reboot"`,
+		`"@every 1h30m"`,
+		`"@daily"`,
+		`* * * * * *`,
+	}
+
+	for _, input := range expectAccept {
+		t.Run(input, func(t *testing.T) {
+			s := fmt.Sprintf("spec: %s\n", input)
+			var v struct {
+				Spec CronSpec
+			}
+			v.Spec.Schedule = nil
+			t.Logf("input:\n%s", s)
+			err := yaml.UnmarshalStrict([]byte(s), &v)
+			t.Logf("error: %T %s", err, err)
+			require.NoError(t, err)
+			require.NotNil(t, v.Spec.Schedule)
+		})
+	}
+
+	for _, input := range expectFail {
+		t.Run(input, func(t *testing.T) {
+			s := fmt.Sprintf("spec: %s\n", input)
+			var v struct {
+				Spec CronSpec
+			}
+			v.Spec.Schedule = nil
+			t.Logf("input: %q", s)
+			err := yaml.UnmarshalStrict([]byte(s), &v)
+			t.Logf("error: %T %s", err, err)
+			require.Error(t, err)
+			require.Nil(t, v.Spec.Schedule)
+		})
+	}
+
+}

config/config_transport_test.go

@@ -0,0 +1,99 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestTransportConnect(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+%s
+  filesystems: {"<": true}
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+
+	mconf := func(s string) string { return fmt.Sprintf(tmpl, s) }
+
+	type test struct {
+		Name        string
+		ExpectError bool
+		Connect     string
+	}
+
+	testTable := []test{
+		{
+			Name:        "tcp_with_address_and_port",
+			ExpectError: false,
+			Connect: `
+			type: tcp
+			address: 10.0.0.23:42
+			`,
+		},
+		{
+			Name:        "tls_with_host_and_port",
+			ExpectError: false,
+			Connect: `
+			type: tls
+			address: "server1.foo.bar:8888"
+			ca:   /etc/zrepl/ca.crt
+			cert: /etc/zrepl/backupserver.fullchain
+			key:  /etc/zrepl/backupserver.key
+			server_cn: "server1"
+			`,
+		},
+		{
+			Name:        "tcp_without_port",
+			ExpectError: true,
+			Connect: `
+			type: tcp
+			address: 10.0.0.23
+			`,
+		},
+		{
+			Name:        "tls_without_port",
+			ExpectError: true,
+			Connect: `
+			type: tls
+			address: 10.0.0.23
+			ca:   /etc/zrepl/ca.crt
+			cert: /etc/zrepl/backupserver.fullchain
+			key:  /etc/zrepl/backupserver.key
+			server_cn: "server1"
+			`,
+		},
+	}
+
+	for _, tc := range testTable {
+		t.Run(tc.Name, func(t *testing.T) {
+			require.NotEmpty(t, tc.Connect)
+			connect := trimSpaceEachLineAndPad(tc.Connect, "    ")
+			conf := mconf(connect)
+			config, err := testConfig(t, conf)
+			if tc.ExpectError && err == nil {
+				t.Errorf("expected test failure, but got valid config %v", config)
+				return
+			}
+			if !tc.ExpectError && err != nil {
+				t.Errorf("not expecint test failure but got error: %s", err)
+				return
+			}
+			t.Logf("error=%v", err)
+			t.Logf("config=%v", config)
+		})
+	}
+
+}

config/retentiongrid.go

@@ -0,0 +1,119 @@
+package config
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type RetentionIntervalList []RetentionInterval
+
+type PruneGrid struct {
+	Type  string                `yaml:"type"`
+	Grid  RetentionIntervalList `yaml:"grid"`
+	Regex string                `yaml:"regex"`
+}
+
+type RetentionInterval struct {
+	length    time.Duration
+	keepCount int
+}
+
+func (i *RetentionInterval) Length() time.Duration {
+	return i.length
+}
+
+func (i *RetentionInterval) KeepCount() int {
+	return i.keepCount
+}
+
+const RetentionGridKeepCountAll int = -1
+
+func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	var in string
+	if err := u(&in, true); err != nil {
+		return err
+	}
+
+	intervals, err := ParseRetentionIntervalSpec(in)
+	if err != nil {
+		return err
+	}
+
+	*t = intervals
+
+	return nil
+}
+
+var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
+
+func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {
+
+	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
+	if comps == nil {
+		err = fmt.Errorf("retention string does not match expected format")
+		return
+	}
+
+	times, err := strconv.Atoi(comps[1])
+	if err != nil {
+		return nil, err
+	} else if times <= 0 {
+		return nil, fmt.Errorf("contains factor <= 0")
+	}
+
+	duration, err := parsePositiveDuration(comps[2])
+	if err != nil {
+		return nil, err
+	}
+
+	keepCount := 1
+	if comps[3] != "" {
+		// Decompose key=value, comma separated
+		// For now, only keep_count is supported
+		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
+		res := re.FindStringSubmatch(comps[4])
+		if res == nil || len(res) != 2 {
+			err = fmt.Errorf("interval parameter contains unknown parameters")
+			return
+		}
+		if res[1] == "all" {
+			keepCount = RetentionGridKeepCountAll
+		} else {
+			keepCount, err = strconv.Atoi(res[1])
+			if err != nil {
+				err = fmt.Errorf("cannot parse keep_count value")
+				return
+			}
+		}
+	}
+
+	intervals = make([]RetentionInterval, times)
+	for i := range intervals {
+		intervals[i] = RetentionInterval{
+			length:    duration,
+			keepCount: keepCount,
+		}
+	}
+
+	return
+
+}
+
+func ParseRetentionIntervalSpec(s string) (intervals []RetentionInterval, err error) {
+
+	ges := strings.Split(s, "|")
+	intervals = make([]RetentionInterval, 0, 7*len(ges))
+
+	for intervalIdx, e := range ges {
+		parsed, err := parseRetentionGridIntervalString(e)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
+		}
+		intervals = append(intervals, parsed...)
+	}
+
+	return
+}

config/samples/bandwidth_limit.yml

@@ -0,0 +1,41 @@
+
+jobs:
+  - type: sink
+    name: "limited_sink"
+    root_fs: "fs0"
+    recv:
+      bandwidth_limit:
+        max: 12345 B
+    serve:
+      type: local
+      listener_name: localsink
+
+  - type: push
+    name: "limited_push"
+    connect:
+      type: local
+      listener_name: localsink
+      client_identity: local_backup
+    filesystems: {
+      "root<": true,
+    }
+    send:
+      bandwidth_limit:
+        max: 54321 B
+        bucket_capacity: 1024 B
+    snapshotting:
+      type: manual
+    pruning:
+      keep_sender:
+        - type: last_n
+          count: 1
+      keep_receiver:
+        - type: last_n
+          count: 1
+
+  - type: sink
+    name: "nolimit_sink"
+    root_fs: "fs1"
+    serve:
+      type: local
+      listener_name: localsink

config/samples/hooks/template.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -e
+
+[ "$ZREPL_DRYRUN" = "true" ] && DRYRUN="echo DRYRUN: "
+
+pre_snapshot() {
+    $DRYRUN date
+}
+
+post_snapshot() {
+    $DRYRUN date
+}
+
+case "$ZREPL_HOOKTYPE" in
+    pre_snapshot|post_snapshot)
+        "$ZREPL_HOOKTYPE"
+        ;;
+    *)
+        printf 'Unrecognized hook type: %s\n' "$ZREPL_HOOKTYPE"
+        exit 255
+        ;;
+esac

config/samples/local.yml

@@ -0,0 +1,31 @@
+
+jobs:
+  - type: sink
+    name: "local_sink"
+    root_fs: "storage/zrepl/sink"
+    serve:
+      type: local
+      listener_name: localsink
+
+  - type: push
+    name: "backup_system"
+    connect:
+      type: local
+      listener_name: localsink
+      client_identity: local_backup
+    filesystems: {
+      "system<": true,
+    }
+    snapshotting:
+      type: periodic
+      interval: 10m
+      prefix: zrepl_
+    pruning:
+      keep_sender:
+      - type: not_replicated
+      - type: last_n
+        count: 10
+      keep_receiver:
+      - type: grid
+        grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+        regex: "zrepl_.*"

config/samples/pull.yml

@@ -0,0 +1,24 @@
+jobs:
+- name: pull_servers
+  type: pull
+  connect:
+    type: tls
+    address: "server1.foo.bar:8888"
+    ca: "/certs/ca.crt"
+    cert: "/certs/cert.crt"
+    key: "/certs/key.pem"
+    server_cn: "server1"
+  root_fs: "pool2/backup_servers"
+  interval: 10m
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    - type: last_n
+      count: 10
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 14x1d
+      regex: "zrepl_.*"
+    keep_receiver:
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+      regex: "zrepl_.*"

config/samples/pull_ssh.yml

@@ -0,0 +1,28 @@
+jobs:
+
+- name: pull_servers
+  type: pull
+  connect:
+    type: ssh+stdinserver
+    host: app-srv.example.com
+    user: root
+    port: 22
+    identity_file: /etc/zrepl/ssh/identity
+    options: # optional, default [], `-o` arguments passed to ssh
+    - "Compression=yes"
+  root_fs: "pool2/backup_servers"
+  interval: 10m
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    - type: last_n
+      count: 10
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 14x1d
+      regex: "^zrepl_.*"
+    keep_receiver:
+    - type: regex
+      regex: keep_
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+      regex: "^zrepl_.*"

config/samples/push.yml

@@ -0,0 +1,26 @@
+jobs:
+  - type: push
+    name: "push"
+    filesystems: {
+      "<": true,
+      "tmp": false
+    }
+    connect:
+      type: tcp
+      address: "backup-server.foo.bar:8888"
+    snapshotting:
+      type: manual
+    send:
+      encrypted: false
+    pruning:
+      keep_sender:
+        - type: not_replicated
+        - type: last_n
+          count: 10
+        - type: grid
+          grid: 1x1h(keep=all) | 24x1h | 14x1d
+          regex: "^zrepl_.*"
+      keep_receiver:
+        - type: grid
+          grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+          regex: "^zrepl_.*"

config/samples/quickstart_backup_to_external_disk.yml

@@ -0,0 +1,88 @@
+# This config serves as an example for a local zrepl installation that
+# backups the entire zpool `system` to `backuppool/zrepl/sink`
+#
+# The requirements covered by this setup are described in the zrepl documentation's
+# quick start section which inlines this example.
+#
+# CUSTOMIZATIONS YOU WILL LIKELY WANT TO APPLY:
+# - adjust the name of the production pool `system` in the `filesystems` filter of jobs `snapjob` and `push_to_drive`
+# - adjust the name of the backup pool `backuppool` in the `backuppool_sink` job
+# - adjust the occurences of `myhostname` to the name of the system you are backing up (cannot be easily changed once you start replicating)
+# - make sure the `zrepl_` prefix is not being used by any other zfs tools you might have installed (it likely isn't)
+
+jobs:
+
+# this job takes care of snapshot creation + pruning
+- name: snapjob
+  type: snap
+  filesystems: {
+      "system<": true,
+  }
+  # create snapshots with prefix `zrepl_` every 15 minutes
+  snapshotting:
+    type: periodic
+    interval: 15m
+    prefix: zrepl_
+  pruning:
+    keep:
+    # fade-out scheme for snapshots starting with `zrepl_`
+    # - keep all created in the last hour
+    # - then destroy snapshots such that we keep 24 each 1 hour apart
+    # - then destroy snapshots such that we keep 14 each 1 day apart
+    # - then destroy all older snapshots
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 14x1d
+      regex: "^zrepl_.*"
+    # keep all snapshots that don't have the `zrepl_` prefix
+    - type: regex
+      negate: true
+      regex: "^zrepl_.*"
+
+# This job pushes to the local sink defined in job `backuppool_sink`.
+# We trigger replication manually from the command line / udev rules using
+#  `zrepl signal wakeup push_to_drive`
+- type: push
+  name: push_to_drive
+  connect:
+    type: local
+    listener_name: backuppool_sink
+    client_identity: myhostname
+  filesystems: {
+      "system<": true
+  }
+  send:
+    encrypted: true
+  replication:
+    protection:
+      initial: guarantee_resumability
+      # Downgrade protection to guarantee_incremental which uses zfs bookmarks instead of zfs holds.
+      # Thus, when we yank out the backup drive during replication
+      # - we might not be able to resume the interrupted replication step because the partially received `to` snapshot of a `from`->`to` step may be pruned any time
+      # - but in exchange we get back the disk space allocated by `to` when we prune it
+      # - and because we still have the bookmarks created by `guarantee_incremental`, we can still do incremental replication of `from`->`to2` in the future
+      incremental: guarantee_incremental
+  snapshotting:
+    type: manual
+  pruning:
+    # no-op prune rule on sender (keep all snapshots), job `snapshot` takes care of this
+    keep_sender:
+    - type: regex
+      regex: ".*"
+    # retain 
+    keep_receiver:
+    # longer retention on the backup drive, we have more space there
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 360x1d
+      regex: "^zrepl_.*"
+    # retain all non-zrepl snapshots on the backup drive
+    - type: regex
+      negate: true
+      regex: "^zrepl_.*"
+
+# This job receives from job `push_to_drive` into `backuppool/zrepl/sink/myhostname`
+- type: sink
+  name: backuppool_sink
+  root_fs: "backuppool/zrepl/sink"
+  serve:
+    type: local
+    listener_name: backuppool_sink

config/samples/quickstart_continuous_server_backup_receiver.yml

@@ -0,0 +1,12 @@
+jobs:
+- name: sink
+  type: sink
+  serve:
+      type: tls
+      listen: ":8888"
+      ca: "/etc/zrepl/prod.crt"
+      cert: "/etc/zrepl/backups.crt"
+      key: "/etc/zrepl/backups.key"
+      client_cns:
+        - "prod"
+  root_fs: "storage/zrepl/sink"

config/samples/quickstart_continuous_server_backup_sender.yml

@@ -0,0 +1,28 @@
+jobs:
+- name: prod_to_backups
+  type: push
+  connect:
+    type: tls
+    address: "backups.example.com:8888"
+    ca: /etc/zrepl/backups.crt
+    cert: /etc/zrepl/prod.crt
+    key:  /etc/zrepl/prod.key
+    server_cn: "backups"
+  filesystems: {
+    "zroot<": true,
+    "zroot/var/tmp<": false,
+    "zroot/usr/home/paranoid": false
+  }
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 10m
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 30x1d | 6x30d
+      regex: "^zrepl_"

config/samples/quickstart_fan_out_replication_source.yml

@@ -0,0 +1,59 @@
+jobs:
+# Separate job for snapshots and pruning
+- name: snapshots
+  type: snap
+  filesystems:
+    'tank<': true # all filesystems
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 10m
+  pruning:
+    keep:
+      # Keep non-zrepl snapshots
+      - type: regex
+        negate: true
+        regex: '^zrepl_'
+      # Time-based snapshot retention
+      - type: grid
+        grid: 1x1h(keep=all) | 24x1h | 30x1d | 12x30d
+        regex: '^zrepl_'
+
+# Source job for target B
+- name: target_b
+  type: source
+  serve:
+    type: tls
+    listen: :8888
+    ca: /etc/zrepl/b.example.com.crt
+    cert: /etc/zrepl/a.example.com.crt
+    key: /etc/zrepl/a.example.com.key
+    client_cns:
+      - b.example.com
+  filesystems:
+    'tank<': true # all filesystems
+  # Snapshots are handled by the separate snap job
+  snapshotting:
+    type: manual
+
+# Source job for target C
+- name: target_c
+  type: source
+  serve:
+    type: tls
+    listen: :8889
+    ca: /etc/zrepl/c.example.com.crt
+    cert: /etc/zrepl/a.example.com.crt
+    key: /etc/zrepl/a.example.com.key
+    client_cns:
+      - c.example.com
+  filesystems:
+    'tank<': true # all filesystems
+  # Snapshots are handled by the separate snap job
+  snapshotting:
+    type: manual
+
+# Source jobs for remaining targets. Each one should listen on a different port
+# and reference the correct certificate and client CN.
+# - name: target_c
+#   ...

config/samples/quickstart_fan_out_replication_target.yml

@@ -0,0 +1,30 @@
+jobs:
+# Pull from source server A
+- name: source_a
+  type: pull
+  connect:
+    type: tls
+    # Use the correct port for this specific client (eg. B is 8888, C is 8889, etc.)
+    address: a.example.com:8888
+    ca: /etc/zrepl/a.example.com.crt
+    # Use the correct key pair for this specific client
+    cert: /etc/zrepl/b.example.com.crt
+    key: /etc/zrepl/b.example.com.key
+    server_cn: a.example.com
+  root_fs: pool0/backup
+  interval: 10m
+  pruning:
+    keep_sender:
+      # Source does the pruning in its snap job
+      - type: regex
+        regex: '.*'
+    # Receiver-side pruning can be configured as desired on each target server
+    keep_receiver:
+      # Keep non-zrepl snapshots
+      - type: regex
+        negate: true
+        regex: '^zrepl_'
+      # Time-based snapshot retention
+      - type: grid
+        grid: 1x1h(keep=all) | 24x1h | 30x1d | 12x30d
+        regex: '^zrepl_'

config/samples/sink.yml

@@ -0,0 +1,13 @@
+jobs:
+  - type: sink
+    name: "laptop_sink"
+    root_fs: "pool2/backup_laptops"
+    serve:
+      type: tls
+      listen: "192.168.122.189:8888"
+      ca: "ca.pem"
+      cert: "cert.pem"
+      key: "key.pem"
+      client_cns:
+        - "laptop1"
+        - "homeserver"

config/samples/snap.yml

@@ -0,0 +1,14 @@
+jobs:
+- name: snapjob
+  type: snap
+  filesystems: {
+    "tank/frequently_changed<": true,
+  }
+  snapshotting:
+    type: periodic
+    interval: 2m
+    prefix: zrepl_snapjob_
+  pruning:
+    keep:
+      - type: last_n
+        count: 60

config/samples/snap_cron.yml

@@ -0,0 +1,14 @@
+jobs:
+- name: snapjob
+  type: snap
+  filesystems: {
+    "tank<": true,
+  }
+  snapshotting:
+    type: cron
+    prefix: zrepl_snapjob_
+    cron: "*/5 * * * *"
+  pruning:
+    keep:
+      - type: last_n
+        count: 60

config/samples/source.yml

@@ -0,0 +1,21 @@
+jobs:
+- name: pull_source
+  type: source
+  serve:
+    type: tcp
+    listen: "0.0.0.0:8888"
+    clients: {
+      "192.168.122.123" :               "mysql01",
+      "192.168.122.42" :                "mx01",
+      "2001:0db8:85a3::8a2e:0370:7334": "gateway",
+      "10.23.42.0/24":       "cluster-*",
+      "fde4:8dba:82e1::/64": "san-*",
+    }
+  filesystems: {
+    "<": true,
+    "secret": false
+  }
+  snapshotting:
+    type: periodic
+    interval: 10m
+    prefix: zrepl_

config/samples/source_ssh.yml

@@ -0,0 +1,17 @@
+jobs:
+- name: pull_source
+  type: source
+  serve:
+    type: stdinserver
+    client_identities:
+      - "client1"
+      - "client2"
+  filesystems: {
+    "<": true,
+    "secret": false
+  }
+  snapshotting:
+    type: periodic
+    interval: 10m
+    prefix: zrepl_
+

daemon/control.go

@@ -0,0 +1,293 @@
+package daemon
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/daemon/nethelpers"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/util/envconst"
+	"github.com/zrepl/zrepl/version"
+	"github.com/zrepl/zrepl/zfs"
+	"github.com/zrepl/zrepl/zfs/zfscmd"
+)
+
+type controlJob struct {
+	sockaddr *net.UnixAddr
+	jobs     *jobs
+}
+
+func newControlJob(sockpath string, jobs *jobs) (j *controlJob, err error) {
+	j = &controlJob{jobs: jobs}
+
+	j.sockaddr, err = net.ResolveUnixAddr("unix", sockpath)
+	if err != nil {
+		err = errors.Wrap(err, "cannot resolve unix address")
+		return
+	}
+
+	return
+}
+
+func (j *controlJob) Name() string { return jobNameControl }
+
+func (j *controlJob) Status() *job.Status { return &job.Status{Type: job.TypeInternal} }
+
+func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
+
+func (j *controlJob) SenderConfig() *endpoint.SenderConfig { return nil }
+
+var promControl struct {
+	requestBegin    *prometheus.CounterVec
+	requestFinished *prometheus.HistogramVec
+}
+
+func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) {
+	promControl.requestBegin = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "zrepl",
+		Subsystem: "control",
+		Name:      "request_begin",
+		Help:      "number of request we started to handle",
+	}, []string{"endpoint"})
+
+	promControl.requestFinished = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "zrepl",
+		Subsystem: "control",
+		Name:      "request_finished",
+		Help:      "time it took a request to finish",
+		Buckets:   []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3, 10e-3, 100e-3, 200e-3, 400e-3, 800e-3, 1, 10, 20},
+	}, []string{"endpoint"})
+	registerer.MustRegister(promControl.requestBegin)
+	registerer.MustRegister(promControl.requestFinished)
+}
+
+const (
+	ControlJobEndpointPProf   string = "/debug/pprof"
+	ControlJobEndpointVersion string = "/version"
+	ControlJobEndpointStatus  string = "/status"
+	ControlJobEndpointSignal  string = "/signal"
+)
+
+func (j *controlJob) Run(ctx context.Context) {
+
+	log := job.GetLogger(ctx)
+	defer log.Info("control job finished")
+
+	l, err := nethelpers.ListenUnixPrivate(j.sockaddr)
+	if err != nil {
+		log.WithError(err).Error("error listening")
+		return
+	}
+
+	pprofServer := NewPProfServer(ctx)
+	if listen := envconst.String("ZREPL_DAEMON_AUTOSTART_PPROF_SERVER", ""); listen != "" {
+		pprofServer.Control(PprofServerControlMsg{
+			Run:               true,
+			HttpListenAddress: listen,
+		})
+	}
+
+	mux := http.NewServeMux()
+	mux.Handle(ControlJobEndpointPProf,
+		requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) {
+			var msg PprofServerControlMsg
+			err := decoder(&msg)
+			if err != nil {
+				return nil, errors.Errorf("decode failed")
+			}
+			pprofServer.Control(msg)
+			return struct{}{}, nil
+		}}})
+
+	mux.Handle(ControlJobEndpointVersion,
+		requestLogger{log: log, handler: jsonResponder{log, func() (interface{}, error) {
+			return version.NewZreplVersionInformation(), nil
+		}}})
+
+	mux.Handle(ControlJobEndpointStatus,
+		// don't log requests to status endpoint, too spammy
+		jsonResponder{log, func() (interface{}, error) {
+			jobs := j.jobs.status()
+			globalZFS := zfscmd.GetReport()
+			envconstReport := envconst.GetReport()
+			s := Status{
+				Jobs: jobs,
+				Global: GlobalStatus{
+					ZFSCmds:   globalZFS,
+					Envconst:  envconstReport,
+					OsEnviron: os.Environ(),
+				}}
+			return s, nil
+		}})
+
+	mux.Handle(ControlJobEndpointSignal,
+		requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) {
+			type reqT struct {
+				Name string
+				Op   string
+			}
+			var req reqT
+			if decoder(&req) != nil {
+				return nil, errors.Errorf("decode failed")
+			}
+
+			var err error
+			switch req.Op {
+			case "wakeup":
+				err = j.jobs.wakeup(req.Name)
+			case "reset":
+				err = j.jobs.reset(req.Name)
+			default:
+				err = fmt.Errorf("operation %q is invalid", req.Op)
+			}
+
+			return struct{}{}, err
+		}}})
+	server := http.Server{
+		Handler: mux,
+		// control socket is local, 1s timeout should be more than sufficient, even on a loaded system
+		WriteTimeout: envconst.Duration("ZREPL_DAEMON_CONTROL_SERVER_WRITE_TIMEOUT", 1*time.Second),
+		ReadTimeout:  envconst.Duration("ZREPL_DAEMON_CONTROL_SERVER_READ_TIMEOUT", 1*time.Second),
+	}
+
+outer:
+	for {
+
+		served := make(chan error)
+		go func() {
+			served <- server.Serve(l)
+			close(served)
+		}()
+
+		select {
+		case <-ctx.Done():
+			log.WithError(ctx.Err()).Info("context done")
+			err := server.Shutdown(context.Background())
+			if err != nil {
+				log.WithError(err).Error("cannot shutdown server")
+			}
+			break outer
+		case err = <-served:
+			if err != nil {
+				log.WithError(err).Error("error serving")
+				break outer
+			}
+		}
+
+	}
+
+}
+
+type jsonResponder struct {
+	log      Logger
+	producer func() (interface{}, error)
+}
+
+func (j jsonResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	logIoErr := func(err error) {
+		if err != nil {
+			j.log.WithError(err).Error("control handler io error")
+		}
+	}
+	res, err := j.producer()
+	if err != nil {
+		j.log.WithError(err).Error("control handler error")
+		w.WriteHeader(http.StatusInternalServerError)
+		_, err = io.WriteString(w, err.Error())
+		logIoErr(err)
+		return
+	}
+	var buf bytes.Buffer
+	err = json.NewEncoder(&buf).Encode(res)
+	if err != nil {
+		j.log.WithError(err).Error("control handler json marshal error")
+		w.WriteHeader(http.StatusInternalServerError)
+		_, err = io.WriteString(w, err.Error())
+	} else {
+		_, err = io.Copy(w, &buf)
+	}
+	logIoErr(err)
+}
+
+type jsonDecoder = func(interface{}) error
+
+type jsonRequestResponder struct {
+	log      Logger
+	producer func(decoder jsonDecoder) (interface{}, error)
+}
+
+func (j jsonRequestResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	logIoErr := func(err error) {
+		if err != nil {
+			j.log.WithError(err).Error("control handler io error")
+		}
+	}
+
+	var decodeError error
+	decoder := func(i interface{}) error {
+		err := json.NewDecoder(r.Body).Decode(&i)
+		decodeError = err
+		return err
+	}
+	res, producerErr := j.producer(decoder)
+
+	//If we had a decode error ignore output of producer and return error
+	if decodeError != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		_, err := io.WriteString(w, decodeError.Error())
+		logIoErr(err)
+		return
+	}
+	if producerErr != nil {
+		j.log.WithError(producerErr).Error("control handler error")
+		w.WriteHeader(http.StatusInternalServerError)
+		_, err := io.WriteString(w, producerErr.Error())
+		logIoErr(err)
+		return
+	}
+
+	var buf bytes.Buffer
+	encodeErr := json.NewEncoder(&buf).Encode(res)
+	if encodeErr != nil {
+		j.log.WithError(producerErr).Error("control handler json marshal error")
+		w.WriteHeader(http.StatusInternalServerError)
+		_, err := io.WriteString(w, encodeErr.Error())
+		logIoErr(err)
+	} else {
+		_, err := io.Copy(w, &buf)
+		logIoErr(err)
+	}
+}
+
+type requestLogger struct {
+	log         logger.Logger
+	handler     http.Handler
+	handlerFunc http.HandlerFunc
+}
+
+func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	log := l.log.WithField("method", r.Method).WithField("url", r.URL)
+	log.Debug("start")
+	promControl.requestBegin.WithLabelValues(r.URL.Path).Inc()
+	defer prometheus.NewTimer(promControl.requestFinished.WithLabelValues(r.URL.Path)).ObserveDuration()
+	if l.handlerFunc != nil {
+		l.handlerFunc(w, r)
+	} else if l.handler != nil {
+		l.handler.ServeHTTP(w, r)
+	} else {
+		log.Error("no handler or handlerFunc configured")
+	}
+	log.Debug("finish")
+}

daemon/daemon.go

@@ -0,0 +1,258 @@
+package daemon
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"os"
+	"os/signal"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/util/envconst"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/daemon/job/reset"
+	"github.com/zrepl/zrepl/daemon/job/wakeup"
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/version"
+	"github.com/zrepl/zrepl/zfs/zfscmd"
+)
+
+func Run(ctx context.Context, conf *config.Config) error {
+	ctx, cancel := context.WithCancel(ctx)
+
+	defer cancel()
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		<-sigChan
+		cancel()
+	}()
+
+	// The math/rand package is used presently for generating trace IDs, we
+	// seed it with the current time and pid so that the IDs are mostly
+	// unique.
+	rand.Seed(time.Now().UnixNano())
+	rand.Seed(int64(os.Getpid()))
+
+	outlets, err := logging.OutletsFromConfig(*conf.Global.Logging)
+	if err != nil {
+		return errors.Wrap(err, "cannot build logging from config")
+	}
+	outlets.Add(newPrometheusLogOutlet(), logger.Debug)
+
+	confJobs, err := job.JobsFromConfig(conf, config.ParseFlagsNone)
+	if err != nil {
+		return errors.Wrap(err, "cannot build jobs from config")
+	}
+
+	log := logger.NewLogger(outlets, 1*time.Second)
+	log.Info(version.NewZreplVersionInformation().String())
+
+	ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
+	trace.RegisterCallback(trace.Callback{
+		OnBegin: func(ctx context.Context) { logging.GetLogger(ctx, logging.SubsysTraceData).Debug("begin span") },
+		OnEnd: func(ctx context.Context, spanInfo trace.SpanInfo) {
+			logging.
+				GetLogger(ctx, logging.SubsysTraceData).
+				WithField("duration_s", spanInfo.EndedAt().Sub(spanInfo.StartedAt()).Seconds()).
+				Debug("finished span " + spanInfo.TaskAndSpanStack(trace.SpanStackKindAnnotation))
+		},
+	})
+
+	for _, job := range confJobs {
+		if IsInternalJobName(job.Name()) {
+			panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME
+		}
+	}
+
+	jobs := newJobs()
+
+	// start control socket
+	controlJob, err := newControlJob(conf.Global.Control.SockPath, jobs)
+	if err != nil {
+		panic(err) // FIXME
+	}
+	jobs.start(ctx, controlJob, true)
+
+	for i, jc := range conf.Global.Monitoring {
+		var (
+			job job.Job
+			err error
+		)
+		switch v := jc.Ret.(type) {
+		case *config.PrometheusMonitoring:
+			job, err = newPrometheusJobFromConfig(v)
+		default:
+			return errors.Errorf("unknown monitoring job #%d (type %T)", i, v)
+		}
+		if err != nil {
+			return errors.Wrapf(err, "cannot build monitoring job #%d", i)
+		}
+		jobs.start(ctx, job, true)
+	}
+
+	// register global (=non job-local) metrics
+	version.PrometheusRegister(prometheus.DefaultRegisterer)
+	zfscmd.RegisterMetrics(prometheus.DefaultRegisterer)
+	trace.RegisterMetrics(prometheus.DefaultRegisterer)
+	endpoint.RegisterMetrics(prometheus.DefaultRegisterer)
+
+	log.Info("starting daemon")
+
+	// start regular jobs
+	for _, j := range confJobs {
+		jobs.start(ctx, j, false)
+	}
+
+	select {
+	case <-jobs.wait():
+		log.Info("all jobs finished")
+	case <-ctx.Done():
+		log.WithError(ctx.Err()).Info("context finished")
+	}
+	log.Info("waiting for jobs to finish")
+	<-jobs.wait()
+	log.Info("daemon exiting")
+	return nil
+}
+
+type jobs struct {
+	wg sync.WaitGroup
+
+	// m protects all fields below it
+	m       sync.RWMutex
+	wakeups map[string]wakeup.Func // by Job.Name
+	resets  map[string]reset.Func  // by Job.Name
+	jobs    map[string]job.Job
+}
+
+func newJobs() *jobs {
+	return &jobs{
+		wakeups: make(map[string]wakeup.Func),
+		resets:  make(map[string]reset.Func),
+		jobs:    make(map[string]job.Job),
+	}
+}
+
+func (s *jobs) wait() <-chan struct{} {
+	ch := make(chan struct{})
+	go func() {
+		s.wg.Wait()
+		close(ch)
+	}()
+	return ch
+}
+
+type Status struct {
+	Jobs   map[string]*job.Status
+	Global GlobalStatus
+}
+
+type GlobalStatus struct {
+	ZFSCmds   *zfscmd.Report
+	Envconst  *envconst.Report
+	OsEnviron []string
+}
+
+func (s *jobs) status() map[string]*job.Status {
+	s.m.RLock()
+	defer s.m.RUnlock()
+
+	type res struct {
+		name   string
+		status *job.Status
+	}
+	var wg sync.WaitGroup
+	c := make(chan res, len(s.jobs))
+	for name, j := range s.jobs {
+		wg.Add(1)
+		go func(name string, j job.Job) {
+			defer wg.Done()
+			c <- res{name: name, status: j.Status()}
+		}(name, j)
+	}
+	wg.Wait()
+	close(c)
+	ret := make(map[string]*job.Status, len(s.jobs))
+	for res := range c {
+		ret[res.name] = res.status
+	}
+	return ret
+}
+
+func (s *jobs) wakeup(job string) error {
+	s.m.RLock()
+	defer s.m.RUnlock()
+
+	wu, ok := s.wakeups[job]
+	if !ok {
+		return errors.Errorf("Job %s does not exist", job)
+	}
+	return wu()
+}
+
+func (s *jobs) reset(job string) error {
+	s.m.RLock()
+	defer s.m.RUnlock()
+
+	wu, ok := s.resets[job]
+	if !ok {
+		return errors.Errorf("Job %s does not exist", job)
+	}
+	return wu()
+}
+
+const (
+	jobNamePrometheus = "_prometheus"
+	jobNameControl    = "_control"
+)
+
+func IsInternalJobName(s string) bool {
+	return strings.HasPrefix(s, "_")
+}
+
+func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
+	s.m.Lock()
+	defer s.m.Unlock()
+
+	ctx = logging.WithInjectedField(ctx, logging.JobField, j.Name())
+
+	jobName := j.Name()
+	if !internal && IsInternalJobName(jobName) {
+		panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName))
+	}
+	if internal && !IsInternalJobName(jobName) {
+		panic(fmt.Sprintf("internal job does not use internal job name %s", jobName))
+	}
+	if _, ok := s.jobs[jobName]; ok {
+		panic(fmt.Sprintf("duplicate job name %s", jobName))
+	}
+
+	j.RegisterMetrics(prometheus.DefaultRegisterer)
+
+	s.jobs[jobName] = j
+	ctx = zfscmd.WithJobID(ctx, j.Name())
+	ctx, wakeup := wakeup.Context(ctx)
+	ctx, resetFunc := reset.Context(ctx)
+	s.wakeups[jobName] = wakeup
+	s.resets[jobName] = resetFunc
+
+	s.wg.Add(1)
+	go func() {
+		defer s.wg.Done()
+		job.GetLogger(ctx).Info("starting job")
+		defer job.GetLogger(ctx).Info("job exited")
+		j.Run(ctx)
+	}()
+}

daemon/filters/fsmapfilter.go

@@ -0,0 +1,283 @@
+package filters
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type DatasetMapFilter struct {
+	entries []datasetMapFilterEntry
+
+	// if set, only valid filter entries can be added using Add()
+	// and Map() will always return an error
+	filterMode bool
+}
+
+type datasetMapFilterEntry struct {
+	path *zfs.DatasetPath
+	// the mapping. since this datastructure acts as both mapping and filter
+	// we have to convert it to the desired rep dynamically
+	mapping      string
+	subtreeMatch bool
+}
+
+func NewDatasetMapFilter(capacity int, filterMode bool) *DatasetMapFilter {
+	return &DatasetMapFilter{
+		entries:    make([]datasetMapFilterEntry, 0, capacity),
+		filterMode: filterMode,
+	}
+}
+
+func (m *DatasetMapFilter) Add(pathPattern, mapping string) (err error) {
+
+	if m.filterMode {
+		if _, err = m.parseDatasetFilterResult(mapping); err != nil {
+			return
+		}
+	}
+
+	// assert path glob adheres to spec
+	const SUBTREE_PATTERN string = "<"
+	patternCount := strings.Count(pathPattern, SUBTREE_PATTERN)
+	switch {
+	case patternCount > 1:
+	case patternCount == 1 && !strings.HasSuffix(pathPattern, SUBTREE_PATTERN):
+		err = fmt.Errorf("pattern invalid: only one '<' at end of string allowed")
+		return
+	}
+
+	pathStr := strings.TrimSuffix(pathPattern, SUBTREE_PATTERN)
+	path, err := zfs.NewDatasetPath(pathStr)
+	if err != nil {
+		return fmt.Errorf("pattern is not a dataset path: %s", err)
+	}
+
+	entry := datasetMapFilterEntry{
+		path:         path,
+		mapping:      mapping,
+		subtreeMatch: patternCount > 0,
+	}
+	m.entries = append(m.entries, entry)
+	return
+
+}
+
+// find the most specific prefix mapping we have
+//
+// longer prefix wins over shorter prefix, direct wins over glob
+func (m DatasetMapFilter) mostSpecificPrefixMapping(path *zfs.DatasetPath) (idx int, found bool) {
+	lcp, lcp_entry_idx := -1, -1
+	direct_idx := -1
+	for e := range m.entries {
+		entry := m.entries[e]
+		ep := m.entries[e].path
+		lep := ep.Length()
+
+		switch {
+		case !entry.subtreeMatch && ep.Equal(path):
+			direct_idx = e
+			continue
+		case entry.subtreeMatch && path.HasPrefix(ep) && lep > lcp:
+			lcp = lep
+			lcp_entry_idx = e
+		default:
+			continue
+		}
+	}
+
+	if lcp_entry_idx >= 0 || direct_idx >= 0 {
+		found = true
+		switch {
+		case direct_idx >= 0:
+			idx = direct_idx
+		case lcp_entry_idx >= 0:
+			idx = lcp_entry_idx
+		}
+	}
+	return
+}
+
+// Returns target == nil if there is no mapping
+func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error) {
+
+	if m.filterMode {
+		err = fmt.Errorf("using a filter for mapping simply does not work")
+		return
+	}
+
+	mi, hasMapping := m.mostSpecificPrefixMapping(source)
+	if !hasMapping {
+		return nil, nil
+	}
+	me := m.entries[mi]
+
+	if me.mapping == "" {
+		// Special case treatment: 'foo/bar<' => ''
+		if !me.subtreeMatch {
+			return nil, fmt.Errorf("mapping to '' must be a subtree match")
+		}
+		// ok...
+	} else {
+		if strings.HasPrefix("!", me.mapping) {
+			// reject mapping
+			return nil, nil
+		}
+	}
+
+	target, err = zfs.NewDatasetPath(me.mapping)
+	if err != nil {
+		err = fmt.Errorf("mapping target is not a dataset path: %s", err)
+		return
+	}
+	if me.subtreeMatch {
+		// strip common prefix ('<' wildcards are no special case here)
+		extendComps := source.Copy()
+		extendComps.TrimPrefix(me.path)
+		target.Extend(extendComps)
+	}
+	return
+}
+
+func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
+
+	if !m.filterMode {
+		err = fmt.Errorf("using a mapping as a filter does not work")
+		return
+	}
+
+	mi, hasMapping := m.mostSpecificPrefixMapping(p)
+	if !hasMapping {
+		pass = false
+		return
+	}
+	me := m.entries[mi]
+	pass, err = m.parseDatasetFilterResult(me.mapping)
+	return
+}
+
+func (m DatasetMapFilter) UserSpecifiedDatasets() (datasets zfs.UserSpecifiedDatasetsSet) {
+	datasets = make(zfs.UserSpecifiedDatasetsSet)
+	for i := range m.entries {
+		datasets[m.entries[i].path.ToString()] = true
+	}
+	return
+}
+
+// Construct a new filter-only DatasetMapFilter from a mapping
+// The new filter allows exactly those paths that were not forbidden by the mapping.
+func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
+
+	if m.filterMode {
+		err = errors.Errorf("can only invert mappings")
+		return
+	}
+
+	inv = &DatasetMapFilter{
+		make([]datasetMapFilterEntry, len(m.entries)),
+		true,
+	}
+
+	for i, e := range m.entries {
+		inv.entries[i].path, err = zfs.NewDatasetPath(e.mapping)
+		if err != nil {
+			err = errors.Wrapf(err, "mapping cannot be inverted: '%s' is not a dataset path", e.mapping)
+			return
+		}
+		inv.entries[i].mapping = MapFilterResultOk
+		inv.entries[i].subtreeMatch = e.subtreeMatch
+	}
+
+	return inv, nil
+}
+
+// FIXME investigate whether we can support more...
+func (m DatasetMapFilter) Invert() (endpoint.FSMap, error) {
+
+	if m.filterMode {
+		return nil, errors.Errorf("can only invert mappings")
+	}
+
+	if len(m.entries) != 1 {
+		return nil, errors.Errorf("inversion of complicated mappings is not implemented") // FIXME
+	}
+
+	e := m.entries[0]
+
+	inv := &DatasetMapFilter{
+		make([]datasetMapFilterEntry, len(m.entries)),
+		false,
+	}
+	mp, err := zfs.NewDatasetPath(e.mapping)
+	if err != nil {
+		return nil, err
+	}
+
+	inv.entries[0] = datasetMapFilterEntry{
+		path:         mp,
+		mapping:      e.path.ToString(),
+		subtreeMatch: e.subtreeMatch,
+	}
+
+	return inv, nil
+}
+
+// Creates a new DatasetMapFilter in filter mode from a mapping
+// All accepting mapping results are mapped to accepting filter results
+// All rejecting mapping results are mapped to rejecting filter results
+func (m DatasetMapFilter) AsFilter() endpoint.FSFilter {
+
+	f := &DatasetMapFilter{
+		make([]datasetMapFilterEntry, len(m.entries)),
+		true,
+	}
+
+	for i, e := range m.entries {
+		var newe datasetMapFilterEntry = e
+		if strings.HasPrefix(newe.mapping, "!") {
+			newe.mapping = MapFilterResultOmit
+		} else {
+			newe.mapping = MapFilterResultOk
+		}
+		f.entries[i] = newe
+	}
+
+	return f
+}
+
+const (
+	MapFilterResultOk   string = "ok"
+	MapFilterResultOmit string = "!"
+)
+
+// Parse a dataset filter result
+func (m DatasetMapFilter) parseDatasetFilterResult(result string) (pass bool, err error) {
+	l := strings.ToLower(result)
+	if l == MapFilterResultOk {
+		return true, nil
+	}
+	if l == MapFilterResultOmit {
+		return false, nil
+	}
+	return false, fmt.Errorf("'%s' is not a valid filter result", result)
+}
+
+func DatasetMapFilterFromConfig(in map[string]bool) (f *DatasetMapFilter, err error) {
+
+	f = NewDatasetMapFilter(len(in), true)
+	for pathPattern, accept := range in {
+		mapping := MapFilterResultOmit
+		if accept {
+			mapping = MapFilterResultOk
+		}
+		if err = f.Add(pathPattern, mapping); err != nil {
+			err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err)
+			return
+		}
+	}
+	return
+}

daemon/filters/fsmapfilter_test.go

@@ -0,0 +1,89 @@
+package filters
+
+import (
+	"testing"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func TestDatasetMapFilter(t *testing.T) {
+
+	type testCase struct {
+		name      string
+		filter    map[string]string
+		checkPass map[string]bool // each entry is checked to match the filter's `pass` return value
+	}
+
+	tcs := []testCase{
+		{
+			"default_no_match",
+			map[string]string{},
+			map[string]bool{
+				"":      false,
+				"foo":   false,
+				"zroot": false,
+			},
+		},
+		{
+			"more_specific_path_has_precedence",
+			map[string]string{
+				"tank<":         "ok",
+				"tank/tmp<":     "!",
+				"tank/home/x<":  "!",
+				"tank/home/x/1": "ok",
+			},
+			map[string]bool{
+				"zroot":         false,
+				"tank":          true,
+				"tank/tmp":      false,
+				"tank/tmp/foo":  false,
+				"tank/home/x":   false,
+				"tank/home/y":   true,
+				"tank/home/x/1": true,
+				"tank/home/x/2": false,
+			},
+		},
+		{
+			"precedence_of_specific_over_subtree_wildcard_on_same_path",
+			map[string]string{
+				"tank/home/bob":  "ok",
+				"tank/home/bob<": "!",
+			},
+			map[string]bool{
+				"tank/home/bob":           true,
+				"tank/home/bob/downloads": false,
+			},
+		},
+	}
+
+	for tc := range tcs {
+		t.Run(tcs[tc].name, func(t *testing.T) {
+			c := tcs[tc]
+			f := NewDatasetMapFilter(len(c.filter), true)
+			for p, a := range c.filter {
+				err := f.Add(p, a)
+				if err != nil {
+					t.Fatalf("incorrect filter spec: %s", err)
+				}
+			}
+			for p, checkPass := range c.checkPass {
+				zp, err := zfs.NewDatasetPath(p)
+				if err != nil {
+					t.Fatalf("incorrect path spec: %s", err)
+				}
+				pass, err := f.Filter(zp)
+				if err != nil {
+					t.Fatalf("unexpected filter error: %s", err)
+				}
+				ok := pass == checkPass
+				failstr := "OK"
+				if !ok {
+					failstr = "FAIL"
+					t.Fail()
+				}
+				t.Logf("%-40q  %5v  (exp=%v act=%v)", p, failstr, checkPass, pass)
+			}
+		})
+	}
+
+}

daemon/hooks/edge_string.go

@@ -0,0 +1,35 @@
+// Code generated by "stringer -type=Edge"; DO NOT EDIT.
+
+package hooks
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[Pre-1]
+	_ = x[Callback-2]
+	_ = x[Post-4]
+}
+
+const (
+	_Edge_name_0 = "PreCallback"
+	_Edge_name_1 = "Post"
+)
+
+var (
+	_Edge_index_0 = [...]uint8{0, 3, 11}
+)
+
+func (i Edge) String() string {
+	switch {
+	case 1 <= i && i <= 2:
+		i -= 1
+		return _Edge_name_0[_Edge_index_0[i]:_Edge_index_0[i+1]]
+	case i == 4:
+		return _Edge_name_1
+	default:
+		return "Edge(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+}

daemon/hooks/hook_config.go

@@ -0,0 +1,52 @@
+package hooks
+
+import (
+	"fmt"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type List []Hook
+
+func HookFromConfig(in config.HookEnum) (Hook, error) {
+	switch v := in.Ret.(type) {
+	case *config.HookCommand:
+		return NewCommandHook(v)
+	case *config.HookPostgresCheckpoint:
+		return PgChkptHookFromConfig(v)
+	case *config.HookMySQLLockTables:
+		return MyLockTablesFromConfig(v)
+	default:
+		return nil, fmt.Errorf("unknown hook type %T", v)
+	}
+}
+
+func ListFromConfig(in *config.HookList) (r *List, err error) {
+	hl := make(List, len(*in))
+
+	for i, h := range *in {
+		hl[i], err = HookFromConfig(h)
+		if err != nil {
+			return nil, fmt.Errorf("create hook #%d: %s", i+1, err)
+		}
+	}
+
+	return &hl, nil
+}
+
+func (l List) CopyFilteredForFilesystem(fs *zfs.DatasetPath) (ret List, err error) {
+	ret = make(List, 0, len(l))
+
+	for _, h := range l {
+		var passFilesystem bool
+		if passFilesystem, err = h.Filesystems().Filter(fs); err != nil {
+			return nil, err
+		}
+		if passFilesystem {
+			ret = append(ret, h)
+		}
+	}
+
+	return ret, nil
+}

daemon/hooks/hook_docs.go

@@ -0,0 +1,33 @@
+// Package hooks implements pre- and post snapshot hooks.
+//
+// Plan is a generic executor for ExpectStepReports before and after an activity specified in a callback.
+// It provides a reporting facility that can be polled while the plan is executing to gather progress information.
+//
+// This package also provides all supported hook type implementations and abstractions around them.
+//
+// # Use For Other Kinds Of ExpectStepReports
+//
+// This package REQUIRES REFACTORING before it can be used for other activities than snapshots, e.g. pre- and post-replication:
+//
+// The Hook interface requires a hook to provide a Filesystems() filter, which doesn't make sense for
+// all kinds of activities.
+//
+// The hook implementations should move out of this package.
+// However, there is a lot of tight coupling which to untangle isn't worth it ATM.
+//
+// # How This Package Is Used By Package Snapper
+//
+// Deserialize a config.List using ListFromConfig().
+// Then it MUST filter the list to only contain hooks for a particular filesystem using
+// hooksList.CopyFilteredForFilesystem(fs).
+//
+// Then create a CallbackHook using NewCallbackHookForFilesystem().
+//
+// Pass all of the above to NewPlan() which provides a Report() and Run() method:
+//
+// Plan.Run(ctx context.Context,dryRun bool) executes the plan and take a context as argument that should contain a logger added using hooks.WithLogger()).
+// The value of dryRun is passed through to the hooks' Run() method.
+// Command hooks make it available in the environment variable ZREPL_DRYRUN.
+//
+// Plan.Report() can be called while Plan.Run() is executing to give an overview of plan execution progress (future use in "zrepl status").
+package hooks

daemon/hooks/hook_exec.go

@@ -0,0 +1,292 @@
+package hooks
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+// Re-export type here so that
+// every file in package hooks doesn't
+// have to import github.com/zrepl/zrepl/zfs
+type Filter zfs.DatasetFilter
+
+type Hook interface {
+	Filesystems() Filter
+
+	// If true and the Pre edge invocation of Run fails, Post edge will not run and other Pre edges will not run.
+	ErrIsFatal() bool
+
+	// Run is invoked by HookPlan for a Pre edge.
+	// If HookReport.HadError() == false, the Post edge will be invoked, too.
+	Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport
+
+	String() string
+}
+
+type Phase string
+
+const (
+	PhaseSnapshot = Phase("snapshot")
+	PhaseTesting  = Phase("testing")
+)
+
+func (p Phase) String() string {
+	return string(p)
+}
+
+//go:generate stringer -type=Edge
+type Edge uint
+
+const (
+	Pre = Edge(1 << iota)
+	Callback
+	Post
+)
+
+func (e Edge) StringForPhase(phase Phase) string {
+	return fmt.Sprintf("%s_%s", e.String(), phase.String())
+}
+
+//go:generate enumer -type=StepStatus -trimprefix=Step
+type StepStatus int
+
+const (
+	StepPending StepStatus = 1 << iota
+	StepExec
+	StepOk
+	StepErr
+	StepSkippedDueToFatalErr
+	StepSkippedDueToPreErr
+)
+
+type HookReport interface {
+	String() string
+	HadError() bool
+	Error() string
+}
+
+type Step struct {
+	Hook       Hook
+	Edge       Edge
+	Status     StepStatus
+	Begin, End time.Time
+	// Report may be nil
+	// FIXME cannot serialize this for client status, but contains interesting info (like what error happened)
+	Report HookReport
+	state  map[interface{}]interface{}
+}
+
+func (s Step) String() (out string) {
+	fatal := "~"
+	if s.Hook.ErrIsFatal() && s.Edge == Pre {
+		fatal = "!"
+	}
+	runTime := "..."
+	if s.Status != StepPending {
+		t := s.End.Sub(s.Begin)
+		runTime = t.Round(time.Millisecond).String()
+	}
+	return fmt.Sprintf("[%s] [%5s] %s [%s]  %s", s.Status, runTime, fatal, s.Edge, s.Hook)
+}
+
+type Plan struct {
+	mtx sync.RWMutex
+
+	steps []*Step
+	pre   []*Step // protected by mtx
+	cb    *Step
+	post  []*Step // not reversed, i.e. entry at index i corresponds to pre-edge in pre[i]
+
+	phase Phase
+	env   Env
+}
+
+func NewPlan(hooks *List, phase Phase, cb *CallbackHook, extra Env) (*Plan, error) {
+
+	var pre, post []*Step
+	// TODO sanity check unique name of hook?
+	for _, hook := range *hooks {
+		state := make(map[interface{}]interface{})
+		preE := &Step{
+			Hook:   hook,
+			Edge:   Pre,
+			Status: StepPending,
+			state:  state,
+		}
+		pre = append(pre, preE)
+		postE := &Step{
+			Hook:   hook,
+			Edge:   Post,
+			Status: StepPending,
+			state:  state,
+		}
+		post = append(post, postE)
+	}
+
+	cbE := &Step{
+		Hook:   cb,
+		Edge:   Callback,
+		Status: StepPending,
+	}
+
+	steps := make([]*Step, 0, len(pre)+len(post)+1)
+	steps = append(steps, pre...)
+	steps = append(steps, cbE)
+	for i := len(post) - 1; i >= 0; i-- {
+		steps = append(steps, post[i])
+	}
+
+	plan := &Plan{
+		phase: phase,
+		env:   extra,
+		steps: steps,
+		pre:   pre,
+		post:  post,
+		cb:    cbE,
+	}
+
+	return plan, nil
+}
+
+type PlanReport []Step
+
+func (p *Plan) Report() PlanReport {
+	p.mtx.RLock()
+	defer p.mtx.RUnlock()
+	rep := make([]Step, len(p.steps))
+	for i := range rep {
+		rep[i] = *p.steps[i]
+	}
+	return rep
+}
+
+func (r PlanReport) HadError() bool {
+	for _, e := range r {
+		if e.Status == StepErr {
+			return true
+		}
+	}
+	return false
+}
+
+func (r PlanReport) HadFatalError() bool {
+	for _, e := range r {
+		if e.Status == StepSkippedDueToFatalErr {
+			return true
+		}
+	}
+	return false
+}
+
+func (r PlanReport) String() string {
+	stepStrings := make([]string, len(r))
+	for i, e := range r {
+		stepStrings[i] = fmt.Sprintf("%02d %s", i+1, e)
+	}
+	return strings.Join(stepStrings, "\n")
+}
+
+func (p *Plan) Run(ctx context.Context, dryRun bool) {
+	p.mtx.RLock()
+	defer p.mtx.RUnlock()
+	w := func(f func()) {
+		p.mtx.RUnlock()
+		defer p.mtx.RLock()
+		p.mtx.Lock()
+		defer p.mtx.Unlock()
+		f()
+	}
+	runHook := func(s *Step, ctx context.Context, edge Edge) HookReport {
+		w(func() { s.Status = StepExec })
+		begin := time.Now()
+		r := s.Hook.Run(ctx, edge, p.phase, dryRun, p.env, s.state)
+		end := time.Now()
+		w(func() {
+			s.Report = r
+			s.Status = StepOk
+			if r.HadError() {
+				s.Status = StepErr
+			}
+			s.Begin, s.End = begin, end
+		})
+		return r
+	}
+
+	l := getLogger(ctx)
+
+	// it's a stack, execute until we reach the end of the list (last item in)
+	// or fail inbetween
+	l.Info("run pre-edges in configuration order")
+	next := 0
+	for ; next < len(p.pre); next++ {
+		e := p.pre[next]
+		l := l.WithField("hook", e.Hook)
+		r := runHook(e, ctx, Pre)
+		if r.HadError() {
+			l.WithError(r).Error("hook invocation failed for pre-edge")
+			if e.Hook.ErrIsFatal() {
+				l.Error("the hook run was aborted due to a fatal error in this hook")
+				break
+			}
+		}
+	}
+
+	hadFatalErr := next != len(p.pre)
+	if hadFatalErr {
+		l.Error("fatal error in a pre-snapshot hook invocation")
+		l.Error("no snapshot will be taken")
+		l.Error("only running post-edges for successful pre-edges")
+		w(func() {
+			p.post[next].Status = StepSkippedDueToFatalErr
+			for i := next + 1; i < len(p.pre); i++ {
+				p.pre[i].Status = StepSkippedDueToFatalErr
+				p.post[i].Status = StepSkippedDueToFatalErr
+			}
+			p.cb.Status = StepSkippedDueToFatalErr
+		})
+		return
+	}
+
+	l.Info("running callback")
+	cbR := runHook(p.cb, ctx, Callback)
+	if cbR.HadError() {
+		l.WithError(cbR).Error("callback failed")
+	}
+
+	l.Info("run post-edges for successful pre-edges in reverse configuration order")
+
+	// the constructor produces pre and post entries
+	// post is NOT reversed
+	next-- // now at index of last executed pre-edge
+	for ; next >= 0; next-- {
+		e := p.post[next]
+		l := l.WithField("hook", e.Hook)
+
+		if p.pre[next].Status != StepOk {
+			if p.pre[next].Status != StepErr {
+				panic(fmt.Sprintf("expecting a pre-edge hook report to be either Ok or Err, got %s", p.pre[next].Status))
+			}
+			l.Info("skip post-edge because pre-edge failed")
+			w(func() {
+				e.Status = StepSkippedDueToPreErr
+			})
+			continue
+		}
+
+		report := runHook(e, ctx, Post)
+
+		if report.HadError() {
+			l.WithError(report).Error("hook invocation failed for post-edge")
+			l.Error("subsequent post-edges run regardless of this post-edge failure")
+		}
+
+		// ErrIsFatal is only relevant for Pre
+
+	}
+
+}

daemon/hooks/hook_logging.go

@@ -0,0 +1,91 @@
+package hooks
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"sync"
+
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/util/envconst"
+)
+
+type Logger = logger.Logger
+
+func GetLogger(ctx context.Context) Logger { return getLogger(ctx) }
+
+func getLogger(ctx context.Context) Logger {
+	return logging.GetLogger(ctx, logging.SubsysHooks)
+}
+
+const MAX_HOOK_LOG_SIZE_DEFAULT int = 1 << 20
+
+type logWriter struct {
+	/*
+		Mutex prevents:
+			concurrent writes to buf, scanner in Write([]byte)
+			data race on scanner vs Write([]byte)
+				and concurrent write to buf (call to buf.Reset())
+				in Close()
+
+		(Also, Close() should generally block until any Write() call completes.)
+	*/
+	mtx     *sync.Mutex
+	buf     bytes.Buffer
+	scanner *bufio.Scanner
+	logger  Logger
+	level   logger.Level
+	field   string
+}
+
+func NewLogWriter(mtx *sync.Mutex, logger Logger, level logger.Level, field string) *logWriter {
+	w := new(logWriter)
+	w.mtx = mtx
+	w.scanner = bufio.NewScanner(&w.buf)
+	w.logger = logger
+	w.level = level
+	w.field = field
+	return w
+}
+
+func (w *logWriter) log(line string) {
+	w.logger.WithField(w.field, line).Log(w.level, "hook output")
+}
+
+func (w *logWriter) logUnreadBytes() error {
+	for w.scanner.Scan() {
+		w.log(w.scanner.Text())
+	}
+	if w.buf.Cap() > envconst.Int("ZREPL_MAX_HOOK_LOG_SIZE", MAX_HOOK_LOG_SIZE_DEFAULT) {
+		w.buf.Reset()
+	}
+
+	return nil
+}
+
+func (w *logWriter) Write(in []byte) (int, error) {
+	w.mtx.Lock()
+	defer w.mtx.Unlock()
+
+	n, bufErr := w.buf.Write(in)
+	if bufErr != nil {
+		return n, bufErr
+	}
+
+	err := w.logUnreadBytes()
+	if err != nil {
+		return n, err
+	}
+	// Always reset the scanner for the next Write
+	w.scanner = bufio.NewScanner(&w.buf)
+
+	return n, nil
+}
+
+func (w *logWriter) Close() (err error) {
+	w.mtx.Lock()
+	defer w.mtx.Unlock()
+
+	return w.logUnreadBytes()
+}

daemon/hooks/hook_type_callback.go

@@ -0,0 +1,65 @@
+package hooks
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type HookJobCallback func(ctx context.Context) error
+
+type CallbackHook struct {
+	cb            HookJobCallback
+	filter        Filter
+	displayString string
+}
+
+func NewCallbackHookForFilesystem(displayString string, fs *zfs.DatasetPath, cb HookJobCallback) *CallbackHook {
+	filter, _ := filters.DatasetMapFilterFromConfig(map[string]bool{fs.ToString(): true})
+	return NewCallbackHook(displayString, cb, filter)
+}
+
+func NewCallbackHook(displayString string, cb HookJobCallback, filter Filter) *CallbackHook {
+	return &CallbackHook{
+		cb:            cb,
+		filter:        filter,
+		displayString: displayString,
+	}
+}
+
+func (h *CallbackHook) Filesystems() Filter {
+	return h.filter
+}
+
+func (h *CallbackHook) ErrIsFatal() bool {
+	return false // callback is by definition
+}
+
+func (h *CallbackHook) String() string {
+	return h.displayString
+}
+
+type CallbackHookReport struct {
+	Name string
+	Err  error
+}
+
+func (r *CallbackHookReport) String() string {
+	if r.HadError() {
+		return r.Error()
+	}
+	return r.Name
+}
+
+func (r *CallbackHookReport) HadError() bool { return r.Err != nil }
+
+func (r *CallbackHookReport) Error() string {
+	return fmt.Sprintf("%s error: %s", r.Name, r.Err)
+}
+
+func (h *CallbackHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
+	err := h.cb(ctx)
+	return &CallbackHookReport{h.displayString, err}
+}

daemon/hooks/hook_type_command.go

@@ -0,0 +1,199 @@
+package hooks
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"os"
+	"os/exec"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/util/circlog"
+	"github.com/zrepl/zrepl/util/envconst"
+)
+
+type HookEnvVar string
+
+const (
+	EnvType     HookEnvVar = "ZREPL_HOOKTYPE"
+	EnvDryRun   HookEnvVar = "ZREPL_DRYRUN"
+	EnvFS       HookEnvVar = "ZREPL_FS"
+	EnvSnapshot HookEnvVar = "ZREPL_SNAPNAME"
+	EnvTimeout  HookEnvVar = "ZREPL_TIMEOUT"
+)
+
+type Env map[HookEnvVar]string
+
+func NewHookEnv(edge Edge, phase Phase, dryRun bool, timeout time.Duration, extra Env) Env {
+	r := Env{
+		EnvTimeout: fmt.Sprintf("%.f", math.Floor(timeout.Seconds())),
+	}
+
+	edgeString := edge.StringForPhase(phase)
+	r[EnvType] = strings.ToLower(edgeString)
+
+	var dryRunString string
+	if dryRun {
+		dryRunString = "true"
+	} else {
+		dryRunString = ""
+	}
+	r[EnvDryRun] = dryRunString
+
+	for k, v := range extra {
+		r[k] = v
+	}
+
+	return r
+}
+
+type CommandHook struct {
+	edge       Edge
+	filter     Filter
+	errIsFatal bool
+	command    string
+	timeout    time.Duration
+}
+
+type CommandHookReport struct {
+	Command                      string
+	Args                         []string // currently always empty
+	Env                          Env
+	Err                          error
+	CapturedStdoutStderrCombined []byte
+}
+
+func (r *CommandHookReport) String() string {
+	// Reproduces a POSIX shell-compatible command line
+	var cmdLine strings.Builder
+	sep := ""
+
+	// Make sure environment variables are always
+	// printed in the same order
+	var hookEnvKeys []HookEnvVar
+	for k := range r.Env {
+		hookEnvKeys = append(hookEnvKeys, k)
+	}
+	sort.Slice(hookEnvKeys, func(i, j int) bool { return string(hookEnvKeys[i]) < string(hookEnvKeys[j]) })
+
+	for _, k := range hookEnvKeys {
+		cmdLine.WriteString(fmt.Sprintf("%s%s='%s'", sep, k, r.Env[k]))
+		sep = " "
+	}
+
+	cmdLine.WriteString(fmt.Sprintf("%s%s", sep, r.Command))
+	for _, a := range r.Args {
+		cmdLine.WriteString(fmt.Sprintf("%s'%s'", sep, a))
+	}
+
+	var msg string
+	if r.Err == nil {
+		msg = "command hook"
+	} else {
+		msg = fmt.Sprintf("command hook failed with %q", r.Err)
+	}
+
+	return fmt.Sprintf("%s: \"%s\"", msg, cmdLine.String()) // no %q to make copy-pastable
+}
+func (r *CommandHookReport) Error() string {
+	if r.Err == nil {
+		return ""
+	}
+	return fmt.Sprintf("%s FAILED with error: %s", r.String(), r.Err)
+}
+
+func (r *CommandHookReport) HadError() bool {
+	return r.Err != nil
+}
+
+func NewCommandHook(in *config.HookCommand) (r *CommandHook, err error) {
+	r = &CommandHook{
+		errIsFatal: in.ErrIsFatal,
+		command:    in.Path,
+		timeout:    in.Timeout,
+	}
+
+	r.filter, err = filters.DatasetMapFilterFromConfig(in.Filesystems)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse filesystem filter: %s", err)
+	}
+
+	r.edge = Pre | Post
+
+	return r, nil
+}
+
+func (h *CommandHook) Filesystems() Filter {
+	return h.filter
+}
+
+func (h *CommandHook) ErrIsFatal() bool {
+	return h.errIsFatal
+}
+
+func (h *CommandHook) String() string {
+	return h.command
+}
+
+func (h *CommandHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
+	l := getLogger(ctx).WithField("command", h.command)
+
+	cmdCtx, cancel := context.WithTimeout(ctx, h.timeout)
+	defer cancel()
+
+	cmdExec := exec.CommandContext(cmdCtx, h.command)
+
+	hookEnv := NewHookEnv(edge, phase, dryRun, h.timeout, extra)
+	cmdEnv := os.Environ()
+	for k, v := range hookEnv {
+		cmdEnv = append(cmdEnv, fmt.Sprintf("%s=%s", k, v))
+	}
+	cmdExec.Env = cmdEnv
+
+	var scanMutex sync.Mutex
+	combinedOutput, err := circlog.NewCircularLog(envconst.Int("ZREPL_MAX_HOOK_LOG_SIZE", MAX_HOOK_LOG_SIZE_DEFAULT))
+	if err != nil {
+		return &CommandHookReport{Err: err}
+	}
+	logErrWriter := NewLogWriter(&scanMutex, l, logger.Warn, "stderr")
+	logOutWriter := NewLogWriter(&scanMutex, l, logger.Info, "stdout")
+	defer logErrWriter.Close()
+	defer logOutWriter.Close()
+
+	cmdExec.Stderr = io.MultiWriter(logErrWriter, combinedOutput)
+	cmdExec.Stdout = io.MultiWriter(logOutWriter, combinedOutput)
+
+	report := &CommandHookReport{
+		Command: h.command,
+		Env:     hookEnv,
+		// no report.Args
+	}
+
+	err = cmdExec.Start()
+	if err != nil {
+		report.Err = err
+		return report
+	}
+
+	err = cmdExec.Wait()
+	combinedOutputBytes := combinedOutput.Bytes()
+	report.CapturedStdoutStderrCombined = make([]byte, len(combinedOutputBytes))
+	copy(report.CapturedStdoutStderrCombined, combinedOutputBytes)
+	if err != nil {
+		if cmdCtx.Err() == context.DeadlineExceeded {
+			report.Err = fmt.Errorf("timed out after %s: %s", h.timeout, err)
+			return report
+		}
+		report.Err = err
+		return report
+	}
+
+	return report
+}

daemon/hooks/hook_type_mysql_lock_tables.go

@@ -0,0 +1,148 @@
+package hooks
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strings"
+
+	sqldriver "database/sql/driver"
+
+	"github.com/go-sql-driver/mysql"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+// Hook to implement the following recommmendation from MySQL docs
+// https://dev.mysql.com/doc/mysql-backup-excerpt/5.7/en/backup-methods.html
+//
+//	Making Backups Using a File System Snapshot:
+//
+//	If you are using a Veritas file system, you can make a backup like this:
+//
+//	   	From a client program, execute FLUSH TABLES WITH READ LOCK.
+//	   	From another shell, execute mount vxfs snapshot.
+//	   	From the first client, execute UNLOCK TABLES.
+//	   	Copy files from the snapshot.
+//	   	Unmount the snapshot.
+//
+//		Similar snapshot capabilities may be available in other file systems, such as LVM or ZFS.
+//
+
+type MySQLLockTables struct {
+	errIsFatal  bool
+	connector   sqldriver.Connector
+	filesystems Filter
+}
+
+type myLockTablesStateKey int
+
+const (
+	myLockTablesConnection myLockTablesStateKey = 1 + iota
+)
+
+func MyLockTablesFromConfig(in *config.HookMySQLLockTables) (*MySQLLockTables, error) {
+	conf, err := mysql.ParseDSN(in.DSN)
+	if err != nil {
+		return nil, errors.Wrap(err, "`dsn` invalid")
+	}
+	cn, err := mysql.NewConnector(conf)
+	if err != nil {
+		return nil, errors.Wrap(err, "`connect` invalid")
+	}
+
+	filesystems, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
+	if err != nil {
+		return nil, errors.Wrap(err, "`filesystems` invalid")
+	}
+
+	return &MySQLLockTables{
+		in.ErrIsFatal,
+		cn,
+		filesystems,
+	}, nil
+}
+
+func (h *MySQLLockTables) ErrIsFatal() bool    { return h.errIsFatal }
+func (h *MySQLLockTables) Filesystems() Filter { return h.filesystems }
+func (h *MySQLLockTables) String() string      { return "MySQL FLUSH TABLES WITH READ LOCK" }
+
+type MyLockTablesReport struct {
+	What string
+	Err  error
+}
+
+func (r *MyLockTablesReport) HadError() bool { return r.Err != nil }
+func (r *MyLockTablesReport) Error() string  { return r.String() }
+func (r *MyLockTablesReport) String() string {
+	var s strings.Builder
+	s.WriteString(r.What)
+	if r.Err != nil {
+		fmt.Fprintf(&s, ": %s", r.Err)
+	}
+	return s.String()
+}
+
+func (h *MySQLLockTables) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
+	fs, ok := extra[EnvFS]
+	if !ok {
+		panic(extra)
+	}
+	dp, err := zfs.NewDatasetPath(fs)
+	if err != nil {
+		panic(err)
+	}
+	if pass, err := h.filesystems.Filter(dp); err != nil {
+		return &MyLockTablesReport{What: "filesystem filter", Err: err}
+	} else if !pass {
+		getLogger(ctx).Debug("filesystem does not match filter, skipping")
+		return &MyLockTablesReport{What: "filesystem filter skipped this filesystem", Err: nil}
+	}
+
+	switch edge {
+	case Pre:
+		err := h.doRunPre(ctx, dp, dryRun, state)
+		return &MyLockTablesReport{"FLUSH TABLES WITH READ LOCK", err}
+	case Post:
+		err := h.doRunPost(ctx, dp, dryRun, state)
+		return &MyLockTablesReport{"UNLOCK TABLES", err}
+	}
+	return &MyLockTablesReport{What: "skipped this edge", Err: nil}
+}
+
+func (h *MySQLLockTables) doRunPre(ctx context.Context, fs *zfs.DatasetPath, dry bool, state map[interface{}]interface{}) (err error) {
+	db := sql.OpenDB(h.connector)
+	defer func(err *error) {
+		if *err != nil {
+			db.Close()
+		}
+	}(&err)
+
+	getLogger(ctx).Debug("do FLUSH TABLES WITH READ LOCK")
+	_, err = db.ExecContext(ctx, "FLUSH TABLES WITH READ LOCK")
+	if err != nil {
+		return
+	}
+
+	state[myLockTablesConnection] = db
+
+	return nil
+}
+
+func (h *MySQLLockTables) doRunPost(ctx context.Context, fs *zfs.DatasetPath, dry bool, state map[interface{}]interface{}) error {
+
+	db := state[myLockTablesConnection].(*sql.DB)
+	defer db.Close()
+
+	getLogger(ctx).Debug("do UNLOCK TABLES")
+	_, err := db.ExecContext(ctx, "UNLOCK TABLES")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

daemon/hooks/hook_type_postgres_checkpoint.go

@@ -0,0 +1,98 @@
+package hooks
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"math"
+	"time"
+
+	"github.com/lib/pq"
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type PgChkptHook struct {
+	errIsFatal  bool
+	connector   *pq.Connector
+	filesystems Filter
+}
+
+func PgChkptHookFromConfig(in *config.HookPostgresCheckpoint) (*PgChkptHook, error) {
+	filesystems, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
+	if err != nil {
+		return nil, errors.Wrap(err, "`filesystems` invalid")
+	}
+	cn, err := pq.NewConnector(in.DSN)
+	if err != nil {
+		return nil, errors.Wrap(err, "`dsn` invalid")
+	}
+
+	return &PgChkptHook{
+		in.ErrIsFatal,
+		cn,
+		filesystems,
+	}, nil
+}
+
+func (h *PgChkptHook) ErrIsFatal() bool    { return h.errIsFatal }
+func (h *PgChkptHook) Filesystems() Filter { return h.filesystems }
+func (h *PgChkptHook) String() string      { return "postgres checkpoint" }
+
+type PgChkptHookReport struct{ Err error }
+
+func (r *PgChkptHookReport) HadError() bool { return r.Err != nil }
+func (r *PgChkptHookReport) Error() string  { return r.Err.Error() }
+func (r *PgChkptHookReport) String() string {
+	if r.Err != nil {
+		return fmt.Sprintf("postgres CHECKPOINT failed: %s", r.Err)
+	} else {
+		return "postgres CHECKPOINT completed"
+	}
+}
+
+func (h *PgChkptHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
+	if edge != Pre {
+		return &PgChkptHookReport{nil}
+	}
+	fs, ok := extra[EnvFS]
+	if !ok {
+		panic(extra)
+	}
+	dp, err := zfs.NewDatasetPath(fs)
+	if err != nil {
+		panic(err)
+	}
+	err = h.doRunPre(ctx, dp, dryRun)
+	return &PgChkptHookReport{err}
+}
+
+func (h *PgChkptHook) doRunPre(ctx context.Context, fs *zfs.DatasetPath, dry bool) error {
+
+	if pass, err := h.filesystems.Filter(fs); err != nil || !pass {
+		getLogger(ctx).Debug("filesystem does not match filter, skipping")
+		return err
+	}
+
+	db := sql.OpenDB(h.connector)
+	defer db.Close()
+	dl, ok := ctx.Deadline()
+	if ok {
+		timeout := uint64(math.Floor(time.Until(dl).Seconds() * 1000)) // TODO go1.13 milliseconds
+		getLogger(ctx).WithField("statement_timeout", timeout).Debug("setting statement timeout for CHECKPOINT")
+		_, err := db.ExecContext(ctx, "SET statement_timeout TO ?", timeout)
+		if err != nil {
+			return err
+		}
+	}
+	if dry {
+		getLogger(ctx).Debug("dry-run - use ping instead of CHECKPOINT")
+		return db.PingContext(ctx)
+	}
+	getLogger(ctx).Info("execute CHECKPOINT command")
+	_, err := db.ExecContext(ctx, "CHECKPOINT")
+	return err
+}

daemon/hooks/hooks_test.go

@@ -0,0 +1,489 @@
+package hooks_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"regexp"
+	"testing"
+	"text/template"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/hooks"
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type comparisonAssertionFunc func(require.TestingT, interface{}, interface{}, ...interface{})
+type valueAssertionFunc func(require.TestingT, interface{}, ...interface{})
+
+type expectStep struct {
+	ExpectedEdge hooks.Edge
+	ExpectStatus hooks.StepStatus
+	OutputTest   valueAssertionFunc
+	ErrorTest    valueAssertionFunc
+}
+
+type testCase struct {
+	Name           string
+	Config         []string
+	IsSlow         bool
+	SuppressOutput bool
+
+	ExpectCallbackSkipped bool
+	ExpectHadFatalErr     bool
+	ExpectHadError        bool
+	ExpectStepReports     []expectStep
+}
+
+func curryLeft(f comparisonAssertionFunc, expected interface{}) valueAssertionFunc {
+	return curry(f, expected, false)
+}
+
+func curryRight(f comparisonAssertionFunc, expected interface{}) valueAssertionFunc {
+	return curry(f, expected, true)
+}
+
+func curry(f comparisonAssertionFunc, expected interface{}, right bool) (ret valueAssertionFunc) {
+	ret = func(t require.TestingT, s interface{}, v ...interface{}) {
+		var x interface{}
+		var y interface{}
+		if right {
+			x = s
+			y = expected
+		} else {
+			x = expected
+			y = s
+		}
+
+		if len(v) > 0 {
+			f(t, x, y, v)
+		} else {
+			f(t, x, y)
+		}
+	}
+	return
+}
+
+func TestHooks(t *testing.T) {
+	ctx, end := trace.WithTaskFromStack(context.Background())
+	defer end()
+
+	testFSName := "testpool/testdataset"
+	testSnapshotName := "testsnap"
+
+	tmpl, err := template.New("TestHooks").Parse(`
+jobs:
+- name: TestHooks
+  type: snap
+  filesystems: {"<": true}
+  snapshotting:
+    type: periodic
+    interval: 1m
+    prefix: zrepl_snapjob_
+    hooks:
+    {{- template "List" . }}
+  pruning:
+    keep:
+      - type: last_n
+        count: 10
+`)
+	if err != nil {
+		panic(err)
+	}
+
+	regexpTest := func(s string) valueAssertionFunc {
+		return curryLeft(require.Regexp, regexp.MustCompile(s))
+	}
+
+	containsTest := func(s string) valueAssertionFunc {
+		return curryRight(require.Contains, s)
+	}
+
+	testTable := []testCase{
+		testCase{
+			Name: "no_hooks",
+			ExpectStepReports: []expectStep{
+				{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+			},
+		},
+		testCase{
+			Name:           "timeout",
+			IsSlow:         true,
+			ExpectHadError: true,
+			Config:         []string{`{type: command, path: {{.WorkDir}}/test/test-timeout.sh, timeout: 2s}`},
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepErr,
+					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s ZREPL_TIMEOUT=2", testFSName, testSnapshotName)),
+					ErrorTest:    regexpTest(`timed out after 2(.\d+)?s`),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepSkippedDueToPreErr,
+				},
+			},
+		},
+		testCase{
+			Name:           "check_env",
+			Config:         []string{`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`},
+			ExpectHadError: false,
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
+				},
+			},
+		},
+
+		testCase{
+			Name:                  "nonfatal_pre_error_continues",
+			ExpectCallbackSkipped: false,
+			ExpectHadError:        true,
+			Config:                []string{`{type: command, path: {{.WorkDir}}/test/test-error.sh}`},
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepErr,
+					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
+					ErrorTest:    regexpTest("^command hook failed.*exit status 1$"),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepSkippedDueToPreErr, // post-edge is not executed for failing pre-edge
+				},
+			},
+		},
+
+		testCase{
+			Name:                  "pre_error_fatal_skips_subsequent_pre_edges_and_callback_and_its_post_edge_and_post_edges",
+			ExpectCallbackSkipped: true,
+			ExpectHadFatalErr:     true,
+			ExpectHadError:        true,
+			Config: []string{
+				`{type: command, path: {{.WorkDir}}/test/test-error.sh, err_is_fatal: true}`,
+				`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`,
+			},
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepErr,
+					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
+					ErrorTest:    regexpTest("^command hook failed.*exit status 1$"),
+				},
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepSkippedDueToFatalErr,
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepSkippedDueToFatalErr},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepSkippedDueToFatalErr,
+				},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepSkippedDueToFatalErr,
+				},
+			},
+		},
+
+		testCase{
+			Name:              "post_error_fails_are_ignored_even_if_fatal",
+			ExpectHadFatalErr: false, // only occurs during Post, so it's not a fatal error
+			ExpectHadError:    true,
+			Config: []string{
+				`{type: command, path: {{.WorkDir}}/test/test-post-error.sh, err_is_fatal: true}`,
+				`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`,
+			},
+			ExpectStepReports: []expectStep{
+				expectStep{
+					// No-action run of test-post-error.sh
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   require.Empty,
+				},
+				expectStep{
+					// Pre run of test-report-env.sh
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
+				},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepErr,
+					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR post_testing %s@%s", testFSName, testSnapshotName)),
+					ErrorTest:    regexpTest("^command hook failed.*exit status 1$"),
+				},
+			},
+		},
+
+		testCase{
+			Name:   "cleanup_check_env",
+			Config: []string{`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`},
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
+				},
+			},
+		},
+
+		testCase{
+			Name:              "pre_error_cancels_post",
+			Config:            []string{`{type: command, path: {{.WorkDir}}/test/test-pre-error-post-ok.sh}`},
+			ExpectHadError:    true,
+			ExpectHadFatalErr: false,
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepErr,
+					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
+					ErrorTest:    regexpTest("^command hook failed.*exit status 1$"),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepSkippedDueToPreErr,
+				},
+			},
+		},
+
+		testCase{
+			Name: "pre_error_does_not_cancel_other_posts_but_itself",
+			Config: []string{
+				`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`,
+				`{type: command, path: {{.WorkDir}}/test/test-pre-error-post-ok.sh}`,
+			},
+			ExpectHadError:    true,
+			ExpectHadFatalErr: false,
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
+				},
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepErr,
+					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
+					ErrorTest:    regexpTest("^command hook failed.*exit status 1$"),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepSkippedDueToPreErr,
+				},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
+				},
+			},
+		},
+
+		testCase{
+			Name:              "exceed_buffer_limit",
+			SuppressOutput:    true,
+			Config:            []string{`{type: command, path: {{.WorkDir}}/test/test-large-stdout.sh}`},
+			ExpectHadError:    false,
+			ExpectHadFatalErr: false,
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepOk,
+					OutputTest: func(t require.TestingT, s interface{}, v ...interface{}) {
+						require.Len(t, s, 1<<20)
+					},
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					// No-action run of above hook
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   require.Empty,
+				},
+			},
+		},
+
+		/*
+			Following not intended to test functionality of
+			filter package. Only to demonstrate that hook
+			filters are being applied. The following should
+			result in NO hooks running. If it does run, a
+			fatal hooks.RunReport will be returned.
+		*/
+		testCase{
+			Name: "exclude_all_filesystems",
+			Config: []string{
+				`{type: command, path: {{.WorkDir}}/test/test-error.sh, err_is_fatal: true, filesystems: {"<": false}}`,
+			},
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Callback,
+					ExpectStatus: hooks.StepOk,
+				},
+			},
+		},
+	}
+
+	parseHookConfig := func(t *testing.T, in string) *config.Config {
+		t.Helper()
+		conf, err := config.ParseConfigBytes([]byte(in))
+		require.NoError(t, err)
+		require.NotNil(t, conf)
+		return conf
+	}
+
+	fillHooks := func(tt *testCase) string {
+		// make hook path absolute
+		cwd, err := os.Getwd()
+		if err != nil {
+			panic("os.Getwd() failed")
+		}
+		var hooksTmpl string = "\n"
+		for _, l := range tt.Config {
+			hooksTmpl += fmt.Sprintf("    - %s\n", l)
+		}
+		tmpl.New("List").Parse(hooksTmpl)
+
+		var outBytes bytes.Buffer
+		data := struct {
+			WorkDir string
+		}{
+			WorkDir: cwd,
+		}
+		if err := tmpl.Execute(&outBytes, data); err != nil {
+			panic(err)
+		}
+
+		return outBytes.String()
+	}
+
+	var c *config.Config
+	fs, err := zfs.NewDatasetPath(testFSName)
+	require.NoError(t, err)
+
+	log := logger.NewTestLogger(t)
+
+	var cbReached bool
+	cb := hooks.NewCallbackHookForFilesystem("testcallback", fs, func(_ context.Context) error {
+		cbReached = true
+		return nil
+	})
+
+	hookEnvExtra := hooks.Env{
+		hooks.EnvFS:       fs.ToString(),
+		hooks.EnvSnapshot: testSnapshotName,
+	}
+
+	for _, tt := range testTable {
+		if testing.Short() && tt.IsSlow {
+			continue
+		}
+
+		t.Run(tt.Name, func(t *testing.T) {
+			c = parseHookConfig(t, fillHooks(&tt))
+			snp := c.Jobs[0].Ret.(*config.SnapJob).Snapshotting.Ret.(*config.SnapshottingPeriodic)
+			hookList, err := hooks.ListFromConfig(&snp.Hooks)
+			require.NoError(t, err)
+
+			filteredHooks, err := hookList.CopyFilteredForFilesystem(fs)
+			require.NoError(t, err)
+			plan, err := hooks.NewPlan(&filteredHooks, hooks.PhaseTesting, cb, hookEnvExtra)
+			require.NoError(t, err)
+			t.Logf("REPORT PRE EXECUTION:\n%s", plan.Report())
+
+			cbReached = false
+
+			if testing.Verbose() && !tt.SuppressOutput {
+				ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
+			}
+			plan.Run(ctx, false)
+			report := plan.Report()
+
+			t.Logf("REPORT POST EXECUTION:\n%s", report)
+
+			/*
+			 * TEST ASSERTIONS
+			 */
+
+			t.Logf("len(runReports)=%v", len(report))
+			t.Logf("len(tt.ExpectStepReports)=%v", len(tt.ExpectStepReports))
+			require.Equal(t, len(tt.ExpectStepReports), len(report), "ExpectStepReports must be same length as expected number of hook runs, excluding possible Callback")
+
+			// Check if callback ran, when required
+			if tt.ExpectCallbackSkipped {
+				require.False(t, cbReached, "callback ran but should not have run")
+			} else {
+				require.True(t, cbReached, "callback should have run but did not")
+			}
+
+			// Check if a fatal run error occurred and was expected
+			require.Equal(t, tt.ExpectHadFatalErr, report.HadFatalError(), "non-matching HadFatalError")
+			require.Equal(t, tt.ExpectHadError, report.HadError(), "non-matching HadError")
+
+			if tt.ExpectHadFatalErr {
+				require.True(t, tt.ExpectHadError, "ExpectHadFatalErr implies ExpectHadError")
+			}
+			if !tt.ExpectHadError {
+				require.False(t, tt.ExpectHadFatalErr, "!ExpectHadError implies !ExpectHadFatalErr")
+			}
+
+			// Iterate through each expected hook run
+			for i, hook := range tt.ExpectStepReports {
+				t.Logf("expecting report conforming to %v", hook)
+
+				exp, act := hook.ExpectStatus, report[i].Status
+				require.Equal(t, exp, act, "%s != %s", exp, act)
+
+				// Check for required ExpectedEdge
+				require.NotZero(t, hook.ExpectedEdge, "each hook must have an ExpectedEdge")
+				require.Equal(t, hook.ExpectedEdge, report[i].Edge,
+					"incorrect edge: expected %q, actual %q", hook.ExpectedEdge.String(), report[i].Edge.String(),
+				)
+
+				// Check for expected output
+				if hook.OutputTest != nil {
+					require.IsType(t, (*hooks.CommandHookReport)(nil), report[i].Report)
+					chr := report[i].Report.(*hooks.CommandHookReport)
+					hook.OutputTest(t, string(chr.CapturedStdoutStderrCombined))
+				}
+
+				// Check for expected errors
+				if hook.ErrorTest != nil {
+					hook.ErrorTest(t, string(report[i].Report.Error()))
+				}
+			}
+
+		})
+	}
+}

daemon/hooks/stepstatus_enumer.go

@@ -0,0 +1,77 @@
+// Code generated by "enumer -type=StepStatus -trimprefix=Step"; DO NOT EDIT.
+
+//
+package hooks
+
+import (
+	"fmt"
+)
+
+const (
+	_StepStatusName_0 = "PendingExec"
+	_StepStatusName_1 = "Ok"
+	_StepStatusName_2 = "Err"
+	_StepStatusName_3 = "SkippedDueToFatalErr"
+	_StepStatusName_4 = "SkippedDueToPreErr"
+)
+
+var (
+	_StepStatusIndex_0 = [...]uint8{0, 7, 11}
+	_StepStatusIndex_1 = [...]uint8{0, 2}
+	_StepStatusIndex_2 = [...]uint8{0, 3}
+	_StepStatusIndex_3 = [...]uint8{0, 20}
+	_StepStatusIndex_4 = [...]uint8{0, 18}
+)
+
+func (i StepStatus) String() string {
+	switch {
+	case 1 <= i && i <= 2:
+		i -= 1
+		return _StepStatusName_0[_StepStatusIndex_0[i]:_StepStatusIndex_0[i+1]]
+	case i == 4:
+		return _StepStatusName_1
+	case i == 8:
+		return _StepStatusName_2
+	case i == 16:
+		return _StepStatusName_3
+	case i == 32:
+		return _StepStatusName_4
+	default:
+		return fmt.Sprintf("StepStatus(%d)", i)
+	}
+}
+
+var _StepStatusValues = []StepStatus{1, 2, 4, 8, 16, 32}
+
+var _StepStatusNameToValueMap = map[string]StepStatus{
+	_StepStatusName_0[0:7]:  1,
+	_StepStatusName_0[7:11]: 2,
+	_StepStatusName_1[0:2]:  4,
+	_StepStatusName_2[0:3]:  8,
+	_StepStatusName_3[0:20]: 16,
+	_StepStatusName_4[0:18]: 32,
+}
+
+// StepStatusString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func StepStatusString(s string) (StepStatus, error) {
+	if val, ok := _StepStatusNameToValueMap[s]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to StepStatus values", s)
+}
+
+// StepStatusValues returns all values of the enum
+func StepStatusValues() []StepStatus {
+	return _StepStatusValues
+}
+
+// IsAStepStatus returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i StepStatus) IsAStepStatus() bool {
+	for _, v := range _StepStatusValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}

daemon/hooks/test/test-error.sh

@@ -0,0 +1,5 @@
+#!/bin/sh -eu
+
+>&2 echo "TEST ERROR $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
+
+exit 1

daemon/hooks/test/test-large-stdout.sh

@@ -0,0 +1,18 @@
+#!/bin/sh -eu
+
+# Exceed default hook log size of 1<<20 bytes by double
+# The Go test should fail if buffer size exceeds 1MB
+
+pre_testing() {
+	ln=0
+	while :; do printf '%06d: 012345678901234567890123456789012345678901234567890123456789012345678901\n' "$ln"; ln="$(($ln + 1))"; done \
+	| head -c$(( 2 << 20 ))
+}
+
+case "$ZREPL_HOOKTYPE" in
+	pre_testing)
+		"$ZREPL_HOOKTYPE";;
+	*)
+		# Not handled by this script
+		exit 0;;
+esac

daemon/hooks/test/test-post-error.sh

@@ -0,0 +1,15 @@
+#!/bin/sh -eu
+
+post_testing() {
+	>&2 echo "TEST ERROR $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
+
+	exit 1
+}
+
+case "$ZREPL_HOOKTYPE" in
+	post_testing)
+		"$ZREPL_HOOKTYPE";;
+	*)
+		# Not handled by this script
+		exit 0;;
+esac

daemon/hooks/test/test-pre-error-post-ok.sh

@@ -0,0 +1,11 @@
+#!/bin/sh -eu
+
+if [ "$ZREPL_HOOKTYPE" = "pre_testing" ]; then
+	>&2 echo "TEST ERROR $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
+	exit 1
+elif [ "$ZREPL_HOOKTYPE" = "post_testing" ]; then
+	echo "TEST $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
+else
+	printf "Unknown hook type: %s" "$ZREPL_HOOKTYPE" 
+	exit 255
+fi