Revert "DROP THIS don't run tests in CI"

This reverts commit a3c6173b85dede625f51ca7172a53a5aff024fae.

.circleci/config.yml

@@ -0,0 +1,181 @@
+version: 2.0
+workflows:
+  version: 2
+  build:
+    jobs:
+      - build-1.11
+      - build-1.12
+      - build-1.13
+      - build-1.14
+      - build-latest
+      - test-build-in-docker
+jobs:
+
+  # build-latest serves as the template
+  # we use YAML anchors & aliases to exchange the docker image (and hence Go version used for the build)
+  build-latest: &build-latest
+    description: Builds zrepl
+    parameters:
+      image:
+        description: "the docker image that the job should use"
+        type: string
+    docker:
+      - image: circleci/golang:latest
+    environment:
+      # required by lazy.sh
+      TERM: xterm
+    working_directory: /go/src/github.com/zrepl/zrepl
+    steps:
+      - run:
+          name: Setup environment variables
+          command: |
+            # used by pip (for docs)
+            echo 'export PATH="$HOME/.local/bin:$PATH"' >> $BASH_ENV
+            # we use modules
+            echo 'export GO111MODULE=on' >> $BASH_ENV
+
+      - restore_cache:
+          keys:
+            - source
+            - protobuf
+
+      - checkout
+
+      - save_cache:
+          key: source
+          paths:
+            - ".git"
+      
+      # install deps
+      - run: wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
+      - run: echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807  protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
+      - run: sudo unzip -d /usr protoc-3.6.1-linux-x86_64.zip
+      - save_cache:
+          key: protobuf
+          paths:
+            - "/usr/include/google/protobuf"
+            
+      - run: sudo apt update && sudo apt install python3 python3-pip libgirepository1.0-dev gawk
+      - run: ./lazy.sh devsetup
+
+      - run: make zrepl-bin
+      - run: make test-platform-bin
+      - run: make vet
+      - run: make lint
+      - run: make release
+      - run: make test-go
+      # cannot run test-platform because circle-ci runs in linux containers
+
+      - store_artifacts:
+          path: ./artifacts/release
+          when: always
+
+      - run:
+          shell: /bin/bash -eo pipefail
+          when: always
+          command: |
+            if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
+              echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
+              exit 0
+            fi
+            set -u # from now on
+            
+            # Download and install minio
+            curl -sSL https://dl.minio.io/client/mc/release/linux-amd64/mc -o ${GOPATH}/bin/mc
+            chmod +x ${GOPATH}/bin/mc
+            mc config host add --api s3v4 zrepl-minio https://minio.cschwarz.com ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
+            
+            # Upload artifacts
+            mkdir -p artifacts/release
+            echo "$CIRCLE_BUILD_URL" > ./artifacts/release/cirlceci_build_url
+            mc cp -r artifacts/release "zrepl-minio/zrepl-ci-artifacts/${CIRCLE_SHA1}/${CIRCLE_JOB}/"
+
+            # Push Artifact Link to GitHub
+            REPO="zrepl/zrepl"
+            COMMIT="${CIRCLE_SHA1}"
+            JOB_NAME="${CIRCLE_JOB}"
+            curl "https://api.github.com/repos/$REPO/statuses/$COMMIT" \
+              -H "Content-Type: application/json" \
+              -H "Authorization: token $GITHUB_COMMIT_STATUS_TOKEN" \
+              -X POST \
+              -d '{"context":"zrepl/publish-ci-artifacts", "state": "success", "description":"CI Build Artifacts for '"$JOB_NAME"'", "target_url":"https://minio.cschwarz.com/minio/zrepl-ci-artifacts/'"$COMMIT"'/"}'
+
+      - run:
+          shell: /bin/bash -eo pipefail
+          when: always
+          command: |
+            if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
+              echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
+              exit 0
+            fi
+            set -u # from now on
+
+            # Trigger Platform Tests
+            # ZREPL_BOT_GIT_PRWN_EU_ACCESS_TOKEN from secrets
+            ## Sync this commit to the GitLab
+            git remote add prwn-gitlab "https://zrepl-bot:$ZREPL_BOT_GIT_PRWN_EU_ACCESS_TOKEN@git.prwn.eu/problame/zrepl.git"
+            REMOTE_REF=circleci/$CIRCLE_BRANCH/$CIRCLE_BUILD_NUM
+            git push -f prwn-gitlab "HEAD:$REMOTE_REF"
+            ## Trigger Tests
+            curl -v --request POST \
+              --header "PRIVATE-TOKEN: $ZREPL_BOT_GIT_PRWN_EU_ACCESS_TOKEN" \
+              --header "Content-Type: application/json" \
+              --data '{ "ref":"'"$REMOTE_REF"'", "variables": [ {"key":"ZREPL_MAIN_REPO_COMMIT", "value": "'"$CIRCLE_SHA1"'" }, { "key":"ZREPL_CI_BUILDJOB", "value": "'"$CIRCLE_JOB"'" } ] } ' \
+              "https://git.prwn.eu/api/v4/projects/13/pipeline"
+
+
+      - run:
+          shell: /bin/bash -euo pipefail
+          command: |
+            # Trigger Debian Package Build
+            curl -v -X POST https://api.github.com/repos/zrepl/debian-binary-packaging/dispatches \
+              -H 'Accept: application/vnd.github.v3+json' \
+              -H "Authorization: token $ZREPL_DEBIAN_BINARYPACKAGIN_TRIGGER_BUILD_GITHUB_TOKEN" \
+              --data '{"event_type": "push", "client_payload": { "zrepl_main_repo_commit": "'"$CIRCLE_SHA1"'", "go_version": "'"${CIRCLE_JOB##build-}"'" }}'
+
+  build-1.11:
+    <<: *build-latest
+    docker:
+    - image: circleci/golang:1.11
+
+  build-1.12:
+    <<: *build-latest
+    docker:
+    - image: circleci/golang:1.12
+
+  build-1.13:
+    <<: *build-latest
+    docker:
+    - image: circleci/golang:1.13
+
+  build-1.14:
+    <<: *build-latest
+    docker:
+    - image: circleci/golang:1.14
+
+  # this job tries to mimic the build-in-docker instructions
+  # given in docs/installation.rst
+  #
+  # However, CircleCI doesn't support volume mounts, so we have to copy
+  # the source into the build-container by modifying the Dockerfile
+  test-build-in-docker:
+    description: Check that build-in-docker works
+    docker:
+      - image: circleci/golang:latest
+    environment:
+    working_directory: /go/src/github.com/zrepl/zrepl
+    steps:
+    - checkout
+    - setup_remote_docker
+    - run:
+        name: (hacky) circleci doesn't allow volume mounts, so copy src to container
+        command: echo "ADD . /src" >> build.Dockerfile
+    - run:
+        name: (hacky) commit modified Dockerfile to avoid failing git clean check in Makefile
+        command: git -c user.name='circleci' -c user.email='circleci@localhost' commit -m 'CIRCLECI modified Dockerfile with zrepl src' --author 'autoauthor <circleci@localhost>' -- build.Dockerfile
+    - run:
+        name: build the build image (build deps)
+        command: docker build -t zrepl_build -f build.Dockerfile .
+    - run:
+        name: try compiling
+        command: docker run -it zrepl_build make release

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+patreon: zrepl
+liberapay: zrepl
+custom: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96
+ko_fi: problame

.gitlab-ci.yml

@@ -0,0 +1,17 @@
+build-and-run-platformtests:
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - mkdir artifacts
+    # platformtests are installed by the CI environment
+    - /opt/platformtest-linux-amd64
+      -test.coverprofile artifacts/platformtest.cover
+      -test.v
+      __DEVEL--i-heard-you-like-tests
+      -poolname zreplplatformtest
+      -imagepath /tmp/zreplplatformtest.pool.img
+      -mountpoint /tmp/zreplplatformtest.pool
+  artifacts:
+    paths:
+      - artifacts
+  image: "images:ubuntu/focal/cloud"

.golangci.yml

@@ -0,0 +1,10 @@
+linters:
+  enable:
+  - goimports
+
+issues:
+  exclude-rules:
+    - path: _test\.go
+      linters:
+        - errcheck
+

.platformtest-ci/download_ci_file.bash

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+commit="$1"
+build="$2"
+src="$3"
+dst="$4"
+
+[ -n "$commit" -a -n "$build" -a -n "$src" -a -n "$dst" ] || ( echo "arguments must not be empty"; exit 1 )
+
+aws="aws --endpoint-url https://minio.cschwarz.com --no-sign-request"
+
+
+path="s3://zrepl-ci-artifacts/$commit/$build/$src"
+$aws s3 cp "$path" "$dst"

.travis.yml

@@ -0,0 +1,70 @@
+dist: xenial
+services:
+- docker
+
+env: # for allow_failures: https://docs.travis-ci.com/user/customizing-the-build/
+matrix:
+  include:
+
+  - language: go
+    name: "Build in Docker (docs/installation.rst)"
+    script:
+    - sudo docker build -t zrepl_build -f build.Dockerfile .
+    - |
+      sudo docker run -it --rm \
+        -v "${PWD}:/go/src/github.com/zrepl/zrepl" \
+        --user "$(id -u):$(id -g)" \
+        zrepl_build make vendordeps release
+
+  - &zrepl_build_template
+    language: go
+    go_import_path: github.com/zrepl/zrepl
+    before_install:
+        - wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
+        - echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807  protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
+        - sudo unzip -d /usr protoc-3.6.1-linux-x86_64.zip
+        - ./lazy.sh godep
+        - make vendordeps
+    script:
+      - make
+      - make vet
+      - make test
+      - make lint
+      - make artifacts/zrepl-freebsd-amd64
+      - make artifacts/zrepl-linux-amd64
+      - make artifacts/zrepl-darwin-amd64
+    go:
+    - "1.11"
+
+  - <<: *zrepl_build_template
+    go:
+    - "1.12"
+
+  - <<: *zrepl_build_template
+    go:
+    - "master"
+
+  - &zrepl_docs_template 
+    language: python
+    python:
+    - "3.4"
+    install:
+    - sudo apt-get install libgirepository1.0-dev
+    - pip install -r docs/requirements.txt
+    script:
+    - make docs
+  - <<: *zrepl_docs_template
+    python:
+    - "3.5"
+  - <<: *zrepl_docs_template
+    python:
+    - "3.6"
+  - <<: *zrepl_docs_template
+    python:
+    - "3.7"
+  
+
+  allow_failures:
+  - <<: *zrepl_build_template
+    go:
+    - "master"

Gopkg.lock

@@ -1,313 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  branch = "master"
-  digest = "1:c0bec5f9b98d0bc872ff5e834fac186b807b656683bd29cb82fb207a1513fabb"
-  name = "github.com/beorn7/perks"
-  packages = ["quantile"]
-  pruneopts = ""
-  revision = "3a771d992973f24aa725d07868b467d1ddfceafb"
-
-[[projects]]
-  digest = "1:56c130d885a4aacae1dd9c7b71cfe39912c7ebc1ff7d2b46083c8812996dc43b"
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  pruneopts = ""
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:e988ed0ca0d81f4d28772760c02ee95084961311291bdfefc1b04617c178b722"
-  name = "github.com/fatih/color"
-  packages = ["."]
-  pruneopts = ""
-  revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4"
-  version = "v1.7.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:5d0a2385edf4ba44f3b7b76bc0436ceb8f62bf55aa5d540a9eb9ec6c58d86809"
-  name = "github.com/ftrvxmtrx/fd"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c6d800382fff6dc1412f34269f71b7f83bd059ad"
-
-[[projects]]
-  branch = "master"
-  digest = "1:4416ad04caf950d004921268f777e418f8c328080ccf6a91e799f977dc128808"
-  name = "github.com/gdamore/encoding"
-  packages = ["."]
-  pruneopts = ""
-  revision = "b23993cbb6353f0e6aa98d0ee318a34728f628b9"
-
-[[projects]]
-  digest = "1:d8088c614a5bffaf38a40c116226e7d230750bafb77942dca25eab779ceeaf1f"
-  name = "github.com/gdamore/tcell"
-  packages = [
-    ".",
-    "termbox",
-    "terminfo",
-  ]
-  pruneopts = ""
-  revision = "de7e78efa4a71b3f36c7154989c529dbdf9ae623"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:6a4a01d58b227c4b6b11111b9f172ec5c17682b82724e58e6daf3f19f4faccd8"
-  name = "github.com/go-logfmt/logfmt"
-  packages = ["."]
-  pruneopts = ""
-  revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5"
-  version = "v0.3.0"
-
-[[projects]]
-  digest = "1:3dd078fda7500c341bc26cfbc6c6a34614f295a2457149fc1045cab767cbcf18"
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  pruneopts = ""
-  revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5"
-  version = "v1.2.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:cb09475f771b9167fb9333629f5d6a7161572602ea040f1094602b0dc8709878"
-  name = "github.com/jinzhu/copier"
-  packages = ["."]
-  pruneopts = ""
-  revision = "db4671f3a9b8df855e993f7c94ec5ef1ffb0a23b"
-
-[[projects]]
-  branch = "master"
-  digest = "1:1ed9eeebdf24aadfbca57eb50e6455bd1d2474525e0f0d4454de8c8e9bc7ee9a"
-  name = "github.com/kr/logfmt"
-  packages = ["."]
-  pruneopts = ""
-  revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:591a2778aa6e896980757ea87e659b3aa13d8c0e790310614028463a31c0998b"
-  name = "github.com/kr/pretty"
-  packages = ["."]
-  pruneopts = ""
-  revision = "cfb55aafdaf3ec08f0db22699ab822c50091b1c4"
-
-[[projects]]
-  branch = "master"
-  digest = "1:109e0a9b9e74c5c8adf94a2dc4dd4a9ca2a183d4d87ffecd4e62db69a5ede55a"
-  name = "github.com/kr/text"
-  packages = ["."]
-  pruneopts = ""
-  revision = "7cafcd837844e784b526369c9bce262804aebc60"
-
-[[projects]]
-  digest = "1:863ba095ae0954100963826333bd1503c169424bb798d9ce800d9f6651f4c234"
-  name = "github.com/lucasb-eyer/go-colorful"
-  packages = ["."]
-  pruneopts = ""
-  revision = "345fbb3dbcdb252d9985ee899a84963c0fa24c82"
-  version = "v1.0"
-
-[[projects]]
-  digest = "1:9ea83adf8e96d6304f394d40436f2eb44c1dc3250d223b74088cc253a6cd0a1c"
-  name = "github.com/mattn/go-colorable"
-  packages = ["."]
-  pruneopts = ""
-  revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072"
-  version = "v0.0.9"
-
-[[projects]]
-  digest = "1:78229b46ddb7434f881390029bd1af7661294af31f6802e0e1bedaad4ab0af3c"
-  name = "github.com/mattn/go-isatty"
-  packages = ["."]
-  pruneopts = ""
-  revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
-  version = "v0.0.3"
-
-[[projects]]
-  digest = "1:82b912465c1da0668582a7d1117339c278e786c2536b3c3623029a0c7141c2d0"
-  name = "github.com/mattn/go-runewidth"
-  packages = ["."]
-  pruneopts = ""
-  revision = "ce7b0b5c7b45a81508558cd1dba6bb1e4ddb51bb"
-  version = "v0.0.3"
-
-[[projects]]
-  digest = "1:4c23ced97a470b17d9ffd788310502a077b9c1f60221a85563e49696276b4147"
-  name = "github.com/matttproud/golang_protobuf_extensions"
-  packages = ["pbutil"]
-  pruneopts = ""
-  revision = "3247c84500bff8d9fb6d579d800f20b3e091582c"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:7365acd48986e205ccb8652cc746f09c8b7876030d53710ea6ef7d0bd0dcd7ca"
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  pruneopts = ""
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  digest = "1:256484dbbcd271f9ecebc6795b2df8cad4c458dd0f5fd82a8c2fa0c29f233411"
-  name = "github.com/pmezard/go-difflib"
-  packages = ["difflib"]
-  pruneopts = ""
-  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:1392748e290ca66ac8447ef24961f8ae9e1d846a53af0f58a5a0256982ce0577"
-  name = "github.com/problame/go-netssh"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c56ad38d2c91397ad3c8dd9443d7448e328a9e9e"
-
-[[projects]]
-  branch = "master"
-  digest = "1:8c63c44f018bd52b03ebad65c9df26aabbc6793138e421df1c8c84c285a45bc6"
-  name = "github.com/problame/go-rwccmd"
-  packages = ["."]
-  pruneopts = ""
-  revision = "391d2c78c8404a9683d79f75dd24ab53040f89f7"
-
-[[projects]]
-  digest = "1:1bcbb0a7ad8d3392d446eb583ae5415ff987838a8f7331a36877789be20667e6"
-  name = "github.com/problame/go-streamrpc"
-  packages = [
-    ".",
-    "internal/pdu",
-  ]
-  pruneopts = ""
-  revision = "d5d111e014342fe1c37f0b71cc37ec5f2afdfd13"
-  version = "v0.5"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ebf8ffdde9bdbf9c83e22121875c68c01d821776523546554b2a3ff6f72773ab"
-  name = "github.com/prometheus/client_golang"
-  packages = [
-    "prometheus",
-    "prometheus/promhttp",
-  ]
-  pruneopts = ""
-  revision = "e11c6ff8170beca9d5fd8b938e71165eeec53ac6"
-
-[[projects]]
-  branch = "master"
-  digest = "1:60aca47f4eeeb972f1b9da7e7db51dee15ff6c59f7b401c1588b8e6771ba15ef"
-  name = "github.com/prometheus/client_model"
-  packages = ["go"]
-  pruneopts = ""
-  revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c"
-
-[[projects]]
-  branch = "master"
-  digest = "1:af21ee3e0a8212f17bb317cd7237f9920bcb2641a291ac111f30f63b3cab817f"
-  name = "github.com/prometheus/common"
-  packages = [
-    "expfmt",
-    "internal/bitbucket.org/ww/goautoneg",
-    "model",
-  ]
-  pruneopts = ""
-  revision = "d0f7cd64bda49e08b22ae8a730aa57aa0db125d6"
-
-[[projects]]
-  branch = "master"
-  digest = "1:61df0898746840afc7be5dc2c3eeec83022fab70df11ecee5b16c85e912cf5ed"
-  name = "github.com/prometheus/procfs"
-  packages = [
-    ".",
-    "internal/util",
-    "nfs",
-    "xfs",
-  ]
-  pruneopts = ""
-  revision = "8b1c2da0d56deffdbb9e48d4414b4e674bd8083e"
-
-[[projects]]
-  branch = "master"
-  digest = "1:146327ce93be37e68bd3ff8541090d96da8cb3adc9e35d57570e9170a29f6bf6"
-  name = "github.com/spf13/cobra"
-  packages = ["."]
-  pruneopts = ""
-  revision = "b78744579491c1ceeaaa3b40205e56b0591b93a3"
-
-[[projects]]
-  digest = "1:261bc565833ef4f02121450d74eb88d5ae4bd74bfe5d0e862cddb8550ec35000"
-  name = "github.com/spf13/pflag"
-  packages = ["."]
-  pruneopts = ""
-  revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:3926a4ec9a4ff1a072458451aa2d9b98acd059a45b38f7335d31e06c3d6a0159"
-  name = "github.com/stretchr/testify"
-  packages = [
-    "assert",
-    "require",
-  ]
-  pruneopts = ""
-  revision = "69483b4bd14f5845b5a1e55bca19e954e827f1d0"
-  version = "v1.1.4"
-
-[[projects]]
-  branch = "v2"
-  digest = "1:9d92186f609a73744232323416ddafd56fae67cb552162cc190ab903e36900dd"
-  name = "github.com/zrepl/yaml-config"
-  packages = ["."]
-  pruneopts = ""
-  revision = "af27d27978ad95808723a62d87557d63c3ff0605"
-
-[[projects]]
-  branch = "master"
-  digest = "1:9c286cf11d0ca56368185bada5dd6d97b6be4648fc26c354fcba8df7293718f7"
-  name = "golang.org/x/sys"
-  packages = ["unix"]
-  pruneopts = ""
-  revision = "bf42f188b9bc6f2cf5b8ee5a912ef1aedd0eba4c"
-
-[[projects]]
-  digest = "1:5acd3512b047305d49e8763eef7ba423901e85d5dd2fd1e71778a0ea8de10bd4"
-  name = "golang.org/x/text"
-  packages = [
-    "encoding",
-    "encoding/internal/identifier",
-    "internal/gen",
-    "transform",
-    "unicode/cldr",
-  ]
-  pruneopts = ""
-  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
-  version = "v0.3.0"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  input-imports = [
-    "github.com/fatih/color",
-    "github.com/gdamore/tcell/termbox",
-    "github.com/go-logfmt/logfmt",
-    "github.com/golang/protobuf/proto",
-    "github.com/jinzhu/copier",
-    "github.com/kr/pretty",
-    "github.com/mattn/go-isatty",
-    "github.com/pkg/errors",
-    "github.com/problame/go-netssh",
-    "github.com/problame/go-rwccmd",
-    "github.com/problame/go-streamrpc",
-    "github.com/prometheus/client_golang/prometheus",
-    "github.com/prometheus/client_golang/prometheus/promhttp",
-    "github.com/spf13/cobra",
-    "github.com/spf13/pflag",
-    "github.com/stretchr/testify/assert",
-    "github.com/stretchr/testify/require",
-    "github.com/zrepl/yaml-config",
-    "golang.org/x/sys/unix",
-  ]
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,75 +0,0 @@
-ignored = [ "github.com/inconshreveable/mousetrap" ]
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/ftrvxmtrx/fd"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/jinzhu/copier"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/kr/pretty"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/mitchellh/go-homedir"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/mitchellh/mapstructure"
-
-[[constraint]]
-  name = "github.com/pkg/errors"
-  version = "0.8.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/spf13/cobra"
-
-[[constraint]]
-  name = "github.com/spf13/viper"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "github.com/stretchr/testify"
-  version = "1.1.4"
-
-[[constraint]]
-  branch = "v2"
-  name = "github.com/zrepl/yaml-config"
-
-[[constraint]]
-  name = "github.com/go-logfmt/logfmt"
-  version = "*"
-
-[[constraint]]
-  name = "github.com/problame/go-rwccmd"
-  branch = "master"
-
-[[constraint]]
-  name = "github.com/problame/go-netssh"
-  branch = "master"
-
-[[constraint]]
-  name = "github.com/prometheus/client_golang"
-  branch = "master"
-
-[[constraint]]
-  name = "github.com/golang/protobuf"
-  version = "1.2.0"
-
-[[constraint]]
-  name = "github.com/fatih/color"
-  version = "1.7.0"
-
-[[constraint]]
-  name = "github.com/problame/go-streamrpc"
-  version = "0.5.0"
-
-
-
-[[constraint]]
-  name = "github.com/gdamore/tcell"
-  version = "1.0.0"

Makefile

@@ -1,37 +1,6 @@
-.PHONY: generate build test vet cover release docs docs-clean clean vendordeps
-.DEFAULT_GOAL := build
-
-ROOT := github.com/zrepl/zrepl
-SUBPKGS += client
-SUBPKGS += config
-SUBPKGS += daemon
-SUBPKGS += daemon/filters
-SUBPKGS += daemon/job
-SUBPKGS += daemon/logging
-SUBPKGS += daemon/nethelpers
-SUBPKGS += daemon/pruner
-SUBPKGS += daemon/snapper
-SUBPKGS += daemon/streamrpcconfig
-SUBPKGS += daemon/transport
-SUBPKGS += daemon/transport/connecter
-SUBPKGS += daemon/transport/serve
-SUBPKGS += endpoint
-SUBPKGS += logger
-SUBPKGS += pruning
-SUBPKGS += pruning/retentiongrid
-SUBPKGS += replication
-SUBPKGS += replication/fsrep
-SUBPKGS += replication/pdu
-SUBPKGS += replication/internal/diff
-SUBPKGS += tlsconf
-SUBPKGS += util
-SUBPKGS += util/socketpair
-SUBPKGS += util/watchdog
-SUBPKGS += util/envconst
-SUBPKGS += version
-SUBPKGS += zfs
-
-_TESTPKGS := $(ROOT) $(foreach p,$(SUBPKGS),$(ROOT)/$(p))
+.PHONY: generate build test vet cover release docs docs-clean clean format lint platformtest
+.PHONY: release bins-all release-noarch
+.DEFAULT_GOAL := zrepl-bin
 
 ARTIFACTDIR := artifacts
 
@@ -39,67 +8,215 @@ ifdef ZREPL_VERSION
     _ZREPL_VERSION := $(ZREPL_VERSION)
 endif
 ifndef _ZREPL_VERSION
-    _ZREPL_VERSION := $(shell git describe --dirty 2>/dev/null || echo "ZREPL_BUILD_INVALID_VERSION" )
+    _ZREPL_VERSION := $(shell git describe --always --dirty 2>/dev/null || echo "ZREPL_BUILD_INVALID_VERSION" )
     ifeq ($(_ZREPL_VERSION),ZREPL_BUILD_INVALID_VERSION) # can't use .SHELLSTATUS because Debian Stretch is still on gmake 4.1
         $(error cannot infer variable ZREPL_VERSION using git and variable is not overriden by make invocation)
     endif
 endif
+
+GO := go
+GOOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOOS"')
+GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"')
+GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"')
+GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"')
+GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
+GO_ENV_VARS := GO111MODULE=on
 GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
+GO_MOD_READONLY := -mod=readonly
+GO_EXTRA_BUILDFLAGS :=
+GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
+GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
+GOLANGCI_LINT := golangci-lint
+GOCOVMERGE := gocovmerge
 
-GO_BUILD := go build -ldflags $(GO_LDFLAGS)
+ifneq ($(GOARM),)
+	ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM)
+else
+	ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)
+endif
 
-RELEASE_BINS := $(ARTIFACTDIR)/zrepl-freebsd-amd64
-RELEASE_BINS += $(ARTIFACTDIR)/zrepl-linux-amd64
-RELEASE_BINS += $(ARTIFACTDIR)/zrepl-darwin-amd64
+.PHONY: printvars
+printvars:
+	@echo GOOS=$(GOOS)
+	@echo GOARCH=$(GOARCH)
+	@echo GOARM=$(GOARM)
 
-RELEASE_NOARCH := $(ARTIFACTDIR)/zrepl-noarch.tar
-THIS_PLATFORM_RELEASE_BIN := $(shell bash -c 'source <(go env) && echo "zrepl-$${GOOS}-$${GOARCH}"' )
 
-vendordeps:
-	dep ensure -v -vendor-only
+##################### PRODUCING A RELEASE #############
+.PHONY: release wrapup-and-checksum check-git-clean sign clean
 
-generate: #not part of the build, must do that manually
-	protoc -I=replication/pdu --go_out=replication/pdu replication/pdu/pdu.proto
-	@for pkg in $(_TESTPKGS); do\
-		go generate "$$pkg" || exit 1; \
-	done;
+release: clean
+	# no cross-platform support for target test
+	$(MAKE) test-go
+	$(MAKE) bins-all
+	$(MAKE) noarch
+	$(MAKE) wrapup-and-checksum
+	$(MAKE) check-git-clean
+ifeq (SIGN, 1)
+	$(make) sign
+endif
+	@echo "ZREPL RELEASE ARTIFACTS AVAILABLE IN artifacts/release"
 
-build:
-	@echo "INFO: In case of missing dependencies, run 'make vendordeps'"
-	$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl"
+# expects `release` target to have run before
+NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
+wrapup-and-checksum:
+	rm -f $(NOARCH_TARBALL)
+	tar --mtime='1970-01-01' --sort=name \
+		--transform 's/$(ARTIFACTDIR)/zrepl-$(_ZREPL_VERSION)-noarch/' \
+		--transform 's#dist#zrepl-$(_ZREPL_VERSION)-noarch/dist#' \
+		--transform 's#config/samples#zrepl-$(_ZREPL_VERSION)-noarch/config#' \
+		-acf $(NOARCH_TARBALL) \
+		$(ARTIFACTDIR)/docs/html \
+		$(ARTIFACTDIR)/bash_completion \
+		$(ARTIFACTDIR)/_zrepl.zsh_completion \
+		$(ARTIFACTDIR)/go_env.txt \
+		dist \
+		config/samples
+	rm -rf "$(ARTIFACTDIR)/release"
+	mkdir -p "$(ARTIFACTDIR)/release"
+	cp -l $(ARTIFACTDIR)/zrepl-* \
+		$(ARTIFACTDIR)/platformtest-* \
+		"$(ARTIFACTDIR)/release"
+	cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
 
-test:
-	@for pkg in $(_TESTPKGS); do \
-		echo "Testing $$pkg"; \
-		go test "$$pkg" || exit 1; \
-	done;
+check-git-clean:
+	@# note that we use ZREPL_VERSION and not _ZREPL_VERSION because we want to detect the override
+	@if git describe --always --dirty 2>/dev/null | grep dirty >/dev/null; then \
+        echo '[INFO] either git reports checkout is dirty or git is not installed or this is not a git checkout'; \
+		if [ "$(ZREPL_VERSION)" = "" ]; then \
+			echo '[WARN] git checkout is dirty and make variable ZREPL_VERSION was not used to override'; \
+			git status; \
+			echo "git diff:";  \
+			git diff | cat; \
+			exit 1; \
+		fi; \
+	fi;
+
+sign:
+	gpg -u "89BC 5D89 C845 568B F578  B306 CDBD 8EC8 E27C A5FC" \
+		--armor \
+		--detach-sign $(ARTIFACTDIR)/release/sha512sum.txt
+
+clean: docs-clean
+	rm -rf "$(ARTIFACTDIR)"
+
+##################### BINARIES #####################
+.PHONY: bins-all lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin generate-platform-test-list
+
+BINS_ALL_TARGETS := zrepl-bin test-platform-bin vet lint
+GO_SUPPORTS_ILLUMOS := $(shell $(GO) version | gawk -F '.' '/^go version /{split($$0, comps, " "); split(comps[3], v, "."); if (v[1] == "go1" && v[2] >= 13) { print "illumos"; } else { print "noillumos"; }}')
+bins-all:
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=amd64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=386
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=amd64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm     GOARM=7
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=386
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=darwin    GOARCH=amd64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=solaris   GOARCH=amd64
+ifeq ($(GO_SUPPORTS_ILLUMOS), illumos)
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=illumos   GOARCH=amd64
+else ifeq ($(GO_SUPPORTS_ILLUMOS), noillumos)
+	@echo "SKIPPING ILLUMOS BUILD BECAUSE GO VERSION DOESN'T SUPPORT IT"
+else
+	@echo "CANNOT DETERMINE WHETHER GO VERSION SUPPORTS GOOS=illumos"; exit 1
+endif
+
+lint:
+	$(GO_ENV_VARS) $(GOLANGCI_LINT) run ./...
 
 vet:
-	@for pkg in $(_TESTPKGS); do \
-		echo "Vetting $$pkg"; \
-		go vet "$$pkg" || exit 1; \
-	done;
+	$(GO_ENV_VARS) $(GO) vet $(GO_BUILDFLAGS) ./...
+
+test-go: $(ARTIFACTDIR)
+	rm -f "$(ARTIFACTDIR)/gotest.cover"
+ifeq ($(COVER),1)
+	$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
+		-coverpkg github.com/zrepl/zrepl/... \
+		-covermode atomic \
+		-coverprofile "$(ARTIFACTDIR)/gotest.cover" \
+		./...
+else
+	$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
+		./...
+endif
+
+zrepl-bin:
+	$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(ZREPL_TARGET_TUPLE)"
+
+generate-platform-test-list:
+	$(GO_BUILD) -o $(ARTIFACTDIR)/generate-platform-test-list ./platformtest/tests/gen
+
+test-platform-bin:
+	$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
+		-c -o "$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" \
+		-covermode=atomic -cover -coverpkg github.com/zrepl/zrepl/... \
+		./platformtest/harness
+
+ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
+ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
+ZREPL_PLATFORMTEST_MOUNTPOINT := /tmp/zreplplatformtest.pool
+ZREPL_PLATFORMTEST_ZFS_LOG := /tmp/zreplplatformtest.zfs.log
+# ZREPL_PLATFORMTEST_STOP_AND_KEEP := -failure.stop-and-keep-pool
+ZREPL_PLATFORMTEST_ARGS := 
+test-platform: $(ARTIFACTDIR) # do not track dependency on test-platform-bin to allow build of platformtest outside of test VM
+	rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
+	rm -f "$(ARTIFACTDIR)/platformtest.cover"
+	platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
+	"$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" \
+		-test.coverprofile "$(ARTIFACTDIR)/platformtest.cover" \
+		-test.v \
+		__DEVEL--i-heard-you-like-tests \
+		-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
+		-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
+		-mountpoint "$(ZREPL_PLATFORMTEST_MOUNTPOINT)" \
+		$(ZREPL_PLATFORMTEST_STOP_AND_KEEP) \
+		$(ZREPL_PLATFORMTEST_ARGS)
+
+cover-merge: $(ARTIFACTDIR)
+	$(GOCOVMERGE) $(ARTIFACTDIR)/platformtest.cover $(ARTIFACTDIR)/gotest.cover > $(ARTIFACTDIR)/merged.cover
+cover-html: cover-merge
+	$(GO) tool cover -html "$(ARTIFACTDIR)/merged.cover" -o "$(ARTIFACTDIR)/merged.cover.html"
+
+test-full:
+	test "$$(id -u)" = "0" || echo "MUST RUN AS ROOT" 1>&2
+	$(MAKE) test-go COVER=1
+	$(MAKE) test-platform
+	$(MAKE) cover-html
+
+##################### DEV TARGETS #####################
+# not part of the build, must do that manually
+.PHONY: generate format
+
+generate: generate-platform-test-list
+	protoc -I=replication/logic/pdu --go_out=plugins=grpc:replication/logic/pdu replication/logic/pdu/pdu.proto
+	$(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./...
+
+format:
+	goimports -srcdir . -local 'github.com/zrepl/zrepl' -w $(shell find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go')
+
+##################### NOARCH #####################
+.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
 
-cover: artifacts
-	@for pkg in $(_TESTPKGS); do \
-		profile="$(ARTIFACTDIR)/cover-$$(basename $$pkg).out"; \
-		go test -coverprofile "$$profile" $$pkg || exit 1; \
-		if [ -f "$$profile" ]; then \
-   			go tool cover -html="$$profile" -o "$${profile}.html" || exit 2; \
-		fi; \
-	done;
 
 $(ARTIFACTDIR):
 	mkdir -p "$@"
-
 $(ARTIFACTDIR)/docs: $(ARTIFACTDIR)
 	mkdir -p "$@"
 
-$(ARTIFACTDIR)/bash_completion: $(RELEASE_BINS)
-	artifacts/$(THIS_PLATFORM_RELEASE_BIN) bashcomp "$@"
+noarch: $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs
+	# pass
 
-$(ARTIFACTDIR)/go_version.txt:
-	go version > $@
+$(ARTIFACTDIR)/bash_completion:
+	$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
+	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion bash "$@"
+
+$(ARTIFACTDIR)/_zrepl.zsh_completion:
+	$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
+	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion zsh "$@"
+
+$(ARTIFACTDIR)/go_env.txt:
+	$(GO_ENV_VARS) $(GO) env > $@
 
 docs: $(ARTIFACTDIR)/docs
 	make -C docs \
@@ -110,34 +227,3 @@ docs-clean:
 	make -C docs \
 		clean \
 		BUILDDIR=../artifacts/docs
-
-
-.PHONY: $(RELEASE_BINS)
-# TODO: two wildcards possible
-$(RELEASE_BINS): $(ARTIFACTDIR)/zrepl-%-amd64: generate $(ARTIFACTDIR) vet test
-	@echo "INFO: In case of missing dependencies, run 'make vendordeps'"
-	GOOS=$* GOARCH=amd64   $(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$*-amd64"
-
-$(RELEASE_NOARCH): docs $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_version.txt
-	tar --mtime='1970-01-01' --sort=name \
-		--transform 's/$(ARTIFACTDIR)/zrepl-$(_ZREPL_VERSION)-noarch/' \
-		-acf $@ \
-		$(ARTIFACTDIR)/docs/html \
-		$(ARTIFACTDIR)/bash_completion \
-		$(ARTIFACTDIR)/go_version.txt
-
-release: $(RELEASE_BINS) $(RELEASE_NOARCH)
-	rm -rf "$(ARTIFACTDIR)/release"
-	mkdir -p "$(ARTIFACTDIR)/release"
-	cp $^ "$(ARTIFACTDIR)/release"
-	cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
-	@# note that we use ZREPL_VERSION and not _ZREPL_VERSION because we want to detect the override
-	@if git describe --dirty 2>/dev/null | grep dirty >/dev/null; then \
-		if [ "$(ZREPL_VERSION)" == "" ]; then \
-			echo "[WARN] git checkout is dirty and make variable ZREPL_VERSION was not used to override"; \
-			exit 1; \
-		fi; \
-	fi;
-
-clean: docs-clean
-	rm -rf "$(ARTIFACTDIR)"

README.md

@@ -1,3 +1,11 @@
+[![GitHub license](https://img.shields.io/github/license/zrepl/zrepl.svg)](https://github.com/zrepl/zrepl/blob/master/LICENSE)
+[![Language: Go](https://img.shields.io/badge/language-Go-6ad7e5.svg)](https://golang.org/)
+[![User Docs](https://img.shields.io/badge/docs-web-blue.svg)](https://zrepl.github.io)
+[![Donate via Patreon](https://img.shields.io/endpoint.svg?url=https%3A%2F%2Fshieldsio-patreon.herokuapp.com%2Fzrepl%2Fpledges&style=flat&color=yellow)](https://www.patreon.com/zrepl)
+[![Donate via Liberapay](https://img.shields.io/liberapay/receives/zrepl.svg?logo=liberapay)](https://liberapay.com/zrepl/donate)
+[![Donate via PayPal](https://img.shields.io/badge/donate-paypal-yellow.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
+[![Twitter](https://img.shields.io/twitter/url/https/github.com/zrepl/zrepl.svg?style=social)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl)
+
 # zrepl
 zrepl is a one-stop ZFS backup & replication solution.
 
@@ -16,6 +24,7 @@ zrepl is a one-stop ZFS backup & replication solution.
    If so, think of an expressive configuration example.
 2. Think of at least one use case that generalizes from your concrete application.
 3. Open an issue on GitHub with example conf & use case attached.
+4. **Optional**: [Post a bounty](https://www.bountysource.com/teams/zrepl) on the issue, or [contact Christian Schwarz](https://cschwarz.com) for contract work.
 
 The above does not apply if you already implemented everything.
 Check out the *Coding Workflow* section below for details.
@@ -26,40 +35,81 @@ Check out the *Coding Workflow* section below for details.
 * Make sure your distro is compatible with the paths in `docs/installation.rst`.
 * Ship a default config that adheres to your distro's `hier` and logging system.
 * Ship a service manager file and _please_ try to upstream it to this repository.
+  * `dist/systemd` contains a Systemd unit template.
+* Ship other material provided in `./dist`, e.g. in `/usr/share/zrepl/`.
 * Use `make release ZREPL_VERSION='mydistro-1.2.3_1'`
     * Your distro's name and any versioning supplemental to zrepl's (e.g. package revision) should be in this string
+* Use `sudo make test-platform` **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
 * Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
 
 ## Developer Documentation
 
-zrepl is written in [Go](https://golang.org) and uses [`dep`](https://github.com/golang/dep) to manage dependencies.
+zrepl is written in [Go](https://golang.org) and uses [Go modules](https://github.com/golang/go/wiki/Modules) to manage dependencies.
 The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
 
-To get started, run `./lazy.sh devsetup` to easily install build dependencies and read `docs/installation.rst -> Compiling from Source`.
+Install **build dependencies** using  `./lazy.sh devsetup`.
+`lazy.sh` uses `python3-pip` to fetch the build dependencies for the docs - you might want to use a [venv](https://docs.python.org/3/library/venv.html).
+If you just want to install the Go dependencies, run `./lazy.sh godep`.
 
-### Overall Architecture
+The **test suite** is split into pure **Go tests** (`make test-go`) and **platform tests** that interact with ZFS and thus generally **require root privileges** (`sudo make test-platform`).
+Platform tests run on their own pool with the name `zreplplatformtest`, which is created using the file vdev in `/tmp`.
 
-The application architecture is documented as part of the user docs in the *Implementation* section (`docs/content/impl`).
-Make sure to develop an understanding how zrepl is typically used by studying the user docs first.
+For a full **code coverage** profile, run `make test-go COVER=1 && sudo make test-platform && make cover-merge`.
+An HTML report can be generated using `make cover-html`.
+
+**Code generation** is triggered by `make generate`. Generated code is committed to the source tree.
 
 ### Project Structure
 
 ```
-├── cmd
-│   ├── endpoint            # implementations of endpoints for package replication
-│   ├── sampleconf          # example configuration
+├── artifacts               # build artifcats generate by make
+├── cli                     # wrapper around CLI package cobra
+├── client                  # all subcommands that are not `daemon`
+├── config                  # config data types (=> package yaml-config)
+│   └── samples
+├── daemon                  # the implementation of `zrepl daemon` subcommand
+│   ├── filters
+│   ├── hooks               # snapshot hooks
+│   ├── job                 # job implementations
+│   ├── logging             # logging outlets + formatters
+│   ├── nethelpers
+│   ├── prometheus
+│   ├── pruner              # pruner implementation
+│   ├── snapper             # snapshotter implementation
+├── dist                    # supplemental material for users & package maintainers
 ├── docs                    # sphinx-based documentation
 │   ├── **/*.rst            # documentation in reStructuredText
 │   ├── sphinxconf
 │   │   └── conf.py         # sphinx config (see commit 445a280 why its not in docs/)
 │   ├── requirements.txt    # pip3 requirements to build documentation
 │   ├── publish.sh          # shell script for automated rendering & deploy to zrepl.github.io repo
-│   ├── public_git          # checkout of zrepl.github.io managed by above shell script
-├── logger                  # logger package used by zrepl
-├── replication             # replication functionality
-├── rpc                     # rpc protocol implementation
+│   └── public_git          # checkout of zrepl.github.io managed by above shell script
+├── endpoint                # implementation of replication endpoints (=> package replication)
+├── logger                  # our own logger package
+├── platformtest            # test suite for our zfs abstractions (error classification, etc)
+├── pruning                 # pruning rules (the logic, not the actual execution)
+│   └── retentiongrid
+├── replication
+│   ├── driver              # the driver of the replication logic (status reporting, error handling)
+│   ├── logic               # planning & executing replication steps via rpc
+|   |   └── pdu             # the generated gRPC & protobuf code used in replication (and endpoints)
+│   └── report              # the JSON-serializable report datastructures exposed to the client
+├── rpc                     # the hybrid gRPC + ./dataconn RPC client: connects to a remote replication.Endpoint
+│   ├── dataconn            # Bulk data-transfer RPC protocol
+│   ├── grpcclientidentity  # adaptor to inject package transport's 'client identity' concept into gRPC contexts
+│   ├── netadaptor          # adaptor to convert a package transport's Connecter and Listener into net.* primitives
+│   ├── transportmux        # TCP connecter and listener used to split control & data traffic
+│   └── versionhandshake    # replication protocol version handshake perfomed on newly established connections
+├── tlsconf                 # abstraction for Go TLS server + client config
+├── transport               # transport implementations
+│   ├── fromconfig
+│   ├── local
+│   ├── ssh
+│   ├── tcp
+│   └── tls
 ├── util
-└── zfs                     # ZFS wrappers, filesystemm diffing
+├── version                 # abstraction for versions (filled during build by Makefile)
+└── zfs                     # zfs(8) wrappers
 ```
 
 ### Coding Workflow
@@ -92,3 +142,15 @@ There will not be a big refactoring (an attempt was made, but it's destroying to
 
 However, new contributions & patches should fix naming without further notice in the commit message.
 
+### RPC debugging
+
+Optionally, there are various RPC-related environment variables, that if set to something != `""` will produce additional debug output on stderr:
+
+https://github.com/zrepl/zrepl/blob/master/rpc/rpc_debug.go#L11
+
+https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/dataconn_debug.go#L11
+
+https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/stream/stream_debug.go#L11
+
+https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/heartbeatconn/heartbeatconn_debug.go#L11
+

build.Dockerfile

@@ -2,21 +2,29 @@ FROM golang:latest
 
 RUN apt-get update && apt-get install -y \
     python3-pip \
-    unzip
+    unzip \
+    gawk
 
-RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
-RUN echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807  protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
-RUN unzip -d /usr protoc-3.6.1-linux-x86_64.zip
+ADD build.installprotoc.bash ./
+RUN bash build.installprotoc.bash
 
 ADD lazy.sh /tmp/lazy.sh
 ADD docs/requirements.txt /tmp/requirements.txt
 ENV ZREPL_LAZY_DOCS_REQPATH=/tmp/requirements.txt
-RUN /tmp/lazy.sh devsetup
+RUN /tmp/lazy.sh docdep
 
 # prepare volume mount of git checkout to /zrepl
-RUN mkdir -p /go/src/github.com/zrepl/zrepl
-RUN chmod -R 0777 /go
+RUN mkdir -p /src/github.com/zrepl/zrepl
 RUN mkdir -p /.cache && chmod -R 0777 /.cache
-WORKDIR /go/src/github.com/zrepl/zrepl
 
+# $GOPATH is /go
+# Go 1.12 doesn't use modules within GOPATH, but 1.13 and later do
+# => store source outside of GOPATH
+WORKDIR /src
+
+# Install build tools (e.g. protobuf generator, stringer) into $GOPATH/bin
+ADD build/ /tmp/build
+RUN /tmp/lazy.sh godep
+
+RUN chmod -R 0777 /go
 

build.installprotoc.bash

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+
+MACH=$(uname -m)
+MACH="${MACH/aarch64/aarch_64}"
+
+VERSION=3.6.1
+FILENAME=protoc-"$VERSION"-linux-"$MACH".zip
+
+if [ -e "$FILENAME" ]; then
+    echo "$FILENAME" already exists 1>&2
+    exit 1
+fi
+
+wget https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
+
+stat "$FILENAME"
+
+sha256sum -c --ignore-missing <<EOF
+6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807  protoc-3.6.1-linux-x86_64.zip
+af8e5aaaf39ddec62ec8dd2be1b8d9602c6da66564883a16393ade5f71170922  protoc-3.6.1-linux-aarch_64.zip
+EOF
+
+unzip -d /usr "$FILENAME"

build/go.mod

@@ -0,0 +1,52 @@
+module github.com/zrepl/zrepl/build
+
+go 1.12
+
+require (
+	github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 // indirect
+	github.com/alvaroloes/enumer v1.1.1
+	github.com/fatih/color v1.7.0 // indirect
+	github.com/go-critic/go-critic v0.3.4 // indirect
+	github.com/gogo/protobuf v1.2.1 // indirect
+	github.com/golang/mock v1.2.0 // indirect
+	github.com/golang/protobuf v1.2.0
+	github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 // indirect
+	github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 // indirect
+	github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d // indirect
+	github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 // indirect
+	github.com/golangci/golangci-lint v1.17.1
+	github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb // indirect
+	github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 // indirect
+	github.com/golangci/misspell v0.3.4 // indirect
+	github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 // indirect
+	github.com/google/go-cmp v0.3.0 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/pkg/errors v0.8.1 // indirect
+	github.com/sirupsen/logrus v1.4.2 // indirect
+	github.com/spf13/afero v1.2.2 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/viper v1.3.2 // indirect
+	github.com/stretchr/testify v1.4.0 // indirect
+	github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad // indirect
+	golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 // indirect
+	golang.org/x/sys v0.0.0-20191010194322-b09406accb47 // indirect
+	golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b
+	mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe // indirect
+	sourcegraph.com/sqs/pbtypes v1.0.0 // indirect
+)
+
+// invalid dates in transitive dependencies (first validated in Go 1.13, didn't fail in earlier Go versions)
+replace (
+	github.com/go-critic/go-critic v0.0.0-20181204210945-1df300866540 => github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540
+	github.com/go-critic/go-critic v0.0.0-20181204210945-ee9bf5809ead => github.com/go-critic/go-critic v0.3.5-0.20190210220443-ee9bf5809ead
+	github.com/golangci/errcheck v0.0.0-20181003203344-ef45e06d44b6 => github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6
+	github.com/golangci/go-tools v0.0.0-20180109140146-35a9f45a5db0 => github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0
+	github.com/golangci/go-tools v0.0.0-20180109140146-af6baa5dc196 => github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196
+	github.com/golangci/gofmt v0.0.0-20181105071733-0b8337e80d98 => github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98
+	github.com/golangci/gosec v0.0.0-20180901114220-66fb7fc33547 => github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547
+	github.com/golangci/ineffassign v0.0.0-20180808204949-42439a7714cc => github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc
+	github.com/golangci/lint-1 v0.0.0-20180610141402-ee948d087217 => github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217
+	golang.org/x/tools v0.0.0-20190125232054-379209517ffe => golang.org/x/tools v0.0.0-20190205201329-379209517ffe
+	mvdan.cc/unparam v0.0.0-20190124213536-fbb59629db34 => mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34
+)

build/go.sum

@@ -0,0 +1,282 @@
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/OpenPeeDeeP/depguard v0.0.0-20180806142446-a69c782687b2/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
+github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 h1:pFks+oaqVWDFq0KsLQZFB2pQGB5Us0HfMSBENtieXOs=
+github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
+github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/alvaroloes/enumer v1.1.1 h1:v+1scNqEhSFAPb9tLwblQegeTXJhnw8hf9O0amTbOvQ=
+github.com/alvaroloes/enumer v1.1.1/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/go-critic/go-critic v0.3.4 h1:FYaiaLjX0Nqei80KPhm4CyFQUBbmJwSrHxQ73taaGBc=
+github.com/go-critic/go-critic v0.3.4/go.mod h1:AHR42Lk/E/aOznsrYdMYeIQS5RH10HZHSqP+rD6AJrc=
+github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
+github.com/go-lintpack/lintpack v0.5.1/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
+github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0=
+github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
+github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
+github.com/go-toolsmith/astcast v0.0.0-20181028201508-b7a89ed70af1/go.mod h1:TEo3Ghaj7PsZawQHxT/oBvo4HK/sl1RcuUHDKTTju+o=
+github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=
+github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
+github.com/go-toolsmith/astcopy v0.0.0-20180903214859-79b422d080c4/go.mod h1:c9CPdq2AzM8oPomdlPniEfPAC6g1s7NqZzODt8y6ib8=
+github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8=
+github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
+github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
+github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ=
+github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
+github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
+github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k=
+github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
+github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
+github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
+github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg=
+github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
+github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8 h1:vVouagbdmqTVlCIAxpyYsNNTbkKZ3V66VpKOLU/s6W4=
+github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
+github.com/go-toolsmith/pkgload v1.0.0 h1:4DFWWMXVfbcN5So1sBNW9+yeiMqLFGl1wFLTL5R0Tgg=
+github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
+github.com/go-toolsmith/strparse v0.0.0-20180903215201-830b6daa1241/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
+github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4=
+github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
+github.com/go-toolsmith/typep v0.0.0-20181030061450-d63dc7650676/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
+github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA=
+github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0=
+github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
+github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
+github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
+github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w=
+github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
+github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw=
+github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
+github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 h1:MRhC9XbUjE6XDOInSJ8pwHuPagqsyO89QDU9IdVhe3o=
+github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
+github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
+github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8=
+github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
+github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
+github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d h1:pXTK/gkVNs7Zyy7WKgLXmpQ5bHTrq5GDsp8R9Qs67g0=
+github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
+github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU=
+github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
+github.com/golangci/golangci-lint v1.17.1 h1:lc8Hf9GPCjIr0hg3S/xhvFT1+Hydass8F1xchr8jkME=
+github.com/golangci/golangci-lint v1.17.1/go.mod h1:+5sJSl2h3aly+fpmL2meSP8CaSKua2E4Twi9LPy7b1g=
+github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb h1:Bi7BYmZVg4C+mKGi8LeohcP2GGUl2XJD4xCkJoZSaYc=
+github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb/go.mod h1:ON/c2UR0VAAv6ZEAFKhjCLplESSmRFfZcDLASbI1GWo=
+github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
+github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
+github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
+github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 h1:utua3L2IbQJmauC5IXdEA547bcoU5dozgQAfc8Onsg4=
+github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
+github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
+github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
+github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
+github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
+github.com/golangci/misspell v0.3.4 h1:kAD5J0611Zo2VirUn9KFP15RjEqkmfEfnFxyIVxZCYg=
+github.com/golangci/misspell v0.3.4/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
+github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us=
+github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
+github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
+github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 h1:XQKc8IYQOeRwVs36tDrEmTgDgP88d5iEURwpmtiAlOM=
+github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
+github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
+github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
+github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw=
+github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
+github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM=
+github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
+github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
+github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
+github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663 h1:Ri1EhipkbhWsffPJ3IPlrb4SkTOPa2PfRXp3jchBczw=
+github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
+github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 h1:/I3lTljEEDNYLho3/FUB7iD/oc2cEFgVmbHzV+O0PtU=
+github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ=
+github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
+github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.2.1/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
+github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
+github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs=
+github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
+github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
+github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.2 h1:NfkwRbgViGoyjBKsLI0QMDcuMnhM+SBg3T0cGfpvKDE=
+github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
+github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
+github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb h1:lI9ufgFfvuqRctP9Ny8lDDLbSWCMxBPletcSqrnyFYM=
+github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
+github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad h1:W0LEBv82YCGEtcmPA3uNZBI33/qF//HAAs3MawDjRa0=
+github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad/go.mod h1:Hy8o65+MXnS6EwGElrSRjUzQDLXreJlzYLlWiHtt8hM=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190313220215-9f648a60d977 h1:actzWV6iWn3GLqN8dZjzsB+CLt+gaV2+wsxroxiQI8I=
+golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181205014116-22934f0fdb62/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8 h1:b0PLhFjEMqrIqsD4rS5sp0YxBYgdqKzX0KkkGGKLV8I=
+golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b h1:iEAPfYPbYbxG/2lNN4cMOHkmgKNsCuUwkxlDCK46UlU=
+golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
+mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
+mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=
+mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
+mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
+mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe h1:Ekmnp+NcP2joadI9pbK4Bva87QKZSeY7le//oiMrc9g=
+mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe/go.mod h1:BnhuWBAqxH3+J5bDybdxgw5ZfS+DsVd4iylsKQePN8o=
+sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
+sourcegraph.com/sqs/pbtypes v1.0.0 h1:f7lAwqviDEGvON4kRv0o5V7FT/IQK+tbkF664XMbP3o=
+sourcegraph.com/sqs/pbtypes v1.0.0/go.mod h1:3AciMUv4qUuRHRHhOG4TZOB+72GdPVz5k+c648qsFS4=

build/tools.go

@@ -0,0 +1,12 @@
+// +build tools
+
+package main
+
+import (
+	_ "github.com/alvaroloes/enumer"
+	_ "github.com/golang/protobuf/protoc-gen-go"
+	_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
+	_ "github.com/wadey/gocovmerge"
+	_ "golang.org/x/tools/cmd/goimports"
+	_ "golang.org/x/tools/cmd/stringer"
+)

cli/cli.go

@@ -1,11 +1,15 @@
 package cli
 
 import (
+	"context"
 	"fmt"
+	"os"
+
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+
 	"github.com/zrepl/zrepl/config"
-	"os"
 )
 
 var rootArgs struct {
@@ -17,37 +21,67 @@ var rootCmd = &cobra.Command{
 	Short: "One-stop ZFS replication solution",
 }
 
-var bashcompCmd = &cobra.Command{
-	Use:   "bashcomp path/to/out/file",
-	Short: "generate bash completions",
-	Run: func(cmd *cobra.Command, args []string) {
-		if len(args) != 1 {
-			fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
-			cmd.Usage()
-			os.Exit(1)
-		}
-		if err := rootCmd.GenBashCompletionFile(args[0]); err != nil {
-			fmt.Fprintf(os.Stderr, "error generating bash completion: %s", err)
-			os.Exit(1)
-		}
+func init() {
+	rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
+}
+
+var genCompletionCmd = &cobra.Command{
+	Use:   "gencompletion",
+	Short: "generate shell auto-completions",
+}
+
+type completionCmdInfo struct {
+	genFunc func(outpath string) error
+	help    string
+}
+
+var completionCmdMap = map[string]completionCmdInfo{
+	"zsh": {
+		rootCmd.GenZshCompletionFile,
+		"  save to file `_zrepl` in your zsh's $fpath",
+	},
+	"bash": {
+		rootCmd.GenBashCompletionFile,
+		"  save to a path and source that path in your .bashrc",
 	},
-	Hidden: true,
 }
 
 func init() {
-	rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
-	rootCmd.AddCommand(bashcompCmd)
+	for sh, info := range completionCmdMap {
+		sh, info := sh, info
+		genCompletionCmd.AddCommand(&cobra.Command{
+			Use:     fmt.Sprintf("%s path/to/out/file", sh),
+			Short:   fmt.Sprintf("generate %s completions", sh),
+			Example: info.help,
+			Run: func(cmd *cobra.Command, args []string) {
+				if len(args) != 1 {
+					fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
+					err := cmd.Usage()
+					if err != nil {
+						panic(err)
+					}
+					os.Exit(1)
+				}
+				if err := info.genFunc(args[0]); err != nil {
+					fmt.Fprintf(os.Stderr, "error generating %s completion: %s", sh, err)
+					os.Exit(1)
+				}
+			},
+		})
+	}
+	rootCmd.AddCommand(genCompletionCmd)
 }
 
 type Subcommand struct {
-	Use             string
-	Short           string
-	NoRequireConfig bool
-	Run             func(subcommand *Subcommand, args []string) error
-	SetupFlags      func(f *pflag.FlagSet)
-	SetupSubcommands  func() []*Subcommand
+	Use              string
+	Short            string
+	Example          string
+	NoRequireConfig  bool
+	Run              func(ctx context.Context, subcommand *Subcommand, args []string) error
+	SetupFlags       func(f *pflag.FlagSet)
+	SetupSubcommands func() []*Subcommand
 
-	config *config.Config
+	config    *config.Config
 	configErr error
 }
 
@@ -64,7 +98,11 @@ func (s *Subcommand) Config() *config.Config {
 
 func (s *Subcommand) run(cmd *cobra.Command, args []string) {
 	s.tryParseConfig()
-	err := s.Run(s, args)
+	ctx := context.Background()
+	endTask := trace.WithTaskFromStackUpdateCtx(&ctx)
+	defer endTask()
+	err := s.Run(ctx, s, args)
+	endTask()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%s\n", err)
 		os.Exit(1)
@@ -92,8 +130,9 @@ func AddSubcommand(s *Subcommand) {
 
 func addSubcommandToCobraCmd(c *cobra.Command, s *Subcommand) {
 	cmd := cobra.Command{
-		Use: s.Use,
-		Short: s.Short,
+		Use:     s.Use,
+		Short:   s.Short,
+		Example: s.Example,
 	}
 	if s.SetupSubcommands == nil {
 		cmd.Run = s.run
@@ -108,9 +147,8 @@ func addSubcommandToCobraCmd(c *cobra.Command, s *Subcommand) {
 	c.AddCommand(&cmd)
 }
 
-
 func Run() {
 	if err := rootCmd.Execute(); err != nil {
 		os.Exit(1)
 	}
-}
+}

client/configcheck.go

@@ -1,41 +1,52 @@
 package client
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
+	"os"
+
 	"github.com/kr/pretty"
 	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
 	"github.com/zrepl/yaml-config"
+
 	"github.com/zrepl/zrepl/cli"
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/job"
 	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
-	"os"
 )
 
 var configcheckArgs struct {
 	format string
-	what string
+	what   string
 }
 
 var ConfigcheckCmd = &cli.Subcommand{
-	Use: "configcheck",
+	Use:   "configcheck",
 	Short: "check if config can be parsed without errors",
 	SetupFlags: func(f *pflag.FlagSet) {
 		f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]")
 		f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]")
 	},
-	Run: func(subcommand *cli.Subcommand, args []string) error {
-		formatMap := map[string]func(interface{}) {
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		formatMap := map[string]func(interface{}){
 			"": func(i interface{}) {},
-			"pretty": func(i interface{}) { pretty.Println(i) },
+			"pretty": func(i interface{}) {
+				if _, err := pretty.Println(i); err != nil {
+					panic(err)
+				}
+			},
 			"json": func(i interface{}) {
-				json.NewEncoder(os.Stdout).Encode(subcommand.Config())
+				if err := json.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
+					panic(err)
+				}
 			},
 			"yaml": func(i interface{}) {
-				yaml.NewEncoder(os.Stdout).Encode(subcommand.Config())
+				if err := yaml.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
+					panic(err)
+				}
 			},
 		}
 
@@ -71,12 +82,11 @@ var ConfigcheckCmd = &cli.Subcommand{
 			}
 		}
 
-
-		whatMap := map[string]func() {
+		whatMap := map[string]func(){
 			"all": func() {
 				o := struct {
-					config *config.Config
-					jobs []job.Job
+					config  *config.Config
+					jobs    []job.Job
 					logging *logger.Outlets
 				}{
 					subcommand.Config(),
@@ -109,4 +119,3 @@ var ConfigcheckCmd = &cli.Subcommand{
 		}
 	},
 }
-

client/jsonclient.go

@@ -4,10 +4,11 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"github.com/pkg/errors"
 	"io"
 	"net"
 	"net/http"
+
+	"github.com/pkg/errors"
 )
 
 func controlHttpClient(sockpath string) (client http.Client, err error) {
@@ -35,7 +36,7 @@ func jsonRequestResponse(c http.Client, endpoint string, req interface{}, res in
 
 	if resp.StatusCode != http.StatusOK {
 		var msg bytes.Buffer
-		io.CopyN(&msg, resp.Body, 4096)
+		_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
 		return errors.Errorf("%s", msg.String())
 	}
 

client/migrate.go

@@ -0,0 +1,268 @@
+package client
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/fatih/color"
+	"github.com/kr/pretty"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+)
+
+var (
+	MigrateCmd = &cli.Subcommand{
+		Use:   "migrate",
+		Short: "perform migration of the on-disk / zfs properties",
+		SetupSubcommands: func() []*cli.Subcommand {
+			return migrations
+		},
+	}
+)
+
+var migrations = []*cli.Subcommand{
+	&cli.Subcommand{
+		Use: "0.0.X:0.1:placeholder",
+		Run: doMigratePlaceholder0_1,
+		SetupFlags: func(f *pflag.FlagSet) {
+			f.BoolVar(&migratePlaceholder0_1Args.dryRun, "dry-run", false, "dry run")
+		},
+	},
+	&cli.Subcommand{
+		Use: "replication-cursor:v1-v2",
+		Run: doMigrateReplicationCursor,
+		SetupFlags: func(f *pflag.FlagSet) {
+			f.BoolVar(&migrateReplicationCursorArgs.dryRun, "dry-run", false, "dry run")
+		},
+	},
+}
+
+var migratePlaceholder0_1Args struct {
+	dryRun bool
+}
+
+func doMigratePlaceholder0_1(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	if len(args) != 0 {
+		return fmt.Errorf("migration does not take arguments, got %v", args)
+	}
+
+	cfg := sc.Config()
+
+	allFSS, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
+	if err != nil {
+		return errors.Wrap(err, "cannot list filesystems")
+	}
+
+	type workItem struct {
+		jobName string
+		rootFS  *zfs.DatasetPath
+		fss     []*zfs.DatasetPath
+	}
+	var wis []workItem
+	for i, j := range cfg.Jobs {
+		var rfsS string
+		switch job := j.Ret.(type) {
+		case *config.SinkJob:
+			rfsS = job.RootFS
+		case *config.PullJob:
+			rfsS = job.RootFS
+		default:
+			fmt.Printf("ignoring job %q (%d/%d, type %T)\n", j.Name(), i, len(cfg.Jobs), j.Ret)
+			continue
+		}
+		rfs, err := zfs.NewDatasetPath(rfsS)
+		if err != nil {
+			return errors.Wrapf(err, "root fs for job %q is not a valid dataset path", j.Name())
+		}
+		var fss []*zfs.DatasetPath
+		for _, fs := range allFSS {
+			if fs.HasPrefix(rfs) {
+				fss = append(fss, fs)
+			}
+		}
+		wis = append(wis, workItem{j.Name(), rfs, fss})
+	}
+
+	for _, wi := range wis {
+		fmt.Printf("job %q => migrate filesystems below root_fs %q\n", wi.jobName, wi.rootFS.ToString())
+		if len(wi.fss) == 0 {
+			fmt.Printf("\tno filesystems\n")
+			continue
+		}
+		for _, fs := range wi.fss {
+			fmt.Printf("\t%q ... ", fs.ToString())
+			r, err := zfs.ZFSMigrateHashBasedPlaceholderToCurrent(ctx, fs, migratePlaceholder0_1Args.dryRun)
+			if err != nil {
+				fmt.Printf("error: %s\n", err)
+			} else if !r.NeedsModification {
+				fmt.Printf("unchanged (placeholder=%v)\n", r.OriginalState.IsPlaceholder)
+			} else {
+				fmt.Printf("migrate (placeholder=%v) (old value = %q)\n",
+					r.OriginalState.IsPlaceholder, r.OriginalState.RawLocalPropertyValue)
+			}
+		}
+	}
+
+	return nil
+}
+
+var migrateReplicationCursorArgs struct {
+	dryRun bool
+}
+
+var bold = color.New(color.Bold)
+var succ = color.New(color.FgGreen)
+var fail = color.New(color.FgRed)
+
+var migrateReplicationCursorSkipSentinel = fmt.Errorf("skipping this filesystem")
+
+func doMigrateReplicationCursor(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	if len(args) != 0 {
+		return fmt.Errorf("migration does not take arguments, got %v", args)
+	}
+
+	cfg := sc.Config()
+	jobs, err := job.JobsFromConfig(cfg)
+	if err != nil {
+		fmt.Printf("cannot parse config:\n%s\n\n", err)
+		fmt.Printf("NOTE: this migration was released together with a change in job name requirements.\n")
+		return fmt.Errorf("exiting migration after error")
+	}
+
+	v1cursorJobs := make([]job.Job, 0, len(cfg.Jobs))
+	for i, j := range cfg.Jobs {
+		if jobs[i].Name() != j.Name() {
+			panic("implementation error")
+		}
+		switch j.Ret.(type) {
+		case *config.PushJob:
+			v1cursorJobs = append(v1cursorJobs, jobs[i])
+		case *config.SourceJob:
+			v1cursorJobs = append(v1cursorJobs, jobs[i])
+		default:
+			fmt.Printf("ignoring job %q (%d/%d, type %T), not supposed to create v1 replication cursors\n", j.Name(), i, len(cfg.Jobs), j.Ret)
+			continue
+		}
+	}
+
+	// scan all filesystems for v1 replication cursors
+
+	fss, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
+	if err != nil {
+		return errors.Wrap(err, "list filesystems")
+	}
+
+	var hadError bool
+	for _, fs := range fss {
+
+		bold.Printf("INSPECT FILESYSTEM %q\n", fs.ToString())
+
+		err := doMigrateReplicationCursorFS(ctx, v1cursorJobs, fs)
+		if err == migrateReplicationCursorSkipSentinel {
+			bold.Printf("FILESYSTEM SKIPPED\n")
+		} else if err != nil {
+			hadError = true
+			fail.Printf("MIGRATION FAILED: %s\n", err)
+		} else {
+			succ.Printf("FILESYSTEM %q COMPLETE\n", fs.ToString())
+		}
+	}
+
+	if hadError {
+		fail.Printf("\n\none or more filesystems could not be migrated, please inspect output and or re-run migration")
+		return errors.Errorf("")
+	}
+	return nil
+}
+
+func doMigrateReplicationCursorFS(ctx context.Context, v1CursorJobs []job.Job, fs *zfs.DatasetPath) error {
+
+	var owningJob job.Job = nil
+	for _, job := range v1CursorJobs {
+		conf := job.SenderConfig()
+		if conf == nil {
+			continue
+		}
+		pass, err := conf.FSF.Filter(fs)
+		if err != nil {
+			return errors.Wrapf(err, "filesystem filter error in job %q for fs %q", job.Name(), fs.ToString())
+		}
+		if !pass {
+			continue
+		}
+		if owningJob != nil {
+			return errors.Errorf("jobs %q and %q both match %q\ncannot attribute replication cursor to either one", owningJob.Name(), job.Name(), fs)
+		}
+		owningJob = job
+	}
+	if owningJob == nil {
+		fmt.Printf("no job's Filesystems filter matches\n")
+		return migrateReplicationCursorSkipSentinel
+	}
+	fmt.Printf("identified owning job %q\n", owningJob.Name())
+
+	bookmarks, err := zfs.ZFSListFilesystemVersions(ctx, fs, zfs.ListFilesystemVersionsOptions{
+		Types: zfs.Bookmarks,
+	})
+	if err != nil {
+		return errors.Wrapf(err, "list filesystem versions of %q", fs.ToString())
+	}
+
+	var oldCursor *zfs.FilesystemVersion
+	for i, fsv := range bookmarks {
+		_, _, err := endpoint.ParseReplicationCursorBookmarkName(fsv.ToAbsPath(fs))
+		if err != endpoint.ErrV1ReplicationCursor {
+			continue
+		}
+
+		if oldCursor != nil {
+			fmt.Printf("unexpected v1 replication cursor candidate: %q", fsv.ToAbsPath(fs))
+			return errors.Wrap(err, "multiple filesystem versions identified as v1 replication cursors")
+		}
+
+		oldCursor = &bookmarks[i]
+
+	}
+
+	if oldCursor == nil {
+		bold.Printf("no v1 replication cursor found for filesystem %q\n", fs.ToString())
+		return migrateReplicationCursorSkipSentinel
+	}
+
+	fmt.Printf("found v1 replication cursor:\n%s\n", pretty.Sprint(oldCursor))
+
+	mostRecentNew, err := endpoint.GetMostRecentReplicationCursorOfJob(ctx, fs.ToString(), owningJob.SenderConfig().JobID)
+	if err != nil {
+		return errors.Wrapf(err, "get most recent v2 replication cursor")
+	}
+
+	if mostRecentNew == nil {
+		return errors.Errorf("no v2 replication cursor found for job %q on filesystem %q", owningJob.SenderConfig().JobID, fs.ToString())
+	}
+
+	fmt.Printf("most recent v2 replication cursor:\n%#v", oldCursor)
+
+	if !(mostRecentNew.CreateTXG >= oldCursor.CreateTXG) {
+		return errors.Errorf("v1 replication cursor createtxg is higher than v2 cursor's, skipping this filesystem")
+	}
+
+	fmt.Printf("determined that v2 cursor is bookmark of same or newer version than v1 cursor\n")
+	fmt.Printf("destroying v1 cursor %q\n", oldCursor.ToAbsPath(fs))
+
+	if migrateReplicationCursorArgs.dryRun {
+		succ.Printf("DRY RUN\n")
+	} else {
+		if err := zfs.ZFSDestroyFilesystemVersion(ctx, fs, oldCursor); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

client/migrate_test.go

@@ -0,0 +1,16 @@
+package client
+
+import "testing"
+
+func TestMigrationsUnambiguousNames(t *testing.T) {
+	names := make(map[string]bool)
+	for _, mig := range migrations {
+		if _, ok := names[mig.Use]; ok {
+			t.Errorf("duplicate migration name %q", mig.Use)
+			t.FailNow()
+			return
+		} else {
+			names[mig.Use] = true
+		}
+	}
+}

client/pprof.go

@@ -1,66 +1,10 @@
 package client
 
-import (
-	"errors"
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon"
-	"log"
-	"os"
-)
-
-var pprofArgs struct {
-	daemon.PprofServerControlMsg
-}
+import "github.com/zrepl/zrepl/cli"
 
 var PprofCmd = &cli.Subcommand{
-	Use:   "pprof off | [on TCP_LISTEN_ADDRESS]",
-	Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
-	Run: func(subcommand *cli.Subcommand, args []string) error {
-		if len(args) < 1 {
-			goto enargs
-		}
-		switch args[0] {
-		case "on":
-			pprofArgs.Run = true
-			if len(args) != 2 {
-				return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
-			}
-			pprofArgs.HttpListenAddress = args[1]
-		case "off":
-			if len(args) != 1 {
-				goto enargs
-			}
-			pprofArgs.Run = false
-		}
-
-		RunPProf(subcommand.Config())
-		return nil
-	enargs:
-		return errors.New("invalid number of positional arguments")
-
+	Use: "pprof",
+	SetupSubcommands: func() []*cli.Subcommand {
+		return []*cli.Subcommand{PprofListenCmd, pprofActivityTraceCmd}
 	},
 }
-
-func RunPProf(conf *config.Config) {
-	log := log.New(os.Stderr, "", 0)
-
-	die := func() {
-		log.Printf("exiting after error")
-		os.Exit(1)
-	}
-
-	log.Printf("connecting to zrepl daemon")
-
-	httpc, err := controlHttpClient(conf.Global.Control.SockPath)
-	if err != nil {
-		log.Printf("error creating http client: %s", err)
-		die()
-	}
-	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofArgs.PprofServerControlMsg, struct{}{})
-	if err != nil {
-		log.Printf("error sending control message: %s", err)
-		die()
-	}
-	log.Printf("finished")
-}

client/pprof_activity_trace.go

@@ -0,0 +1,44 @@
+package client
+
+import (
+	"context"
+	"io"
+	"log"
+	"os"
+
+	"golang.org/x/net/websocket"
+
+	"github.com/zrepl/zrepl/cli"
+)
+
+var pprofActivityTraceCmd = &cli.Subcommand{
+	Use:   "activity-trace ZREPL_PPROF_HOST:ZREPL_PPROF_PORT",
+	Short: "attach to zrepl daemon with activated pprof listener and dump an activity-trace to stdout",
+	Run:   runPProfActivityTrace,
+}
+
+func runPProfActivityTrace(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+	log := log.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Printf("exiting after error")
+		os.Exit(1)
+	}
+
+	if len(args) != 1 {
+		log.Printf("exactly one positional argument is required")
+		die()
+	}
+
+	url := "ws://" + args[0] + "/debug/zrepl/activity-trace" // FIXME dont' repeat that
+
+	log.Printf("attaching to activity trace stream %s", url)
+	ws, err := websocket.Dial(url, "", url)
+	if err != nil {
+		log.Printf("error: %s", err)
+		die()
+	}
+
+	_, err = io.Copy(os.Stdout, ws)
+	return err
+}

client/pprof_listen.go

@@ -0,0 +1,68 @@
+package client
+
+import (
+	"context"
+	"errors"
+	"log"
+	"os"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+)
+
+var pprofListenCmd struct {
+	daemon.PprofServerControlMsg
+}
+
+var PprofListenCmd = &cli.Subcommand{
+	Use:   "listen off | [on TCP_LISTEN_ADDRESS]",
+	Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		if len(args) < 1 {
+			goto enargs
+		}
+		switch args[0] {
+		case "on":
+			pprofListenCmd.Run = true
+			if len(args) != 2 {
+				return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
+			}
+			pprofListenCmd.HttpListenAddress = args[1]
+		case "off":
+			if len(args) != 1 {
+				goto enargs
+			}
+			pprofListenCmd.Run = false
+		}
+
+		RunPProf(subcommand.Config())
+		return nil
+	enargs:
+		return errors.New("invalid number of positional arguments")
+
+	},
+}
+
+func RunPProf(conf *config.Config) {
+	log := log.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Printf("exiting after error")
+		os.Exit(1)
+	}
+
+	log.Printf("connecting to zrepl daemon")
+
+	httpc, err := controlHttpClient(conf.Global.Control.SockPath)
+	if err != nil {
+		log.Printf("error creating http client: %s", err)
+		die()
+	}
+	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofListenCmd.PprofServerControlMsg, struct{}{})
+	if err != nil {
+		log.Printf("error sending control message: %s", err)
+		die()
+	}
+	log.Printf("finished")
+}

client/signal.go

@@ -1,7 +1,10 @@
 package client
 
 import (
+	"context"
+
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/cli"
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon"
@@ -10,7 +13,7 @@ import (
 var SignalCmd = &cli.Subcommand{
 	Use:   "signal [wakeup|reset] JOB",
 	Short: "wake up a job from wait state or abort its current invocation",
-	Run: func(subcommand *cli.Subcommand, args []string) error {
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
 		return runSignalCmd(subcommand.Config(), args)
 	},
 }
@@ -28,10 +31,10 @@ func runSignalCmd(config *config.Config, args []string) error {
 	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointSignal,
 		struct {
 			Name string
-			Op string
+			Op   string
 		}{
 			Name: args[1],
-			Op: args[0],
+			Op:   args[0],
 		},
 		struct{}{},
 	)

client/status.go

@@ -1,17 +1,8 @@
 package client
 
 import (
+	"context"
 	"fmt"
-	"github.com/gdamore/tcell/termbox"
-	"github.com/pkg/errors"
-	"github.com/spf13/pflag"
-	"github.com/zrepl/yaml-config"
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/daemon"
-	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/daemon/pruner"
-	"github.com/zrepl/zrepl/replication"
-	"github.com/zrepl/zrepl/replication/fsrep"
 	"io"
 	"math"
 	"net/http"
@@ -20,41 +11,70 @@ import (
 	"strings"
 	"sync"
 	"time"
+
+	// tcell is the termbox-compatible library for abstracting away escape sequences, etc.
+	// as of tcell#252, the number of default distributed terminals is relatively limited
+	// additional terminal definitions can be included via side-effect import
+	// See https://github.com/gdamore/tcell/blob/master/terminfo/base/base.go
+	// See https://github.com/gdamore/tcell/issues/252#issuecomment-533836078
+	"github.com/gdamore/tcell/termbox"
+	_ "github.com/gdamore/tcell/terminfo/s/screen" // tmux on FreeBSD 11 & 12 without ncurses
+
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+	"github.com/zrepl/yaml-config"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/daemon"
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/daemon/pruner"
+	"github.com/zrepl/zrepl/daemon/snapper"
+	"github.com/zrepl/zrepl/replication/report"
 )
 
+type byteProgressMeasurement struct {
+	time time.Time
+	val  int64
+}
+
 type bytesProgressHistory struct {
-	changeCounter     int
-	lastChangeAt      time.Time
-	last              int64
-	bpsIncreaseExpAvg float64
+	last        *byteProgressMeasurement // pointer as poor man's optional
+	changeCount int
+	lastChange  time.Time
+	bpsAvg      float64
 }
 
 func (p *bytesProgressHistory) Update(currentVal int64) (bytesPerSecondAvg int64, changeCount int) {
-	if currentVal < p.last {
-		*p = bytesProgressHistory{
-			last: currentVal,
-			lastChangeAt: time.Now(),
+
+	if p.last == nil {
+		p.last = &byteProgressMeasurement{
+			time: time.Now(),
+			val:  currentVal,
 		}
-	}
-	defer func() {
-		p.last = currentVal
-	}()
-	if time.Now().Sub(p.lastChangeAt) > 3 *time.Second { // FIXME depends on refresh frequency
-		p.changeCounter = 0
-		p.bpsIncreaseExpAvg = 0
-	}
-	if currentVal != p.last {
-		p.changeCounter++
-		p.lastChangeAt = time.Now()
+		return 0, 0
 	}
 
-	byteIncrease := float64(currentVal - p.last)
-	if byteIncrease < 0 {
-		byteIncrease = 0
+	if p.last.val != currentVal {
+		p.changeCount++
+		p.lastChange = time.Now()
 	}
-	const factor = 0.1
-	p.bpsIncreaseExpAvg = (1-factor) * p.bpsIncreaseExpAvg + factor *byteIncrease
-	return int64(p.bpsIncreaseExpAvg), p.changeCounter
+
+	if time.Since(p.lastChange) > 3*time.Second {
+		p.last = nil
+		return 0, 0
+	}
+
+	deltaV := currentVal - p.last.val
+	deltaT := time.Since(p.last.time)
+	rate := float64(deltaV) / deltaT.Seconds()
+
+	factor := 0.3
+	p.bpsAvg = (1-factor)*p.bpsAvg + factor*rate
+
+	p.last.time = time.Now()
+	p.last.val = currentVal
+
+	return int64(p.bpsAvg), p.changeCount
 }
 
 type tui struct {
@@ -62,23 +82,20 @@ type tui struct {
 	indent int
 
 	lock   sync.Mutex //For report and error
-	report map[string]job.Status
+	report map[string]*job.Status
 	err    error
 
+	jobFilter string
+
 	replicationProgress map[string]*bytesProgressHistory // by job name
 }
 
 func newTui() tui {
 	return tui{
-		replicationProgress: make(map[string]*bytesProgressHistory, 0),
+		replicationProgress: make(map[string]*bytesProgressHistory),
 	}
 }
 
-func (t *tui) moveCursor(x, y int) {
-	t.x += x
-	t.y += y
-}
-
 const INDENT_MULTIPLIER = 4
 
 func (t *tui) moveLine(dl int, col int) {
@@ -109,9 +126,9 @@ func wrap(s string, width int) string {
 			rem = len(s)
 		}
 		if idx := strings.IndexAny(s, "\n\r"); idx != -1 && idx < rem {
-			rem = idx+1
+			rem = idx + 1
 		}
-		untilNewline := strings.TrimSpace(s[:rem])
+		untilNewline := strings.TrimRight(s[:rem], "\n\r")
 		s = s[rem:]
 		if len(untilNewline) == 0 {
 			continue
@@ -119,18 +136,18 @@ func wrap(s string, width int) string {
 		b.WriteString(untilNewline)
 		b.WriteString("\n")
 	}
-	return strings.TrimSpace(b.String())
+	return strings.TrimRight(b.String(), "\n\r")
 }
 
 func (t *tui) printfDrawIndentedAndWrappedIfMultiline(format string, a ...interface{}) {
 	whole := fmt.Sprintf(format, a...)
 	width, _ := termbox.Size()
-	if !strings.ContainsAny(whole, "\n\r") && t.x + len(whole) <= width {
+	if !strings.ContainsAny(whole, "\n\r") && t.x+len(whole) <= width {
 		t.printf(format, a...)
 	} else {
 		t.addIndent(1)
 		t.newline()
-		t.write(wrap(whole, width - INDENT_MULTIPLIER*t.indent))
+		t.write(wrap(whole, width-INDENT_MULTIPLIER*t.indent))
 		t.addIndent(-1)
 	}
 }
@@ -149,9 +166,9 @@ func (t *tui) addIndent(indent int) {
 	t.moveLine(0, 0)
 }
 
-
 var statusFlags struct {
 	Raw bool
+	Job string
 }
 
 var StatusCmd = &cli.Subcommand{
@@ -159,25 +176,29 @@ var StatusCmd = &cli.Subcommand{
 	Short: "show job activity or dump as JSON for monitoring",
 	SetupFlags: func(f *pflag.FlagSet) {
 		f.BoolVar(&statusFlags.Raw, "raw", false, "dump raw status description from zrepl daemon")
+		f.StringVar(&statusFlags.Job, "job", "", "only dump specified job")
 	},
 	Run: runStatus,
 }
 
-func runStatus(s *cli.Subcommand, args []string) error {
+func runStatus(ctx context.Context, s *cli.Subcommand, args []string) error {
 	httpc, err := controlHttpClient(s.Config().Global.Control.SockPath)
 	if err != nil {
 		return err
 	}
 
 	if statusFlags.Raw {
-		resp, err := httpc.Get("http://unix"+daemon.ControlJobEndpointStatus)
+		resp, err := httpc.Get("http://unix" + daemon.ControlJobEndpointStatus)
 		if err != nil {
 			return err
 		}
 		defer resp.Body.Close()
 		if resp.StatusCode != http.StatusOK {
 			fmt.Fprintf(os.Stderr, "Received error response:\n")
-			io.CopyN(os.Stderr, resp.Body, 4096)
+			_, err := io.CopyN(os.Stderr, resp.Body, 4096)
+			if err != nil {
+				return err
+			}
 			return errors.Errorf("exit")
 		}
 		if _, err := io.Copy(os.Stdout, resp.Body); err != nil {
@@ -190,6 +211,7 @@ func runStatus(s *cli.Subcommand, args []string) error {
 	t.lock.Lock()
 	t.err = errors.New("Got no report yet")
 	t.lock.Unlock()
+	t.jobFilter = statusFlags.Job
 
 	err = termbox.Init()
 	if err != nil {
@@ -198,7 +220,7 @@ func runStatus(s *cli.Subcommand, args []string) error {
 	defer termbox.Close()
 
 	update := func() {
-		m := make(map[string]job.Status)
+		var m daemon.Status
 
 		err2 := jsonRequestResponse(httpc, daemon.ControlJobEndpointStatus,
 			struct{}{},
@@ -207,7 +229,7 @@ func runStatus(s *cli.Subcommand, args []string) error {
 
 		t.lock.Lock()
 		t.err = err2
-		t.report = m
+		t.report = m.Jobs
 		t.lock.Unlock()
 		t.draw()
 	}
@@ -216,7 +238,7 @@ func runStatus(s *cli.Subcommand, args []string) error {
 	ticker := time.NewTicker(500 * time.Millisecond)
 	defer ticker.Stop()
 	go func() {
-		for _ = range ticker.C {
+		for range ticker.C {
 			update()
 		}
 	}()
@@ -243,7 +265,7 @@ loop:
 
 }
 
-func (t *tui) getReplicationProgresHistory(jobName string) *bytesProgressHistory {
+func (t *tui) getReplicationProgressHistory(jobName string) *bytesProgressHistory {
 	p, ok := t.replicationProgress[jobName]
 	if !ok {
 		p = &bytesProgressHistory{}
@@ -265,19 +287,29 @@ func (t *tui) draw() {
 		t.write(t.err.Error())
 	} else {
 		//Iterate over map in alphabetical order
-		keys := make([]string, len(t.report))
-		i := 0
-		for k, _ := range t.report {
-			keys[i] = k
-			i++
-		}
-		sort.Strings(keys)
-
-		for _, k := range keys {
-			v := t.report[k]
+		keys := make([]string, 0, len(t.report))
+		for k := range t.report {
 			if len(k) == 0 || daemon.IsInternalJobName(k) { //Internal job
 				continue
 			}
+			if t.jobFilter != "" && k != t.jobFilter {
+				continue
+			}
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+
+		if len(keys) == 0 {
+			t.setIndent(0)
+			t.printf("no jobs to display")
+			t.newline()
+			termbox.Flush()
+			return
+		}
+
+		for _, k := range keys {
+			v := t.report[k]
+
 			t.setIndent(0)
 
 			t.printf("Job: %s", k)
@@ -287,7 +319,66 @@ func (t *tui) draw() {
 			t.setIndent(1)
 			t.newline()
 
-			if v.Type != job.TypePush && v.Type != job.TypePull {
+			if v.Type == job.TypePush || v.Type == job.TypePull {
+				activeStatus, ok := v.JobSpecific.(*job.ActiveSideStatus)
+				if !ok || activeStatus == nil {
+					t.printf("ActiveSideStatus is null")
+					t.newline()
+					continue
+				}
+
+				t.printf("Replication:")
+				t.newline()
+				t.addIndent(1)
+				t.renderReplicationReport(activeStatus.Replication, t.getReplicationProgressHistory(k))
+				t.addIndent(-1)
+
+				t.printf("Pruning Sender:")
+				t.newline()
+				t.addIndent(1)
+				t.renderPrunerReport(activeStatus.PruningSender)
+				t.addIndent(-1)
+
+				t.printf("Pruning Receiver:")
+				t.newline()
+				t.addIndent(1)
+				t.renderPrunerReport(activeStatus.PruningReceiver)
+				t.addIndent(-1)
+
+				if v.Type == job.TypePush {
+					t.printf("Snapshotting:")
+					t.newline()
+					t.addIndent(1)
+					t.renderSnapperReport(activeStatus.Snapshotting)
+					t.addIndent(-1)
+				}
+
+			} else if v.Type == job.TypeSnap {
+				snapStatus, ok := v.JobSpecific.(*job.SnapJobStatus)
+				if !ok || snapStatus == nil {
+					t.printf("SnapJobStatus is null")
+					t.newline()
+					continue
+				}
+				t.printf("Pruning snapshots:")
+				t.newline()
+				t.addIndent(1)
+				t.renderPrunerReport(snapStatus.Pruning)
+				t.addIndent(-1)
+				t.printf("Snapshotting:")
+				t.newline()
+				t.addIndent(1)
+				t.renderSnapperReport(snapStatus.Snapshotting)
+				t.addIndent(-1)
+			} else if v.Type == job.TypeSource {
+
+				st := v.JobSpecific.(*job.PassiveStatus)
+				t.printf("Snapshotting:\n")
+				t.addIndent(1)
+				t.renderSnapperReport(st.Snapper)
+				t.addIndent(-1)
+
+			} else {
 				t.printf("No status representation for job type '%s', dumping as YAML", v.Type)
 				t.newline()
 				asYaml, err := yaml.Marshal(v.JobSpecific)
@@ -300,105 +391,107 @@ func (t *tui) draw() {
 				t.newline()
 				continue
 			}
-
-			pushStatus, ok := v.JobSpecific.(*job.ActiveSideStatus)
-			if !ok || pushStatus == nil {
-				t.printf("ActiveSideStatus is null")
-				t.newline()
-				continue
-			}
-
-			t.printf("Replication:")
-			t.newline()
-			t.addIndent(1)
-			t.renderReplicationReport(pushStatus.Replication, t.getReplicationProgresHistory(k))
-			t.addIndent(-1)
-
-			t.printf("Pruning Sender:")
-			t.newline()
-			t.addIndent(1)
-			t.renderPrunerReport(pushStatus.PruningSender)
-			t.addIndent(-1)
-
-			t.printf("Pruning Receiver:")
-			t.newline()
-			t.addIndent(1)
-			t.renderPrunerReport(pushStatus.PruningReceiver)
-			t.addIndent(-1)
-
 		}
 	}
 	termbox.Flush()
 }
 
-func (t *tui) renderReplicationReport(rep *replication.Report, history *bytesProgressHistory) {
+func (t *tui) renderReplicationReport(rep *report.Report, history *bytesProgressHistory) {
 	if rep == nil {
 		t.printf("...\n")
 		return
 	}
 
-	all := make([]*fsrep.Report, 0, len(rep.Completed)+len(rep.Pending) + 1)
-	all = append(all, rep.Completed...)
-	all = append(all, rep.Pending...)
-	if rep.Active != nil {
-		all = append(all, rep.Active)
+	if rep.WaitReconnectError != nil {
+		t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: %s", rep.WaitReconnectError)
+		t.newline()
 	}
-	sort.Slice(all, func(i, j int) bool {
-		return all[i].Filesystem < all[j].Filesystem
+	if !rep.WaitReconnectSince.IsZero() {
+		delta := time.Until(rep.WaitReconnectUntil).Round(time.Second)
+		if rep.WaitReconnectUntil.IsZero() || delta > 0 {
+			var until string
+			if rep.WaitReconnectUntil.IsZero() {
+				until = "waiting indefinitely"
+			} else {
+				until = fmt.Sprintf("hard fail in %s @ %s", delta, rep.WaitReconnectUntil)
+			}
+			t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnecting with exponential backoff (since %s) (%s)",
+				rep.WaitReconnectSince, until)
+		} else {
+			t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnects reached hard-fail timeout @ %s", rep.WaitReconnectUntil)
+		}
+		t.newline()
+	}
+
+	// TODO visualize more than the latest attempt by folding all attempts into one
+	if len(rep.Attempts) == 0 {
+		t.printf("no attempts made yet")
+		return
+	} else {
+		t.printf("Attempt #%d", len(rep.Attempts))
+		if len(rep.Attempts) > 1 {
+			t.printf(". Previous attempts failed with the following statuses:")
+			t.newline()
+			t.addIndent(1)
+			for i, a := range rep.Attempts[:len(rep.Attempts)-1] {
+				t.printfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt))
+				t.newline()
+			}
+			t.addIndent(-1)
+		} else {
+			t.newline()
+		}
+	}
+
+	latest := rep.Attempts[len(rep.Attempts)-1]
+	sort.Slice(latest.Filesystems, func(i, j int) bool {
+		return latest.Filesystems[i].Info.Name < latest.Filesystems[j].Info.Name
 	})
 
-	state, err := replication.StateString(rep.Status)
-	if err != nil {
-		t.printf("Status: %q (parse error: %q)\n", rep.Status, err)
-		return
-	}
-
-	t.printf("Status: %s", state)
+	t.printf("Status: %s", latest.State)
 	t.newline()
-	if rep.Problem != "" {
+	if latest.State == report.AttemptPlanningError {
 		t.printf("Problem: ")
-		t.printfDrawIndentedAndWrappedIfMultiline("%s", rep.Problem)
+		t.printfDrawIndentedAndWrappedIfMultiline("%s", latest.PlanError)
+		t.newline()
+	} else if latest.State == report.AttemptFanOutError {
+		t.printf("Problem: one or more of the filesystems encountered errors")
 		t.newline()
 	}
-	if rep.SleepUntil.After(time.Now()) && !state.IsTerminal() {
-		t.printf("Sleeping until %s (%s left)\n", rep.SleepUntil, rep.SleepUntil.Sub(time.Now()))
-	}
 
-	if state != replication.Planning && state != replication.PlanningError {
+	if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
+		// Draw global progress bar
 		// Progress: [---------------]
-		sumUpFSRep := func(rep *fsrep.Report) (transferred, total int64) {
-			for _, s := range rep.Pending {
-				transferred += s.Bytes
-				total  += s.ExpectedBytes
-			}
-			for _, s := range rep.Completed {
-				transferred += s.Bytes
-				total += s.ExpectedBytes
-			}
-			return
+		expected, replicated, containsInvalidSizeEstimates := latest.BytesSum()
+		rate, changeCount := history.Update(replicated)
+		eta := time.Duration(0)
+		if rate > 0 {
+			eta = time.Duration((expected-replicated)/rate) * time.Second
 		}
-		var transferred, total int64
-		for _, fs := range all {
-			fstx, fstotal := sumUpFSRep(fs)
-			transferred += fstx
-			total += fstotal
-		}
-		rate, changeCount := history.Update(transferred)
 		t.write("Progress: ")
-		t.drawBar(50, transferred, total, changeCount)
-		t.write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinary(transferred), ByteCountBinary(total), ByteCountBinary(rate)))
+		t.drawBar(50, replicated, expected, changeCount)
+		t.write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinary(replicated), ByteCountBinary(expected), ByteCountBinary(rate)))
+		if eta != 0 {
+			t.write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
+		}
 		t.newline()
+		if containsInvalidSizeEstimates {
+			t.write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
+			t.newline()
+		}
+
+		var maxFSLen int
+		for _, fs := range latest.Filesystems {
+			if len(fs.Info.Name) > maxFSLen {
+				maxFSLen = len(fs.Info.Name)
+			}
+		}
+		for _, fs := range latest.Filesystems {
+			t.printFilesystemStatus(fs, false, maxFSLen) // FIXME bring 'active' flag back
+		}
+
 	}
 
-	var maxFSLen int
-	for _, fs := range all {
-		if len(fs.Filesystem) > maxFSLen {
-			maxFSLen = len(fs.Filesystem)
-		}
-	}
-	for _, fs := range all {
-		t.printFilesystemStatus(fs, fs == rep.Active, maxFSLen)
-	}
 }
 
 func (t *tui) renderPrunerReport(r *pruner.Report) {
@@ -419,15 +512,12 @@ func (t *tui) renderPrunerReport(r *pruner.Report) {
 	if r.Error != "" {
 		t.printf("Error: %s\n", r.Error)
 	}
-	if r.SleepUntil.After(time.Now()) {
-		t.printf("Sleeping until %s (%s left)\n", r.SleepUntil, r.SleepUntil.Sub(time.Now()))
-	}
 
 	type commonFS struct {
 		*pruner.FSReport
 		completed bool
 	}
-	all := make([]commonFS, 0, len(r.Pending) + len(r.Completed))
+	all := make([]commonFS, 0, len(r.Pending)+len(r.Completed))
 	for i := range r.Pending {
 		all = append(all, commonFS{&r.Pending[i], false})
 	}
@@ -436,9 +526,9 @@ func (t *tui) renderPrunerReport(r *pruner.Report) {
 	}
 
 	switch state {
-	case pruner.Plan: fallthrough
-	case pruner.PlanWait: fallthrough
-	case pruner.ErrPerm:
+	case pruner.Plan:
+		fallthrough
+	case pruner.PlanErr:
 		return
 	}
 
@@ -465,7 +555,7 @@ func (t *tui) renderPrunerReport(r *pruner.Report) {
 	t.write("[")
 	t.write(times("=", progress))
 	t.write(">")
-	t.write(times("-", 80 - progress))
+	t.write(times("-", 80-progress))
 	t.write("]")
 	t.printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
 	t.newline()
@@ -478,8 +568,18 @@ func (t *tui) renderPrunerReport(r *pruner.Report) {
 	for _, fs := range all {
 		t.write(rightPad(fs.Filesystem, maxFSname, " "))
 		t.write(" ")
+		if !fs.SkipReason.NotSkipped() {
+			t.printf("skipped: %s\n", fs.SkipReason)
+			continue
+		}
 		if fs.LastError != "" {
-			t.printf("ERROR (%d): %s\n", fs.ErrorCount, fs.LastError) // whitespace is padding
+			if strings.ContainsAny(fs.LastError, "\r\n") {
+				t.printf("ERROR:")
+				t.printfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError)
+			} else {
+				t.printfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError)
+			}
+			t.newline()
 			continue
 		}
 
@@ -487,7 +587,7 @@ func (t *tui) renderPrunerReport(r *pruner.Report) {
 			len(fs.DestroyList), len(fs.SnapshotList))
 
 		if fs.completed {
-			t.printf( "Completed  %s\n", pruneRuleActionStr)
+			t.printf("Completed  %s\n", pruneRuleActionStr)
 			continue
 		}
 
@@ -502,23 +602,84 @@ func (t *tui) renderPrunerReport(r *pruner.Report) {
 
 }
 
-const snapshotIndent = 1
-func calculateMaxFSLength(all []*fsrep.Report) (maxFS, maxStatus int) {
-	for _, e := range all {
-		if len(e.Filesystem) > maxFS {
-			maxFS = len(e.Filesystem)
+func (t *tui) renderSnapperReport(r *snapper.Report) {
+	if r == nil {
+		t.printf("<snapshot type does not have a report>\n")
+		return
+	}
+
+	t.printf("Status: %s", r.State)
+	t.newline()
+
+	if r.Error != "" {
+		t.printf("Error: %s\n", r.Error)
+	}
+	if !r.SleepUntil.IsZero() {
+		t.printf("Sleep until: %s\n", r.SleepUntil)
+	}
+
+	sort.Slice(r.Progress, func(i, j int) bool {
+		return strings.Compare(r.Progress[i].Path, r.Progress[j].Path) == -1
+	})
+
+	t.addIndent(1)
+	defer t.addIndent(-1)
+	dur := func(d time.Duration) string {
+		return d.Round(100 * time.Millisecond).String()
+	}
+
+	type row struct {
+		path, state, duration, remainder, hookReport string
+	}
+	var widths struct {
+		path, state, duration int
+	}
+	rows := make([]*row, len(r.Progress))
+	for i, fs := range r.Progress {
+		r := &row{
+			path:  fs.Path,
+			state: fs.State.String(),
 		}
-		all2 := make([]*fsrep.StepReport, 0, len(e.Pending) + len(e.Completed))
-		all2 = append(all2, e.Pending...)
-		all2 = append(all2, e.Completed...)
-		for _, e2 := range all2 {
-			elen := len(e2.Problem) + len(e2.From) + len(e2.To) + 60 // random spacing, units, labels, etc
-			if elen > maxStatus {
-				maxStatus = elen
-			}
+		if fs.HooksHadError {
+			r.hookReport = fs.Hooks // FIXME render here, not in daemon
+		}
+		switch fs.State {
+		case snapper.SnapPending:
+			r.duration = "..."
+			r.remainder = ""
+		case snapper.SnapStarted:
+			r.duration = dur(time.Since(fs.StartAt))
+			r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
+		case snapper.SnapDone:
+			fallthrough
+		case snapper.SnapError:
+			r.duration = dur(fs.DoneAt.Sub(fs.StartAt))
+			r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
+		}
+		rows[i] = r
+		if len(r.path) > widths.path {
+			widths.path = len(r.path)
+		}
+		if len(r.state) > widths.state {
+			widths.state = len(r.state)
+		}
+		if len(r.duration) > widths.duration {
+			widths.duration = len(r.duration)
 		}
 	}
-	return
+
+	for _, r := range rows {
+		path := rightPad(r.path, widths.path, " ")
+		state := rightPad(r.state, widths.state, " ")
+		duration := rightPad(r.duration, widths.duration, " ")
+		t.printf("%s %s %s", path, state, duration)
+		t.printfDrawIndentedAndWrappedIfMultiline(" %s", r.remainder)
+		if r.hookReport != "" {
+			t.printfDrawIndentedAndWrappedIfMultiline("%s", r.hookReport)
+		}
+		t.newline()
+	}
+
 }
 
 func times(str string, n int) (out string) {
@@ -532,20 +693,12 @@ func rightPad(str string, length int, pad string) string {
 	if len(str) > length {
 		return str[:length]
 	}
-	return str + times(pad, length-len(str))
-}
-
-
-func leftPad(str string, length int, pad string) string {
-	if len(str) > length {
-		return str[len(str)-length:]
-	}
-	return times(pad, length-len(str)) + str
+	return str + strings.Repeat(pad, length-len(str))
 }
 
 var arrowPositions = `>\|/`
 
-// changeCount = 0 indicates stall / no progresss
+// changeCount = 0 indicates stall / no progress
 func (t *tui) drawBar(length int, bytes, totalBytes int64, changeCount int) {
 	var completedLength int
 	if totalBytes > 0 {
@@ -559,40 +712,27 @@ func (t *tui) drawBar(length int, bytes, totalBytes int64, changeCount int) {
 
 	t.write("[")
 	t.write(times("=", completedLength))
-	t.write( string(arrowPositions[changeCount%len(arrowPositions)]))
+	t.write(string(arrowPositions[changeCount%len(arrowPositions)]))
 	t.write(times("-", length-completedLength))
 	t.write("]")
 }
 
-func StringStepState(s fsrep.StepState) string {
-	switch s {
-	case fsrep.StepReplicationReady: return "Ready"
-	case fsrep.StepMarkReplicatedReady: return "MarkReady"
-	case fsrep.StepCompleted: return "Completed"
-	default:
-		return fmt.Sprintf("UNKNOWN %d", s)
-	}
-}
+func (t *tui) printFilesystemStatus(rep *report.FilesystemReport, active bool, maxFS int) {
 
-func (t *tui) printFilesystemStatus(rep *fsrep.Report, active bool, maxFS int) {
-
-	bytes := int64(0)
-	totalBytes := int64(0)
-	for _, s := range rep.Pending {
-		bytes += s.Bytes
-		totalBytes += s.ExpectedBytes
+	expected, replicated, containsInvalidSizeEstimates := rep.BytesSum()
+	sizeEstimationImpreciseNotice := ""
+	if containsInvalidSizeEstimates {
+		sizeEstimationImpreciseNotice = " (some steps lack size estimation)"
 	}
-	for _, s := range rep.Completed {
-		bytes += s.Bytes
-		totalBytes += s.ExpectedBytes
+	if rep.CurrentStep < len(rep.Steps) && rep.Steps[rep.CurrentStep].Info.BytesExpected == 0 {
+		sizeEstimationImpreciseNotice = " (step lacks size estimation)"
 	}
 
-
-	status := fmt.Sprintf("%s (step %d/%d, %s/%s)",
-		rep.Status,
-		len(rep.Completed), len(rep.Pending) + len(rep.Completed),
-		ByteCountBinary(bytes), ByteCountBinary(totalBytes),
-
+	status := fmt.Sprintf("%s (step %d/%d, %s/%s)%s",
+		strings.ToUpper(string(rep.State)),
+		rep.CurrentStep, len(rep.Steps),
+		ByteCountBinary(replicated), ByteCountBinary(expected),
+		sizeEstimationImpreciseNotice,
 	)
 
 	activeIndicator := " "
@@ -601,18 +741,32 @@ func (t *tui) printFilesystemStatus(rep *fsrep.Report, active bool, maxFS int) {
 	}
 	t.printf("%s %s %s ",
 		activeIndicator,
-		rightPad(rep.Filesystem, maxFS, " "),
+		rightPad(rep.Info.Name, maxFS, " "),
 		status)
 
 	next := ""
-	if rep.Problem != "" {
-		next = rep.Problem
-	} else if len(rep.Pending) > 0 {
-		if rep.Pending[0].From != "" {
-			next = fmt.Sprintf("next: %s => %s", rep.Pending[0].From, rep.Pending[0].To)
+	if err := rep.Error(); err != nil {
+		next = err.Err
+	} else if rep.State != report.FilesystemDone {
+		if nextStep := rep.NextStep(); nextStep != nil {
+			if nextStep.IsIncremental() {
+				next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
+			} else {
+				next = fmt.Sprintf("next: full send %s", nextStep.Info.To)
+			}
+			attribs := []string{}
+
+			if nextStep.Info.Resumed {
+				attribs = append(attribs, "resumed")
+			}
+
+			attribs = append(attribs, fmt.Sprintf("encrypted=%s", nextStep.Info.Encrypted))
+
+			next += fmt.Sprintf(" (%s)", strings.Join(attribs, ", "))
 		} else {
-			next = fmt.Sprintf("next: %s (full)", rep.Pending[0].To)
+			next = "" // individual FSes may still be in planning state
 		}
+
 	}
 	t.printfDrawIndentedAndWrappedIfMultiline("%s", next)
 
@@ -631,3 +785,27 @@ func ByteCountBinary(b int64) string {
 	}
 	return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
 }
+
+func humanizeDuration(duration time.Duration) string {
+	days := int64(duration.Hours() / 24)
+	hours := int64(math.Mod(duration.Hours(), 24))
+	minutes := int64(math.Mod(duration.Minutes(), 60))
+	seconds := int64(math.Mod(duration.Seconds(), 60))
+
+	var parts []string
+
+	force := false
+	chunks := []int64{days, hours, minutes, seconds}
+	for i, chunk := range chunks {
+		if force || chunk > 0 {
+			padding := 0
+			if force {
+				padding = 2
+			}
+			parts = append(parts, fmt.Sprintf("%*d%c", padding, chunk, "dhms"[i]))
+			force = true
+		}
+	}
+
+	return strings.Join(parts, " ")
+}

client/stdinserver.go

@@ -1,13 +1,15 @@
 package client
 
 import (
-	"github.com/zrepl/zrepl/cli"
 	"os"
 
+	"github.com/problame/go-netssh"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+
 	"context"
 	"errors"
-	"github.com/problame/go-netssh"
-	"github.com/zrepl/zrepl/config"
 	"log"
 	"path"
 )
@@ -15,7 +17,7 @@ import (
 var StdinserverCmd = &cli.Subcommand{
 	Use:   "stdinserver CLIENT_IDENTITY",
 	Short: "stdinserver transport mode (started from authorized_keys file as forced command)",
-	Run: func(subcommand *cli.Subcommand, args []string) error {
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
 		return runStdinserver(subcommand.Config(), args)
 	},
 }

client/testcmd.go

@@ -1,29 +1,35 @@
 package client
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
+	"os"
+
+	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
+
 	"github.com/zrepl/zrepl/cli"
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/filters"
 	"github.com/zrepl/zrepl/zfs"
 )
 
-var TestCmd = &cli.Subcommand {
+var TestCmd = &cli.Subcommand{
 	Use: "test",
 	SetupSubcommands: func() []*cli.Subcommand {
-		return []*cli.Subcommand{testFilter}
+		return []*cli.Subcommand{testFilter, testPlaceholder, testDecodeResumeToken}
 	},
 }
 
 var testFilterArgs struct {
-	job string
-	all bool
+	job   string
+	all   bool
 	input string
 }
 
 var testFilter = &cli.Subcommand{
-	Use: "filesystems --job JOB [--all | --input INPUT]",
+	Use:   "filesystems --job JOB [--all | --input INPUT]",
 	Short: "test filesystems filter specified in push or source job",
 	SetupFlags: func(f *pflag.FlagSet) {
 		f.StringVar(&testFilterArgs.job, "job", "", "the name of the push or source job")
@@ -33,7 +39,7 @@ var testFilter = &cli.Subcommand{
 	Run: runTestFilterCmd,
 }
 
-func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
+func runTestFilterCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
 
 	if testFilterArgs.job == "" {
 		return fmt.Errorf("must specify --job flag")
@@ -50,8 +56,12 @@ func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
 		return err
 	}
 	switch j := job.Ret.(type) {
-	case *config.SourceJob: confFilter = j.Filesystems
-	case *config.PushJob: confFilter = j.Filesystems
+	case *config.SourceJob:
+		confFilter = j.Filesystems
+	case *config.PushJob:
+		confFilter = j.Filesystems
+	case *config.SnapJob:
+		confFilter = j.Filesystems
 	default:
 		return fmt.Errorf("job type %T does not have filesystems filter", j)
 	}
@@ -65,7 +75,7 @@ func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
 	if testFilterArgs.input != "" {
 		fsnames = []string{testFilterArgs.input}
 	} else {
-		out, err := zfs.ZFSList([]string{"name"})
+		out, err := zfs.ZFSList(ctx, []string{"name"})
 		if err != nil {
 			return fmt.Errorf("could not list ZFS filesystems: %s", err)
 		}
@@ -105,4 +115,98 @@ func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
 		return fmt.Errorf("filter errors occurred")
 	}
 	return nil
-}
+}
+
+var testPlaceholderArgs struct {
+	ds  string
+	all bool
+}
+
+var testPlaceholder = &cli.Subcommand{
+	Use:   "placeholder [--all | --dataset DATASET]",
+	Short: fmt.Sprintf("list received placeholder filesystems (zfs property %q)", zfs.PlaceholderPropertyName),
+	Example: `
+	placeholder --all
+	placeholder --dataset path/to/sink/clientident/fs`,
+	NoRequireConfig: true,
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&testPlaceholderArgs.ds, "dataset", "", "dataset path (not required to exist)")
+		f.BoolVar(&testPlaceholderArgs.all, "all", false, "list tab-separated placeholder status of all filesystems")
+	},
+	Run: runTestPlaceholder,
+}
+
+func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+
+	var checkDPs []*zfs.DatasetPath
+
+	// all actions first
+	if testPlaceholderArgs.all {
+		out, err := zfs.ZFSList(ctx, []string{"name"})
+		if err != nil {
+			return errors.Wrap(err, "could not list ZFS filesystems")
+		}
+		for _, row := range out {
+			dp, err := zfs.NewDatasetPath(row[0])
+			if err != nil {
+				panic(err)
+			}
+			checkDPs = append(checkDPs, dp)
+		}
+	} else {
+		dp, err := zfs.NewDatasetPath(testPlaceholderArgs.ds)
+		if err != nil {
+			return err
+		}
+		if dp.Empty() {
+			return fmt.Errorf("must specify --dataset DATASET or --all")
+		}
+		checkDPs = append(checkDPs, dp)
+	}
+
+	fmt.Printf("IS_PLACEHOLDER\tDATASET\tzrepl:placeholder\n")
+	for _, dp := range checkDPs {
+		ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, dp)
+		if err != nil {
+			return errors.Wrap(err, "cannot get placeholder state")
+		}
+		if !ph.FSExists {
+			panic("placeholder state inconsistent: filesystem " + ph.FS + " must exist in this context")
+		}
+		is := "yes"
+		if !ph.IsPlaceholder {
+			is = "no"
+		}
+		fmt.Printf("%s\t%s\t%s\n", is, dp.ToString(), ph.RawLocalPropertyValue)
+	}
+	return nil
+}
+
+var testDecodeResumeTokenArgs struct {
+	token string
+}
+
+var testDecodeResumeToken = &cli.Subcommand{
+	Use:   "decoderesumetoken --token TOKEN",
+	Short: "decode resume token",
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&testDecodeResumeTokenArgs.token, "token", "", "the resume token obtained from the receive_resume_token property")
+	},
+	Run: runTestDecodeResumeTokenCmd,
+}
+
+func runTestDecodeResumeTokenCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+	if testDecodeResumeTokenArgs.token == "" {
+		return fmt.Errorf("token argument must be specified")
+	}
+	token, err := zfs.ParseResumeToken(ctx, testDecodeResumeTokenArgs.token)
+	if err != nil {
+		return err
+	}
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", " ")
+	if err := enc.Encode(&token); err != nil {
+		panic(err)
+	}
+	return nil
+}

client/version.go

@@ -1,29 +1,32 @@
 package client
 
 import (
+	"context"
 	"fmt"
+	"os"
+
 	"github.com/spf13/pflag"
+
 	"github.com/zrepl/zrepl/cli"
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon"
 	"github.com/zrepl/zrepl/version"
-	"os"
 )
 
 var versionArgs struct {
-	Show   string
-	Config *config.Config
+	Show      string
+	Config    *config.Config
 	ConfigErr error
 }
 
 var VersionCmd = &cli.Subcommand{
-	Use:   "version",
-	Short: "print version of zrepl binary and running daemon",
+	Use:             "version",
+	Short:           "print version of zrepl binary and running daemon",
 	NoRequireConfig: true,
 	SetupFlags: func(f *pflag.FlagSet) {
 		f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)")
 	},
-	Run: func(subcommand *cli.Subcommand, args []string) error {
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
 		versionArgs.Config = subcommand.Config()
 		versionArgs.ConfigErr = subcommand.ConfigParsingError()
 		return runVersionCmd()

client/zfsabstractions.go

@@ -0,0 +1,147 @@
+package client
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var (
+	ZFSAbstractionsCmd = &cli.Subcommand{
+		Use:   "zfs-abstraction",
+		Short: "manage abstractions that zrepl builds on top of ZFS",
+		SetupSubcommands: func() []*cli.Subcommand {
+			return []*cli.Subcommand{
+				zabsCmdList,
+				zabsCmdReleaseAll,
+				zabsCmdReleaseStale,
+				zabsCmdCreate,
+			}
+		},
+	}
+)
+
+// a common set of CLI flags that map to the fields of an
+// endpoint.ListZFSHoldsAndBookmarksQuery
+type zabsFilterFlags struct {
+	Filesystems FilesystemsFilterFlag
+	Job         JobIDFlag
+	Types       AbstractionTypesFlag
+	Concurrency int64
+}
+
+// produce a query from the CLI flags
+func (f zabsFilterFlags) Query() (endpoint.ListZFSHoldsAndBookmarksQuery, error) {
+	q := endpoint.ListZFSHoldsAndBookmarksQuery{
+		FS:          f.Filesystems.FlagValue(),
+		What:        f.Types.FlagValue(),
+		JobID:       f.Job.FlagValue(),
+		Concurrency: f.Concurrency,
+	}
+	return q, q.Validate()
+}
+
+func (f *zabsFilterFlags) registerZabsFilterFlags(s *pflag.FlagSet, verb string) {
+	// Note: the default value is defined in the .FlagValue methods
+	s.Var(&f.Filesystems, "fs", fmt.Sprintf("only %s holds on the specified filesystem [default: all filesystems] [comma-separated list of <dataset-pattern>:<ok|!> pairs]", verb))
+	s.Var(&f.Job, "job", fmt.Sprintf("only %s holds created by the specified job [default: any job]", verb))
+
+	variants := make([]string, 0, len(endpoint.AbstractionTypesAll))
+	for v := range endpoint.AbstractionTypesAll {
+		variants = append(variants, string(v))
+	}
+	variants = sort.StringSlice(variants)
+	variantsJoined := strings.Join(variants, "|")
+	s.Var(&f.Types, "type", fmt.Sprintf("only %s holds of the specified type [default: all] [comma-separated list of %s]", verb, variantsJoined))
+
+	s.Int64VarP(&f.Concurrency, "concurrency", "p", 1, "number of concurrently queried filesystems")
+}
+
+type JobIDFlag struct{ J *endpoint.JobID }
+
+func (f *JobIDFlag) Set(s string) error {
+	if len(s) == 0 {
+		*f = JobIDFlag{J: nil}
+		return nil
+	}
+
+	jobID, err := endpoint.MakeJobID(s)
+	if err != nil {
+		return err
+	}
+	*f = JobIDFlag{J: &jobID}
+	return nil
+}
+func (f JobIDFlag) Type() string               { return "job-ID" }
+func (f JobIDFlag) String() string             { return fmt.Sprint(f.J) }
+func (f JobIDFlag) FlagValue() *endpoint.JobID { return f.J }
+
+type AbstractionTypesFlag map[endpoint.AbstractionType]bool
+
+func (f *AbstractionTypesFlag) Set(s string) error {
+	ats, err := endpoint.AbstractionTypeSetFromStrings(strings.Split(s, ","))
+	if err != nil {
+		return err
+	}
+	*f = AbstractionTypesFlag(ats)
+	return nil
+}
+func (f AbstractionTypesFlag) Type() string { return "abstraction-type" }
+func (f AbstractionTypesFlag) String() string {
+	return endpoint.AbstractionTypeSet(f).String()
+}
+func (f AbstractionTypesFlag) FlagValue() map[endpoint.AbstractionType]bool {
+	if len(f) > 0 {
+		return f
+	}
+	return endpoint.AbstractionTypesAll
+}
+
+type FilesystemsFilterFlag struct {
+	F endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter
+}
+
+func (flag *FilesystemsFilterFlag) Set(s string) error {
+	mappings := strings.Split(s, ",")
+	if len(mappings) == 1 && !strings.Contains(mappings[0], ":") {
+		flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &mappings[0],
+		}
+		return nil
+	}
+
+	f := filters.NewDatasetMapFilter(len(mappings), true)
+	for _, m := range mappings {
+		thisMappingErr := fmt.Errorf("expecting comma-separated list of <dataset-pattern>:<ok|!> pairs, got %q", m)
+		lhsrhs := strings.SplitN(m, ":", 2)
+		if len(lhsrhs) != 2 {
+			return thisMappingErr
+		}
+		err := f.Add(lhsrhs[0], lhsrhs[1])
+		if err != nil {
+			return fmt.Errorf("%s: %s", thisMappingErr, err)
+		}
+	}
+	flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+		Filter: f,
+	}
+	return nil
+}
+func (flag FilesystemsFilterFlag) Type() string { return "filesystem filter spec" }
+func (flag FilesystemsFilterFlag) String() string {
+	return fmt.Sprintf("%v", flag.F)
+}
+func (flag FilesystemsFilterFlag) FlagValue() endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter {
+	var z FilesystemsFilterFlag
+	if flag == z {
+		return endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{Filter: zfs.NoFilter()}
+	}
+	return flag.F
+}

client/zfsabstractions_create.go

@@ -0,0 +1,14 @@
+package client
+
+import "github.com/zrepl/zrepl/cli"
+
+var zabsCmdCreate = &cli.Subcommand{
+	Use:             "create",
+	NoRequireConfig: true,
+	Short:           `create zrepl ZFS abstractions (mostly useful for debugging & development, users should not need to use this command)`,
+	SetupSubcommands: func() []*cli.Subcommand {
+		return []*cli.Subcommand{
+			zabsCmdCreateStepHold,
+		}
+	},
+}

client/zfsabstractions_create_step_hold.go

@@ -0,0 +1,58 @@
+package client
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var zabsCreateStepHoldFlags struct {
+	target string
+	jobid  JobIDFlag
+}
+
+var zabsCmdCreateStepHold = &cli.Subcommand{
+	Use:             "step",
+	Run:             doZabsCreateStep,
+	NoRequireConfig: true,
+	Short:           `create a step hold or bookmark`,
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVarP(&zabsCreateStepHoldFlags.target, "target", "t", "", "snapshot to be held / bookmark to be held")
+		f.VarP(&zabsCreateStepHoldFlags.jobid, "jobid", "j", "jobid for which the hold is installed")
+	},
+}
+
+func doZabsCreateStep(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	if len(args) > 0 {
+		return errors.New("subcommand takes no arguments")
+	}
+
+	f := &zabsCreateStepHoldFlags
+
+	fs, _, _, err := zfs.DecomposeVersionString(f.target)
+	if err != nil {
+		return errors.Wrapf(err, "%q invalid target", f.target)
+	}
+
+	if f.jobid.FlagValue() == nil {
+		return errors.Errorf("jobid must be set")
+	}
+
+	v, err := zfs.ZFSGetFilesystemVersion(ctx, f.target)
+	if err != nil {
+		return errors.Wrapf(err, "get info about target %q", f.target)
+	}
+
+	step, err := endpoint.HoldStep(ctx, fs, v, *f.jobid.FlagValue())
+	if err != nil {
+		return errors.Wrap(err, "create step hold")
+	}
+	fmt.Println(step.String())
+	return nil
+}

client/zfsabstractions_list.go

@@ -0,0 +1,99 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/util/chainlock"
+)
+
+var zabsListFlags struct {
+	Filter zabsFilterFlags
+	Json   bool
+}
+
+var zabsCmdList = &cli.Subcommand{
+	Use:             "list",
+	Short:           `list zrepl ZFS abstractions`,
+	Run:             doZabsList,
+	NoRequireConfig: true,
+	SetupFlags: func(f *pflag.FlagSet) {
+		zabsListFlags.Filter.registerZabsFilterFlags(f, "list")
+		f.BoolVar(&zabsListFlags.Json, "json", false, "emit JSON")
+	},
+}
+
+func doZabsList(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsListFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	abstractions, errors, err := endpoint.ListAbstractionsStreamed(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+
+	var line chainlock.L
+	var wg sync.WaitGroup
+	defer wg.Wait()
+	wg.Add(1)
+
+	// print results
+	go func() {
+		defer wg.Done()
+		enc := json.NewEncoder(os.Stdout)
+		for a := range abstractions {
+			func() {
+				defer line.Lock().Unlock()
+				if zabsListFlags.Json {
+					enc.SetIndent("", "  ")
+					if err := enc.Encode(abstractions); err != nil {
+						panic(err)
+					}
+					fmt.Println()
+				} else {
+					fmt.Println(a)
+				}
+			}()
+		}
+	}()
+
+	// print errors to stderr
+	errorColor := color.New(color.FgRed)
+	var errorsSlice []endpoint.ListAbstractionsError
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for err := range errors {
+			func() {
+				defer line.Lock().Unlock()
+				errorsSlice = append(errorsSlice, err)
+				errorColor.Fprintf(os.Stderr, "%s\n", err)
+			}()
+		}
+	}()
+	wg.Wait()
+	if len(errorsSlice) > 0 {
+		errorColor.Add(color.Bold).Fprintf(os.Stderr, "there were errors in listing the abstractions")
+		return fmt.Errorf("")
+	} else {
+		return nil
+	}
+
+}

client/zfsabstractions_release.go

@@ -0,0 +1,143 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+)
+
+// shared between release-all and release-step
+var zabsReleaseFlags struct {
+	Filter zabsFilterFlags
+	Json   bool
+	DryRun bool
+}
+
+func registerZabsReleaseFlags(s *pflag.FlagSet) {
+	zabsReleaseFlags.Filter.registerZabsFilterFlags(s, "release")
+	s.BoolVar(&zabsReleaseFlags.Json, "json", false, "emit json instead of pretty-printed")
+	s.BoolVar(&zabsReleaseFlags.DryRun, "dry-run", false, "do a dry-run")
+}
+
+var zabsCmdReleaseAll = &cli.Subcommand{
+	Use:             "release-all",
+	Run:             doZabsReleaseAll,
+	NoRequireConfig: true,
+	Short:           `(DANGEROUS) release ALL zrepl ZFS abstractions (mostly useful for uninstalling zrepl completely or for "de-zrepl-ing" a filesystem)`,
+	SetupFlags:      registerZabsReleaseFlags,
+}
+
+var zabsCmdReleaseStale = &cli.Subcommand{
+	Use:             "release-stale",
+	Run:             doZabsReleaseStale,
+	NoRequireConfig: true,
+	Short:           `release stale zrepl ZFS abstractions (useful if zrepl has a bug and does not do it by itself)`,
+	SetupFlags:      registerZabsReleaseFlags,
+}
+
+func doZabsReleaseAll(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsReleaseFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	abstractions, listErrors, err := endpoint.ListAbstractions(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+	if len(listErrors) > 0 {
+		color.New(color.FgRed).Fprintf(os.Stderr, "there were errors in listing the abstractions:\n%s\n", listErrors)
+		// proceed anyways with rest of abstractions
+	}
+
+	return doZabsRelease_Common(ctx, abstractions)
+}
+
+func doZabsReleaseStale(ctx context.Context, sc *cli.Subcommand, args []string) error {
+
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsReleaseFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	stalenessInfo, err := endpoint.ListStale(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+
+	return doZabsRelease_Common(ctx, stalenessInfo.Stale)
+}
+
+func doZabsRelease_Common(ctx context.Context, destroy []endpoint.Abstraction) error {
+
+	if zabsReleaseFlags.DryRun {
+		if zabsReleaseFlags.Json {
+			m, err := json.MarshalIndent(destroy, "", "  ")
+			if err != nil {
+				panic(err)
+			}
+			if _, err := os.Stdout.Write(m); err != nil {
+				panic(err)
+			}
+			fmt.Println()
+		} else {
+			for _, a := range destroy {
+				fmt.Printf("would destroy %s\n", a)
+			}
+		}
+		return nil
+	}
+
+	outcome := endpoint.BatchDestroy(ctx, destroy)
+	hadErr := false
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	colorErr := color.New(color.FgRed)
+	printfSuccess := color.New(color.FgGreen).FprintfFunc()
+	printfSection := color.New(color.Bold).FprintfFunc()
+
+	for res := range outcome {
+		hadErr = hadErr || res.DestroyErr != nil
+		if zabsReleaseFlags.Json {
+			err := enc.Encode(res)
+			if err != nil {
+				colorErr.Fprintf(os.Stderr, "cannot marshal there were errors in destroying the abstractions")
+			}
+		} else {
+			printfSection(os.Stdout, "destroy %s ...", res.Abstraction)
+			if res.DestroyErr != nil {
+				colorErr.Fprintf(os.Stdout, " failed:\n%s\n", res.DestroyErr)
+			} else {
+				printfSuccess(os.Stdout, " OK\n")
+			}
+		}
+	}
+
+	if hadErr {
+		colorErr.Add(color.Bold).Fprintf(os.Stderr, "there were errors in destroying the abstractions")
+		return fmt.Errorf("")
+	} else {
+		return nil
+	}
+}

config/config.go

@@ -2,14 +2,16 @@ package config
 
 import (
 	"fmt"
-	"github.com/pkg/errors"
-	"github.com/zrepl/yaml-config"
 	"io/ioutil"
+	"log/syslog"
 	"os"
 	"reflect"
 	"regexp"
 	"strconv"
 	"time"
+
+	"github.com/pkg/errors"
+	"github.com/zrepl/yaml-config"
 )
 
 type Config struct {
@@ -33,52 +35,131 @@ type JobEnum struct {
 func (j JobEnum) Name() string {
 	var name string
 	switch v := j.Ret.(type) {
-	case *PushJob: name = v.Name
-	case *SinkJob: name = v.Name
-	case *PullJob: name = v.Name
-	case *SourceJob: name = v.Name
+	case *SnapJob:
+		name = v.Name
+	case *PushJob:
+		name = v.Name
+	case *SinkJob:
+		name = v.Name
+	case *PullJob:
+		name = v.Name
+	case *SourceJob:
+		name = v.Name
 	default:
-		panic(fmt.Sprintf("unknownn job type %T", v))
+		panic(fmt.Sprintf("unknown job type %T", v))
 	}
 	return name
 }
 
 type ActiveJob struct {
-	Type         string                `yaml:"type"`
-	Name         string                `yaml:"name"`
-	Connect     ConnectEnum     `yaml:"connect"`
-	Pruning      PruningSenderReceiver `yaml:"pruning"`
-	Debug        JobDebugSettings      `yaml:"debug,optional"`
+	Type    string                `yaml:"type"`
+	Name    string                `yaml:"name"`
+	Connect ConnectEnum           `yaml:"connect"`
+	Pruning PruningSenderReceiver `yaml:"pruning"`
+	Debug   JobDebugSettings      `yaml:"debug,optional"`
 }
 
 type PassiveJob struct {
-	Type        string           `yaml:"type"`
-	Name        string           `yaml:"name"`
-	Serve       ServeEnum `yaml:"serve"`
-	Debug       JobDebugSettings `yaml:"debug,optional"`
+	Type  string           `yaml:"type"`
+	Name  string           `yaml:"name"`
+	Serve ServeEnum        `yaml:"serve"`
+	Debug JobDebugSettings `yaml:"debug,optional"`
+}
+
+type SnapJob struct {
+	Type         string            `yaml:"type"`
+	Name         string            `yaml:"name"`
+	Pruning      PruningLocal      `yaml:"pruning"`
+	Debug        JobDebugSettings  `yaml:"debug,optional"`
+	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
+	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+}
+
+type SendOptions struct {
+	Encrypted bool                 `yaml:"encrypted"`
+	StepHolds SendOptionsStepHolds `yaml:"step_holds,optional"`
+}
+
+type SendOptionsStepHolds struct {
+	DisableIncremental bool `yaml:"disable_incremental,optional"`
+}
+
+var _ yaml.Defaulter = (*SendOptions)(nil)
+
+func (l *SendOptions) SetDefault() {
+	*l = SendOptions{Encrypted: false}
+}
+
+type RecvOptions struct {
+	// Note: we cannot enforce encrypted recv as the ZFS cli doesn't provide a mechanism for it
+	// Encrypted bool `yaml:"may_encrypted"`
+
+	// Future:
+	// Reencrypt bool `yaml:"reencrypt"`
+}
+
+var _ yaml.Defaulter = (*RecvOptions)(nil)
+
+func (l *RecvOptions) SetDefault() {
+	*l = RecvOptions{}
 }
 
 type PushJob struct {
-	ActiveJob `yaml:",inline"`
-	Snapshotting SnapshottingEnum          `yaml:"snapshotting"`
-	Filesystems FilesystemsFilter `yaml:"filesystems"`
+	ActiveJob    `yaml:",inline"`
+	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
+	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+	Send         *SendOptions      `yaml:"send,fromdefaults,optional"`
 }
 
 type PullJob struct {
 	ActiveJob `yaml:",inline"`
-	RootFS    string        `yaml:"root_fs"`
-	Interval  time.Duration `yaml:"interval,positive"`
+	RootFS    string                   `yaml:"root_fs"`
+	Interval  PositiveDurationOrManual `yaml:"interval"`
+	Recv      *RecvOptions             `yaml:"recv,fromdefaults,optional"`
+}
+
+type PositiveDurationOrManual struct {
+	Interval time.Duration
+	Manual   bool
+}
+
+var _ yaml.Unmarshaler = (*PositiveDurationOrManual)(nil)
+
+func (i *PositiveDurationOrManual) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	var s string
+	if err := u(&s, true); err != nil {
+		return err
+	}
+	switch s {
+	case "manual":
+		i.Manual = true
+		i.Interval = 0
+	case "":
+		return fmt.Errorf("value must not be empty")
+	default:
+		i.Manual = false
+		i.Interval, err = time.ParseDuration(s)
+		if err != nil {
+			return err
+		}
+		if i.Interval <= 0 {
+			return fmt.Errorf("value must be a positive duration, got %q", s)
+		}
+	}
+	return nil
 }
 
 type SinkJob struct {
 	PassiveJob `yaml:",inline"`
-	RootFS     string `yaml:"root_fs"`
+	RootFS     string       `yaml:"root_fs"`
+	Recv       *RecvOptions `yaml:"recv,optional,fromdefaults"`
 }
 
 type SourceJob struct {
-	PassiveJob `yaml:",inline"`
-	Snapshotting SnapshottingEnum      `yaml:"snapshotting"`
-	Filesystems FilesystemsFilter `yaml:"filesystems"`
+	PassiveJob   `yaml:",inline"`
+	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
+	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+	Send         *SendOptions      `yaml:"send,optional,fromdefaults"`
 }
 
 type FilesystemsFilter map[string]bool
@@ -88,9 +169,10 @@ type SnapshottingEnum struct {
 }
 
 type SnapshottingPeriodic struct {
-	Type string		`yaml:"type"`
-	Prefix string	`yaml:"prefix"`
+	Type     string        `yaml:"type"`
+	Prefix   string        `yaml:"prefix"`
 	Interval time.Duration `yaml:"interval,positive"`
+	Hooks    HookList      `yaml:"hooks,optional"`
 }
 
 type SnapshottingManual struct {
@@ -130,7 +212,6 @@ type Global struct {
 	Monitoring []MonitoringEnum       `yaml:"monitoring,optional"`
 	Control    *GlobalControl         `yaml:"control,optional,fromdefaults"`
 	Serve      *GlobalServe           `yaml:"serve,optional,fromdefaults"`
-	RPC        *RPCConfig             `yaml:"rpc,optional,fromdefaults"`
 }
 
 func Default(i interface{}) {
@@ -145,39 +226,28 @@ func Default(i interface{}) {
 	}
 }
 
-type RPCConfig struct {
-	Timeout             time.Duration `yaml:"timeout,optional,positive,default=10s"`
-	TxChunkSize         uint32        `yaml:"tx_chunk_size,optional,default=32768"`
-	RxStructuredMaxLen  uint32        `yaml:"rx_structured_max,optional,default=16777216"`
-	RxStreamChunkMaxLen uint32        `yaml:"rx_stream_chunk_max,optional,default=16777216"`
-	RxHeaderMaxLen      uint32        `yaml:"rx_header_max,optional,default=40960"`
-	SendHeartbeatInterval             time.Duration `yaml:"send_heartbeat_interval,optional,positive,default=5s"`
-
-}
-
 type ConnectEnum struct {
 	Ret interface{}
 }
 
 type ConnectCommon struct {
-	Type string     `yaml:"type"`
-	RPC  *RPCConfig `yaml:"rpc,optional"`
+	Type string `yaml:"type"`
 }
 
 type TCPConnect struct {
 	ConnectCommon `yaml:",inline"`
-	Address       string        `yaml:"address"`
-	DialTimeout   time.Duration `yaml:"dial_timeout,positive,default=10s"`
+	Address       string        `yaml:"address,hostport"`
+	DialTimeout   time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
 }
 
 type TLSConnect struct {
 	ConnectCommon `yaml:",inline"`
-	Address       string        `yaml:"address"`
+	Address       string        `yaml:"address,hostport"`
 	Ca            string        `yaml:"ca"`
 	Cert          string        `yaml:"cert"`
 	Key           string        `yaml:"key"`
 	ServerCN      string        `yaml:"server_cn"`
-	DialTimeout   time.Duration `yaml:"dial_timeout,positive,default=10s"`
+	DialTimeout   time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
 }
 
 type SSHStdinserverConnect struct {
@@ -189,13 +259,14 @@ type SSHStdinserverConnect struct {
 	TransportOpenCommand []string      `yaml:"transport_open_command,optional"` //TODO unused
 	SSHCommand           string        `yaml:"ssh_command,optional"`            //TODO unused
 	Options              []string      `yaml:"options,optional"`
-	DialTimeout          time.Duration `yaml:"dial_timeout,positive,default=10s"`
+	DialTimeout          time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
 }
 
 type LocalConnect struct {
-	ConnectCommon `yaml:",inline"`
-	ListenerName string `yaml:"listener_name"`
-	ClientIdentity string `yaml:"client_identity"`
+	ConnectCommon  `yaml:",inline"`
+	ListenerName   string        `yaml:"listener_name"`
+	ClientIdentity string        `yaml:"client_identity"`
+	DialTimeout    time.Duration `yaml:"dial_timeout,zeropositive,default=2s"`
 }
 
 type ServeEnum struct {
@@ -203,33 +274,34 @@ type ServeEnum struct {
 }
 
 type ServeCommon struct {
-	Type string     `yaml:"type"`
-	RPC  *RPCConfig `yaml:"rpc,optional"`
+	Type string `yaml:"type"`
 }
 
 type TCPServe struct {
-	ServeCommon `yaml:",inline"`
-	Listen      string            `yaml:"listen"`
-	Clients     map[string]string `yaml:"clients"`
+	ServeCommon    `yaml:",inline"`
+	Listen         string            `yaml:"listen,hostport"`
+	ListenFreeBind bool              `yaml:"listen_freebind,default=false"`
+	Clients        map[string]string `yaml:"clients"`
 }
 
 type TLSServe struct {
 	ServeCommon      `yaml:",inline"`
-	Listen           string        `yaml:"listen"`
+	Listen           string        `yaml:"listen,hostport"`
+	ListenFreeBind   bool          `yaml:"listen_freebind,default=false"`
 	Ca               string        `yaml:"ca"`
 	Cert             string        `yaml:"cert"`
 	Key              string        `yaml:"key"`
 	ClientCNs        []string      `yaml:"client_cns"`
-	HandshakeTimeout time.Duration `yaml:"handshake_timeout,positive,default=10s"`
+	HandshakeTimeout time.Duration `yaml:"handshake_timeout,zeropositive,default=10s"`
 }
 
 type StdinserverServer struct {
-	ServeCommon    `yaml:",inline"`
+	ServeCommon      `yaml:",inline"`
 	ClientIdentities []string `yaml:"client_identities"`
 }
 
 type LocalServe struct {
-	ServeCommon `yaml:",inline"`
+	ServeCommon  `yaml:",inline"`
 	ListenerName string `yaml:"listener_name"`
 }
 
@@ -238,8 +310,8 @@ type PruningEnum struct {
 }
 
 type PruneKeepNotReplicated struct {
-	Type string `yaml:"type"`
-	KeepSnapshotAtCursor bool `yaml:"keep_snapshot_at_cursor,optional,default=true"`
+	Type                 string `yaml:"type"`
+	KeepSnapshotAtCursor bool   `yaml:"keep_snapshot_at_cursor,optional,default=true"`
 }
 
 type PruneKeepLastN struct {
@@ -248,8 +320,9 @@ type PruneKeepLastN struct {
 }
 
 type PruneKeepRegex struct { // FIXME rename to KeepRegex
-	Type  string `yaml:"type"`
-	Regex string `yaml:"regex"`
+	Type   string `yaml:"type"`
+	Regex  string `yaml:"regex"`
+	Negate bool   `yaml:"negate,optional,default=false"`
 }
 
 type LoggingOutletEnum struct {
@@ -270,12 +343,13 @@ type StdoutLoggingOutlet struct {
 
 type SyslogLoggingOutlet struct {
 	LoggingOutletCommon `yaml:",inline"`
-	RetryInterval       time.Duration `yaml:"retry_interval,positive,default=10s"`
+	Facility            *SyslogFacility `yaml:"facility,optional,fromdefaults"`
+	RetryInterval       time.Duration   `yaml:"retry_interval,positive,default=10s"`
 }
 
 type TCPLoggingOutlet struct {
 	LoggingOutletCommon `yaml:",inline"`
-	Address             string               `yaml:"address"`
+	Address             string               `yaml:"address,hostport"`
 	Net                 string               `yaml:"net,default=tcp"`
 	RetryInterval       time.Duration        `yaml:"retry_interval,positive,default=10s"`
 	TLS                 *TCPLoggingOutletTLS `yaml:"tls,optional"`
@@ -292,10 +366,19 @@ type MonitoringEnum struct {
 }
 
 type PrometheusMonitoring struct {
-	Type   string `yaml:"type"`
-	Listen string `yaml:"listen"`
+	Type           string `yaml:"type"`
+	Listen         string `yaml:"listen,hostport"`
+	ListenFreeBind bool   `yaml:"listen_freebind,default=false"`
 }
 
+type SyslogFacility syslog.Priority
+
+func (f *SyslogFacility) SetDefault() {
+	*f = SyslogFacility(syslog.LOG_LOCAL0)
+}
+
+var _ yaml.Defaulter = (*SyslogFacility)(nil)
+
 type GlobalControl struct {
 	SockPath string `yaml:"sockpath,default=/var/run/zrepl/control"`
 }
@@ -316,6 +399,38 @@ type JobDebugSettings struct {
 	RPCLog bool `yaml:"rpc_log,optional,default=false"`
 }
 
+type HookList []HookEnum
+
+type HookEnum struct {
+	Ret interface{}
+}
+
+type HookCommand struct {
+	Path               string            `yaml:"path"`
+	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
+	Filesystems        FilesystemsFilter `yaml:"filesystems,optional,default={'<': true}"`
+	HookSettingsCommon `yaml:",inline"`
+}
+
+type HookPostgresCheckpoint struct {
+	HookSettingsCommon `yaml:",inline"`
+	DSN                string            `yaml:"dsn"`
+	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
+	Filesystems        FilesystemsFilter `yaml:"filesystems"` // required, user should not CHECKPOINT for every FS
+}
+
+type HookMySQLLockTables struct {
+	HookSettingsCommon `yaml:",inline"`
+	DSN                string            `yaml:"dsn"`
+	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
+	Filesystems        FilesystemsFilter `yaml:"filesystems"`
+}
+
+type HookSettingsCommon struct {
+	Type       string `yaml:"type"`
+	ErrIsFatal bool   `yaml:"err_is_fatal,optional,default=false"`
+}
+
 func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) {
 	var in struct {
 		Type string
@@ -339,6 +454,7 @@ func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}
 
 func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
 	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"snap":   &SnapJob{},
 		"push":   &PushJob{},
 		"sink":   &SinkJob{},
 		"pull":   &PullJob{},
@@ -352,7 +468,7 @@ func (t *ConnectEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error)
 		"tcp":             &TCPConnect{},
 		"tls":             &TLSConnect{},
 		"ssh+stdinserver": &SSHStdinserverConnect{},
-		"local": 		   &LocalConnect{},
+		"local":           &LocalConnect{},
 	})
 	return
 }
@@ -362,7 +478,7 @@ func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
 		"tcp":         &TCPServe{},
 		"tls":         &TLSServe{},
 		"stdinserver": &StdinserverServer{},
-		"local"      : &LocalServe{},
+		"local":       &LocalServe{},
 	})
 	return
 }
@@ -380,7 +496,7 @@ func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error)
 func (t *SnapshottingEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
 	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
 		"periodic": &SnapshottingPeriodic{},
-		"manual": &SnapshottingManual{},
+		"manual":   &SnapshottingManual{},
 	})
 	return
 }
@@ -401,6 +517,69 @@ func (t *MonitoringEnum) UnmarshalYAML(u func(interface{}, bool) error) (err err
 	return
 }
 
+func (t *SyslogFacility) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	var s string
+	if err := u(&s, true); err != nil {
+		return err
+	}
+	var level syslog.Priority
+	switch s {
+	case "kern":
+		level = syslog.LOG_KERN
+	case "user":
+		level = syslog.LOG_USER
+	case "mail":
+		level = syslog.LOG_MAIL
+	case "daemon":
+		level = syslog.LOG_DAEMON
+	case "auth":
+		level = syslog.LOG_AUTH
+	case "syslog":
+		level = syslog.LOG_SYSLOG
+	case "lpr":
+		level = syslog.LOG_LPR
+	case "news":
+		level = syslog.LOG_NEWS
+	case "uucp":
+		level = syslog.LOG_UUCP
+	case "cron":
+		level = syslog.LOG_CRON
+	case "authpriv":
+		level = syslog.LOG_AUTHPRIV
+	case "ftp":
+		level = syslog.LOG_FTP
+	case "local0":
+		level = syslog.LOG_LOCAL0
+	case "local1":
+		level = syslog.LOG_LOCAL1
+	case "local2":
+		level = syslog.LOG_LOCAL2
+	case "local3":
+		level = syslog.LOG_LOCAL3
+	case "local4":
+		level = syslog.LOG_LOCAL4
+	case "local5":
+		level = syslog.LOG_LOCAL5
+	case "local6":
+		level = syslog.LOG_LOCAL6
+	case "local7":
+		level = syslog.LOG_LOCAL7
+	default:
+		return fmt.Errorf("invalid syslog level: %q", s)
+	}
+	*t = SyslogFacility(level)
+	return nil
+}
+
+func (t *HookEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"command":             &HookCommand{},
+		"postgres-checkpoint": &HookPostgresCheckpoint{},
+		"mysql-lock-tables":   &HookMySQLLockTables{},
+	})
+	return
+}
+
 var ConfigFileDefaultLocations = []string{
 	"/etc/zrepl/zrepl.yml",
 	"/usr/local/etc/zrepl/zrepl.yml",
@@ -446,7 +625,7 @@ func ParseConfigBytes(bytes []byte) (*Config, error) {
 
 var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
 
-func parsePostitiveDuration(e string) (d time.Duration, err error) {
+func parsePositiveDuration(e string) (d time.Duration, err error) {
 	comps := durationStringRegex.FindStringSubmatch(e)
 	if len(comps) != 3 {
 		err = fmt.Errorf("does not match regex: %s %#v", e, comps)

config/config_global_test.go

@@ -1,10 +1,13 @@
 package config
 
 import (
+	"fmt"
+	"log/syslog"
+	"testing"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/zrepl/yaml-config"
-	"testing"
 )
 
 func testValidGlobalSection(t *testing.T, s string) *Config {
@@ -18,11 +21,11 @@ jobs:
     clients: {
       "10.0.0.1":"foo"
     }
-  root_fs: zoot/foo
+  root_fs: zroot/foo
 `
 	_, err := ParseConfigBytes([]byte(jobdef))
 	require.NoError(t, err)
-	return testValidConfig(t, s + jobdef)
+	return testValidConfig(t, s+jobdef)
 }
 
 func TestOutletTypes(t *testing.T) {
@@ -69,7 +72,37 @@ global:
     - type: prometheus
       listen: ':9091'
 `)
-	assert.Equal(t, ":9091", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)	
+	assert.Equal(t, ":9091", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
+}
+
+func TestSyslogLoggingOutletFacility(t *testing.T) {
+	type SyslogFacilityPriority struct {
+		Facility string
+		Priority syslog.Priority
+	}
+	syslogFacilitiesPriorities := []SyslogFacilityPriority{
+		{"", syslog.LOG_LOCAL0}, // default
+		{"kern", syslog.LOG_KERN}, {"daemon", syslog.LOG_DAEMON}, {"auth", syslog.LOG_AUTH},
+		{"syslog", syslog.LOG_SYSLOG}, {"lpr", syslog.LOG_LPR}, {"news", syslog.LOG_NEWS},
+		{"uucp", syslog.LOG_UUCP}, {"cron", syslog.LOG_CRON}, {"authpriv", syslog.LOG_AUTHPRIV},
+		{"ftp", syslog.LOG_FTP}, {"local0", syslog.LOG_LOCAL0}, {"local1", syslog.LOG_LOCAL1},
+		{"local2", syslog.LOG_LOCAL2}, {"local3", syslog.LOG_LOCAL3}, {"local4", syslog.LOG_LOCAL4},
+		{"local5", syslog.LOG_LOCAL5}, {"local6", syslog.LOG_LOCAL6}, {"local7", syslog.LOG_LOCAL7},
+	}
+
+	for _, sFP := range syslogFacilitiesPriorities {
+		logcfg := fmt.Sprintf(`
+global:
+  logging:
+  - type: syslog
+    level: info
+    format: human
+    facility: %s
+`, sFP.Facility)
+		conf := testValidGlobalSection(t, logcfg)
+		assert.Equal(t, 1, len(*conf.Global.Logging))
+		assert.True(t, SyslogFacility(sFP.Priority) == *(*conf.Global.Logging)[0].Ret.(*SyslogLoggingOutlet).Facility)
+	}
 }
 
 func TestLoggingOutletEnumList_SetDefaults(t *testing.T) {

config/config_jobid_test.go

@@ -0,0 +1 @@
+package config

config/config_minimal_test.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"testing"
+
 	"github.com/stretchr/testify/assert"
 )
 
@@ -36,4 +37,4 @@ jobs:
     - type: last_n 
       count: 1
 `)
-}
+}

config/config_positiveintervalormanual_test.go

@@ -0,0 +1,41 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/zrepl/yaml-config"
+)
+
+func TestPositiveDurationOrManual(t *testing.T) {
+	cases := []struct {
+		Comment, Input string
+		Result         *PositiveDurationOrManual
+	}{
+		{"empty is error", "", nil},
+		{"negative is error", "-1s", nil},
+		{"zero seconds is error", "0s", nil},
+		{"zero is error", "0", nil},
+		{"non-manual is error", "something", nil},
+		{"positive seconds works", "1s", &PositiveDurationOrManual{Manual: false, Interval: 1 * time.Second}},
+		{"manual works", "manual", &PositiveDurationOrManual{Manual: true, Interval: 0}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.Comment, func(t *testing.T) {
+			var out struct {
+				FieldName PositiveDurationOrManual `yaml:"fieldname"`
+			}
+			input := fmt.Sprintf("\nfieldname: %s\n", tc.Input)
+			err := yaml.UnmarshalStrict([]byte(input), &out)
+			if tc.Result == nil {
+				assert.Error(t, err)
+				t.Logf("%#v", out)
+			} else {
+				assert.Equal(t, *tc.Result, out.FieldName)
+			}
+		})
+	}
+
+}

config/config_rpc_test.go

@@ -1,86 +0,0 @@
-package config
-
-import (
-	"github.com/stretchr/testify/assert"
-	"testing"
-	"time"
-)
-
-func TestRPC(t *testing.T) {
-	conf := testValidConfig(t, `
-jobs:
-- name: pull_servers
-  type: pull
-  connect:
-    type: tcp
-    address: "server1.foo.bar:8888"
-    rpc:
-      timeout: 20s # different form default, should merge
-  root_fs: "pool2/backup_servers"
-  interval: 10m
-  pruning:
-    keep_sender:
-      - type: not_replicated
-    keep_receiver:
-      - type: last_n
-        count: 100
-
-- name: pull_servers2
-  type: pull
-  connect:
-    type: tcp
-    address: "server1.foo.bar:8888"
-    rpc:
-      tx_chunk_size: 0xabcd # different from default, should merge
-  root_fs: "pool2/backup_servers"
-  interval: 10m
-  pruning:
-    keep_sender:
-    - type: not_replicated
-    keep_receiver:
-    - type: last_n
-      count: 100
-
-- type: sink
-  name: "laptop_sink"
-  root_fs: "pool2/backup_laptops"
-  serve:
-    type: tcp
-    listen: "192.168.122.189:8888"
-    clients: {
-	"10.23.42.23":"client1"
-    }
-    rpc:
-      rx_structured_max: 0x2342
-
-- type: sink
-  name: "other_sink"
-  root_fs: "pool2/backup_laptops"
-  serve:
-    type: tcp
-    listen: "192.168.122.189:8888"
-    clients: {
-	"10.23.42.23":"client1"
-    }
-    rpc:
-      send_heartbeat_interval: 10s
-
-`)
-
-	assert.Equal(t, 20*time.Second, conf.Jobs[0].Ret.(*PullJob).Connect.Ret.(*TCPConnect).RPC.Timeout)
-	assert.Equal(t, uint32(0xabcd), conf.Jobs[1].Ret.(*PullJob).Connect.Ret.(*TCPConnect).RPC.TxChunkSize)
-	assert.Equal(t, uint32(0x2342), conf.Jobs[2].Ret.(*SinkJob).Serve.Ret.(*TCPServe).RPC.RxStructuredMaxLen)
-	assert.Equal(t, 10*time.Second, conf.Jobs[3].Ret.(*SinkJob).Serve.Ret.(*TCPServe).RPC.SendHeartbeatInterval)
-	defConf := RPCConfig{}
-	Default(&defConf)
-	assert.Equal(t, defConf.Timeout, conf.Global.RPC.Timeout)
-}
-
-func TestGlobal_DefaultRPCConfig(t *testing.T) {
-	assert.NotPanics(t, func() {
-		var c RPCConfig
-		Default(&c)
-		assert.NotNil(t, c)
-		assert.Equal(t, c.TxChunkSize, uint32(1)<<15)
-	})
-}

config/config_send_test.go

@@ -0,0 +1,74 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSendOptions(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  %s
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+	encrypted_false := `
+  send:
+    encrypted: false
+`
+
+	encrypted_true := `
+  send:
+    encrypted: true
+`
+
+	encrypted_unspecified := `
+  send: {}
+`
+
+	send_not_specified := `
+`
+
+	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
+	var c *Config
+
+	t.Run("encrypted_false", func(t *testing.T) {
+		c = testValidConfig(t, fill(encrypted_false))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, false, encrypted)
+	})
+	t.Run("encrypted_true", func(t *testing.T) {
+		c = testValidConfig(t, fill(encrypted_true))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, true, encrypted)
+	})
+
+	t.Run("encrypted_unspecified", func(t *testing.T) {
+		c, err := testConfig(t, fill(encrypted_unspecified))
+		assert.Error(t, err)
+		assert.Nil(t, c)
+	})
+
+	t.Run("send_not_specified", func(t *testing.T) {
+		c, err := testConfig(t, fill(send_not_specified))
+		assert.NoError(t, err)
+		assert.NotNil(t, c)
+	})
+
+}

config/config_snapshotting_test.go

@@ -2,9 +2,10 @@ package config
 
 import (
 	"fmt"
-	"github.com/stretchr/testify/assert"
 	"testing"
 	"time"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestSnapshotting(t *testing.T) {
@@ -37,7 +38,30 @@ jobs:
     interval: 10m
 `
 
-	fillSnapshotting := func(s string) string {return fmt.Sprintf(tmpl, s)}
+	hooks := `
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 10m
+    hooks:
+    - type: command
+      path: /tmp/path/to/command
+    - type: command
+      path: /tmp/path/to/command
+      filesystems: { "zroot<": true, "<": false }
+    - type: postgres-checkpoint
+      dsn: "host=localhost port=5432 user=postgres sslmode=disable"
+      filesystems: {
+          "tank/postgres/data11": true
+      }
+    - type: mysql-lock-tables
+      dsn: "root@tcp(localhost)/"
+      filesystems: {
+        "tank/mysql": true
+      }
+`
+
+	fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
 	var c *Config
 
 	t.Run("manual", func(t *testing.T) {
@@ -51,7 +75,16 @@ jobs:
 		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
 		assert.Equal(t, "periodic", snp.Type)
 		assert.Equal(t, 10*time.Minute, snp.Interval)
-		assert.Equal(t, "zrepl_" , snp.Prefix)
+		assert.Equal(t, "zrepl_", snp.Prefix)
+	})
+
+	t.Run("hooks", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(hooks))
+		hs := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic).Hooks
+		assert.Equal(t, hs[0].Ret.(*HookCommand).Filesystems["<"], true)
+		assert.Equal(t, hs[1].Ret.(*HookCommand).Filesystems["zroot<"], true)
+		assert.Equal(t, hs[2].Ret.(*HookPostgresCheckpoint).Filesystems["tank/postgres/data11"], true)
+		assert.Equal(t, hs[3].Ret.(*HookMySQLLockTables).Filesystems["tank/mysql"], true)
 	})
 
 }

config/config_test.go

@@ -1,11 +1,18 @@
 package config
 
 import (
-	"github.com/kr/pretty"
-	"github.com/stretchr/testify/require"
+	"bufio"
+	"bytes"
+	"fmt"
 	"path"
 	"path/filepath"
+	"strings"
 	"testing"
+	"text/template"
+
+	"github.com/kr/pretty"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
@@ -35,8 +42,22 @@ func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
 
 }
 
+// template must be a template/text template with a single '{{ . }}' as placeholder for val
+//nolint[:deadcode,unused]
+func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
+	tmp, err := template.New("master").Parse(tmpl)
+	if err != nil {
+		panic(err)
+	}
+	var buf bytes.Buffer
+	err = tmp.Execute(&buf, val)
+	if err != nil {
+		panic(err)
+	}
+	return testValidConfig(t, buf.String())
+}
 
-func testValidConfig(t *testing.T, input string) (*Config) {
+func testValidConfig(t *testing.T, input string) *Config {
 	t.Helper()
 	conf, err := testConfig(t, input)
 	require.NoError(t, err)
@@ -47,4 +68,21 @@ func testValidConfig(t *testing.T, input string) (*Config) {
 func testConfig(t *testing.T, input string) (*Config, error) {
 	t.Helper()
 	return ParseConfigBytes([]byte(input))
-}
+}
+
+func trimSpaceEachLineAndPad(s, pad string) string {
+	var out strings.Builder
+	scan := bufio.NewScanner(strings.NewReader(s))
+	for scan.Scan() {
+		fmt.Fprintf(&out, "%s%s\n", pad, bytes.TrimSpace(scan.Bytes()))
+	}
+	return out.String()
+}
+
+func TestTrimSpaceEachLineAndPad(t *testing.T) {
+	foo := `
+	foo
+	bar baz 
+	`
+	assert.Equal(t, "  \n  foo\n  bar baz\n  \n", trimSpaceEachLineAndPad(foo, "  "))
+}

config/config_transport_test.go

@@ -0,0 +1,99 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestTransportConnect(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+%s
+  filesystems: {"<": true}
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+
+	mconf := func(s string) string { return fmt.Sprintf(tmpl, s) }
+
+	type test struct {
+		Name        string
+		ExpectError bool
+		Connect     string
+	}
+
+	testTable := []test{
+		{
+			Name:        "tcp_with_address_and_port",
+			ExpectError: false,
+			Connect: `
+			type: tcp
+			address: 10.0.0.23:42
+			`,
+		},
+		{
+			Name:        "tls_with_host_and_port",
+			ExpectError: false,
+			Connect: `
+			type: tls
+			address: "server1.foo.bar:8888"
+			ca:   /etc/zrepl/ca.crt
+			cert: /etc/zrepl/backupserver.fullchain
+			key:  /etc/zrepl/backupserver.key
+			server_cn: "server1"
+			`,
+		},
+		{
+			Name:        "tcp_without_port",
+			ExpectError: true,
+			Connect: `
+			type: tcp
+			address: 10.0.0.23
+			`,
+		},
+		{
+			Name:        "tls_without_port",
+			ExpectError: true,
+			Connect: `
+			type: tls
+			address: 10.0.0.23
+			ca:   /etc/zrepl/ca.crt
+			cert: /etc/zrepl/backupserver.fullchain
+			key:  /etc/zrepl/backupserver.key
+			server_cn: "server1"
+			`,
+		},
+	}
+
+	for _, tc := range testTable {
+		t.Run(tc.Name, func(t *testing.T) {
+			require.NotEmpty(t, tc.Connect)
+			connect := trimSpaceEachLineAndPad(tc.Connect, "    ")
+			conf := mconf(connect)
+			config, err := testConfig(t, conf)
+			if tc.ExpectError && err == nil {
+				t.Errorf("expected test failure, but got valid config %v", config)
+				return
+			}
+			if !tc.ExpectError && err != nil {
+				t.Errorf("not expecint test failure but got error: %s", err)
+				return
+			}
+			t.Logf("error=%v", err)
+			t.Logf("config=%v", config)
+		})
+	}
+
+}

config/retentiongrid.go

@@ -11,9 +11,9 @@ import (
 type RetentionIntervalList []RetentionInterval
 
 type PruneGrid struct {
-	Type          string                `yaml:"type"`
-	Grid          RetentionIntervalList `yaml:"grid"`
-	Regex	      string                `yaml:"regex"`
+	Type  string                `yaml:"type"`
+	Grid  RetentionIntervalList `yaml:"grid"`
+	Regex string                `yaml:"regex"`
 }
 
 type RetentionInterval struct {
@@ -31,10 +31,6 @@ func (i *RetentionInterval) KeepCount() int {
 
 const RetentionGridKeepCountAll int = -1
 
-type RetentionGrid struct {
-	intervals []RetentionInterval
-}
-
 func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
 	var in string
 	if err := u(&in, true); err != nil {
@@ -68,7 +64,7 @@ func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval,
 		return nil, fmt.Errorf("contains factor <= 0")
 	}
 
-	duration, err := parsePostitiveDuration(comps[2])
+	duration, err := parsePositiveDuration(comps[2])
 	if err != nil {
 		return nil, err
 	}

config/samples/hooks/template.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -e
+
+[ "$ZREPL_DRYRUN" = "true" ] && DRYRUN="echo DRYRUN: "
+
+pre_snapshot() {
+    $DRYRUN date
+}
+
+post_snapshot() {
+    $DRYRUN date
+}
+
+case "$ZREPL_HOOKTYPE" in
+    pre_snapshot|post_snapshot)
+        "$ZREPL_HOOKTYPE"
+        ;;
+    *)
+        printf 'Unrecognized hook type: %s\n' "$ZREPL_HOOKTYPE"
+        exit 255
+        ;;
+esac

config/samples/pull_ssh.yml

@@ -9,7 +9,7 @@ jobs:
     port: 22
     identity_file: /etc/zrepl/ssh/identity
     options: # optional, default [], `-o` arguments passed to ssh
-    - "Compression=on"
+    - "Compression=yes"
   root_fs: "pool2/backup_servers"
   interval: 10m
   pruning:

config/samples/push.yml

@@ -10,6 +10,8 @@ jobs:
       address: "backup-server.foo.bar:8888"
     snapshotting:
       type: manual
+    send:
+      encrypted: false
     pruning:
       keep_sender:
         - type: not_replicated

config/samples/snap.yml

@@ -0,0 +1,14 @@
+jobs:
+- name: snapjob
+  type: snap
+  filesystems: {
+    "tank/frequently_changed<": true,
+  }
+  snapshotting:
+    type: periodic
+    interval: 2m
+    prefix: zrepl_snapjob_
+  pruning:
+    keep:
+      - type: last_n
+        count: 60

config/samples/source.yml

@@ -5,7 +5,11 @@ jobs:
     type: tcp
     listen: "0.0.0.0:8888"
     clients: {
-      "192.168.122.123" : "client1"
+      "192.168.122.123" :               "mysql01",
+      "192.168.122.42" :                "mx01",
+      "2001:0db8:85a3::8a2e:0370:7334": "gateway",
+      "10.23.42.0/24":       "cluster-*",
+      "fde4:8dba:82e1::/64": "san-*",
     }
   filesystems: {
     "<": true,

daemon/control.go

@@ -5,16 +5,22 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/daemon/nethelpers"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/version"
 	"io"
 	"net"
 	"net/http"
 	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/daemon/nethelpers"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/util/envconst"
+	"github.com/zrepl/zrepl/version"
+	"github.com/zrepl/zrepl/zfs"
+	"github.com/zrepl/zrepl/zfs/zfscmd"
 )
 
 type controlJob struct {
@@ -38,25 +44,29 @@ func (j *controlJob) Name() string { return jobNameControl }
 
 func (j *controlJob) Status() *job.Status { return &job.Status{Type: job.TypeInternal} }
 
+func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
+
+func (j *controlJob) SenderConfig() *endpoint.SenderConfig { return nil }
+
 var promControl struct {
-	requestBegin *prometheus.CounterVec
+	requestBegin    *prometheus.CounterVec
 	requestFinished *prometheus.HistogramVec
 }
 
 func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) {
 	promControl.requestBegin = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Namespace:   "zrepl",
-		Subsystem:   "control",
-		Name:        "request_begin",
-		Help:        "number of request we started to handle",
+		Namespace: "zrepl",
+		Subsystem: "control",
+		Name:      "request_begin",
+		Help:      "number of request we started to handle",
 	}, []string{"endpoint"})
 
 	promControl.requestFinished = prometheus.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace:   "zrepl",
-		Subsystem:   "control",
-		Name:        "request_finished",
-		Help:        "time it took a request to finih",
-		Buckets: []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3,10e-3, 100e-3, 200e-3,400e-3,800e-3, 1, 10, 20},
+		Namespace: "zrepl",
+		Subsystem: "control",
+		Name:      "request_finished",
+		Help:      "time it took a request to finish",
+		Buckets:   []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3, 10e-3, 100e-3, 200e-3, 400e-3, 800e-3, 1, 10, 20},
 	}, []string{"endpoint"})
 	registerer.MustRegister(promControl.requestBegin)
 	registerer.MustRegister(promControl.requestFinished)
@@ -81,10 +91,16 @@ func (j *controlJob) Run(ctx context.Context) {
 	}
 
 	pprofServer := NewPProfServer(ctx)
+	if listen := envconst.String("ZREPL_DAEMON_AUTOSTART_PPROF_SERVER", ""); listen != "" {
+		pprofServer.Control(PprofServerControlMsg{
+			Run:               true,
+			HttpListenAddress: listen,
+		})
+	}
 
 	mux := http.NewServeMux()
 	mux.Handle(ControlJobEndpointPProf,
-		requestLogger{log: log, handler: jsonRequestResponder{func(decoder jsonDecoder) (interface{}, error) {
+		requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) {
 			var msg PprofServerControlMsg
 			err := decoder(&msg)
 			if err != nil {
@@ -95,22 +111,30 @@ func (j *controlJob) Run(ctx context.Context) {
 		}}})
 
 	mux.Handle(ControlJobEndpointVersion,
-		requestLogger{log: log, handler: jsonResponder{func() (interface{}, error) {
+		requestLogger{log: log, handler: jsonResponder{log, func() (interface{}, error) {
 			return version.NewZreplVersionInformation(), nil
 		}}})
 
 	mux.Handle(ControlJobEndpointStatus,
 		// don't log requests to status endpoint, too spammy
-		jsonResponder{func() (interface{}, error) {
-			s := j.jobs.status()
+		jsonResponder{log, func() (interface{}, error) {
+			jobs := j.jobs.status()
+			globalZFS := zfscmd.GetReport()
+			envconstReport := envconst.GetReport()
+			s := Status{
+				Jobs: jobs,
+				Global: GlobalStatus{
+					ZFSCmds:  globalZFS,
+					Envconst: envconstReport,
+				}}
 			return s, nil
 		}})
 
 	mux.Handle(ControlJobEndpointSignal,
-		requestLogger{log: log, handler: jsonRequestResponder{func(decoder jsonDecoder) (interface{}, error) {
+		requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) {
 			type reqT struct {
 				Name string
-				Op string
+				Op   string
 			}
 			var req reqT
 			if decoder(&req) != nil {
@@ -132,8 +156,8 @@ func (j *controlJob) Run(ctx context.Context) {
 	server := http.Server{
 		Handler: mux,
 		// control socket is local, 1s timeout should be more than sufficient, even on a loaded system
-		WriteTimeout: 1*time.Second,
-		ReadTimeout: 1*time.Second,
+		WriteTimeout: 1 * time.Second,
+		ReadTimeout:  1 * time.Second,
 	}
 
 outer:
@@ -148,7 +172,10 @@ outer:
 		select {
 		case <-ctx.Done():
 			log.WithError(ctx.Err()).Info("context done")
-			server.Shutdown(context.Background())
+			err := server.Shutdown(context.Background())
+			if err != nil {
+				log.WithError(err).Error("cannot shutdown server")
+			}
 			break outer
 		case err = <-served:
 			if err != nil {
@@ -162,33 +189,50 @@ outer:
 }
 
 type jsonResponder struct {
+	log      Logger
 	producer func() (interface{}, error)
 }
 
 func (j jsonResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	logIoErr := func(err error) {
+		if err != nil {
+			j.log.WithError(err).Error("control handler io error")
+		}
+	}
 	res, err := j.producer()
 	if err != nil {
+		j.log.WithError(err).Error("control handler error")
 		w.WriteHeader(http.StatusInternalServerError)
-		io.WriteString(w, err.Error())
+		_, err = io.WriteString(w, err.Error())
+		logIoErr(err)
 		return
 	}
 	var buf bytes.Buffer
 	err = json.NewEncoder(&buf).Encode(res)
 	if err != nil {
+		j.log.WithError(err).Error("control handler json marshal error")
 		w.WriteHeader(http.StatusInternalServerError)
-		io.WriteString(w, err.Error())
+		_, err = io.WriteString(w, err.Error())
 	} else {
-		io.Copy(w, &buf)
+		_, err = io.Copy(w, &buf)
 	}
+	logIoErr(err)
 }
 
 type jsonDecoder = func(interface{}) error
 
 type jsonRequestResponder struct {
+	log      Logger
 	producer func(decoder jsonDecoder) (interface{}, error)
 }
 
 func (j jsonRequestResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	logIoErr := func(err error) {
+		if err != nil {
+			j.log.WithError(err).Error("control handler io error")
+		}
+	}
+
 	var decodeError error
 	decoder := func(i interface{}) error {
 		err := json.NewDecoder(r.Body).Decode(&i)
@@ -200,22 +244,28 @@ func (j jsonRequestResponder) ServeHTTP(w http.ResponseWriter, r *http.Request)
 	//If we had a decode error ignore output of producer and return error
 	if decodeError != nil {
 		w.WriteHeader(http.StatusBadRequest)
-		io.WriteString(w, decodeError.Error())
+		_, err := io.WriteString(w, decodeError.Error())
+		logIoErr(err)
 		return
 	}
 	if producerErr != nil {
+		j.log.WithError(producerErr).Error("control handler error")
 		w.WriteHeader(http.StatusInternalServerError)
-		io.WriteString(w, producerErr.Error())
+		_, err := io.WriteString(w, producerErr.Error())
+		logIoErr(err)
 		return
 	}
 
 	var buf bytes.Buffer
 	encodeErr := json.NewEncoder(&buf).Encode(res)
 	if encodeErr != nil {
+		j.log.WithError(producerErr).Error("control handler json marshal error")
 		w.WriteHeader(http.StatusInternalServerError)
-		io.WriteString(w, encodeErr.Error())
+		_, err := io.WriteString(w, encodeErr.Error())
+		logIoErr(err)
 	} else {
-		io.Copy(w, &buf)
+		_, err := io.Copy(w, &buf)
+		logIoErr(err)
 	}
 }
 

daemon/daemon.go

@@ -3,8 +3,19 @@ package daemon
 import (
 	"context"
 	"fmt"
+	"os"
+	"os/signal"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/util/envconst"
+
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/job"
 	"github.com/zrepl/zrepl/daemon/job/reset"
@@ -12,17 +23,11 @@ import (
 	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/version"
-	"os"
-	"os/signal"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
+	"github.com/zrepl/zrepl/zfs/zfscmd"
 )
 
-func Run(conf *config.Config) error {
-
-	ctx, cancel := context.WithCancel(context.Background())
+func Run(ctx context.Context, conf *config.Config) error {
+	ctx, cancel := context.WithCancel(ctx)
 
 	defer cancel()
 	sigChan := make(chan os.Signal, 1)
@@ -36,6 +41,7 @@ func Run(conf *config.Config) error {
 	if err != nil {
 		return errors.Wrap(err, "cannot build logging from config")
 	}
+	outlets.Add(newPrometheusLogOutlet(), logger.Debug)
 
 	confJobs, err := job.JobsFromConfig(conf)
 	if err != nil {
@@ -45,14 +51,23 @@ func Run(conf *config.Config) error {
 	log := logger.NewLogger(outlets, 1*time.Second)
 	log.Info(version.NewZreplVersionInformation().String())
 
+	ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
+	trace.RegisterCallback(trace.Callback{
+		OnBegin: func(ctx context.Context) { logging.GetLogger(ctx, logging.SubsysTraceData).Debug("begin span") },
+		OnEnd: func(ctx context.Context, spanInfo trace.SpanInfo) {
+			logging.
+				GetLogger(ctx, logging.SubsysTraceData).
+				WithField("duration_s", spanInfo.EndedAt().Sub(spanInfo.StartedAt()).Seconds()).
+				Debug("finished span " + spanInfo.TaskAndSpanStack(trace.SpanStackKindAnnotation))
+		},
+	})
+
 	for _, job := range confJobs {
 		if IsInternalJobName(job.Name()) {
 			panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME
 		}
 	}
 
-	ctx = job.WithLogger(ctx, log)
-
 	jobs := newJobs()
 
 	// start control socket
@@ -74,11 +89,15 @@ func Run(conf *config.Config) error {
 			return errors.Errorf("unknown monitoring job #%d (type %T)", i, v)
 		}
 		if err != nil {
-			return errors.Wrapf(err,"cannot build monitorin gjob #%d", i)
+			return errors.Wrapf(err, "cannot build monitoring job #%d", i)
 		}
 		jobs.start(ctx, job, true)
 	}
 
+	// register global (=non job-local) metrics
+	zfscmd.RegisterMetrics(prometheus.DefaultRegisterer)
+	trace.RegisterMetrics(prometheus.DefaultRegisterer)
+	endpoint.RegisterMetrics(prometheus.DefaultRegisterer)
 
 	log.Info("starting daemon")
 
@@ -93,6 +112,8 @@ func Run(conf *config.Config) error {
 	case <-ctx.Done():
 		log.WithError(ctx.Err()).Info("context finished")
 	}
+	log.Info("waiting for jobs to finish")
+	<-jobs.wait()
 	log.Info("daemon exiting")
 	return nil
 }
@@ -103,7 +124,7 @@ type jobs struct {
 	// m protects all fields below it
 	m       sync.RWMutex
 	wakeups map[string]wakeup.Func // by Job.Name
-	resets map[string]reset.Func // by Job.Name
+	resets  map[string]reset.Func  // by Job.Name
 	jobs    map[string]job.Job
 }
 
@@ -115,20 +136,25 @@ func newJobs() *jobs {
 	}
 }
 
-const (
-	logJobField    string = "job"
-	logTaskField   string = "task"
-	logSubsysField string = "subsystem"
-)
-
 func (s *jobs) wait() <-chan struct{} {
 	ch := make(chan struct{})
 	go func() {
 		s.wg.Wait()
+		close(ch)
 	}()
 	return ch
 }
 
+type Status struct {
+	Jobs   map[string]*job.Status
+	Global GlobalStatus
+}
+
+type GlobalStatus struct {
+	ZFSCmds  *zfscmd.Report
+	Envconst *envconst.Report
+}
+
 func (s *jobs) status() map[string]*job.Status {
 	s.m.RLock()
 	defer s.m.RUnlock()
@@ -190,9 +216,8 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
 	s.m.Lock()
 	defer s.m.Unlock()
 
-	jobLog := job.GetLogger(ctx).
-		WithField(logJobField, j.Name()).
-		WithOutlet(newPrometheusLogOutlet(j.Name()), logger.Debug)
+	ctx = logging.WithInjectedField(ctx, logging.JobField, j.Name())
+
 	jobName := j.Name()
 	if !internal && IsInternalJobName(jobName) {
 		panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName))
@@ -207,7 +232,7 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
 	j.RegisterMetrics(prometheus.DefaultRegisterer)
 
 	s.jobs[jobName] = j
-	ctx = job.WithLogger(ctx, jobLog)
+	ctx = zfscmd.WithJobID(ctx, j.Name())
 	ctx, wakeup := wakeup.Context(ctx)
 	ctx, resetFunc := reset.Context(ctx)
 	s.wakeups[jobName] = wakeup
@@ -216,8 +241,8 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
 	s.wg.Add(1)
 	go func() {
 		defer s.wg.Done()
-		jobLog.Info("starting job")
-		defer jobLog.Info("job exited")
+		job.GetLogger(ctx).Info("starting job")
+		defer job.GetLogger(ctx).Info("job exited")
 		j.Run(ctx)
 	}()
 }

daemon/filters/fsmapfilter.go

@@ -2,10 +2,12 @@ package filters
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/zfs"
-	"strings"
 )
 
 type DatasetMapFilter struct {
@@ -159,7 +161,7 @@ func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
 }
 
 // Construct a new filter-only DatasetMapFilter from a mapping
-// The new filter allows excactly those paths that were not forbidden by the mapping.
+// The new filter allows exactly those paths that were not forbidden by the mapping.
 func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
 
 	if m.filterMode {

daemon/filters/fsmapfilter_test.go

@@ -0,0 +1,89 @@
+package filters
+
+import (
+	"testing"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func TestDatasetMapFilter(t *testing.T) {
+
+	type testCase struct {
+		name      string
+		filter    map[string]string
+		checkPass map[string]bool // each entry is checked to match the filter's `pass` return value
+	}
+
+	tcs := []testCase{
+		{
+			"default_no_match",
+			map[string]string{},
+			map[string]bool{
+				"":      false,
+				"foo":   false,
+				"zroot": false,
+			},
+		},
+		{
+			"more_specific_path_has_precedence",
+			map[string]string{
+				"tank<":         "ok",
+				"tank/tmp<":     "!",
+				"tank/home/x<":  "!",
+				"tank/home/x/1": "ok",
+			},
+			map[string]bool{
+				"zroot":         false,
+				"tank":          true,
+				"tank/tmp":      false,
+				"tank/tmp/foo":  false,
+				"tank/home/x":   false,
+				"tank/home/y":   true,
+				"tank/home/x/1": true,
+				"tank/home/x/2": false,
+			},
+		},
+		{
+			"precedence_of_specific_over_subtree_wildcard_on_same_path",
+			map[string]string{
+				"tank/home/bob":  "ok",
+				"tank/home/bob<": "!",
+			},
+			map[string]bool{
+				"tank/home/bob":           true,
+				"tank/home/bob/downloads": false,
+			},
+		},
+	}
+
+	for tc := range tcs {
+		t.Run(tcs[tc].name, func(t *testing.T) {
+			c := tcs[tc]
+			f := NewDatasetMapFilter(len(c.filter), true)
+			for p, a := range c.filter {
+				err := f.Add(p, a)
+				if err != nil {
+					t.Fatalf("incorrect filter spec: %s", err)
+				}
+			}
+			for p, checkPass := range c.checkPass {
+				zp, err := zfs.NewDatasetPath(p)
+				if err != nil {
+					t.Fatalf("incorrect path spec: %s", err)
+				}
+				pass, err := f.Filter(zp)
+				if err != nil {
+					t.Fatalf("unexpected filter error: %s", err)
+				}
+				ok := pass == checkPass
+				failstr := "OK"
+				if !ok {
+					failstr = "FAIL"
+					t.Fail()
+				}
+				t.Logf("%-40q  %5v  (exp=%v act=%v)", p, failstr, checkPass, pass)
+			}
+		})
+	}
+
+}

daemon/filters/fsvfilter.go

@@ -1,41 +0,0 @@
-package filters
-
-import (
-	"github.com/zrepl/zrepl/zfs"
-	"strings"
-)
-
-type AnyFSVFilter struct{}
-
-func NewAnyFSVFilter() AnyFSVFilter {
-	return AnyFSVFilter{}
-}
-
-var _ zfs.FilesystemVersionFilter = AnyFSVFilter{}
-
-func (AnyFSVFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) {
-	return true, nil
-}
-
-
-type PrefixFilter struct {
-	prefix    string
-	fstype    zfs.VersionType
-	fstypeSet bool // optionals anyone?
-}
-
-var _ zfs.FilesystemVersionFilter = &PrefixFilter{}
-
-func NewPrefixFilter(prefix string) *PrefixFilter {
-	return &PrefixFilter{prefix: prefix}
-}
-
-func NewTypedPrefixFilter(prefix string, versionType zfs.VersionType) *PrefixFilter {
-	return &PrefixFilter{prefix, versionType, true}
-}
-
-func (f *PrefixFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) {
-	fstypeMatches := (!f.fstypeSet || t == f.fstype)
-	prefixMatches := strings.HasPrefix(name, f.prefix)
-	return fstypeMatches && prefixMatches, nil
-}

daemon/hooks/edge_string.go

@@ -0,0 +1,35 @@
+// Code generated by "stringer -type=Edge"; DO NOT EDIT.
+
+package hooks
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[Pre-1]
+	_ = x[Callback-2]
+	_ = x[Post-4]
+}
+
+const (
+	_Edge_name_0 = "PreCallback"
+	_Edge_name_1 = "Post"
+)
+
+var (
+	_Edge_index_0 = [...]uint8{0, 3, 11}
+)
+
+func (i Edge) String() string {
+	switch {
+	case 1 <= i && i <= 2:
+		i -= 1
+		return _Edge_name_0[_Edge_index_0[i]:_Edge_index_0[i+1]]
+	case i == 4:
+		return _Edge_name_1
+	default:
+		return "Edge(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+}

daemon/hooks/hook_config.go

@@ -0,0 +1,52 @@
+package hooks
+
+import (
+	"fmt"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type List []Hook
+
+func HookFromConfig(in config.HookEnum) (Hook, error) {
+	switch v := in.Ret.(type) {
+	case *config.HookCommand:
+		return NewCommandHook(v)
+	case *config.HookPostgresCheckpoint:
+		return PgChkptHookFromConfig(v)
+	case *config.HookMySQLLockTables:
+		return MyLockTablesFromConfig(v)
+	default:
+		return nil, fmt.Errorf("unknown hook type %T", v)
+	}
+}
+
+func ListFromConfig(in *config.HookList) (r *List, err error) {
+	hl := make(List, len(*in))
+
+	for i, h := range *in {
+		hl[i], err = HookFromConfig(h)
+		if err != nil {
+			return nil, fmt.Errorf("create hook #%d: %s", i+1, err)
+		}
+	}
+
+	return &hl, nil
+}
+
+func (l List) CopyFilteredForFilesystem(fs *zfs.DatasetPath) (ret List, err error) {
+	ret = make(List, 0, len(l))
+
+	for _, h := range l {
+		var passFilesystem bool
+		if passFilesystem, err = h.Filesystems().Filter(fs); err != nil {
+			return nil, err
+		}
+		if passFilesystem {
+			ret = append(ret, h)
+		}
+	}
+
+	return ret, nil
+}

daemon/hooks/hook_docs.go

@@ -0,0 +1,34 @@
+// Package hooks implements pre- and post snapshot hooks.
+//
+// Plan is a generic executor for ExpectStepReports before and after an activity specified in a callback.
+// It provides a reporting facility that can be polled while the plan is executing to gather progress information.
+//
+// This package also provides all supported hook type implementations and abstractions around them.
+//
+// Use For Other Kinds Of ExpectStepReports
+//
+// This package REQUIRES REFACTORING before it can be used for other activities than snapshots, e.g. pre- and post-replication:
+//
+// The Hook interface requires a hook to provide a Filesystems() filter, which doesn't make sense for
+// all kinds of activities.
+//
+// The hook implementations should move out of this package.
+// However, there is a lot of tight coupling which to untangle isn't worth it ATM.
+//
+// How This Package Is Used By Package Snapper
+//
+// Deserialize a config.List using ListFromConfig().
+// Then it MUST filter the list to only contain hooks for a particular filesystem using
+// hooksList.CopyFilteredForFilesystem(fs).
+//
+// Then create a CallbackHook using NewCallbackHookForFilesystem().
+//
+// Pass all of the above to NewPlan() which provides a Report() and Run() method:
+//
+// Plan.Run(ctx context.Context,dryRun bool) executes the plan and take a context as argument that should contain a logger added using hooks.WithLogger()).
+// The value of dryRun is passed through to the hooks' Run() method.
+// Command hooks make it available in the environment variable ZREPL_DRYRUN.
+//
+// Plan.Report() can be called while Plan.Run() is executing to give an overview of plan execution progress (future use in "zrepl status").
+//
+package hooks

daemon/hooks/hook_exec.go

@@ -0,0 +1,292 @@
+package hooks
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+// Re-export type here so that
+// every file in package hooks doesn't
+// have to import github.com/zrepl/zrepl/zfs
+type Filter zfs.DatasetFilter
+
+type Hook interface {
+	Filesystems() Filter
+
+	// If true and the Pre edge invocation of Run fails, Post edge will not run and other Pre edges will not run.
+	ErrIsFatal() bool
+
+	// Run is invoked by HookPlan for a Pre edge.
+	// If HookReport.HadError() == false, the Post edge will be invoked, too.
+	Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport
+
+	String() string
+}
+
+type Phase string
+
+const (
+	PhaseSnapshot = Phase("snapshot")
+	PhaseTesting  = Phase("testing")
+)
+
+func (p Phase) String() string {
+	return string(p)
+}
+
+//go:generate stringer -type=Edge
+type Edge uint
+
+const (
+	Pre = Edge(1 << iota)
+	Callback
+	Post
+)
+
+func (e Edge) StringForPhase(phase Phase) string {
+	return fmt.Sprintf("%s_%s", e.String(), phase.String())
+}
+
+//go:generate enumer -type=StepStatus -trimprefix=Step
+type StepStatus int
+
+const (
+	StepPending StepStatus = 1 << iota
+	StepExec
+	StepOk
+	StepErr
+	StepSkippedDueToFatalErr
+	StepSkippedDueToPreErr
+)
+
+type HookReport interface {
+	String() string
+	HadError() bool
+	Error() string
+}
+
+type Step struct {
+	Hook       Hook
+	Edge       Edge
+	Status     StepStatus
+	Begin, End time.Time
+	// Report may be nil
+	// FIXME cannot serialize this for client status, but contains interesting info (like what error happened)
+	Report HookReport
+	state  map[interface{}]interface{}
+}
+
+func (s Step) String() (out string) {
+	fatal := "~"
+	if s.Hook.ErrIsFatal() && s.Edge == Pre {
+		fatal = "!"
+	}
+	runTime := "..."
+	if s.Status != StepPending {
+		t := s.End.Sub(s.Begin)
+		runTime = t.Round(time.Millisecond).String()
+	}
+	return fmt.Sprintf("[%s] [%5s] %s [%s]  %s", s.Status, runTime, fatal, s.Edge, s.Hook)
+}
+
+type Plan struct {
+	mtx sync.RWMutex
+
+	steps []*Step
+	pre   []*Step // protected by mtx
+	cb    *Step
+	post  []*Step // not reversed, i.e. entry at index i corresponds to pre-edge in pre[i]
+
+	phase Phase
+	env   Env
+}
+
+func NewPlan(hooks *List, phase Phase, cb *CallbackHook, extra Env) (*Plan, error) {
+
+	var pre, post []*Step
+	// TODO sanity check unique name of hook?
+	for _, hook := range *hooks {
+		state := make(map[interface{}]interface{})
+		preE := &Step{
+			Hook:   hook,
+			Edge:   Pre,
+			Status: StepPending,
+			state:  state,
+		}
+		pre = append(pre, preE)
+		postE := &Step{
+			Hook:   hook,
+			Edge:   Post,
+			Status: StepPending,
+			state:  state,
+		}
+		post = append(post, postE)
+	}
+
+	cbE := &Step{
+		Hook:   cb,
+		Edge:   Callback,
+		Status: StepPending,
+	}
+
+	steps := make([]*Step, 0, len(pre)+len(post)+1)
+	steps = append(steps, pre...)
+	steps = append(steps, cbE)
+	for i := len(post) - 1; i >= 0; i-- {
+		steps = append(steps, post[i])
+	}
+
+	plan := &Plan{
+		phase: phase,
+		env:   extra,
+		steps: steps,
+		pre:   pre,
+		post:  post,
+		cb:    cbE,
+	}
+
+	return plan, nil
+}
+
+type PlanReport []Step
+
+func (p *Plan) Report() PlanReport {
+	p.mtx.RLock()
+	defer p.mtx.RUnlock()
+	rep := make([]Step, len(p.steps))
+	for i := range rep {
+		rep[i] = *p.steps[i]
+	}
+	return rep
+}
+
+func (r PlanReport) HadError() bool {
+	for _, e := range r {
+		if e.Status == StepErr {
+			return true
+		}
+	}
+	return false
+}
+
+func (r PlanReport) HadFatalError() bool {
+	for _, e := range r {
+		if e.Status == StepSkippedDueToFatalErr {
+			return true
+		}
+	}
+	return false
+}
+
+func (r PlanReport) String() string {
+	stepStrings := make([]string, len(r))
+	for i, e := range r {
+		stepStrings[i] = fmt.Sprintf("%02d %s", i+1, e)
+	}
+	return strings.Join(stepStrings, "\n")
+}
+
+func (p *Plan) Run(ctx context.Context, dryRun bool) {
+	p.mtx.RLock()
+	defer p.mtx.RUnlock()
+	w := func(f func()) {
+		p.mtx.RUnlock()
+		defer p.mtx.RLock()
+		p.mtx.Lock()
+		defer p.mtx.Unlock()
+		f()
+	}
+	runHook := func(s *Step, ctx context.Context, edge Edge) HookReport {
+		w(func() { s.Status = StepExec })
+		begin := time.Now()
+		r := s.Hook.Run(ctx, edge, p.phase, dryRun, p.env, s.state)
+		end := time.Now()
+		w(func() {
+			s.Report = r
+			s.Status = StepOk
+			if r.HadError() {
+				s.Status = StepErr
+			}
+			s.Begin, s.End = begin, end
+		})
+		return r
+	}
+
+	l := getLogger(ctx)
+
+	// it's a stack, execute until we reach the end of the list (last item in)
+	// or fail inbetween
+	l.Info("run pre-edges in configuration order")
+	next := 0
+	for ; next < len(p.pre); next++ {
+		e := p.pre[next]
+		l := l.WithField("hook", e.Hook)
+		r := runHook(e, ctx, Pre)
+		if r.HadError() {
+			l.WithError(r).Error("hook invocation failed for pre-edge")
+			if e.Hook.ErrIsFatal() {
+				l.Error("the hook run was aborted due to a fatal error in this hook")
+				break
+			}
+		}
+	}
+
+	hadFatalErr := next != len(p.pre)
+	if hadFatalErr {
+		l.Error("fatal error in a pre-snapshot hook invocation")
+		l.Error("no snapshot will be taken")
+		l.Error("only running post-edges for successful pre-edges")
+		w(func() {
+			p.post[next].Status = StepSkippedDueToFatalErr
+			for i := next + 1; i < len(p.pre); i++ {
+				p.pre[i].Status = StepSkippedDueToFatalErr
+				p.post[i].Status = StepSkippedDueToFatalErr
+			}
+			p.cb.Status = StepSkippedDueToFatalErr
+		})
+		return
+	}
+
+	l.Info("running callback")
+	cbR := runHook(p.cb, ctx, Callback)
+	if cbR.HadError() {
+		l.WithError(cbR).Error("callback failed")
+	}
+
+	l.Info("run post-edges for successful pre-edges in reverse configuration order")
+
+	// the constructor produces pre and post entries
+	// post is NOT reversed
+	next-- // now at index of last executed pre-edge
+	for ; next >= 0; next-- {
+		e := p.post[next]
+		l := l.WithField("hook", e.Hook)
+
+		if p.pre[next].Status != StepOk {
+			if p.pre[next].Status != StepErr {
+				panic(fmt.Sprintf("expecting a pre-edge hook report to be either Ok or Err, got %s", p.pre[next].Status))
+			}
+			l.Info("skip post-edge because pre-edge failed")
+			w(func() {
+				e.Status = StepSkippedDueToPreErr
+			})
+			continue
+		}
+
+		report := runHook(e, ctx, Post)
+
+		if report.HadError() {
+			l.WithError(report).Error("hook invocation failed for post-edge")
+			l.Error("subsequent post-edges run regardless of this post-edge failure")
+		}
+
+		// ErrIsFatal is only relevant for Pre
+
+	}
+
+}

daemon/hooks/hook_logging.go

@@ -0,0 +1,91 @@
+package hooks
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"sync"
+
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/util/envconst"
+)
+
+type Logger = logger.Logger
+
+func GetLogger(ctx context.Context) Logger { return getLogger(ctx) }
+
+func getLogger(ctx context.Context) Logger {
+	return logging.GetLogger(ctx, logging.SubsysHooks)
+}
+
+const MAX_HOOK_LOG_SIZE_DEFAULT int = 1 << 20
+
+type logWriter struct {
+	/*
+		Mutex prevents:
+			concurrent writes to buf, scanner in Write([]byte)
+			data race on scanner vs Write([]byte)
+				and concurrent write to buf (call to buf.Reset())
+				in Close()
+
+		(Also, Close() should generally block until any Write() call completes.)
+	*/
+	mtx     *sync.Mutex
+	buf     bytes.Buffer
+	scanner *bufio.Scanner
+	logger  Logger
+	level   logger.Level
+	field   string
+}
+
+func NewLogWriter(mtx *sync.Mutex, logger Logger, level logger.Level, field string) *logWriter {
+	w := new(logWriter)
+	w.mtx = mtx
+	w.scanner = bufio.NewScanner(&w.buf)
+	w.logger = logger
+	w.level = level
+	w.field = field
+	return w
+}
+
+func (w *logWriter) log(line string) {
+	w.logger.WithField(w.field, line).Log(w.level, "hook output")
+}
+
+func (w *logWriter) logUnreadBytes() error {
+	for w.scanner.Scan() {
+		w.log(w.scanner.Text())
+	}
+	if w.buf.Cap() > envconst.Int("ZREPL_MAX_HOOK_LOG_SIZE", MAX_HOOK_LOG_SIZE_DEFAULT) {
+		w.buf.Reset()
+	}
+
+	return nil
+}
+
+func (w *logWriter) Write(in []byte) (int, error) {
+	w.mtx.Lock()
+	defer w.mtx.Unlock()
+
+	n, bufErr := w.buf.Write(in)
+	if bufErr != nil {
+		return n, bufErr
+	}
+
+	err := w.logUnreadBytes()
+	if err != nil {
+		return n, err
+	}
+	// Always reset the scanner for the next Write
+	w.scanner = bufio.NewScanner(&w.buf)
+
+	return n, nil
+}
+
+func (w *logWriter) Close() (err error) {
+	w.mtx.Lock()
+	defer w.mtx.Unlock()
+
+	return w.logUnreadBytes()
+}

daemon/hooks/hook_type_callback.go

@@ -0,0 +1,65 @@
+package hooks
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type HookJobCallback func(ctx context.Context) error
+
+type CallbackHook struct {
+	cb            HookJobCallback
+	filter        Filter
+	displayString string
+}
+
+func NewCallbackHookForFilesystem(displayString string, fs *zfs.DatasetPath, cb HookJobCallback) *CallbackHook {
+	filter, _ := filters.DatasetMapFilterFromConfig(map[string]bool{fs.ToString(): true})
+	return NewCallbackHook(displayString, cb, filter)
+}
+
+func NewCallbackHook(displayString string, cb HookJobCallback, filter Filter) *CallbackHook {
+	return &CallbackHook{
+		cb:            cb,
+		filter:        filter,
+		displayString: displayString,
+	}
+}
+
+func (h *CallbackHook) Filesystems() Filter {
+	return h.filter
+}
+
+func (h *CallbackHook) ErrIsFatal() bool {
+	return false // callback is by definition
+}
+
+func (h *CallbackHook) String() string {
+	return h.displayString
+}
+
+type CallbackHookReport struct {
+	Name string
+	Err  error
+}
+
+func (r *CallbackHookReport) String() string {
+	if r.HadError() {
+		return r.Error()
+	}
+	return r.Name
+}
+
+func (r *CallbackHookReport) HadError() bool { return r.Err != nil }
+
+func (r *CallbackHookReport) Error() string {
+	return fmt.Sprintf("%s error: %s", r.Name, r.Err)
+}
+
+func (h *CallbackHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
+	err := h.cb(ctx)
+	return &CallbackHookReport{h.displayString, err}
+}

daemon/hooks/hook_type_command.go

@@ -0,0 +1,192 @@
+package hooks
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"os"
+	"os/exec"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/util/circlog"
+	"github.com/zrepl/zrepl/util/envconst"
+)
+
+type HookEnvVar string
+
+const (
+	EnvType     HookEnvVar = "ZREPL_HOOKTYPE"
+	EnvDryRun   HookEnvVar = "ZREPL_DRYRUN"
+	EnvFS       HookEnvVar = "ZREPL_FS"
+	EnvSnapshot HookEnvVar = "ZREPL_SNAPNAME"
+	EnvTimeout  HookEnvVar = "ZREPL_TIMEOUT"
+)
+
+type Env map[HookEnvVar]string
+
+func NewHookEnv(edge Edge, phase Phase, dryRun bool, timeout time.Duration, extra Env) Env {
+	r := Env{
+		EnvTimeout: fmt.Sprintf("%.f", math.Floor(timeout.Seconds())),
+	}
+
+	edgeString := edge.StringForPhase(phase)
+	r[EnvType] = strings.ToLower(edgeString)
+
+	var dryRunString string
+	if dryRun {
+		dryRunString = "true"
+	} else {
+		dryRunString = ""
+	}
+	r[EnvDryRun] = dryRunString
+
+	for k, v := range extra {
+		r[k] = v
+	}
+
+	return r
+}
+
+type CommandHook struct {
+	edge       Edge
+	filter     Filter
+	errIsFatal bool
+	command    string
+	timeout    time.Duration
+}
+
+type CommandHookReport struct {
+	Command                      string
+	Args                         []string // currently always empty
+	Env                          Env
+	Err                          error
+	CapturedStdoutStderrCombined []byte
+}
+
+func (r *CommandHookReport) String() string {
+	// Reproduces a POSIX shell-compatible command line
+	var cmdLine strings.Builder
+	sep := ""
+
+	// Make sure environment variables are always
+	// printed in the same order
+	var hookEnvKeys []HookEnvVar
+	for k := range r.Env {
+		hookEnvKeys = append(hookEnvKeys, k)
+	}
+	sort.Slice(hookEnvKeys, func(i, j int) bool { return string(hookEnvKeys[i]) < string(hookEnvKeys[j]) })
+
+	for _, k := range hookEnvKeys {
+		cmdLine.WriteString(fmt.Sprintf("%s%s='%s'", sep, k, r.Env[k]))
+		sep = " "
+	}
+
+	cmdLine.WriteString(fmt.Sprintf("%s%s", sep, r.Command))
+	for _, a := range r.Args {
+		cmdLine.WriteString(fmt.Sprintf("%s'%s'", sep, a))
+	}
+
+	return fmt.Sprintf("command hook invocation: \"%s\"", cmdLine.String()) // no %q to make copy-pastable
+}
+func (r *CommandHookReport) Error() string {
+	if r.Err == nil {
+		return ""
+	}
+	return fmt.Sprintf("%s FAILED with error: %s", r.String(), r.Err)
+}
+
+func (r *CommandHookReport) HadError() bool {
+	return r.Err != nil
+}
+
+func NewCommandHook(in *config.HookCommand) (r *CommandHook, err error) {
+	r = &CommandHook{
+		errIsFatal: in.ErrIsFatal,
+		command:    in.Path,
+		timeout:    in.Timeout,
+	}
+
+	r.filter, err = filters.DatasetMapFilterFromConfig(in.Filesystems)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse filesystem filter: %s", err)
+	}
+
+	r.edge = Pre | Post
+
+	return r, nil
+}
+
+func (h *CommandHook) Filesystems() Filter {
+	return h.filter
+}
+
+func (h *CommandHook) ErrIsFatal() bool {
+	return h.errIsFatal
+}
+
+func (h *CommandHook) String() string {
+	return h.command
+}
+
+func (h *CommandHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
+	l := getLogger(ctx).WithField("command", h.command)
+
+	cmdCtx, cancel := context.WithTimeout(ctx, h.timeout)
+	defer cancel()
+
+	cmdExec := exec.CommandContext(cmdCtx, h.command)
+
+	hookEnv := NewHookEnv(edge, phase, dryRun, h.timeout, extra)
+	cmdEnv := os.Environ()
+	for k, v := range hookEnv {
+		cmdEnv = append(cmdEnv, fmt.Sprintf("%s=%s", k, v))
+	}
+	cmdExec.Env = cmdEnv
+
+	var scanMutex sync.Mutex
+	combinedOutput, err := circlog.NewCircularLog(envconst.Int("ZREPL_MAX_HOOK_LOG_SIZE", MAX_HOOK_LOG_SIZE_DEFAULT))
+	if err != nil {
+		return &CommandHookReport{Err: err}
+	}
+	logErrWriter := NewLogWriter(&scanMutex, l, logger.Warn, "stderr")
+	logOutWriter := NewLogWriter(&scanMutex, l, logger.Info, "stdout")
+	defer logErrWriter.Close()
+	defer logOutWriter.Close()
+
+	cmdExec.Stderr = io.MultiWriter(logErrWriter, combinedOutput)
+	cmdExec.Stdout = io.MultiWriter(logOutWriter, combinedOutput)
+
+	report := &CommandHookReport{
+		Command: h.command,
+		Env:     hookEnv,
+		// no report.Args
+	}
+
+	err = cmdExec.Start()
+	if err != nil {
+		report.Err = err
+		return report
+	}
+
+	err = cmdExec.Wait()
+	combinedOutputBytes := combinedOutput.Bytes()
+	report.CapturedStdoutStderrCombined = make([]byte, len(combinedOutputBytes))
+	copy(report.CapturedStdoutStderrCombined, combinedOutputBytes)
+	if err != nil {
+		if cmdCtx.Err() == context.DeadlineExceeded {
+			report.Err = fmt.Errorf("timed out after %s: %s", h.timeout, err)
+			return report
+		}
+		report.Err = err
+		return report
+	}
+
+	return report
+}

daemon/hooks/hook_type_mysql_lock_tables.go

@@ -0,0 +1,145 @@
+package hooks
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strings"
+
+	sqldriver "database/sql/driver"
+
+	"github.com/go-sql-driver/mysql"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+// https://dev.mysql.com/doc/mysql-backup-excerpt/5.7/en/backup-methods.html
+//
+// Making Backups Using a File System Snapshot:
+//
+//  If you are using a Veritas file system, you can make a backup like this:
+//
+//    	From a client program, execute FLUSH TABLES WITH READ LOCK.
+//    	From another shell, execute mount vxfs snapshot.
+//    	From the first client, execute UNLOCK TABLES.
+//    	Copy files from the snapshot.
+//    	Unmount the snapshot.
+//
+//	Similar snapshot capabilities may be available in other file systems, such as LVM or ZFS.
+type MySQLLockTables struct {
+	errIsFatal  bool
+	connector   sqldriver.Connector
+	filesystems Filter
+}
+
+type myLockTablesStateKey int
+
+const (
+	myLockTablesConnection myLockTablesStateKey = 1 + iota
+)
+
+func MyLockTablesFromConfig(in *config.HookMySQLLockTables) (*MySQLLockTables, error) {
+	conf, err := mysql.ParseDSN(in.DSN)
+	if err != nil {
+		return nil, errors.Wrap(err, "`dsn` invalid")
+	}
+	cn, err := mysql.NewConnector(conf)
+	if err != nil {
+		return nil, errors.Wrap(err, "`connect` invalid")
+	}
+
+	filesystems, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
+	if err != nil {
+		return nil, errors.Wrap(err, "`filesystems` invalid")
+	}
+
+	return &MySQLLockTables{
+		in.ErrIsFatal,
+		cn,
+		filesystems,
+	}, nil
+}
+
+func (h *MySQLLockTables) ErrIsFatal() bool    { return h.errIsFatal }
+func (h *MySQLLockTables) Filesystems() Filter { return h.filesystems }
+func (h *MySQLLockTables) String() string      { return "MySQL FLUSH TABLES WITH READ LOCK" }
+
+type MyLockTablesReport struct {
+	What string
+	Err  error
+}
+
+func (r *MyLockTablesReport) HadError() bool { return r.Err != nil }
+func (r *MyLockTablesReport) Error() string  { return r.String() }
+func (r *MyLockTablesReport) String() string {
+	var s strings.Builder
+	s.WriteString(r.What)
+	if r.Err != nil {
+		fmt.Fprintf(&s, ": %s", r.Err)
+	}
+	return s.String()
+}
+
+func (h *MySQLLockTables) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
+	fs, ok := extra[EnvFS]
+	if !ok {
+		panic(extra)
+	}
+	dp, err := zfs.NewDatasetPath(fs)
+	if err != nil {
+		panic(err)
+	}
+	if pass, err := h.filesystems.Filter(dp); err != nil {
+		return &MyLockTablesReport{What: "filesystem filter", Err: err}
+	} else if !pass {
+		getLogger(ctx).Debug("filesystem does not match filter, skipping")
+		return &MyLockTablesReport{What: "filesystem filter skipped this filesystem", Err: nil}
+	}
+
+	switch edge {
+	case Pre:
+		err := h.doRunPre(ctx, dp, dryRun, state)
+		return &MyLockTablesReport{"FLUSH TABLES WITH READ LOCK", err}
+	case Post:
+		err := h.doRunPost(ctx, dp, dryRun, state)
+		return &MyLockTablesReport{"UNLOCK TABLES", err}
+	}
+	return &MyLockTablesReport{What: "skipped this edge", Err: nil}
+}
+
+func (h *MySQLLockTables) doRunPre(ctx context.Context, fs *zfs.DatasetPath, dry bool, state map[interface{}]interface{}) (err error) {
+	db := sql.OpenDB(h.connector)
+	defer func(err *error) {
+		if *err != nil {
+			db.Close()
+		}
+	}(&err)
+
+	getLogger(ctx).Debug("do FLUSH TABLES WITH READ LOCK")
+	_, err = db.ExecContext(ctx, "FLUSH TABLES WITH READ LOCK")
+	if err != nil {
+		return
+	}
+
+	state[myLockTablesConnection] = db
+
+	return nil
+}
+
+func (h *MySQLLockTables) doRunPost(ctx context.Context, fs *zfs.DatasetPath, dry bool, state map[interface{}]interface{}) error {
+
+	db := state[myLockTablesConnection].(*sql.DB)
+	defer db.Close()
+
+	getLogger(ctx).Debug("do UNLOCK TABLES")
+	_, err := db.ExecContext(ctx, "UNLOCK TABLES")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

daemon/hooks/hook_type_postgres_checkpoint.go

@@ -0,0 +1,98 @@
+package hooks
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"math"
+	"time"
+
+	"github.com/lib/pq"
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type PgChkptHook struct {
+	errIsFatal  bool
+	connector   *pq.Connector
+	filesystems Filter
+}
+
+func PgChkptHookFromConfig(in *config.HookPostgresCheckpoint) (*PgChkptHook, error) {
+	filesystems, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
+	if err != nil {
+		return nil, errors.Wrap(err, "`filesystems` invalid")
+	}
+	cn, err := pq.NewConnector(in.DSN)
+	if err != nil {
+		return nil, errors.Wrap(err, "`dsn` invalid")
+	}
+
+	return &PgChkptHook{
+		in.ErrIsFatal,
+		cn,
+		filesystems,
+	}, nil
+}
+
+func (h *PgChkptHook) ErrIsFatal() bool    { return h.errIsFatal }
+func (h *PgChkptHook) Filesystems() Filter { return h.filesystems }
+func (h *PgChkptHook) String() string      { return "postgres checkpoint" }
+
+type PgChkptHookReport struct{ Err error }
+
+func (r *PgChkptHookReport) HadError() bool { return r.Err != nil }
+func (r *PgChkptHookReport) Error() string  { return r.Err.Error() }
+func (r *PgChkptHookReport) String() string {
+	if r.Err != nil {
+		return fmt.Sprintf("postgres CHECKPOINT failed: %s", r.Err)
+	} else {
+		return "postgres CHECKPOINT completed"
+	}
+}
+
+func (h *PgChkptHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
+	if edge != Pre {
+		return &PgChkptHookReport{nil}
+	}
+	fs, ok := extra[EnvFS]
+	if !ok {
+		panic(extra)
+	}
+	dp, err := zfs.NewDatasetPath(fs)
+	if err != nil {
+		panic(err)
+	}
+	err = h.doRunPre(ctx, dp, dryRun)
+	return &PgChkptHookReport{err}
+}
+
+func (h *PgChkptHook) doRunPre(ctx context.Context, fs *zfs.DatasetPath, dry bool) error {
+
+	if pass, err := h.filesystems.Filter(fs); err != nil || !pass {
+		getLogger(ctx).Debug("filesystem does not match filter, skipping")
+		return err
+	}
+
+	db := sql.OpenDB(h.connector)
+	defer db.Close()
+	dl, ok := ctx.Deadline()
+	if ok {
+		timeout := uint64(math.Floor(time.Until(dl).Seconds() * 1000)) // TODO go1.13 milliseconds
+		getLogger(ctx).WithField("statement_timeout", timeout).Debug("setting statement timeout for CHECKPOINT")
+		_, err := db.ExecContext(ctx, "SET statement_timeout TO ?", timeout)
+		if err != nil {
+			return err
+		}
+	}
+	if dry {
+		getLogger(ctx).Debug("dry-run - use ping instead of CHECKPOINT")
+		return db.PingContext(ctx)
+	}
+	getLogger(ctx).Info("execute CHECKPOINT command")
+	_, err := db.ExecContext(ctx, "CHECKPOINT")
+	return err
+}

daemon/hooks/hooks_test.go

@@ -0,0 +1,488 @@
+package hooks_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"regexp"
+	"testing"
+	"text/template"
+
+	"github.com/stretchr/testify/require"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/hooks"
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type comparisonAssertionFunc func(require.TestingT, interface{}, interface{}, ...interface{})
+type valueAssertionFunc func(require.TestingT, interface{}, ...interface{})
+
+type expectStep struct {
+	ExpectedEdge hooks.Edge
+	ExpectStatus hooks.StepStatus
+	OutputTest   valueAssertionFunc
+	ErrorTest    valueAssertionFunc
+}
+
+type testCase struct {
+	Name           string
+	Config         []string
+	IsSlow         bool
+	SuppressOutput bool
+
+	ExpectCallbackSkipped bool
+	ExpectHadFatalErr     bool
+	ExpectHadError        bool
+	ExpectStepReports     []expectStep
+}
+
+func curryLeft(f comparisonAssertionFunc, expected interface{}) valueAssertionFunc {
+	return curry(f, expected, false)
+}
+
+func curryRight(f comparisonAssertionFunc, expected interface{}) valueAssertionFunc {
+	return curry(f, expected, true)
+}
+
+func curry(f comparisonAssertionFunc, expected interface{}, right bool) (ret valueAssertionFunc) {
+	ret = func(t require.TestingT, s interface{}, v ...interface{}) {
+		var x interface{}
+		var y interface{}
+		if right {
+			x = s
+			y = expected
+		} else {
+			x = expected
+			y = s
+		}
+
+		if len(v) > 0 {
+			f(t, x, y, v)
+		} else {
+			f(t, x, y)
+		}
+	}
+	return
+}
+
+func TestHooks(t *testing.T) {
+	ctx, end := trace.WithTaskFromStack(context.Background())
+	defer end()
+
+	testFSName := "testpool/testdataset"
+	testSnapshotName := "testsnap"
+
+	tmpl, err := template.New("TestHooks").Parse(`
+jobs:
+- name: TestHooks
+  type: snap
+  filesystems: {"<": true}
+  snapshotting:
+    type: periodic
+    interval: 1m
+    prefix: zrepl_snapjob_
+    hooks:
+    {{- template "List" . }}
+  pruning:
+    keep:
+      - type: last_n
+        count: 10
+`)
+	if err != nil {
+		panic(err)
+	}
+
+	regexpTest := func(s string) valueAssertionFunc {
+		return curryLeft(require.Regexp, regexp.MustCompile(s))
+	}
+
+	containsTest := func(s string) valueAssertionFunc {
+		return curryRight(require.Contains, s)
+	}
+
+	testTable := []testCase{
+		testCase{
+			Name: "no_hooks",
+			ExpectStepReports: []expectStep{
+				{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+			},
+		},
+		testCase{
+			Name:           "timeout",
+			IsSlow:         true,
+			ExpectHadError: true,
+			Config:         []string{`{type: command, path: {{.WorkDir}}/test/test-timeout.sh, timeout: 2s}`},
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepErr,
+					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s ZREPL_TIMEOUT=2", testFSName, testSnapshotName)),
+					ErrorTest:    regexpTest(`timed out after 2(.\d+)?s`),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepSkippedDueToPreErr,
+				},
+			},
+		},
+		testCase{
+			Name:           "check_env",
+			Config:         []string{`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`},
+			ExpectHadError: false,
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
+				},
+			},
+		},
+
+		testCase{
+			Name:                  "nonfatal_pre_error_continues",
+			ExpectCallbackSkipped: false,
+			ExpectHadError:        true,
+			Config:                []string{`{type: command, path: {{.WorkDir}}/test/test-error.sh}`},
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepErr,
+					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
+					ErrorTest:    regexpTest("^command hook invocation.*exit status 1$"),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepSkippedDueToPreErr, // post-edge is not executed for failing pre-edge
+				},
+			},
+		},
+
+		testCase{
+			Name:                  "pre_error_fatal_skips_subsequent_pre_edges_and_callback_and_its_post_edge_and_post_edges",
+			ExpectCallbackSkipped: true,
+			ExpectHadFatalErr:     true,
+			ExpectHadError:        true,
+			Config: []string{
+				`{type: command, path: {{.WorkDir}}/test/test-error.sh, err_is_fatal: true}`,
+				`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`,
+			},
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepErr,
+					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
+					ErrorTest:    regexpTest("^command hook invocation.*exit status 1$"),
+				},
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepSkippedDueToFatalErr,
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepSkippedDueToFatalErr},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepSkippedDueToFatalErr,
+				},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepSkippedDueToFatalErr,
+				},
+			},
+		},
+
+		testCase{
+			Name:              "post_error_fails_are_ignored_even_if_fatal",
+			ExpectHadFatalErr: false, // only occurs during Post, so it's not a fatal error
+			ExpectHadError:    true,
+			Config: []string{
+				`{type: command, path: {{.WorkDir}}/test/test-post-error.sh, err_is_fatal: true}`,
+				`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`,
+			},
+			ExpectStepReports: []expectStep{
+				expectStep{
+					// No-action run of test-post-error.sh
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   require.Empty,
+				},
+				expectStep{
+					// Pre run of test-report-env.sh
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
+				},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepErr,
+					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR post_testing %s@%s", testFSName, testSnapshotName)),
+					ErrorTest:    regexpTest("^command hook invocation.*exit status 1$"),
+				},
+			},
+		},
+
+		testCase{
+			Name:   "cleanup_check_env",
+			Config: []string{`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`},
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
+				},
+			},
+		},
+
+		testCase{
+			Name:              "pre_error_cancels_post",
+			Config:            []string{`{type: command, path: {{.WorkDir}}/test/test-pre-error-post-ok.sh}`},
+			ExpectHadError:    true,
+			ExpectHadFatalErr: false,
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepErr,
+					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
+					ErrorTest:    regexpTest("^command hook invocation.*exit status 1$"),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepSkippedDueToPreErr,
+				},
+			},
+		},
+
+		testCase{
+			Name: "pre_error_does_not_cancel_other_posts_but_itself",
+			Config: []string{
+				`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`,
+				`{type: command, path: {{.WorkDir}}/test/test-pre-error-post-ok.sh}`,
+			},
+			ExpectHadError:    true,
+			ExpectHadFatalErr: false,
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
+				},
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepErr,
+					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
+					ErrorTest:    regexpTest("^command hook invocation.*exit status 1$"),
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepSkippedDueToPreErr,
+				},
+				expectStep{
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
+				},
+			},
+		},
+
+		testCase{
+			Name:              "exceed_buffer_limit",
+			SuppressOutput:    true,
+			Config:            []string{`{type: command, path: {{.WorkDir}}/test/test-large-stdout.sh}`},
+			ExpectHadError:    false,
+			ExpectHadFatalErr: false,
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Pre,
+					ExpectStatus: hooks.StepOk,
+					OutputTest: func(t require.TestingT, s interface{}, v ...interface{}) {
+						require.Len(t, s, 1<<20)
+					},
+				},
+				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
+				expectStep{
+					// No-action run of above hook
+					ExpectedEdge: hooks.Post,
+					ExpectStatus: hooks.StepOk,
+					OutputTest:   require.Empty,
+				},
+			},
+		},
+
+		/*
+			Following not intended to test functionality of
+			filter package. Only to demonstrate that hook
+			filters are being applied. The following should
+			result in NO hooks running. If it does run, a
+			fatal hooks.RunReport will be returned.
+		*/
+		testCase{
+			Name: "exclude_all_filesystems",
+			Config: []string{
+				`{type: command, path: {{.WorkDir}}/test/test-error.sh, err_is_fatal: true, filesystems: {"<": false}}`,
+			},
+			ExpectStepReports: []expectStep{
+				expectStep{
+					ExpectedEdge: hooks.Callback,
+					ExpectStatus: hooks.StepOk,
+				},
+			},
+		},
+	}
+
+	parseHookConfig := func(t *testing.T, in string) *config.Config {
+		t.Helper()
+		conf, err := config.ParseConfigBytes([]byte(in))
+		require.NoError(t, err)
+		require.NotNil(t, conf)
+		return conf
+	}
+
+	fillHooks := func(tt *testCase) string {
+		// make hook path absolute
+		cwd, err := os.Getwd()
+		if err != nil {
+			panic("os.Getwd() failed")
+		}
+		var hooksTmpl string = "\n"
+		for _, l := range tt.Config {
+			hooksTmpl += fmt.Sprintf("    - %s\n", l)
+		}
+		tmpl.New("List").Parse(hooksTmpl)
+
+		var outBytes bytes.Buffer
+		data := struct {
+			WorkDir string
+		}{
+			WorkDir: cwd,
+		}
+		if err := tmpl.Execute(&outBytes, data); err != nil {
+			panic(err)
+		}
+
+		return outBytes.String()
+	}
+
+	var c *config.Config
+	fs, err := zfs.NewDatasetPath(testFSName)
+	require.NoError(t, err)
+
+	log := logger.NewTestLogger(t)
+
+	var cbReached bool
+	cb := hooks.NewCallbackHookForFilesystem("testcallback", fs, func(_ context.Context) error {
+		cbReached = true
+		return nil
+	})
+
+	hookEnvExtra := hooks.Env{
+		hooks.EnvFS:       fs.ToString(),
+		hooks.EnvSnapshot: testSnapshotName,
+	}
+
+	for _, tt := range testTable {
+		if testing.Short() && tt.IsSlow {
+			continue
+		}
+
+		t.Run(tt.Name, func(t *testing.T) {
+			c = parseHookConfig(t, fillHooks(&tt))
+			snp := c.Jobs[0].Ret.(*config.SnapJob).Snapshotting.Ret.(*config.SnapshottingPeriodic)
+			hookList, err := hooks.ListFromConfig(&snp.Hooks)
+			require.NoError(t, err)
+
+			filteredHooks, err := hookList.CopyFilteredForFilesystem(fs)
+			require.NoError(t, err)
+			plan, err := hooks.NewPlan(&filteredHooks, hooks.PhaseTesting, cb, hookEnvExtra)
+			require.NoError(t, err)
+			t.Logf("REPORT PRE EXECUTION:\n%s", plan.Report())
+
+			cbReached = false
+
+			if testing.Verbose() && !tt.SuppressOutput {
+				ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
+			}
+			plan.Run(ctx, false)
+			report := plan.Report()
+
+			t.Logf("REPORT POST EXECUTION:\n%s", report)
+
+			/*
+			 * TEST ASSERTIONS
+			 */
+
+			t.Logf("len(runReports)=%v", len(report))
+			t.Logf("len(tt.ExpectStepReports)=%v", len(tt.ExpectStepReports))
+			require.Equal(t, len(tt.ExpectStepReports), len(report), "ExpectStepReports must be same length as expected number of hook runs, excluding possible Callback")
+
+			// Check if callback ran, when required
+			if tt.ExpectCallbackSkipped {
+				require.False(t, cbReached, "callback ran but should not have run")
+			} else {
+				require.True(t, cbReached, "callback should have run but did not")
+			}
+
+			// Check if a fatal run error occurred and was expected
+			require.Equal(t, tt.ExpectHadFatalErr, report.HadFatalError(), "non-matching HadFatalError")
+			require.Equal(t, tt.ExpectHadError, report.HadError(), "non-matching HadError")
+
+			if tt.ExpectHadFatalErr {
+				require.True(t, tt.ExpectHadError, "ExpectHadFatalErr implies ExpectHadError")
+			}
+			if !tt.ExpectHadError {
+				require.False(t, tt.ExpectHadFatalErr, "!ExpectHadError implies !ExpectHadFatalErr")
+			}
+
+			// Iterate through each expected hook run
+			for i, hook := range tt.ExpectStepReports {
+				t.Logf("expecting report conforming to %v", hook)
+
+				exp, act := hook.ExpectStatus, report[i].Status
+				require.Equal(t, exp, act, "%s != %s", exp, act)
+
+				// Check for required ExpectedEdge
+				require.NotZero(t, hook.ExpectedEdge, "each hook must have an ExpectedEdge")
+				require.Equal(t, hook.ExpectedEdge, report[i].Edge,
+					"incorrect edge: expected %q, actual %q", hook.ExpectedEdge.String(), report[i].Edge.String(),
+				)
+
+				// Check for expected output
+				if hook.OutputTest != nil {
+					require.IsType(t, (*hooks.CommandHookReport)(nil), report[i].Report)
+					chr := report[i].Report.(*hooks.CommandHookReport)
+					hook.OutputTest(t, string(chr.CapturedStdoutStderrCombined))
+				}
+
+				// Check for expected errors
+				if hook.ErrorTest != nil {
+					hook.ErrorTest(t, string(report[i].Report.Error()))
+				}
+			}
+
+		})
+	}
+}

daemon/hooks/stepstatus_enumer.go

@@ -0,0 +1,77 @@
+// Code generated by "enumer -type=StepStatus -trimprefix=Step"; DO NOT EDIT.
+
+//
+package hooks
+
+import (
+	"fmt"
+)
+
+const (
+	_StepStatusName_0 = "PendingExec"
+	_StepStatusName_1 = "Ok"
+	_StepStatusName_2 = "Err"
+	_StepStatusName_3 = "SkippedDueToFatalErr"
+	_StepStatusName_4 = "SkippedDueToPreErr"
+)
+
+var (
+	_StepStatusIndex_0 = [...]uint8{0, 7, 11}
+	_StepStatusIndex_1 = [...]uint8{0, 2}
+	_StepStatusIndex_2 = [...]uint8{0, 3}
+	_StepStatusIndex_3 = [...]uint8{0, 20}
+	_StepStatusIndex_4 = [...]uint8{0, 18}
+)
+
+func (i StepStatus) String() string {
+	switch {
+	case 1 <= i && i <= 2:
+		i -= 1
+		return _StepStatusName_0[_StepStatusIndex_0[i]:_StepStatusIndex_0[i+1]]
+	case i == 4:
+		return _StepStatusName_1
+	case i == 8:
+		return _StepStatusName_2
+	case i == 16:
+		return _StepStatusName_3
+	case i == 32:
+		return _StepStatusName_4
+	default:
+		return fmt.Sprintf("StepStatus(%d)", i)
+	}
+}
+
+var _StepStatusValues = []StepStatus{1, 2, 4, 8, 16, 32}
+
+var _StepStatusNameToValueMap = map[string]StepStatus{
+	_StepStatusName_0[0:7]:  1,
+	_StepStatusName_0[7:11]: 2,
+	_StepStatusName_1[0:2]:  4,
+	_StepStatusName_2[0:3]:  8,
+	_StepStatusName_3[0:20]: 16,
+	_StepStatusName_4[0:18]: 32,
+}
+
+// StepStatusString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func StepStatusString(s string) (StepStatus, error) {
+	if val, ok := _StepStatusNameToValueMap[s]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to StepStatus values", s)
+}
+
+// StepStatusValues returns all values of the enum
+func StepStatusValues() []StepStatus {
+	return _StepStatusValues
+}
+
+// IsAStepStatus returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i StepStatus) IsAStepStatus() bool {
+	for _, v := range _StepStatusValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}

daemon/hooks/test/test-error.sh

@@ -0,0 +1,5 @@
+#!/bin/sh -eu
+
+>&2 echo "TEST ERROR $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
+
+exit 1

daemon/hooks/test/test-large-stdout.sh

@@ -0,0 +1,18 @@
+#!/bin/sh -eu
+
+# Exceed default hook log size of 1<<20 bytes by double
+# The Go test should fail if buffer size exceeds 1MB
+
+pre_testing() {
+	ln=0
+	while :; do printf '%06d: 012345678901234567890123456789012345678901234567890123456789012345678901\n' "$ln"; ln="$(($ln + 1))"; done \
+	| head -c$(( 2 << 20 ))
+}
+
+case "$ZREPL_HOOKTYPE" in
+	pre_testing)
+		"$ZREPL_HOOKTYPE";;
+	*)
+		# Not handled by this script
+		exit 0;;
+esac

daemon/hooks/test/test-post-error.sh

@@ -0,0 +1,15 @@
+#!/bin/sh -eu
+
+post_testing() {
+	>&2 echo "TEST ERROR $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
+
+	exit 1
+}
+
+case "$ZREPL_HOOKTYPE" in
+	post_testing)
+		"$ZREPL_HOOKTYPE";;
+	*)
+		# Not handled by this script
+		exit 0;;
+esac

daemon/hooks/test/test-pre-error-post-ok.sh

@@ -0,0 +1,11 @@
+#!/bin/sh -eu
+
+if [ "$ZREPL_HOOKTYPE" = "pre_testing" ]; then
+	>&2 echo "TEST ERROR $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
+	exit 1
+elif [ "$ZREPL_HOOKTYPE" = "post_testing" ]; then
+	echo "TEST $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
+else
+	printf "Unknown hook type: %s" "$ZREPL_HOOKTYPE" 
+	exit 255
+fi

daemon/hooks/test/test-report-env.sh

@@ -0,0 +1,3 @@
+#!/bin/sh -eu
+
+echo "TEST $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME ${ZREPL_DRYRUN:+(dry run)}"

daemon/hooks/test/test-timeout.sh

@@ -0,0 +1,5 @@
+#!/bin/sh -eu
+
+echo "TEST $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME ZREPL_TIMEOUT=$ZREPL_TIMEOUT"
+
+exec sleep $(($ZREPL_TIMEOUT + 1))

daemon/job/active.go

@@ -2,42 +2,47 @@ package job
 
 import (
 	"context"
+	"fmt"
+	"sync"
+	"time"
+
 	"github.com/pkg/errors"
-	"github.com/problame/go-streamrpc"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/log"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/filters"
 	"github.com/zrepl/zrepl/daemon/job/reset"
 	"github.com/zrepl/zrepl/daemon/job/wakeup"
-	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/daemon/pruner"
 	"github.com/zrepl/zrepl/daemon/snapper"
-	"github.com/zrepl/zrepl/daemon/transport/connecter"
 	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/replication"
-	"github.com/zrepl/zrepl/util/envconst"
+	"github.com/zrepl/zrepl/replication/driver"
+	"github.com/zrepl/zrepl/replication/logic"
+	"github.com/zrepl/zrepl/replication/report"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/transport"
+	"github.com/zrepl/zrepl/transport/fromconfig"
 	"github.com/zrepl/zrepl/zfs"
-	"sync"
-	"time"
 )
 
 type ActiveSide struct {
-	mode          activeMode
-	name          string
-	clientFactory *connecter.ClientFactory
+	mode      activeMode
+	name      endpoint.JobID
+	connecter transport.Connecter
 
 	prunerFactory *pruner.PrunerFactory
 
-
-	promRepStateSecs *prometheus.HistogramVec // labels: state
-	promPruneSecs *prometheus.HistogramVec // labels: prune_side
-	promBytesReplicated *prometheus.CounterVec // labels: filesystem
+	promRepStateSecs    *prometheus.HistogramVec // labels: state
+	promPruneSecs       *prometheus.HistogramVec // labels: prune_side
+	promBytesReplicated *prometheus.CounterVec   // labels: filesystem
 
 	tasksMtx sync.Mutex
 	tasks    activeSideTasks
 }
 
-
 //go:generate enumer -type=ActiveSideState
 type ActiveSideState int
 
@@ -48,12 +53,11 @@ const (
 	ActiveSideDone // also errors
 )
 
-
 type activeSideTasks struct {
 	state ActiveSideState
 
 	// valid for state ActiveSideReplicating, ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
-	replication *replication.Replication
+	replicationReport driver.ReportFunc
 	replicationCancel context.CancelFunc
 
 	// valid for state ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
@@ -66,8 +70,7 @@ type activeSideTasks struct {
 func (a *ActiveSide) updateTasks(u func(*activeSideTasks)) activeSideTasks {
 	a.tasksMtx.Lock()
 	defer a.tasksMtx.Unlock()
-	var copy activeSideTasks
-	copy = a.tasks
+	copy := a.tasks
 	if u == nil {
 		return copy
 	}
@@ -77,36 +80,86 @@ func (a *ActiveSide) updateTasks(u func(*activeSideTasks)) activeSideTasks {
 }
 
 type activeMode interface {
-	SenderReceiver(client *streamrpc.Client) (replication.Sender, replication.Receiver, error)
+	ConnectEndpoints(ctx context.Context, connecter transport.Connecter)
+	DisconnectEndpoints()
+	SenderReceiver() (logic.Sender, logic.Receiver)
 	Type() Type
+	PlannerPolicy() logic.PlannerPolicy
 	RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{})
+	SnapperReport() *snapper.Report
+	ResetConnectBackoff()
 }
 
 type modePush struct {
-	fsfilter         endpoint.FSFilter
-	snapper *snapper.PeriodicOrManual
+	setupMtx      sync.Mutex
+	sender        *endpoint.Sender
+	receiver      *rpc.Client
+	senderConfig  *endpoint.SenderConfig
+	plannerPolicy *logic.PlannerPolicy
+	snapper       *snapper.PeriodicOrManual
 }
 
-func (m *modePush) SenderReceiver(client *streamrpc.Client) (replication.Sender, replication.Receiver, error) {
-	sender := endpoint.NewSender(m.fsfilter)
-	receiver := endpoint.NewRemote(client)
-	return sender, receiver, nil
+func (m *modePush) ConnectEndpoints(ctx context.Context, connecter transport.Connecter) {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	if m.receiver != nil || m.sender != nil {
+		panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
+	}
+	m.sender = endpoint.NewSender(*m.senderConfig)
+	m.receiver = rpc.NewClient(connecter, rpc.GetLoggersOrPanic(ctx))
+}
+
+func (m *modePush) DisconnectEndpoints() {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	m.receiver.Close()
+	m.sender = nil
+	m.receiver = nil
+}
+
+func (m *modePush) SenderReceiver() (logic.Sender, logic.Receiver) {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	return m.sender, m.receiver
 }
 
 func (m *modePush) Type() Type { return TypePush }
 
-func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan <- struct{}) {
+func (m *modePush) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
+
+func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
 	m.snapper.Run(ctx, wakeUpCommon)
 }
 
+func (m *modePush) SnapperReport() *snapper.Report {
+	return m.snapper.Report()
+}
 
-func modePushFromConfig(g *config.Global, in *config.PushJob) (*modePush, error) {
+func (m *modePush) ResetConnectBackoff() {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	if m.receiver != nil {
+		m.receiver.ResetConnectBackoff()
+	}
+}
+
+func modePushFromConfig(g *config.Global, in *config.PushJob, jobID endpoint.JobID) (*modePush, error) {
 	m := &modePush{}
+
 	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
 	if err != nil {
-		return nil, errors.Wrap(err, "cannnot build filesystem filter")
+		return nil, errors.Wrap(err, "cannot build filesystem filter")
+	}
+
+	m.senderConfig = &endpoint.SenderConfig{
+		FSF:                         fsf,
+		Encrypt:                     &zfs.NilBool{B: in.Send.Encrypted},
+		DisableIncrementalStepHolds: in.Send.StepHolds.DisableIncremental,
+		JobID:                       jobID,
+	}
+	m.plannerPolicy = &logic.PlannerPolicy{
+		EncryptedSend: logic.TriFromBool(in.Send.Encrypted),
 	}
-	m.fsfilter = fsf
 
 	if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
 		return nil, errors.Wrap(err, "cannot build snapper")
@@ -116,20 +169,50 @@ func modePushFromConfig(g *config.Global, in *config.PushJob) (*modePush, error)
 }
 
 type modePull struct {
-	rootFS   *zfs.DatasetPath
-	interval time.Duration
+	setupMtx       sync.Mutex
+	receiver       *endpoint.Receiver
+	receiverConfig endpoint.ReceiverConfig
+	sender         *rpc.Client
+	rootFS         *zfs.DatasetPath
+	plannerPolicy  *logic.PlannerPolicy
+	interval       config.PositiveDurationOrManual
 }
 
-func (m *modePull) SenderReceiver(client *streamrpc.Client) (replication.Sender, replication.Receiver, error) {
-	sender := endpoint.NewRemote(client)
-	receiver, err := endpoint.NewReceiver(m.rootFS)
-	return sender, receiver, err
+func (m *modePull) ConnectEndpoints(ctx context.Context, connecter transport.Connecter) {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	if m.receiver != nil || m.sender != nil {
+		panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
+	}
+	m.receiver = endpoint.NewReceiver(m.receiverConfig)
+	m.sender = rpc.NewClient(connecter, rpc.GetLoggersOrPanic(ctx))
+}
+
+func (m *modePull) DisconnectEndpoints() {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	m.sender.Close()
+	m.sender = nil
+	m.receiver = nil
+}
+
+func (m *modePull) SenderReceiver() (logic.Sender, logic.Receiver) {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	return m.sender, m.receiver
 }
 
 func (*modePull) Type() Type { return TypePull }
 
+func (m *modePull) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
+
 func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
-	t := time.NewTicker(m.interval)
+	if m.interval.Manual {
+		GetLogger(ctx).Info("manual pull configured, periodic pull disabled")
+		// "waiting for wakeups" is printed in common ActiveSide.do
+		return
+	}
+	t := time.NewTicker(m.interval.Interval)
 	defer t.Stop()
 	for {
 		select {
@@ -148,11 +231,20 @@ func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}
 	}
 }
 
-func modePullFromConfig(g *config.Global, in *config.PullJob) (m *modePull, err error) {
-	m = &modePull{}
-	if in.Interval <= 0 {
-		return nil, errors.New("interval must be positive")
+func (m *modePull) SnapperReport() *snapper.Report {
+	return nil
+}
+
+func (m *modePull) ResetConnectBackoff() {
+	m.setupMtx.Lock()
+	defer m.setupMtx.Unlock()
+	if m.sender != nil {
+		m.sender.ResetConnectBackoff()
 	}
+}
+
+func modePullFromConfig(g *config.Global, in *config.PullJob, jobID endpoint.JobID) (m *modePull, err error) {
+	m = &modePull{}
 	m.interval = in.Interval
 
 	m.rootFS, err = zfs.NewDatasetPath(in.RootFS)
@@ -163,29 +255,59 @@ func modePullFromConfig(g *config.Global, in *config.PullJob) (m *modePull, err
 		return nil, errors.New("RootFS must not be empty") // duplicates error check of receiver
 	}
 
+	m.plannerPolicy = &logic.PlannerPolicy{
+		EncryptedSend: logic.DontCare,
+	}
+
+	m.receiverConfig = endpoint.ReceiverConfig{
+		JobID:                      jobID,
+		RootWithoutClientComponent: m.rootFS,
+		AppendClientIdentity:       false, // !
+		UpdateLastReceivedHold:     true,
+	}
+	if err := m.receiverConfig.Validate(); err != nil {
+		return nil, errors.Wrap(err, "cannot build receiver config")
+	}
+
 	return m, nil
 }
 
-func activeSide(g *config.Global, in *config.ActiveJob, mode activeMode) (j *ActiveSide, err error) {
+func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}) (j *ActiveSide, err error) {
+
+	j = &ActiveSide{}
+	j.name, err = endpoint.MakeJobID(in.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid job name")
+	}
+
+	switch v := configJob.(type) {
+	case *config.PushJob:
+		j.mode, err = modePushFromConfig(g, v, j.name) // shadow
+	case *config.PullJob:
+		j.mode, err = modePullFromConfig(g, v, j.name) // shadow
+	default:
+		panic(fmt.Sprintf("implementation error: unknown job type %T", v))
+	}
+	if err != nil {
+		return nil, err // no wrapping required
+	}
 
-	j = &ActiveSide{mode: mode}
-	j.name = in.Name
 	j.promRepStateSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace:   "zrepl",
 		Subsystem:   "replication",
 		Name:        "state_time",
 		Help:        "seconds spent during replication",
-		ConstLabels: prometheus.Labels{"zrepl_job":j.name},
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
 	}, []string{"state"})
 	j.promBytesReplicated = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace:   "zrepl",
 		Subsystem:   "replication",
 		Name:        "bytes_replicated",
 		Help:        "number of bytes replicated from sender to receiver per filesystem",
-		ConstLabels: prometheus.Labels{"zrepl_job":j.name},
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
 	}, []string{"filesystem"})
 
-	j.clientFactory, err = connecter.FromConfig(g, in.Connect)
+	j.connecter, err = fromconfig.ConnecterFromConfig(g, in.Connect)
 	if err != nil {
 		return nil, errors.Wrap(err, "cannot build client")
 	}
@@ -195,7 +317,7 @@ func activeSide(g *config.Global, in *config.ActiveJob, mode activeMode) (j *Act
 		Subsystem:   "pruning",
 		Name:        "time",
 		Help:        "seconds spent in pruner",
-		ConstLabels: prometheus.Labels{"zrepl_job":j.name},
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
 	}, []string{"prune_side"})
 	j.prunerFactory, err = pruner.NewPrunerFactory(in.Pruning, j.promPruneSecs)
 	if err != nil {
@@ -211,11 +333,12 @@ func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) {
 	registerer.MustRegister(j.promBytesReplicated)
 }
 
-func (j *ActiveSide) Name() string { return j.name }
+func (j *ActiveSide) Name() string { return j.name.String() }
 
 type ActiveSideStatus struct {
-	Replication *replication.Report
+	Replication                    *report.Report
 	PruningSender, PruningReceiver *pruner.Report
+	Snapshotting                   *snapper.Report
 }
 
 func (j *ActiveSide) Status() *Status {
@@ -223,8 +346,8 @@ func (j *ActiveSide) Status() *Status {
 
 	s := &ActiveSideStatus{}
 	t := j.mode.Type()
-	if tasks.replication != nil {
-		s.Replication = tasks.replication.Report()
+	if tasks.replicationReport != nil {
+		s.Replication = tasks.replicationReport()
 	}
 	if tasks.prunerSender != nil {
 		s.PruningSender = tasks.prunerSender.Report()
@@ -232,19 +355,53 @@ func (j *ActiveSide) Status() *Status {
 	if tasks.prunerReceiver != nil {
 		s.PruningReceiver = tasks.prunerReceiver.Report()
 	}
+	s.Snapshotting = j.mode.SnapperReport()
 	return &Status{Type: t, JobSpecific: s}
 }
 
+func (j *ActiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
+	pull, ok := j.mode.(*modePull)
+	if !ok {
+		_ = j.mode.(*modePush) // make sure we didn't introduce a new job type
+		return nil, false
+	}
+	return pull.rootFS.Copy(), true
+}
+
+func (j *ActiveSide) SenderConfig() *endpoint.SenderConfig {
+	push, ok := j.mode.(*modePush)
+	if !ok {
+		_ = j.mode.(*modePull) // make sure we didn't introduce a new job type
+		return nil
+	}
+	return push.senderConfig
+}
+
+// The active side of a replication uses one end (sender or receiver)
+// directly by method invocation, without going through a transport that
+// provides a client identity.
+// However, in order to avoid the need to distinguish between direct-method-invocating
+// clients and RPC client, we use an invalid client identity as a sentinel value.
+func FakeActiveSideDirectMethodInvocationClientIdentity(jobId endpoint.JobID) string {
+	return fmt.Sprintf("<local><active><job><client><identity><job=%q>", jobId.String())
+}
+
 func (j *ActiveSide) Run(ctx context.Context) {
+	ctx, endTask := trace.WithTaskAndSpan(ctx, "active-side-job", j.Name())
+	defer endTask()
+
+	ctx = context.WithValue(ctx, endpoint.ClientIdentityKey, FakeActiveSideDirectMethodInvocationClientIdentity(j.name))
+
 	log := GetLogger(ctx)
-	ctx = logging.WithSubsystemLoggers(ctx, log)
 
 	defer log.Info("job exiting")
 
 	periodicDone := make(chan struct{})
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
-	go j.mode.RunPeriodic(ctx, periodicDone)
+	periodicCtx, endTask := trace.WithTask(ctx, "periodic")
+	defer endTask()
+	go j.mode.RunPeriodic(periodicCtx, periodicDone)
 
 	invocationCount := 0
 outer:
@@ -256,18 +413,20 @@ outer:
 			break outer
 
 		case <-wakeup.Wait(ctx):
+			j.mode.ResetConnectBackoff()
 		case <-periodicDone:
 		}
 		invocationCount++
-		invLog := log.WithField("invocation", invocationCount)
-		j.do(WithLogger(ctx, invLog))
+		invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", invocationCount))
+		j.do(invocationCtx)
+		endSpan()
 	}
 }
 
 func (j *ActiveSide) do(ctx context.Context) {
 
-	log := GetLogger(ctx)
-	ctx = logging.WithSubsystemLoggers(ctx, log)
+	j.mode.ConnectEndpoints(ctx, j.connecter)
+	defer j.mode.DisconnectEndpoints()
 
 	// allow cancellation of an invocation (this function)
 	ctx, cancelThisRun := context.WithCancel(ctx)
@@ -281,85 +440,7 @@ func (j *ActiveSide) do(ctx context.Context) {
 		}
 	}()
 
-	// The code after this watchdog goroutine is sequential and transitions the state from
-	//   ActiveSideReplicating -> ActiveSidePruneSender -> ActiveSidePruneReceiver -> ActiveSideDone
-	// If any of those sequential tasks 'gets stuck' (livelock, no progress), the watchdog will eventually
-	// cancel its context.
-	// If the task is written to support context cancellation, it will return immediately (in permanent error state),
-	// and the sequential code above transitions to the next state.
-	go func() {
-
-		wdto := envconst.Duration("ZREPL_JOB_WATCHDOG_TIMEOUT", 10*time.Minute)
-		jitter := envconst.Duration("ZREPL_JOB_WATCHDOG_JITTER", 1*time.Second)
-		// shadowing!
-		log := log.WithField("watchdog_timeout", wdto.String())
-
-		log.Debug("starting watchdog")
-		defer log.Debug("watchdog stopped")
-
-		t := time.NewTicker(wdto)
-		defer t.Stop()
-
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-t.C: // fall
-			}
-
-			j.updateTasks(func(tasks *activeSideTasks) {
-				// Since cancelling a task will cause the sequential code to transition to the next state immediately,
-				// we cannot check for its progress right then (no fallthrough).
-				// Instead, we return (not continue because we are in a closure) and give the new state another
-				// ZREPL_JOB_WATCHDOG_TIMEOUT interval to try make some progress.
-
-				log.WithField("state", tasks.state).Debug("watchdog firing")
-
-				const WATCHDOG_ENVCONST_NOTICE = " (adjust ZREPL_JOB_WATCHDOG_TIMEOUT env variable if inappropriate)"
-
-				switch tasks.state {
-				case ActiveSideReplicating:
-					log.WithField("replication_progress", tasks.replication.Progress.String()).
-						Debug("check replication progress")
-					if tasks.replication.Progress.CheckTimeout(wdto, jitter) {
-						log.Error("replication did not make progress, cancelling" + WATCHDOG_ENVCONST_NOTICE)
-						tasks.replicationCancel()
-						return
-					}
-				case ActiveSidePruneSender:
-					log.WithField("prune_sender_progress", tasks.replication.Progress.String()).
-						Debug("check pruner_sender progress")
-					if tasks.prunerSender.Progress.CheckTimeout(wdto, jitter) {
-						log.Error("pruner_sender did not make progress, cancelling" + WATCHDOG_ENVCONST_NOTICE)
-						tasks.prunerSenderCancel()
-						return
-					}
-				case ActiveSidePruneReceiver:
-					log.WithField("prune_receiver_progress", tasks.replication.Progress.String()).
-						Debug("check pruner_receiver progress")
-					if tasks.prunerReceiver.Progress.CheckTimeout(wdto, jitter) {
-						log.Error("pruner_receiver did not make progress, cancelling" + WATCHDOG_ENVCONST_NOTICE)
-						tasks.prunerReceiverCancel()
-						return
-					}
-				case ActiveSideDone:
-					// ignore, ctx will be Done() in a few milliseconds and the watchdog will exit
-				default:
-					log.WithField("state", tasks.state).
-						Error("watchdog implementation error: unknown active side state")
-				}
-			})
-
-		}
-	}()
-
-	client, err := j.clientFactory.NewClient()
-	if err != nil {
-		log.WithError(err).Error("factory cannot instantiate streamrpc client")
-	}
-	defer client.Close(ctx)
-
-	sender, receiver, err := j.mode.SenderReceiver(client)
+	sender, receiver := j.mode.SenderReceiver()
 
 	{
 		select {
@@ -367,17 +448,22 @@ func (j *ActiveSide) do(ctx context.Context) {
 			return
 		default:
 		}
+		ctx, endSpan := trace.WithSpan(ctx, "replication")
 		ctx, repCancel := context.WithCancel(ctx)
-		tasks := j.updateTasks(func(tasks *activeSideTasks) {
+		var repWait driver.WaitFunc
+		j.updateTasks(func(tasks *activeSideTasks) {
 			// reset it
 			*tasks = activeSideTasks{}
-			tasks.replicationCancel = repCancel
-			tasks.replication = replication.NewReplication(j.promRepStateSecs, j.promBytesReplicated)
+			tasks.replicationCancel = func() { repCancel(); endSpan() }
+			tasks.replicationReport, repWait = replication.Do(
+				ctx, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver, j.mode.PlannerPolicy()),
+			)
 			tasks.state = ActiveSideReplicating
 		})
-		log.Info("start replication")
-		tasks.replication.Drive(ctx, sender, receiver)
-		repCancel() // always cancel to free up context resources
+		GetLogger(ctx).Info("start replication")
+		repWait(true) // wait blocking
+		repCancel()   // always cancel to free up context resources
+		endSpan()
 	}
 
 	{
@@ -386,16 +472,18 @@ func (j *ActiveSide) do(ctx context.Context) {
 			return
 		default:
 		}
+		ctx, endSpan := trace.WithSpan(ctx, "prune_sender")
 		ctx, senderCancel := context.WithCancel(ctx)
 		tasks := j.updateTasks(func(tasks *activeSideTasks) {
 			tasks.prunerSender = j.prunerFactory.BuildSenderPruner(ctx, sender, sender)
-			tasks.prunerSenderCancel = senderCancel
+			tasks.prunerSenderCancel = func() { senderCancel(); endSpan() }
 			tasks.state = ActiveSidePruneSender
 		})
-		log.Info("start pruning sender")
+		GetLogger(ctx).Info("start pruning sender")
 		tasks.prunerSender.Prune()
-		log.Info("finished pruning sender")
+		GetLogger(ctx).Info("finished pruning sender")
 		senderCancel()
+		endSpan()
 	}
 	{
 		select {
@@ -403,16 +491,18 @@ func (j *ActiveSide) do(ctx context.Context) {
 			return
 		default:
 		}
+		ctx, endSpan := trace.WithSpan(ctx, "prune_recever")
 		ctx, receiverCancel := context.WithCancel(ctx)
 		tasks := j.updateTasks(func(tasks *activeSideTasks) {
 			tasks.prunerReceiver = j.prunerFactory.BuildReceiverPruner(ctx, receiver, sender)
-			tasks.prunerReceiverCancel = receiverCancel
+			tasks.prunerReceiverCancel = func() { receiverCancel(); endSpan() }
 			tasks.state = ActiveSidePruneReceiver
 		})
-		log.Info("start pruning receiver")
+		GetLogger(ctx).Info("start pruning receiver")
 		tasks.prunerReceiver.Prune()
-		log.Info("finished pruning receiver")
+		GetLogger(ctx).Info("finished pruning receiver")
 		receiverCancel()
+		endSpan()
 	}
 
 	j.updateTasks(func(tasks *activeSideTasks) {

daemon/job/active_test.go

@@ -0,0 +1,19 @@
+package job
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/transport"
+)
+
+func TestFakeActiveSideDirectMethodInvocationClientIdentityDoesNotPassValidityTest(t *testing.T) {
+	jobid, err := endpoint.MakeJobID("validjobname")
+	require.NoError(t, err)
+	clientIdentity := FakeActiveSideDirectMethodInvocationClientIdentity(jobid)
+	t.Logf("%v", clientIdentity)
+	err = transport.ValidateClientIdentity(clientIdentity)
+	assert.Error(t, err)
+}

daemon/job/activesidestate_enumer.go

@@ -1,5 +1,6 @@
 // Code generated by "enumer -type=ActiveSideState"; DO NOT EDIT.
 
+//
 package job
 
 import (

daemon/job/build_jobs.go

@@ -2,7 +2,11 @@ package job
 
 import (
 	"fmt"
+	"sort"
+	"strings"
+
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/config"
 )
 
@@ -18,6 +22,22 @@ func JobsFromConfig(c *config.Config) ([]Job, error) {
 		}
 		js[i] = j
 	}
+
+	// receiving-side root filesystems must not overlap
+	{
+		rfss := make([]string, 0, len(js))
+		for _, j := range js {
+			jrfs, ok := j.OwnedDatasetSubtreeRoot()
+			if !ok {
+				continue
+			}
+			rfss = append(rfss, jrfs.ToString())
+		}
+		if err := validateReceivingSidesDoNotOverlap(rfss); err != nil {
+			return nil, err
+		}
+	}
+
 	return js, nil
 }
 
@@ -28,38 +48,27 @@ func buildJob(c *config.Global, in config.JobEnum) (j Job, err error) {
 	// FIXME prettify this
 	switch v := in.Ret.(type) {
 	case *config.SinkJob:
-		m, err := modeSinkFromConfig(c, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-		j, err = passiveSideFromConfig(c, &v.PassiveJob, m)
+		j, err = passiveSideFromConfig(c, &v.PassiveJob, v)
 		if err != nil {
 			return cannotBuildJob(err, v.Name)
 		}
 	case *config.SourceJob:
-		m, err := modeSourceFromConfig(c, v)
+		j, err = passiveSideFromConfig(c, &v.PassiveJob, v)
 		if err != nil {
 			return cannotBuildJob(err, v.Name)
 		}
-		j, err = passiveSideFromConfig(c, &v.PassiveJob, m)
+	case *config.SnapJob:
+		j, err = snapJobFromConfig(c, v)
 		if err != nil {
 			return cannotBuildJob(err, v.Name)
 		}
 	case *config.PushJob:
-		m, err := modePushFromConfig(c, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-		j, err = activeSide(c, &v.ActiveJob, m)
+		j, err = activeSide(c, &v.ActiveJob, v)
 		if err != nil {
 			return cannotBuildJob(err, v.Name)
 		}
 	case *config.PullJob:
-		m, err := modePullFromConfig(c, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-		j, err = activeSide(c, &v.ActiveJob, m)
+		j, err = activeSide(c, &v.ActiveJob, v)
 		if err != nil {
 			return cannotBuildJob(err, v.Name)
 		}
@@ -69,3 +78,32 @@ func buildJob(c *config.Global, in config.JobEnum) (j Job, err error) {
 	return j, nil
 
 }
+
+func validateReceivingSidesDoNotOverlap(receivingRootFSs []string) error {
+	if len(receivingRootFSs) == 0 {
+		return nil
+	}
+	rfss := make([]string, len(receivingRootFSs))
+	copy(rfss, receivingRootFSs)
+	sort.Slice(rfss, func(i, j int) bool {
+		return strings.Compare(rfss[i], rfss[j]) == -1
+	})
+	// add tailing slash because of hierarchy-simulation
+	// rootfs/ is not root of rootfs2/
+	for i := 0; i < len(rfss); i++ {
+		rfss[i] += "/"
+	}
+	// idea:
+	//   no path in rfss must be prefix of another
+	//
+	// rfss is now lexicographically sorted, which means that
+	// if i is prefix of j, i < j (in lexicographical order)
+	// thus,
+	// if any i is prefix of i+n (n >= 1), there is overlap
+	for i := 0; i < len(rfss)-1; i++ {
+		if strings.HasPrefix(rfss[i+1], rfss[i]) {
+			return fmt.Errorf("receiving jobs with overlapping root filesystems are forbidden")
+		}
+	}
+	return nil
+}

daemon/job/build_jobs_test.go

@@ -0,0 +1,110 @@
+package job
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/config"
+)
+
+func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
+	type testCase struct {
+		err   bool
+		input []string
+	}
+	tcs := []testCase{
+		{false, nil},
+		{false, []string{}},
+		{false, []string{""}}, // not our job to determine valid paths
+		{false, []string{"a"}},
+		{false, []string{"some/path"}},
+		{false, []string{"zroot/sink1", "zroot/sink2", "zroot/sink3"}},
+		{false, []string{"zroot/foo", "zroot/foobar"}},
+		{true, []string{"zroot/b", "zroot/b"}},
+		{true, []string{"zroot/foo", "zroot/foo/bar", "zroot/baz"}},
+		{false, []string{"a/x", "b/x"}},
+		{false, []string{"a", "b"}},
+		{true, []string{"a", "a"}},
+		{true, []string{"a/x/y", "a/x"}},
+		{true, []string{"a/x", "a/x/y"}},
+		{true, []string{"a/x", "b/x", "a/x/y"}},
+		{true, []string{"a", "a/b", "a/c", "a/b"}},
+		{true, []string{"a/b", "a/c", "a/b", "a/d", "a/c"}},
+	}
+
+	for _, tc := range tcs {
+		t.Logf("input: %v", tc.input)
+		err := validateReceivingSidesDoNotOverlap(tc.input)
+		if tc.err {
+			assert.Error(t, err)
+		} else {
+			assert.NoError(t, err)
+		}
+	}
+}
+
+func TestJobIDErrorHandling(t *testing.T) {
+	tmpl := `
+jobs:
+- name: %s
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
+
+	type Case struct {
+		jobName string
+		valid   bool
+	}
+	cases := []Case{
+		{"validjobname", true},
+		{"valid with spaces", true},
+		{"invalid\twith\ttabs", false},
+		{"invalid#withdelimiter", false},
+		{"invalid@withdelimiter", false},
+		{"withnewline\\nmiddle", false},
+		{"withnewline\\n", false},
+		{"withslash/", false},
+		{"withslash/inthemiddle", false},
+		{"/", false},
+	}
+
+	for i := range cases {
+		t.Run(cases[i].jobName, func(t *testing.T) {
+			c := cases[i]
+
+			conf, err := config.ParseConfigBytes([]byte(fill(c.jobName)))
+			require.NoError(t, err, "not expecting yaml-config to know about job ids")
+			require.NotNil(t, conf)
+			jobs, err := JobsFromConfig(conf)
+
+			if c.valid {
+				assert.NoError(t, err)
+				require.Len(t, jobs, 1)
+				assert.Equal(t, c.jobName, jobs[0].Name())
+			} else {
+				t.Logf("error: %s", err)
+				assert.Error(t, err)
+				assert.Nil(t, jobs)
+			}
+
+		})
+	}
+
+}

daemon/job/job.go

@@ -4,49 +4,45 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+
 	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/zfs"
 )
 
 type Logger = logger.Logger
 
-type contextKey int
-
-const (
-	contextKeyLog contextKey = iota
-)
-
 func GetLogger(ctx context.Context) Logger {
-	if l, ok := ctx.Value(contextKeyLog).(Logger); ok {
-		return l
-	}
-	return logger.NewNullLogger()
+	return logging.GetLogger(ctx, logging.SubsysJob)
 }
 
-func WithLogger(ctx context.Context, l Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLog, l)
-}
-
-
 type Job interface {
 	Name() string
 	Run(ctx context.Context)
 	Status() *Status
 	RegisterMetrics(registerer prometheus.Registerer)
+	// Jobs that return a subtree of the dataset hierarchy
+	// must return the root of that subtree as rfs and ok = true
+	OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool)
+	SenderConfig() *endpoint.SenderConfig
 }
 
 type Type string
 
 const (
 	TypeInternal Type = "internal"
-	TypePush Type = "push"
-	TypeSink Type = "sink"
-	TypePull Type  = "pull"
-	TypeSource Type = "source"
+	TypeSnap     Type = "snap"
+	TypePush     Type = "push"
+	TypeSink     Type = "sink"
+	TypePull     Type = "pull"
+	TypeSource   Type = "source"
 )
 
 type Status struct {
-	Type Type
+	Type        Type
 	JobSpecific interface{}
 }
 
@@ -59,8 +55,8 @@ func (s *Status) MarshalJSON() ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	m := map[string]json.RawMessage {
-		"type": typeJson,
+	m := map[string]json.RawMessage{
+		"type":         typeJson,
 		string(s.Type): jobJSON,
 	}
 	return json.Marshal(m)
@@ -84,16 +80,25 @@ func (s *Status) UnmarshalJSON(in []byte) (err error) {
 		return fmt.Errorf("field '%s', not found", key)
 	}
 	switch s.Type {
-	case TypePull: fallthrough
+	case TypeSnap:
+		var st SnapJobStatus
+		err = json.Unmarshal(jobJSON, &st)
+		s.JobSpecific = &st
+
+	case TypePull:
+		fallthrough
 	case TypePush:
 		var st ActiveSideStatus
 		err = json.Unmarshal(jobJSON, &st)
 		s.JobSpecific = &st
-	case TypeSource: fallthrough
+
+	case TypeSource:
+		fallthrough
 	case TypeSink:
 		var st PassiveStatus
 		err = json.Unmarshal(jobJSON, &st)
 		s.JobSpecific = &st
+
 	case TypeInternal:
 		// internal jobs do not report specifics
 	default:

daemon/job/passive.go

@@ -2,87 +2,88 @@ package job
 
 import (
 	"context"
+	"fmt"
+
 	"github.com/pkg/errors"
-	"github.com/problame/go-streamrpc"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/filters"
 	"github.com/zrepl/zrepl/daemon/logging"
-	"github.com/zrepl/zrepl/daemon/transport/serve"
 	"github.com/zrepl/zrepl/daemon/snapper"
 	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/transport"
+	"github.com/zrepl/zrepl/transport/fromconfig"
 	"github.com/zrepl/zrepl/zfs"
-	"path"
 )
 
 type PassiveSide struct {
-	mode passiveMode
-	name     string
-	l        serve.ListenerFactory
-	rpcConf  *streamrpc.ConnConfig
+	mode   passiveMode
+	name   endpoint.JobID
+	listen transport.AuthenticatedListenerFactory
 }
 
 type passiveMode interface {
-	ConnHandleFunc(ctx context.Context, conn serve.AuthenticatedConn) streamrpc.HandlerFunc
+	Handler() rpc.Handler
 	RunPeriodic(ctx context.Context)
+	SnapperReport() *snapper.Report // may be nil
 	Type() Type
 }
 
 type modeSink struct {
-	rootDataset *zfs.DatasetPath
+	receiverConfig endpoint.ReceiverConfig
 }
 
 func (m *modeSink) Type() Type { return TypeSink }
 
-func (m *modeSink) ConnHandleFunc(ctx context.Context, conn serve.AuthenticatedConn) streamrpc.HandlerFunc {
-	log := GetLogger(ctx)
-
-	clientRootStr := path.Join(m.rootDataset.ToString(), conn.ClientIdentity())
-	clientRoot, err := zfs.NewDatasetPath(clientRootStr)
-	if err != nil {
-		log.WithError(err).
-			WithField("client_identity", conn.ClientIdentity()).
-			Error("cannot build client filesystem map (client identity must be a valid ZFS FS name")
-	}
-	log.WithField("client_root", clientRoot).Debug("client root")
-
-	local, err := endpoint.NewReceiver(clientRoot)
-	if err != nil {
-		log.WithError(err).Error("unexpected error: cannot convert mapping to filter")
-		return nil
-	}
-
-	h := endpoint.NewHandler(local)
-	return h.Handle
+func (m *modeSink) Handler() rpc.Handler {
+	return endpoint.NewReceiver(m.receiverConfig)
 }
 
-func (m *modeSink) RunPeriodic(_ context.Context) {}
+func (m *modeSink) RunPeriodic(_ context.Context)  {}
+func (m *modeSink) SnapperReport() *snapper.Report { return nil }
 
-func modeSinkFromConfig(g *config.Global, in *config.SinkJob) (m *modeSink, err error) {
+func modeSinkFromConfig(g *config.Global, in *config.SinkJob, jobID endpoint.JobID) (m *modeSink, err error) {
 	m = &modeSink{}
-	m.rootDataset, err = zfs.NewDatasetPath(in.RootFS)
+
+	rootDataset, err := zfs.NewDatasetPath(in.RootFS)
 	if err != nil {
 		return nil, errors.New("root dataset is not a valid zfs filesystem path")
 	}
-	if m.rootDataset.Length() <= 0 {
-		return nil, errors.New("root dataset must not be empty") // duplicates error check of receiver
+
+	m.receiverConfig = endpoint.ReceiverConfig{
+		JobID:                      jobID,
+		RootWithoutClientComponent: rootDataset,
+		AppendClientIdentity:       true, // !
+		UpdateLastReceivedHold:     true,
 	}
+	if err := m.receiverConfig.Validate(); err != nil {
+		return nil, errors.Wrap(err, "cannot build receiver config")
+	}
+
 	return m, nil
 }
 
 type modeSource struct {
-	fsfilter zfs.DatasetFilter
-	snapper *snapper.PeriodicOrManual
+	senderConfig *endpoint.SenderConfig
+	snapper      *snapper.PeriodicOrManual
 }
 
-func modeSourceFromConfig(g *config.Global, in *config.SourceJob) (m *modeSource, err error) {
+func modeSourceFromConfig(g *config.Global, in *config.SourceJob, jobID endpoint.JobID) (m *modeSource, err error) {
 	// FIXME exact dedup of modePush
 	m = &modeSource{}
 	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
 	if err != nil {
-		return nil, errors.Wrap(err, "cannnot build filesystem filter")
+		return nil, errors.Wrap(err, "cannot build filesystem filter")
+	}
+	m.senderConfig = &endpoint.SenderConfig{
+		FSF:                         fsf,
+		Encrypt:                     &zfs.NilBool{B: in.Send.Encrypted},
+		DisableIncrementalStepHolds: in.Send.StepHolds.DisableIncremental,
+		JobID:                       jobID,
 	}
-	m.fsfilter = fsf
 
 	if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
 		return nil, errors.Wrap(err, "cannot build snapper")
@@ -93,104 +94,113 @@ func modeSourceFromConfig(g *config.Global, in *config.SourceJob) (m *modeSource
 
 func (m *modeSource) Type() Type { return TypeSource }
 
-func (m *modeSource) ConnHandleFunc(ctx context.Context, conn serve.AuthenticatedConn) streamrpc.HandlerFunc {
-	sender := endpoint.NewSender(m.fsfilter)
-	h := endpoint.NewHandler(sender)
-	return h.Handle
+func (m *modeSource) Handler() rpc.Handler {
+	return endpoint.NewSender(*m.senderConfig)
 }
 
 func (m *modeSource) RunPeriodic(ctx context.Context) {
 	m.snapper.Run(ctx, nil)
 }
 
-func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, mode passiveMode) (s *PassiveSide, err error) {
+func (m *modeSource) SnapperReport() *snapper.Report {
+	return m.snapper.Report()
+}
 
-	s = &PassiveSide{mode: mode, name: in.Name}
-	if s.l, s.rpcConf, err = serve.FromConfig(g, in.Serve); err != nil {
-		return nil, errors.Wrap(err, "cannot build server")
+func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, configJob interface{}) (s *PassiveSide, err error) {
+
+	s = &PassiveSide{}
+
+	s.name, err = endpoint.MakeJobID(in.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid job name")
+	}
+
+	switch v := configJob.(type) {
+	case *config.SinkJob:
+		s.mode, err = modeSinkFromConfig(g, v, s.name) // shadow
+	case *config.SourceJob:
+		s.mode, err = modeSourceFromConfig(g, v, s.name) // shadow
+	}
+	if err != nil {
+		return nil, err // no wrapping necessary
+	}
+
+	if s.listen, err = fromconfig.ListenerFactoryFromConfig(g, in.Serve); err != nil {
+		return nil, errors.Wrap(err, "cannot build listener factory")
 	}
 
 	return s, nil
 }
 
-func (j *PassiveSide) Name() string { return j.name }
+func (j *PassiveSide) Name() string { return j.name.String() }
 
-type PassiveStatus struct {}
+type PassiveStatus struct {
+	Snapper *snapper.Report
+}
 
 func (s *PassiveSide) Status() *Status {
-	return &Status{Type: s.mode.Type()} // FIXME PassiveStatus
+	st := &PassiveStatus{
+		Snapper: s.mode.SnapperReport(),
+	}
+	return &Status{Type: s.mode.Type(), JobSpecific: st}
+}
+
+func (j *PassiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
+	sink, ok := j.mode.(*modeSink)
+	if !ok {
+		_ = j.mode.(*modeSource) // make sure we didn't introduce a new job type
+		return nil, false
+	}
+	return sink.receiverConfig.RootWithoutClientComponent.Copy(), true
+}
+
+func (j *PassiveSide) SenderConfig() *endpoint.SenderConfig {
+	source, ok := j.mode.(*modeSource)
+	if !ok {
+		_ = j.mode.(*modeSink) // make sure we didn't introduce a new job type
+		return nil
+	}
+	return source.senderConfig
 }
 
 func (*PassiveSide) RegisterMetrics(registerer prometheus.Registerer) {}
 
 func (j *PassiveSide) Run(ctx context.Context) {
-
+	ctx, endTask := trace.WithTaskAndSpan(ctx, "passive-side-job", j.Name())
+	defer endTask()
 	log := GetLogger(ctx)
 	defer log.Info("job exiting")
-
-	l, err := j.l.Listen()
-	if err != nil {
-		log.WithError(err).Error("cannot listen")
-		return
-	}
-	defer l.Close()
-
 	{
-		ctx, cancel := context.WithCancel(logging.WithSubsystemLoggers(ctx, log)) // shadowing
+		ctx, endTask := trace.WithTask(ctx, "periodic") // shadowing
+		defer endTask()
+		ctx, cancel := context.WithCancel(ctx)
 		defer cancel()
 		go j.mode.RunPeriodic(ctx)
 	}
 
-	log.WithField("addr", l.Addr()).Debug("accepting connections")
-	var connId int
-outer:
-	for {
-
-		select {
-		case res := <-accept(ctx, l):
-			if res.err != nil {
-				log.WithError(res.err).Info("accept error")
-				continue
-			}
-			conn := res.conn
-			connId++
-			connLog := log.
-				WithField("connID", connId)
-			connLog.
-				WithField("addr", conn.RemoteAddr()).
-				WithField("client_identity", conn.ClientIdentity()).
-				Info("handling connection")
-			go func() {
-				defer connLog.Info("finished handling connection")
-				defer conn.Close()
-				ctx := logging.WithSubsystemLoggers(ctx, connLog)
-				handleFunc := j.mode.ConnHandleFunc(ctx, conn)
-				if handleFunc == nil {
-					return
-				}
-				if err := streamrpc.ServeConn(ctx, conn, j.rpcConf, handleFunc); err != nil {
-					log.WithError(err).Error("error serving client")
-				}
-			}()
-
-		case <-ctx.Done():
-			break outer
-		}
-
+	handler := j.mode.Handler()
+	if handler == nil {
+		panic(fmt.Sprintf("implementation error: j.mode.Handler() returned nil: %#v", j))
 	}
 
-}
+	ctxInterceptor := func(handlerCtx context.Context, info rpc.HandlerContextInterceptorData, handler func(ctx context.Context)) {
+		// the handlerCtx is clean => need to inherit logging and tracing config from job context
+		handlerCtx = logging.WithInherit(handlerCtx, ctx)
+		handlerCtx = trace.WithInherit(handlerCtx, ctx)
 
-type acceptResult struct {
-	conn serve.AuthenticatedConn
-	err  error
-}
+		handlerCtx, endTask := trace.WithTaskAndSpan(handlerCtx, "handler", fmt.Sprintf("job=%q client=%q method=%q", j.Name(), info.ClientIdentity(), info.FullMethod()))
+		defer endTask()
+		handler(handlerCtx)
+	}
 
-func accept(ctx context.Context, listener serve.AuthenticatedListener) <-chan acceptResult {
-	c := make(chan acceptResult, 1)
-	go func() {
-		conn, err := listener.Accept(ctx)
-		c <- acceptResult{conn, err}
-	}()
-	return c
+	rpcLoggers := rpc.GetLoggersOrPanic(ctx) // WithSubsystemLoggers above
+	server := rpc.NewServer(handler, rpcLoggers, ctxInterceptor)
+
+	listener, err := j.listen()
+	if err != nil {
+		log.WithError(err).Error("cannot listen")
+		return
+	}
+
+	server.Serve(ctx, listener)
 }

daemon/job/snapjob.go

@@ -0,0 +1,185 @@
+package job
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/daemon/job/wakeup"
+	"github.com/zrepl/zrepl/daemon/pruner"
+	"github.com/zrepl/zrepl/daemon/snapper"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/replication/logic/pdu"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type SnapJob struct {
+	name     endpoint.JobID
+	fsfilter zfs.DatasetFilter
+	snapper  *snapper.PeriodicOrManual
+
+	prunerFactory *pruner.LocalPrunerFactory
+
+	promPruneSecs *prometheus.HistogramVec // labels: prune_side
+
+	pruner *pruner.Pruner
+}
+
+func (j *SnapJob) Name() string { return j.name.String() }
+
+func (j *SnapJob) Type() Type { return TypeSnap }
+
+func snapJobFromConfig(g *config.Global, in *config.SnapJob) (j *SnapJob, err error) {
+	j = &SnapJob{}
+	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot build filesystem filter")
+	}
+	j.fsfilter = fsf
+
+	if j.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
+		return nil, errors.Wrap(err, "cannot build snapper")
+	}
+	j.name, err = endpoint.MakeJobID(in.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid job name")
+	}
+	j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace:   "zrepl",
+		Subsystem:   "pruning",
+		Name:        "time",
+		Help:        "seconds spent in pruner",
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
+	}, []string{"prune_side"})
+	j.prunerFactory, err = pruner.NewLocalPrunerFactory(in.Pruning, j.promPruneSecs)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot build snapjob pruning rules")
+	}
+	return j, nil
+}
+
+func (j *SnapJob) RegisterMetrics(registerer prometheus.Registerer) {
+	registerer.MustRegister(j.promPruneSecs)
+}
+
+type SnapJobStatus struct {
+	Pruning      *pruner.Report
+	Snapshotting *snapper.Report // may be nil
+}
+
+func (j *SnapJob) Status() *Status {
+	s := &SnapJobStatus{}
+	t := j.Type()
+	if j.pruner != nil {
+		s.Pruning = j.pruner.Report()
+	}
+	s.Snapshotting = j.snapper.Report()
+	return &Status{Type: t, JobSpecific: s}
+}
+
+func (j *SnapJob) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
+	return nil, false
+}
+
+func (j *SnapJob) SenderConfig() *endpoint.SenderConfig { return nil }
+
+func (j *SnapJob) Run(ctx context.Context) {
+	ctx, endTask := trace.WithTaskAndSpan(ctx, "snap-job", j.Name())
+	defer endTask()
+	log := GetLogger(ctx)
+
+	defer log.Info("job exiting")
+
+	periodicDone := make(chan struct{})
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	periodicCtx, endTask := trace.WithTask(ctx, "snapshotting")
+	defer endTask()
+	go j.snapper.Run(periodicCtx, periodicDone)
+
+	invocationCount := 0
+outer:
+	for {
+		log.Info("wait for wakeups")
+		select {
+		case <-ctx.Done():
+			log.WithError(ctx.Err()).Info("context")
+			break outer
+
+		case <-wakeup.Wait(ctx):
+		case <-periodicDone:
+		}
+		invocationCount++
+
+		invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", invocationCount))
+		j.doPrune(invocationCtx)
+		endSpan()
+	}
+}
+
+// Adaptor that implements pruner.History around a pruner.Target.
+// The ReplicationCursor method is Get-op only and always returns
+// the filesystem's most recent version's GUID.
+//
+// TODO:
+// This is a work-around for the current package daemon/pruner
+// and package pruning.Snapshot limitation: they require the
+//  `Replicated` getter method be present, but obviously,
+// a local job like SnapJob can't deliver on that.
+// But the pruner.Pruner gives up on an FS if no replication
+// cursor is present, which is why this pruner returns the
+// most recent filesystem version.
+type alwaysUpToDateReplicationCursorHistory struct {
+	// the Target passed as Target to BuildLocalPruner
+	target pruner.Target
+}
+
+var _ pruner.History = (*alwaysUpToDateReplicationCursorHistory)(nil)
+
+func (h alwaysUpToDateReplicationCursorHistory) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
+	fsvReq := &pdu.ListFilesystemVersionsReq{
+		Filesystem: req.GetFilesystem(),
+	}
+	res, err := h.target.ListFilesystemVersions(ctx, fsvReq)
+	if err != nil {
+		return nil, err
+	}
+	fsvs := res.GetVersions()
+	if len(fsvs) <= 0 {
+		return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Notexist{Notexist: true}}, nil
+	}
+	// always return must recent version
+	sort.Slice(fsvs, func(i, j int) bool {
+		return fsvs[i].CreateTXG < fsvs[j].CreateTXG
+	})
+	mostRecent := fsvs[len(fsvs)-1]
+	return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Guid{Guid: mostRecent.GetGuid()}}, nil
+}
+
+func (h alwaysUpToDateReplicationCursorHistory) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error) {
+	return h.target.ListFilesystems(ctx, req)
+}
+
+func (j *SnapJob) doPrune(ctx context.Context) {
+	ctx, endSpan := trace.WithSpan(ctx, "snap-job-do-prune")
+	defer endSpan()
+	log := GetLogger(ctx)
+	sender := endpoint.NewSender(endpoint.SenderConfig{
+		JobID: j.name,
+		FSF:   j.fsfilter,
+		// FIXME encryption setting is irrelevant for SnapJob because the endpoint is only used as pruner.Target
+		Encrypt: &zfs.NilBool{B: true},
+		// FIXME DisableIncrementalStepHolds setting is irrelevant for SnapJob because the endpoint is only used as pruner.Target
+		DisableIncrementalStepHolds: false,
+	})
+	j.pruner = j.prunerFactory.BuildLocalPruner(ctx, sender, alwaysUpToDateReplicationCursorHistory{sender})
+	log.Info("start pruning")
+	j.pruner.Prune()
+	log.Info("finished pruning")
+}

daemon/logging/adaptors.go

@@ -1,32 +0,0 @@
-package logging
-
-import (
-	"fmt"
-	"github.com/problame/go-streamrpc"
-	"github.com/zrepl/zrepl/logger"
-	"strings"
-)
-
-type streamrpcLogAdaptor = twoClassLogAdaptor
-
-type twoClassLogAdaptor struct {
-	logger.Logger
-}
-
-var _ streamrpc.Logger = twoClassLogAdaptor{}
-
-func (a twoClassLogAdaptor) Errorf(fmtStr string, args ...interface{}) {
-	const errorSuffix = ": %s"
-	if len(args) == 1 {
-		if err, ok := args[0].(error); ok && strings.HasSuffix(fmtStr, errorSuffix) {
-			msg := strings.TrimSuffix(fmtStr, errorSuffix)
-			a.WithError(err).Error(msg)
-			return
-		}
-	}
-	a.Logger.Error(fmt.Sprintf(fmtStr, args...))
-}
-
-func (a twoClassLogAdaptor) Infof(fmtStr string, args ...interface{}) {
-	a.Logger.Debug(fmt.Sprintf(fmtStr, args...))
-}

daemon/logging/build_logging.go

@@ -4,18 +4,15 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
+	"log/syslog"
+	"os"
+
 	"github.com/mattn/go-isatty"
 	"github.com/pkg/errors"
-	"github.com/problame/go-streamrpc"
 	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/pruner"
-	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/replication"
 	"github.com/zrepl/zrepl/tlsconf"
-	"os"
-	"github.com/zrepl/zrepl/daemon/snapper"
-	"github.com/zrepl/zrepl/daemon/transport/serve"
 )
 
 func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error) {
@@ -32,7 +29,7 @@ func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error)
 	var syslogOutlets, stdoutOutlets int
 	for lei, le := range in {
 
-		outlet, minLevel, err := parseOutlet(le)
+		outlet, minLevel, err := ParseOutlet(le)
 		if err != nil {
 			return nil, errors.Wrapf(err, "cannot parse outlet #%d", lei)
 		}
@@ -60,20 +57,119 @@ func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error)
 
 }
 
+type Subsystem string
+
 const (
-	SubsysReplication = "repl"
-	SubsysStreamrpc   = "rpc"
-	SubsyEndpoint     = "endpoint"
+	SubsysMeta         Subsystem = "meta"
+	SubsysJob          Subsystem = "job"
+	SubsysReplication  Subsystem = "repl"
+	SubsysEndpoint     Subsystem = "endpoint"
+	SubsysPruning      Subsystem = "pruning"
+	SubsysSnapshot     Subsystem = "snapshot"
+	SubsysHooks        Subsystem = "hook"
+	SubsysTransport    Subsystem = "transport"
+	SubsysTransportMux Subsystem = "transportmux"
+	SubsysRPC          Subsystem = "rpc"
+	SubsysRPCControl   Subsystem = "rpc.ctrl"
+	SubsysRPCData      Subsystem = "rpc.data"
+	SubsysZFSCmd       Subsystem = "zfs.cmd"
+	SubsysTraceData    Subsystem = "trace.data"
+	SubsysPlatformtest Subsystem = "platformtest"
 )
 
-func WithSubsystemLoggers(ctx context.Context, log logger.Logger) context.Context {
-	ctx = replication.WithLogger(ctx, log.WithField(SubsysField, "repl"))
-	ctx = streamrpc.ContextWithLogger(ctx, streamrpcLogAdaptor{log.WithField(SubsysField, "rpc")})
-	ctx = endpoint.WithLogger(ctx, log.WithField(SubsysField, "endpoint"))
-	ctx = pruner.WithLogger(ctx, log.WithField(SubsysField, "pruning"))
-	ctx = snapper.WithLogger(ctx, log.WithField(SubsysField, "snapshot"))
-	ctx = serve.WithLogger(ctx, log.WithField(SubsysField, "serve"))
-	return ctx
+var AllSubsystems = []Subsystem{
+	SubsysMeta,
+	SubsysJob,
+	SubsysReplication,
+	SubsysEndpoint,
+	SubsysPruning,
+	SubsysSnapshot,
+	SubsysHooks,
+	SubsysTransport,
+	SubsysTransportMux,
+	SubsysRPC,
+	SubsysRPCControl,
+	SubsysRPCData,
+	SubsysZFSCmd,
+	SubsysTraceData,
+	SubsysPlatformtest,
+}
+
+type injectedField struct {
+	field  string
+	value  interface{}
+	parent *injectedField
+}
+
+func WithInjectedField(ctx context.Context, field string, value interface{}) context.Context {
+	var parent *injectedField
+	parentI := ctx.Value(contextKeyInjectedField)
+	if parentI != nil {
+		parent = parentI.(*injectedField)
+	}
+	// TODO sanity-check `field` now
+	this := &injectedField{field, value, parent}
+	return context.WithValue(ctx, contextKeyInjectedField, this)
+}
+
+func iterInjectedFields(ctx context.Context, cb func(field string, value interface{})) {
+	injI := ctx.Value(contextKeyInjectedField)
+	if injI == nil {
+		return
+	}
+	inj := injI.(*injectedField)
+	for ; inj != nil; inj = inj.parent {
+		cb(inj.field, inj.value)
+	}
+}
+
+type SubsystemLoggers map[Subsystem]logger.Logger
+
+func SubsystemLoggersWithUniversalLogger(l logger.Logger) SubsystemLoggers {
+	loggers := make(SubsystemLoggers)
+	for _, s := range AllSubsystems {
+		loggers[s] = l
+	}
+	return loggers
+}
+
+func WithLoggers(ctx context.Context, loggers SubsystemLoggers) context.Context {
+	return context.WithValue(ctx, contextKeyLoggers, loggers)
+}
+
+func GetLoggers(ctx context.Context) SubsystemLoggers {
+	loggers, ok := ctx.Value(contextKeyLoggers).(SubsystemLoggers)
+	if !ok {
+		return nil
+	}
+	return loggers
+}
+
+func GetLogger(ctx context.Context, subsys Subsystem) logger.Logger {
+	return getLoggerImpl(ctx, subsys, true)
+}
+
+func getLoggerImpl(ctx context.Context, subsys Subsystem, panicIfEnded bool) logger.Logger {
+	loggers, ok := ctx.Value(contextKeyLoggers).(SubsystemLoggers)
+	if !ok || loggers == nil {
+		return logger.NewNullLogger()
+	}
+	l, ok := loggers[subsys]
+	if !ok {
+		return logger.NewNullLogger()
+	}
+
+	l = l.WithField(SubsysField, subsys)
+
+	l = l.WithField(SpanField, trace.GetSpanStackOrDefault(ctx, *trace.StackKindId, "NOSPAN"))
+
+	fields := make(logger.Fields)
+	iterInjectedFields(ctx, func(field string, value interface{}) {
+		fields[field] = value
+	})
+	l = l.WithFields(fields)
+
+	return l
 }
 
 func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
@@ -98,7 +194,7 @@ func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
 
 }
 
-func parseOutlet(in config.LoggingOutletEnum) (o logger.Outlet, level logger.Level, err error) {
+func ParseOutlet(in config.LoggingOutletEnum) (o logger.Outlet, level logger.Level, err error) {
 
 	parseCommon := func(common config.LoggingOutletCommon) (logger.Level, EntryFormatter, error) {
 		if common.Level == "" || common.Format == "" {
@@ -200,6 +296,7 @@ func parseSyslogOutlet(in *config.SyslogLoggingOutlet, formatter EntryFormatter)
 	out = &SyslogOutlet{}
 	out.Formatter = formatter
 	out.Formatter.SetMetadataFlags(MetadataNone)
+	out.Facility = syslog.Priority(*in.Facility)
 	out.RetryInterval = in.RetryInterval
 	return out, nil
 }

daemon/logging/logging_context.go

@@ -0,0 +1,24 @@
+package logging
+
+import "context"
+
+type contextKey int
+
+const (
+	contextKeyLoggers contextKey = 1 + iota
+	contextKeyInjectedField
+)
+
+var contextKeys = []contextKey{
+	contextKeyLoggers,
+	contextKeyInjectedField,
+}
+
+func WithInherit(ctx, inheritFrom context.Context) context.Context {
+	for _, k := range contextKeys {
+		if v := inheritFrom.Value(k); v != nil {
+			ctx = context.WithValue(ctx, k, v) // no shadow
+		}
+	}
+	return ctx
+}

daemon/logging/logging_formatters.go

@@ -4,11 +4,13 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"time"
+
 	"github.com/fatih/color"
 	"github.com/go-logfmt/logfmt"
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/logger"
-	"time"
 )
 
 const (
@@ -20,6 +22,7 @@ const (
 const (
 	JobField    string = "job"
 	SubsysField string = "subsystem"
+	SpanField   string = "span"
 )
 
 type MetadataFlags int64
@@ -83,10 +86,10 @@ func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
 		fmt.Fprintf(&line, "[%s]", col.Sprint(e.Level.Short()))
 	}
 
-	prefixFields := []string{JobField, SubsysField}
+	prefixFields := []string{JobField, SubsysField, SpanField}
 	prefixed := make(map[string]bool, len(prefixFields)+2)
 	for _, field := range prefixFields {
-		val, ok := e.Fields[field].(string)
+		val, ok := e.Fields[field]
 		if !ok {
 			continue
 		}
@@ -159,15 +162,21 @@ func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
 	enc := logfmt.NewEncoder(&buf)
 
 	if f.metadataFlags&MetadataTime != 0 {
-		enc.EncodeKeyval(FieldTime, e.Time)
+		err := enc.EncodeKeyval(FieldTime, e.Time)
+		if err != nil {
+			return nil, errors.Wrap(err, "logfmt: encode time")
+		}
 	}
 	if f.metadataFlags&MetadataLevel != 0 {
-		enc.EncodeKeyval(FieldLevel, e.Level)
+		err := enc.EncodeKeyval(FieldLevel, e.Level)
+		if err != nil {
+			return nil, errors.Wrap(err, "logfmt: encode level")
+		}
 	}
 
 	// at least try and put job and task in front
-	prefixed := make(map[string]bool, 2)
-	prefix := []string{JobField, SubsysField}
+	prefixed := make(map[string]bool, 3)
+	prefix := []string{JobField, SubsysField, SpanField}
 	for _, pf := range prefix {
 		v, ok := e.Fields[pf]
 		if !ok {
@@ -179,8 +188,10 @@ func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
 		prefixed[pf] = true
 	}
 
-	enc.EncodeKeyval(FieldMessage, e.Message)
-
+	err := enc.EncodeKeyval(FieldMessage, e.Message)
+	if err != nil {
+		return nil, errors.Wrap(err, "logfmt: encode message")
+	}
 	for k, v := range e.Fields {
 		if !prefixed[k] {
 			if err := logfmtTryEncodeKeyval(enc, k, v); err != nil {
@@ -199,7 +210,10 @@ func logfmtTryEncodeKeyval(enc *logfmt.Encoder, field, value interface{}) error
 	case nil: // ok
 		return nil
 	case logfmt.ErrUnsupportedValueType:
-		enc.EncodeKeyval(field, fmt.Sprintf("<%T>", value))
+		err := enc.EncodeKeyval(field, fmt.Sprintf("<%T>", value))
+		if err != nil {
+			return errors.Wrap(err, "cannot encode unsupported value type Go type")
+		}
 		return nil
 	}
 	return errors.Wrapf(err, "cannot encode field '%s'", field)

daemon/logging/logging_outlets.go

@@ -4,12 +4,14 @@ import (
 	"bytes"
 	"context"
 	"crypto/tls"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/logger"
 	"io"
 	"log/syslog"
 	"net"
 	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/logger"
 )
 
 type EntryFormatter interface {
@@ -28,7 +30,10 @@ func (h WriterOutlet) WriteEntry(entry logger.Entry) error {
 		return err
 	}
 	_, err = h.writer.Write(bytes)
-	h.writer.Write([]byte("\n"))
+	if err != nil {
+		return err
+	}
+	_, err = h.writer.Write([]byte("\n"))
 	return err
 }
 
@@ -58,7 +63,7 @@ func NewTCPOutlet(formatter EntryFormatter, network, address string, tlsConfig *
 		return
 	}
 
-	entryChan := make(chan *bytes.Buffer, 1) // allow one message in flight while previos is in io.Copy()
+	entryChan := make(chan *bytes.Buffer, 1) // allow one message in flight while previous is in io.Copy()
 
 	o := &TCPOutlet{
 		formatter: formatter,
@@ -92,8 +97,10 @@ func (h *TCPOutlet) outLoop(retryInterval time.Duration) {
 				conn = nil
 			}
 		}
-		conn.SetWriteDeadline(time.Now().Add(retryInterval))
-		_, err = io.Copy(conn, msg)
+		err = conn.SetWriteDeadline(time.Now().Add(retryInterval))
+		if err == nil {
+			_, err = io.Copy(conn, msg)
+		}
 		if err != nil {
 			retry = time.Now().Add(retryInterval)
 			conn.Close()
@@ -124,6 +131,7 @@ func (h *TCPOutlet) WriteEntry(e logger.Entry) error {
 type SyslogOutlet struct {
 	Formatter          EntryFormatter
 	RetryInterval      time.Duration
+	Facility           syslog.Priority
 	writer             *syslog.Writer
 	lastConnectAttempt time.Time
 }
@@ -142,7 +150,7 @@ func (o *SyslogOutlet) WriteEntry(entry logger.Entry) error {
 		if now.Sub(o.lastConnectAttempt) < o.RetryInterval {
 			return nil // not an error toward logger
 		}
-		o.writer, err = syslog.New(syslog.LOG_LOCAL0, "zrepl")
+		o.writer, err = syslog.New(o.Facility, "zrepl")
 		o.lastConnectAttempt = time.Now()
 		if err != nil {
 			o.writer = nil

daemon/logging/trace/trace.go

@@ -0,0 +1,386 @@
+// package trace provides activity tracing via ctx through Tasks and Spans
+//
+// Basic Concepts
+//
+// Tracing can be used to identify where a piece of code spends its time.
+//
+// The Go standard library provides package runtime/trace which is useful to identify CPU bottlenecks or
+// to understand what happens inside the Go runtime.
+// However, it is not ideal for application level tracing, in particular if those traces should be understandable
+// to tech-savvy users (albeit not developers).
+//
+// This package provides the concept of Tasks and Spans to express what activity is happening within an application:
+//
+//  - Neither task nor span is really tangible but instead contained within the context.Context tree
+//  - Tasks represent concurrent activity (i.e. goroutines).
+//  - Spans represent a semantic stack trace within a task.
+//
+// As a consequence, whenever a context is propagated across goroutine boundary, you need to create a child task:
+//
+//   go func(ctx context.Context) {
+//     ctx, endTask = WithTask(ctx, "what-happens-inside-the-child-task")
+//     defer endTask()
+//     // ...
+//   }(ctx)
+//
+// Within the task, you can open up a hierarchy of spans.
+// In contrast to tasks, which have can multiple concurrently running child tasks,
+// spans must nest and not cross the goroutine boundary.
+//
+//  ctx, endSpan = WithSpan(ctx, "copy-dir")
+//  defer endSpan()
+//  for _, f := range dir.Files() {
+//    func() {
+//      ctx, endSpan := WithSpan(ctx, fmt.Sprintf("copy-file %q", f))
+//      defer endspan()
+//      b, _ := ioutil.ReadFile(f)
+//      _ = ioutil.WriteFile(f + ".copy", b, 0600)
+//    }()
+//  }
+//
+// In combination:
+//  ctx, endTask = WithTask(ctx, "copy-dirs")
+//  defer endTask()
+//  for i := range dirs {
+//    go func(dir string) {
+//      ctx, endTask := WithTask(ctx, "copy-dir")
+//      defer endTask()
+//      for _, f := range filesIn(dir) {
+//        func() {
+//          ctx, endSpan := WithSpan(ctx, fmt.Sprintf("copy-file %q", f))
+//          defer endspan()
+//          b, _ := ioutil.ReadFile(f)
+//          _ = ioutil.WriteFile(f + ".copy", b, 0600)
+//        }()
+//      }
+//    }()
+//  }
+//
+// Note that a span ends at the time you call endSpan - not before and not after that.
+// If you violate the stack-like nesting of spans by forgetting an endSpan() invocation,
+// the out-of-order endSpan() will panic.
+//
+// A similar rule applies to the endTask closure returned by WithTask:
+// If a task has live child tasks at the time you call endTask(), the call will panic.
+//
+// Recovering from endSpan() or endTask() panics will corrupt the trace stack and lead to corrupt tracefile output.
+//
+//
+// Best Practices For Naming Tasks And Spans
+//
+// Tasks should always have string constants as names, and must not contain the `#` character. WHy?
+// First, the visualization by chrome://tracing draws a horizontal bar for each task in the trace.
+// Also, the package appends `#NUM` for each concurrently running instance of a task name.
+// Note that the `#NUM` suffix will be reused if a task has ended, in order to avoid an
+// infinite number of horizontal bars in the visualization.
+//
+//
+// Chrome-compatible Tracefile Support
+//
+// The activity trace generated by usage of WithTask and WithSpan can be rendered to a JSON output file
+// that can be loaded into chrome://tracing .
+// Apart from function GetSpanStackOrDefault, this is the main benefit of this package.
+//
+// First, there is a convenience environment variable 'ZREPL_ACTIVITY_TRACE' that can be set to an output path.
+// From process start onward, a trace is written to that path.
+//
+// More consumers can attach to the activity trace through the ChrometraceClientWebsocketHandler websocket handler.
+//
+// If a write error is encountered with any consumer (including the env-var based one), the consumer is closed and
+// will not receive further trace output.
+package trace
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/zrepl/zrepl/util/chainlock"
+)
+
+var metrics struct {
+	activeTasks prometheus.Gauge
+}
+var taskNamer *uniqueConcurrentTaskNamer = newUniqueTaskNamer()
+
+func init() {
+	metrics.activeTasks = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "zrepl",
+		Subsystem: "trace",
+		Name:      "active_tasks",
+		Help:      "number of active (tracing-level) tasks in the daemon",
+	})
+}
+
+func RegisterMetrics(r prometheus.Registerer) {
+	r.MustRegister(metrics.activeTasks)
+}
+
+type traceNode struct {
+	id         string
+	annotation string
+	parentTask *traceNode
+
+	mtx chainlock.L
+
+	activeChildTasks int32 // only for task nodes, insignificant for span nodes
+	parentSpan       *traceNode
+	activeChildSpan  *traceNode // nil if task or span doesn't have an active child span
+
+	startedAt time.Time
+	endedAt   time.Time
+}
+
+func (s *traceNode) StartedAt() time.Time { return s.startedAt }
+func (s *traceNode) EndedAt() time.Time   { return s.endedAt }
+
+// Returned from WithTask or WithSpan.
+// Must be called once the task or span ends.
+// See package-level docs for nesting rules.
+// Wrong call order / forgetting to call it will result in panics.
+type DoneFunc func()
+
+var ErrTaskStillHasActiveChildTasks = fmt.Errorf("end task: task still has active child tasks")
+
+// Start a new root task or create a child task of an existing task.
+//
+// This is required when starting a new goroutine and
+// passing an existing task context to it.
+//
+// taskName should be a constantand must not contain '#'
+//
+// The implementation ensures that,
+// if multiple tasks with the same name exist simultaneously,
+// a unique suffix is appended to uniquely identify the task opened with this function.
+func WithTask(ctx context.Context, taskName string) (context.Context, DoneFunc) {
+
+	var parentTask *traceNode
+	nodeI := ctx.Value(contextKeyTraceNode)
+	if nodeI != nil {
+		node := nodeI.(*traceNode)
+		if node.parentSpan != nil {
+			parentTask = node.parentTask
+		} else {
+			parentTask = node
+		}
+	}
+	// find the first ancestor that hasn't ended yet (nil if need be)
+	if parentTask != nil {
+		parentTask.mtx.Lock()
+	}
+	for parentTask != nil && !parentTask.endedAt.IsZero() {
+		thisParent := parentTask
+		parentTask = parentTask.parentTask
+		// lock parent first such that it isn't modified by other callers to thisParent
+		if parentTask != nil {
+			parentTask.mtx.Lock()
+		}
+		thisParent.mtx.Unlock()
+	}
+	// invariant: either parentTask != nil and we hold the lock on parentTask, or parentTask is nil
+
+	taskName, taskNameDone := taskNamer.UniqueConcurrentTaskName(taskName)
+
+	this := &traceNode{
+		id:               genID(),
+		annotation:       taskName,
+		parentTask:       parentTask,
+		activeChildTasks: 0,
+		parentSpan:       nil,
+		activeChildSpan:  nil,
+
+		startedAt: time.Now(),
+		endedAt:   time.Time{},
+	}
+
+	if parentTask != nil {
+		this.parentTask.activeChildTasks++
+		parentTask.mtx.Unlock()
+	}
+
+	ctx = context.WithValue(ctx, contextKeyTraceNode, this)
+
+	chrometraceBeginTask(this)
+
+	metrics.activeTasks.Inc()
+
+	endTaskFunc := func() {
+
+		// only hold locks while manipulating the tree
+		// (trace writer might block too long and unlike spans, tasks are updated concurrently)
+		alreadyEnded := func() (alreadyEnded bool) {
+			if this.parentTask != nil {
+				defer this.parentTask.mtx.Lock().Unlock()
+			}
+			defer this.mtx.Lock().Unlock()
+
+			if this.activeChildTasks != 0 {
+				panic(errors.Wrapf(ErrTaskStillHasActiveChildTasks, "end task: %v active child tasks", this.activeChildTasks))
+			}
+
+			// support idempotent task ends
+			if !this.endedAt.IsZero() {
+				return true
+			}
+			this.endedAt = time.Now()
+
+			if this.parentTask != nil {
+				this.parentTask.activeChildTasks--
+				if this.parentTask.activeChildTasks < 0 {
+					panic("impl error: parent task with negative activeChildTasks count")
+				}
+			}
+			return false
+		}()
+		if alreadyEnded {
+			return
+		}
+
+		chrometraceEndTask(this)
+
+		metrics.activeTasks.Dec()
+
+		taskNameDone()
+	}
+
+	return ctx, endTaskFunc
+}
+
+var ErrAlreadyActiveChildSpan = fmt.Errorf("create child span: span already has an active child span")
+var ErrSpanStillHasActiveChildSpan = fmt.Errorf("end span: span still has active child spans")
+
+// Start a new span.
+// Important: ctx must have an active task (see WithTask)
+func WithSpan(ctx context.Context, annotation string) (context.Context, DoneFunc) {
+	var parentSpan, parentTask *traceNode
+	nodeI := ctx.Value(contextKeyTraceNode)
+	if nodeI != nil {
+		parentSpan = nodeI.(*traceNode)
+		if parentSpan.parentSpan == nil {
+			parentTask = parentSpan
+		} else {
+			parentTask = parentSpan.parentTask
+		}
+	} else {
+		panic("must be called from within a task")
+	}
+
+	this := &traceNode{
+		id:              genID(),
+		annotation:      annotation,
+		parentTask:      parentTask,
+		parentSpan:      parentSpan,
+		activeChildSpan: nil,
+
+		startedAt: time.Now(),
+		endedAt:   time.Time{},
+	}
+
+	parentSpan.mtx.HoldWhile(func() {
+		if parentSpan.activeChildSpan != nil {
+			panic(ErrAlreadyActiveChildSpan)
+		}
+		parentSpan.activeChildSpan = this
+	})
+
+	ctx = context.WithValue(ctx, contextKeyTraceNode, this)
+	chrometraceBeginSpan(this)
+	callbackEndSpan := callbackBeginSpan(ctx)
+
+	endTaskFunc := func() {
+
+		defer parentSpan.mtx.Lock().Unlock()
+		if parentSpan.activeChildSpan != this && this.endedAt.IsZero() {
+			panic("impl error: activeChildSpan should not change while != nil because there can only be one")
+		}
+
+		defer this.mtx.Lock().Unlock()
+		if this.activeChildSpan != nil {
+			panic(ErrSpanStillHasActiveChildSpan)
+		}
+
+		if !this.endedAt.IsZero() {
+			return // support idempotent span ends
+		}
+
+		parentSpan.activeChildSpan = nil
+		this.endedAt = time.Now()
+
+		chrometraceEndSpan(this)
+		callbackEndSpan(this)
+	}
+
+	return ctx, endTaskFunc
+}
+
+type StackKind struct {
+	symbolizeTask func(t *traceNode) string
+	symbolizeSpan func(s *traceNode) string
+}
+
+var (
+	StackKindId = &StackKind{
+		symbolizeTask: func(t *traceNode) string { return t.id },
+		symbolizeSpan: func(s *traceNode) string { return s.id },
+	}
+	SpanStackKindCombined = &StackKind{
+		symbolizeTask: func(t *traceNode) string { return fmt.Sprintf("(%s %q)", t.id, t.annotation) },
+		symbolizeSpan: func(s *traceNode) string { return fmt.Sprintf("(%s %q)", s.id, s.annotation) },
+	}
+	SpanStackKindAnnotation = &StackKind{
+		symbolizeTask: func(t *traceNode) string { return t.annotation },
+		symbolizeSpan: func(s *traceNode) string { return s.annotation },
+	}
+)
+
+func (n *traceNode) task() *traceNode {
+	task := n.parentTask
+	if n.parentSpan == nil {
+		task = n
+	}
+	return task
+}
+
+func (n *traceNode) TaskName() string {
+	task := n.task()
+	return task.annotation
+}
+
+func (this *traceNode) TaskAndSpanStack(kind *StackKind) (spanIdStack string) {
+	task := this.task()
+
+	var spansInTask []*traceNode
+	for s := this; s != nil; s = s.parentSpan {
+		spansInTask = append(spansInTask, s)
+	}
+
+	var tasks []*traceNode
+	for t := task; t != nil; t = t.parentTask {
+		tasks = append(tasks, t)
+	}
+
+	var taskIdsRev []string
+	for i := len(tasks) - 1; i >= 0; i-- {
+		taskIdsRev = append(taskIdsRev, kind.symbolizeTask(tasks[i]))
+	}
+
+	var spanIdsRev []string
+	for i := len(spansInTask) - 1; i >= 0; i-- {
+		spanIdsRev = append(spanIdsRev, kind.symbolizeSpan(spansInTask[i]))
+	}
+
+	taskStack := strings.Join(taskIdsRev, "$")
+	return fmt.Sprintf("%s$%s", taskStack, strings.Join(spanIdsRev, "."))
+}
+
+func GetSpanStackOrDefault(ctx context.Context, kind StackKind, def string) string {
+	if nI := ctx.Value(contextKeyTraceNode); nI != nil {
+		n := nI.(*traceNode)
+		return n.TaskAndSpanStack(StackKindId)
+	} else {
+		return def
+	}
+}

daemon/logging/trace/trace_callback.go

@@ -0,0 +1,54 @@
+package trace
+
+import (
+	"context"
+	"time"
+
+	"github.com/zrepl/zrepl/util/chainlock"
+)
+
+type SpanInfo interface {
+	StartedAt() time.Time
+	EndedAt() time.Time
+	TaskAndSpanStack(kind *StackKind) string
+}
+
+type Callback struct {
+	OnBegin func(ctx context.Context)
+	OnEnd   func(ctx context.Context, spanInfo SpanInfo)
+}
+
+var callbacks struct {
+	mtx chainlock.L
+	cs  []Callback
+}
+
+func RegisterCallback(c Callback) {
+	callbacks.mtx.HoldWhile(func() {
+		callbacks.cs = append(callbacks.cs, c)
+	})
+}
+
+func callbackBeginSpan(ctx context.Context) func(SpanInfo) {
+	// capture the current state of callbacks into a local variable
+	// this is safe because the slice is append-only and immutable
+
+	// (it is important that a callback registered _after_ callbackBeginSpin is called does not get called on OnEnd)
+
+	var cbs []Callback
+	callbacks.mtx.HoldWhile(func() {
+		cbs = callbacks.cs
+	})
+	for _, cb := range cbs {
+		if cb.OnBegin != nil {
+			cb.OnBegin(ctx)
+		}
+	}
+	return func(spanInfo SpanInfo) {
+		for _, cb := range cbs {
+			if cb.OnEnd != nil {
+				cb.OnEnd(ctx, spanInfo)
+			}
+		}
+	}
+}

daemon/logging/trace/trace_chrometrace.go

@@ -0,0 +1,230 @@
+package trace
+
+// The functions in this file are concerned with the generation
+// of trace files based on the information from WithTask and WithSpan.
+//
+// The emitted trace files are open-ended array of JSON objects
+// that follow the Chrome trace file format:
+//   https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview
+//
+// The emitted JSON can be loaded into Chrome's chrome://tracing view.
+//
+// The trace file can be written to a file whose path is specified in an env file,
+// and be written to web sockets established on ChrometraceHttpHandler
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"sync"
+	"sync/atomic"
+
+	"golang.org/x/net/websocket"
+
+	"github.com/zrepl/zrepl/util/envconst"
+)
+
+var chrometracePID string
+
+func init() {
+	var err error
+	chrometracePID, err = os.Hostname()
+	if err != nil {
+		panic(err)
+	}
+	chrometracePID = fmt.Sprintf("%q", chrometracePID)
+}
+
+type chrometraceEvent struct {
+	Cat                       string   `json:"cat,omitempty"`
+	Name                      string   `json:"name"`
+	Stack                     []string `json:"stack,omitempty"`
+	Phase                     string   `json:"ph"`
+	TimestampUnixMicroseconds int64    `json:"ts"`
+	DurationMicroseconds      int64    `json:"dur,omitempty"`
+	Pid                       string   `json:"pid"`
+	Tid                       string   `json:"tid"`
+	Id                        string   `json:"id,omitempty"`
+}
+
+func chrometraceBeginSpan(s *traceNode) {
+	taskName := s.TaskName()
+	chrometraceWrite(chrometraceEvent{
+		Name:                      s.annotation,
+		Phase:                     "B",
+		TimestampUnixMicroseconds: s.startedAt.UnixNano() / 1000,
+		Pid:                       chrometracePID,
+		Tid:                       taskName,
+	})
+}
+
+func chrometraceEndSpan(s *traceNode) {
+	taskName := s.TaskName()
+	chrometraceWrite(chrometraceEvent{
+		Name:                      s.annotation,
+		Phase:                     "E",
+		TimestampUnixMicroseconds: s.endedAt.UnixNano() / 1000,
+		Pid:                       chrometracePID,
+		Tid:                       taskName,
+	})
+}
+
+var chrometraceFlowId uint64
+
+func chrometraceBeginTask(s *traceNode) {
+	chrometraceBeginSpan(s)
+
+	if s.parentTask == nil {
+		return
+	}
+	// beginning of a task that has a parent
+	// => use flow events to link parent and child
+
+	flowId := atomic.AddUint64(&chrometraceFlowId, 1)
+	flowIdStr := fmt.Sprintf("%x", flowId)
+
+	parentTask := s.parentTask.TaskName()
+	chrometraceWrite(chrometraceEvent{
+		Cat:                       "task", // seems to be necessary, otherwise the GUI shows some `indexOf` JS error
+		Name:                      "child-task",
+		Phase:                     "s",
+		TimestampUnixMicroseconds: s.startedAt.UnixNano() / 1000, // yes, the child's timestamp (=> from-point of the flow line is at right x-position of parent's bar)
+		Pid:                       chrometracePID,
+		Tid:                       parentTask,
+		Id:                        flowIdStr,
+	})
+
+	childTask := s.TaskName()
+	if parentTask == childTask {
+		panic(parentTask)
+	}
+	chrometraceWrite(chrometraceEvent{
+		Cat:                       "task", // seems to be necessary, otherwise the GUI shows some `indexOf` JS error
+		Name:                      "child-task",
+		Phase:                     "f",
+		TimestampUnixMicroseconds: s.startedAt.UnixNano() / 1000,
+		Pid:                       chrometracePID,
+		Tid:                       childTask,
+		Id:                        flowIdStr,
+	})
+}
+
+func chrometraceEndTask(s *traceNode) {
+	chrometraceEndSpan(s)
+}
+
+type chrometraceConsumerRegistration struct {
+	w io.Writer
+	// errored must have capacity 1, the writer thread will send to it non-blocking, then close it
+	errored chan error
+}
+
+var chrometraceConsumers struct {
+	register  chan chrometraceConsumerRegistration
+	consumers map[chrometraceConsumerRegistration]bool
+	write     chan []byte
+}
+
+func init() {
+	chrometraceConsumers.register = make(chan chrometraceConsumerRegistration)
+	chrometraceConsumers.consumers = make(map[chrometraceConsumerRegistration]bool)
+	chrometraceConsumers.write = make(chan []byte)
+	go func() {
+		kickConsumer := func(c chrometraceConsumerRegistration, err error) {
+			debug("chrometrace kicking consumer %#v after error %v", c, err)
+			select {
+			case c.errored <- err:
+			default:
+			}
+			close(c.errored)
+			delete(chrometraceConsumers.consumers, c)
+		}
+		for {
+			select {
+			case reg := <-chrometraceConsumers.register:
+				debug("registered chrometrace consumer %#v", reg)
+				chrometraceConsumers.consumers[reg] = true
+				n, err := reg.w.Write([]byte("[\n"))
+				if err != nil {
+					kickConsumer(reg, err)
+				} else if n != 2 {
+					kickConsumer(reg, fmt.Errorf("short write: %v", n))
+				}
+				// successfully registered
+
+			case buf := <-chrometraceConsumers.write:
+				debug("chrometrace write request: %s", string(buf))
+				var r bytes.Reader
+				for c := range chrometraceConsumers.consumers {
+					r.Reset(buf)
+					n, err := io.Copy(c.w, &r)
+					debug("chrometrace wrote n=%v bytes to consumer %#v", n, c)
+					if err != nil {
+						kickConsumer(c, err)
+					}
+				}
+			}
+		}
+	}()
+}
+
+func chrometraceWrite(i interface{}) {
+	var buf bytes.Buffer
+	err := json.NewEncoder(&buf).Encode(i)
+	if err != nil {
+		panic(err)
+	}
+	buf.WriteString(",")
+	chrometraceConsumers.write <- buf.Bytes()
+}
+
+func ChrometraceClientWebsocketHandler(conn *websocket.Conn) {
+	defer conn.Close()
+
+	var wg sync.WaitGroup
+	defer wg.Wait()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		r := bufio.NewReader(conn)
+		_, _, _ = r.ReadLine() // ignore errors
+		conn.Close()
+	}()
+
+	errored := make(chan error, 1)
+	chrometraceConsumers.register <- chrometraceConsumerRegistration{
+		w:       conn,
+		errored: errored,
+	}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		<-errored
+		conn.Close()
+	}()
+}
+
+var chrometraceFileConsumerPath = envconst.String("ZREPL_ACTIVITY_TRACE", "")
+
+func init() {
+	if chrometraceFileConsumerPath != "" {
+		var err error
+		f, err := os.Create(chrometraceFileConsumerPath)
+		if err != nil {
+			panic(err)
+		}
+		errored := make(chan error, 1)
+		chrometraceConsumers.register <- chrometraceConsumerRegistration{
+			w:       f,
+			errored: errored,
+		}
+		go func() {
+			<-errored
+			f.Close()
+		}()
+	}
+}

daemon/logging/trace/trace_context.go

@@ -0,0 +1,27 @@
+package trace
+
+import "context"
+
+type contextKey int
+
+const (
+	contextKeyTraceNode contextKey = 1 + iota
+)
+
+var contextKeys = []contextKey{
+	contextKeyTraceNode,
+}
+
+// WithInherit inherits the task hierarchy from inheritFrom into ctx.
+// The returned context is a child of ctx, but its task and span are those of inheritFrom.
+//
+// Note that in most use cases, callers most likely want to call WithTask since it will most likely
+// be in some sort of connection handler context.
+func WithInherit(ctx, inheritFrom context.Context) context.Context {
+	for _, k := range contextKeys {
+		if v := inheritFrom.Value(k); v != nil {
+			ctx = context.WithValue(ctx, k, v) // no shadow
+		}
+	}
+	return ctx
+}

daemon/logging/trace/trace_convenience.go

@@ -0,0 +1,74 @@
+package trace
+
+import (
+	"context"
+	"fmt"
+	"runtime"
+	"strings"
+	"sync"
+)
+
+// use like this:
+//
+//	defer WithSpanFromStackUpdateCtx(&existingCtx)()
+//
+//
+func WithSpanFromStackUpdateCtx(ctx *context.Context) DoneFunc {
+	childSpanCtx, end := WithSpan(*ctx, getMyCallerOrPanic())
+	*ctx = childSpanCtx
+	return end
+}
+
+// derive task name from call stack (caller's name)
+func WithTaskFromStack(ctx context.Context) (context.Context, DoneFunc) {
+	return WithTask(ctx, getMyCallerOrPanic())
+}
+
+// derive task name from call stack (caller's name) and update *ctx
+// to point to be the child task ctx
+func WithTaskFromStackUpdateCtx(ctx *context.Context) DoneFunc {
+	child, end := WithTask(*ctx, getMyCallerOrPanic())
+	*ctx = child
+	return end
+}
+
+// create a task and a span within it in one call
+func WithTaskAndSpan(ctx context.Context, task string, span string) (context.Context, DoneFunc) {
+	ctx, endTask := WithTask(ctx, task)
+	ctx, endSpan := WithSpan(ctx, fmt.Sprintf("%s %s", task, span))
+	return ctx, func() {
+		endSpan()
+		endTask()
+	}
+}
+
+// create a span during which several child tasks are spawned using the `add` function
+func WithTaskGroup(ctx context.Context, taskGroup string) (_ context.Context, add func(f func(context.Context)), waitEnd DoneFunc) {
+	var wg sync.WaitGroup
+	ctx, endSpan := WithSpan(ctx, taskGroup)
+	add = func(f func(context.Context)) {
+		wg.Add(1)
+		defer wg.Done()
+		ctx, endTask := WithTask(ctx, taskGroup)
+		defer endTask()
+		f(ctx)
+	}
+	waitEnd = func() {
+		wg.Wait()
+		endSpan()
+	}
+	return ctx, add, waitEnd
+}
+
+func getMyCallerOrPanic() string {
+	pc, _, _, ok := runtime.Caller(2)
+	if !ok {
+		panic("cannot get caller")
+	}
+	details := runtime.FuncForPC(pc)
+	if ok && details != nil {
+		const prefix = "github.com/zrepl/zrepl"
+		return strings.TrimPrefix(strings.TrimPrefix(details.Name(), prefix), "/")
+	}
+	return ""
+}

daemon/logging/trace/trace_convenience_test.go

@@ -0,0 +1,16 @@
+package trace
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetCallerOrPanic(t *testing.T) {
+	withStackFromCtxMock := func() string {
+		return getMyCallerOrPanic()
+	}
+	ret := withStackFromCtxMock()
+	// zrepl prefix is stripped
+	assert.Equal(t, "daemon/logging/trace.TestGetCallerOrPanic", ret)
+}

daemon/logging/trace/trace_debug.go

@@ -0,0 +1,15 @@
+package trace
+
+import (
+	"fmt"
+	"os"
+)
+
+const debugEnabled = false
+
+func debug(format string, args ...interface{}) {
+	if !debugEnabled {
+		return
+	}
+	fmt.Fprintf(os.Stderr, format+"\n", args...)
+}

daemon/logging/trace/trace_genID.go

@@ -0,0 +1,47 @@
+package trace
+
+import (
+	"encoding/base64"
+	"math/rand"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/zrepl/zrepl/util/envconst"
+)
+
+var genIdPRNG = rand.New(rand.NewSource(1))
+
+func init() {
+	genIdPRNG.Seed(time.Now().UnixNano())
+	genIdPRNG.Seed(int64(os.Getpid()))
+}
+
+var genIdNumBytes = envconst.Int("ZREPL_TRACE_ID_NUM_BYTES", 3)
+
+func init() {
+	if genIdNumBytes < 1 {
+		panic("trace node id byte length must be at least 1")
+	}
+}
+
+func genID() string {
+	var out strings.Builder
+	enc := base64.NewEncoder(base64.RawStdEncoding, &out)
+	buf := make([]byte, genIdNumBytes)
+	for i := 0; i < len(buf); {
+		n, err := genIdPRNG.Read(buf[i:])
+		if err != nil {
+			panic(err)
+		}
+		i += n
+	}
+	n, err := enc.Write(buf[:])
+	if err != nil || n != len(buf) {
+		panic(err)
+	}
+	if err := enc.Close(); err != nil {
+		panic(err)
+	}
+	return out.String()
+}

daemon/logging/trace/trace_test.go

@@ -0,0 +1,205 @@
+package trace
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/gitchander/permutation"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRegularSpanUsage(t *testing.T) {
+	root, endRoot := WithTask(context.Background(), "root")
+	defer endRoot()
+
+	s1, endS1 := WithSpan(root, "parent")
+	s2, endS2 := WithSpan(s1, "child")
+	_, endS3 := WithSpan(s2, "grand-child")
+	require.NotPanics(t, func() { endS3() })
+	require.NotPanics(t, func() { endS2() })
+
+	// reuse
+	_, endS4 := WithSpan(s1, "child-2")
+	require.NotPanics(t, func() { endS4() })
+
+	// close parent
+	require.NotPanics(t, func() { endS1() })
+}
+
+func TestMultipleActiveChildSpansNotAllowed(t *testing.T) {
+	root, endRoot := WithTask(context.Background(), "root")
+	defer endRoot()
+
+	s1, _ := WithSpan(root, "s1")
+	_, endS2 := WithSpan(s1, "s1-child1")
+
+	require.PanicsWithValue(t, ErrAlreadyActiveChildSpan, func() {
+		_, _ = WithSpan(s1, "s1-child2")
+	})
+
+	endS2()
+
+	require.NotPanics(t, func() {
+		_, _ = WithSpan(s1, "s1-child2")
+	})
+}
+
+func TestForkingChildSpansNotAllowed(t *testing.T) {
+	root, endRoot := WithTask(context.Background(), "root")
+	defer endRoot()
+
+	s1, _ := WithSpan(root, "s1")
+	sc, endSC := WithSpan(s1, "s1-child")
+	_, _ = WithSpan(sc, "s1-child-child")
+
+	require.PanicsWithValue(t, ErrSpanStillHasActiveChildSpan, func() {
+		endSC()
+	})
+}
+
+func TestRegularTaskUsage(t *testing.T) {
+	// assert concurrent activities on different tasks can end in any order
+	closeOrder := []int{0, 1, 2}
+	closeOrders := permutation.New(permutation.IntSlice(closeOrder))
+	for closeOrders.Next() {
+		t.Run(fmt.Sprintf("%v", closeOrder), func(t *testing.T) {
+			root, endRoot := WithTask(context.Background(), "root")
+			defer endRoot()
+
+			c1, endC1 := WithTask(root, "c1")
+			defer endC1()
+			c2, endC2 := WithTask(root, "c2")
+			defer endC2()
+
+			// begin 3 concurrent activities
+			_, endAR := WithSpan(root, "aR")
+			_, endAC1 := WithSpan(c1, "aC1")
+			_, endAC2 := WithSpan(c2, "aC2")
+
+			endFuncs := []DoneFunc{endAR, endAC1, endAC2}
+			for _, i := range closeOrder {
+				require.NotPanics(t, func() {
+					endFuncs[i]()
+				}, "%v", i)
+			}
+		})
+	}
+}
+
+func TestTaskEndWithActiveChildTaskNotAllowed(t *testing.T) {
+	root, _ := WithTask(context.Background(), "root")
+	c, endC := WithTask(root, "child")
+	_, _ = WithTask(c, "grand-child")
+	func() {
+		defer func() {
+			r := recover()
+			require.NotNil(t, r)
+			err, ok := r.(error)
+			require.True(t, ok)
+			require.Equal(t, ErrTaskStillHasActiveChildTasks, errors.Cause(err))
+		}()
+		endC()
+	}()
+
+}
+
+func TestIdempotentEndTask(t *testing.T) {
+	_, end := WithTask(context.Background(), "root")
+	end()
+	require.NotPanics(t, func() { end() })
+}
+
+func TestSpansPanicIfNoParentTask(t *testing.T) {
+	require.Panics(t, func() { WithSpan(context.Background(), "taskless-span") })
+}
+
+func TestIdempotentEndSpan(t *testing.T) {
+	root, _ := WithTask(context.Background(), "root")
+	_, end := WithSpan(root, "span")
+	end()
+	require.NotPanics(t, func() { end() })
+}
+
+func logAndGetTraceNode(t *testing.T, descr string, ctx context.Context) *traceNode {
+	n, ok := ctx.Value(contextKeyTraceNode).(*traceNode)
+	require.True(t, ok)
+	t.Logf("% 20s %p %#v", descr, n, n)
+	return n
+}
+
+func TestWhiteboxHierachy(t *testing.T) {
+	root, e1 := WithTask(context.Background(), "root")
+	rootN := logAndGetTraceNode(t, "root", root)
+	assert.Nil(t, rootN.parentTask)
+	assert.Nil(t, rootN.parentSpan)
+
+	child, e2 := WithSpan(root, "child")
+	childN := logAndGetTraceNode(t, "child", child)
+	assert.Equal(t, rootN, childN.parentTask)
+	assert.Equal(t, rootN, childN.parentSpan)
+
+	grandchild, e3 := WithSpan(child, "grandchild")
+	grandchildN := logAndGetTraceNode(t, "grandchild", grandchild)
+	assert.Equal(t, rootN, grandchildN.parentTask)
+	assert.Equal(t, childN, grandchildN.parentSpan)
+
+	gcTask, e4 := WithTask(grandchild, "grandchild-task")
+	gcTaskN := logAndGetTraceNode(t, "grandchild-task", gcTask)
+	assert.Equal(t, rootN, gcTaskN.parentTask)
+	assert.Nil(t, gcTaskN.parentSpan)
+
+	// it is allowed that a child task outlives the _span_ in which it was created
+	// (albeit not its parent task)
+	e3()
+	e2()
+	gcTaskSpan, e5 := WithSpan(gcTask, "granschild-task-span")
+	gcTaskSpanN := logAndGetTraceNode(t, "granschild-task-span", gcTaskSpan)
+	assert.Equal(t, gcTaskN, gcTaskSpanN.parentTask)
+	assert.Equal(t, gcTaskN, gcTaskSpanN.parentSpan)
+	e5()
+
+	e4()
+	e1()
+}
+
+func TestOrphanTasksBecomeNewRootWhenAllAncestorsAreDead(t *testing.T) {
+	parent, e1 := WithTask(context.Background(), "parent")
+	_ = logAndGetTraceNode(t, "parent-task", parent)
+	e1()
+
+	var child context.Context
+	var e2 DoneFunc
+	require.NotPanics(t, func() {
+		child, e2 = WithTask(parent, "child")
+	})
+	childN := logAndGetTraceNode(t, "child-task", child)
+	assert.Nil(t, childN.parentTask)
+
+	grandchild, _ := WithTask(child, "grandchild")
+	grandchildN := logAndGetTraceNode(t, "grandchild-task", grandchild)
+	assert.Equal(t, childN, grandchildN.parentTask)
+
+	require.Panics(t, func() { e2() }, "if the parent was alive at creation of child task, wrong termination order remains a panicable offense though")
+}
+
+func TestOrphanTaskBecomesChildToNearestLiveAncestor(t *testing.T) {
+	parent, e1 := WithTask(context.Background(), "parent")
+	parentN := logAndGetTraceNode(t, "parent-task", parent)
+
+	child, e2 := WithTask(parent, "child")
+	childN := logAndGetTraceNode(t, "child-task", child)
+	assert.Equal(t, parentN, childN.parentTask)
+	e2()
+
+	grandchild, e3 := WithTask(child, "grandchild")
+	grandchildN := logAndGetTraceNode(t, "grandchild-task", grandchild)
+	assert.Equal(t, parentN, grandchildN.parentTask) // child is already dead
+
+	require.NotPanics(t, func() {
+		e3()
+		e1()
+	})
+}

daemon/logging/trace/trace_unique_concurrent_task_namer.go

@@ -0,0 +1,53 @@
+package trace
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/willf/bitset"
+)
+
+type uniqueConcurrentTaskNamer struct {
+	mtx    sync.Mutex
+	active map[string]*bitset.BitSet
+}
+
+// bitvecLengthGauge may be nil
+func newUniqueTaskNamer() *uniqueConcurrentTaskNamer {
+	return &uniqueConcurrentTaskNamer{
+		active: make(map[string]*bitset.BitSet),
+	}
+}
+
+// appends `#%d` to `name` such that until `done` is called,
+// it is guaranteed that `#%d` is not returned a second time for the same `name`
+func (namer *uniqueConcurrentTaskNamer) UniqueConcurrentTaskName(name string) (uniqueName string, done func()) {
+	if strings.Contains(name, "#") {
+		panic(name)
+	}
+	namer.mtx.Lock()
+	act, ok := namer.active[name]
+	if !ok {
+		act = bitset.New(64) // FIXME magic const
+		namer.active[name] = act
+	}
+	id, ok := act.NextClear(0)
+	if !ok {
+		// if !ok, all bits are 1 and act.Len() returns the next bit
+		id = act.Len()
+		// FIXME unbounded growth without reclamation
+	}
+	act.Set(id)
+	namer.mtx.Unlock()
+
+	return fmt.Sprintf("%s#%d", name, id), func() {
+		namer.mtx.Lock()
+		defer namer.mtx.Unlock()
+		act, ok := namer.active[name]
+		if !ok {
+			panic("must be initialized upon entry")
+		}
+		act.Clear(id)
+	}
+}

daemon/logging/trace/trace_unique_concurrent_task_namer_test.go

@@ -0,0 +1,58 @@
+package trace
+
+import (
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/willf/bitset"
+)
+
+func TestBitsetFeaturesForUniqueConcurrentTaskNamer(t *testing.T) {
+	var b bitset.BitSet
+	require.Equal(t, uint(0), b.Len())
+	require.Equal(t, uint(0), b.Count())
+
+	b.Set(0)
+	require.Equal(t, uint(1), b.Len())
+	require.Equal(t, uint(1), b.Count())
+
+	b.Set(8)
+	require.Equal(t, uint(9), b.Len())
+	require.Equal(t, uint(2), b.Count())
+
+	b.Set(1)
+	require.Equal(t, uint(9), b.Len())
+	require.Equal(t, uint(3), b.Count())
+}
+
+func TestUniqueConcurrentTaskNamer(t *testing.T) {
+	namer := newUniqueTaskNamer()
+
+	var wg sync.WaitGroup
+
+	const N = 8128
+	const Q = 23
+
+	var fails uint32
+	var m sync.Map
+	wg.Add(N)
+	for i := 0; i < N; i++ {
+		go func(i int) {
+			defer wg.Done()
+			name := fmt.Sprintf("%d", i/Q)
+			uniqueName, done := namer.UniqueConcurrentTaskName(name)
+			act, _ := m.LoadOrStore(uniqueName, i)
+			if act.(int) != i {
+				atomic.AddUint32(&fails, 1)
+			}
+			m.Delete(uniqueName)
+			done()
+		}(i)
+	}
+	wg.Wait()
+
+	require.Equal(t, uint32(0), fails)
+}

daemon/main.go

@@ -1,16 +1,18 @@
 package daemon
 
 import (
+	"context"
+
 	"github.com/zrepl/zrepl/cli"
 	"github.com/zrepl/zrepl/logger"
 )
 
 type Logger = logger.Logger
 
-var DaemonCmd = &cli.Subcommand {
+var DaemonCmd = &cli.Subcommand{
 	Use:   "daemon",
 	Short: "run the zrepl daemon",
-	Run: func(subcommand *cli.Subcommand, args []string) error {
-		return Run(subcommand.Config())
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		return Run(ctx, subcommand.Config())
 	},
 }

daemon/nethelpers/helpers.go

@@ -1,10 +1,11 @@
 package nethelpers
 
 import (
-	"github.com/pkg/errors"
 	"net"
 	"os"
 	"path/filepath"
+
+	"github.com/pkg/errors"
 )
 
 func PreparePrivateSockpath(sockpath string) error {

daemon/pprof.go

@@ -7,11 +7,16 @@ import (
 	"context"
 	"net"
 	"net/http/pprof"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"golang.org/x/net/websocket"
+
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 )
 
 type pprofServer struct {
 	cc       chan PprofServerControlMsg
-	state    PprofServerControlMsg
 	listener net.Listener
 }
 
@@ -63,7 +68,16 @@ outer:
 			mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
 			mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
 			mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
-			go http.Serve(s.listener, mux)
+			mux.Handle("/metrics", promhttp.Handler())
+			mux.Handle("/debug/zrepl/activity-trace", websocket.Handler(trace.ChrometraceClientWebsocketHandler))
+			go func() {
+				err := http.Serve(s.listener, mux)
+				if ctx.Err() != nil {
+					return
+				} else if err != nil {
+					job.GetLogger(ctx).WithError(err).Error("pprof server serve error")
+				}
+			}()
 			continue
 		}
 

daemon/prometheus.go

@@ -2,29 +2,36 @@ package daemon
 
 import (
 	"context"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/zfs"
 	"net"
 	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/rpc/dataconn/frameconn"
+	"github.com/zrepl/zrepl/util/tcpsock"
+	"github.com/zrepl/zrepl/zfs"
 )
 
 type prometheusJob struct {
-	listen string
+	listen   string
+	freeBind bool
 }
 
 func newPrometheusJobFromConfig(in *config.PrometheusMonitoring) (*prometheusJob, error) {
 	if _, _, err := net.SplitHostPort(in.Listen); err != nil {
 		return nil, err
 	}
-	return &prometheusJob{in.Listen}, nil
+	return &prometheusJob{in.Listen, in.ListenFreeBind}, nil
 }
 
 var prom struct {
-	taskLogEntries         *prometheus.CounterVec
+	taskLogEntries *prometheus.CounterVec
 }
 
 func init() {
@@ -41,6 +48,10 @@ func (j *prometheusJob) Name() string { return jobNamePrometheus }
 
 func (j *prometheusJob) Status() *job.Status { return &job.Status{Type: job.TypeInternal} }
 
+func (j *prometheusJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
+
+func (j *prometheusJob) SenderConfig() *endpoint.SenderConfig { return nil }
+
 func (j *prometheusJob) RegisterMetrics(registerer prometheus.Registerer) {}
 
 func (j *prometheusJob) Run(ctx context.Context) {
@@ -49,41 +60,46 @@ func (j *prometheusJob) Run(ctx context.Context) {
 		panic(err)
 	}
 
+	if err := frameconn.PrometheusRegister(prometheus.DefaultRegisterer); err != nil {
+		panic(err)
+	}
+
 	log := job.GetLogger(ctx)
 
-	l, err := net.Listen("tcp", j.listen)
+	l, err := tcpsock.Listen(j.listen, j.freeBind)
 	if err != nil {
 		log.WithError(err).Error("cannot listen")
+		return
 	}
 	go func() {
-		select {
-		case <-ctx.Done():
-			l.Close()
-		}
+		<-ctx.Done()
+		l.Close()
 	}()
 
 	mux := http.NewServeMux()
 	mux.Handle("/metrics", promhttp.Handler())
 
 	err = http.Serve(l, mux)
-	if err != nil {
+	if err != nil && ctx.Err() == nil {
 		log.WithError(err).Error("error while serving")
 	}
 
 }
 
 type prometheusJobOutlet struct {
-	jobName string
 }
 
 var _ logger.Outlet = prometheusJobOutlet{}
 
-func newPrometheusLogOutlet(jobName string) prometheusJobOutlet {
-	return prometheusJobOutlet{jobName}
+func newPrometheusLogOutlet() prometheusJobOutlet {
+	return prometheusJobOutlet{}
 }
 
 func (o prometheusJobOutlet) WriteEntry(entry logger.Entry) error {
-	prom.taskLogEntries.WithLabelValues(o.jobName, entry.Level.String()).Inc()
+	jobFieldVal, ok := entry.Fields[logging.JobField].(string)
+	if !ok {
+		jobFieldVal = "_nojobid"
+	}
+	prom.taskLogEntries.WithLabelValues(jobFieldVal, entry.Level.String()).Inc()
 	return nil
 }
-

daemon/pruner/pruner.go

@@ -3,30 +3,32 @@ package pruner
 import (
 	"context"
 	"fmt"
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/pruning"
-	"github.com/zrepl/zrepl/replication/pdu"
-	"github.com/zrepl/zrepl/util/envconst"
-	"github.com/zrepl/zrepl/util/watchdog"
-	"github.com/problame/go-streamrpc"
-	"net"
 	"sort"
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/pruning"
+	"github.com/zrepl/zrepl/replication/logic/pdu"
+	"github.com/zrepl/zrepl/util/envconst"
 )
 
-// Try to keep it compatible with gitub.com/zrepl/zrepl/replication.Endpoint
+// Try to keep it compatible with github.com/zrepl/zrepl/endpoint.Endpoint
 type History interface {
 	ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error)
+	ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
 }
 
+// Try to keep it compatible with github.com/zrepl/zrepl/endpoint.Endpoint
 type Target interface {
-	ListFilesystems(ctx context.Context) ([]*pdu.Filesystem, error)
-	ListFilesystemVersions(ctx context.Context, fs string) ([]*pdu.FilesystemVersion, error) // fix depS
+	ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
+	ListFilesystemVersions(ctx context.Context, req *pdu.ListFilesystemVersionsReq) (*pdu.ListFilesystemVersionsRes, error)
 	DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error)
 }
 
@@ -34,17 +36,13 @@ type Logger = logger.Logger
 
 type contextKey int
 
-const contextKeyLogger contextKey = 0
-
-func WithLogger(ctx context.Context, log Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLogger, log)
-}
+const (
+	contextKeyPruneSide contextKey = 1 + iota
+)
 
 func GetLogger(ctx context.Context) Logger {
-	if l, ok := ctx.Value(contextKeyLogger).(Logger); ok {
-		return l
-	}
-	return logger.NewNullLogger()
+	pruneSide := ctx.Value(contextKeyPruneSide).(string)
+	return logging.GetLogger(ctx, logging.SubsysPruning).WithField("prune_side", pruneSide)
 }
 
 type args struct {
@@ -54,21 +52,18 @@ type args struct {
 	rules                          []pruning.KeepRule
 	retryWait                      time.Duration
 	considerSnapAtCursorReplicated bool
-	promPruneSecs prometheus.Observer
+	promPruneSecs                  prometheus.Observer
 }
 
 type Pruner struct {
 	args args
 
-	Progress watchdog.KeepAlive
-
 	mtx sync.RWMutex
 
 	state State
 
-	// State ErrWait|ErrPerm
-	sleepUntil time.Time
-	err        error
+	// State PlanErr
+	err error
 
 	// State Exec
 	execQueue *execQueue
@@ -79,20 +74,33 @@ type PrunerFactory struct {
 	receiverRules                  []pruning.KeepRule
 	retryWait                      time.Duration
 	considerSnapAtCursorReplicated bool
+	promPruneSecs                  *prometheus.HistogramVec
+}
+
+type LocalPrunerFactory struct {
+	keepRules     []pruning.KeepRule
+	retryWait     time.Duration
 	promPruneSecs *prometheus.HistogramVec
 }
 
-func checkContainsKeep1(rules []pruning.KeepRule) error {
-	if len(rules) == 0 {
-		return nil //No keep rules means keep all - ok
+func NewLocalPrunerFactory(in config.PruningLocal, promPruneSecs *prometheus.HistogramVec) (*LocalPrunerFactory, error) {
+	rules, err := pruning.RulesFromConfig(in.Keep)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot build pruning rules")
 	}
-	for _, e := range rules {
-		switch e.(type) {
-		case *pruning.KeepLastN:
-			return nil
+	for _, r := range in.Keep {
+		if _, ok := r.Ret.(*config.PruneKeepNotReplicated); ok {
+			// rule NotReplicated  for a local pruner doesn't make sense
+			// because no replication happens with that job type
+			return nil, fmt.Errorf("single-site pruner cannot support `not_replicated` keep rule")
 		}
 	}
-	return errors.New("sender keep rules must contain last_n or be empty so that the last snapshot is definitely kept")
+	f := &LocalPrunerFactory{
+		keepRules:     rules,
+		retryWait:     envconst.Duration("ZREPL_PRUNER_RETRY_INTERVAL", 10*time.Second),
+		promPruneSecs: promPruneSecs,
+	}
+	return f, nil
 }
 
 func NewPrunerFactory(in config.PruningSenderReceiver, promPruneSecs *prometheus.HistogramVec) (*PrunerFactory, error) {
@@ -115,11 +123,11 @@ func NewPrunerFactory(in config.PruningSenderReceiver, promPruneSecs *prometheus
 		considerSnapAtCursorReplicated = considerSnapAtCursorReplicated || !knr.KeepSnapshotAtCursor
 	}
 	f := &PrunerFactory{
-		senderRules: keepRulesSender,
-		receiverRules: keepRulesReceiver,
-		retryWait: envconst.Duration("ZREPL_PRUNER_RETRY_INTERVAL", 10 * time.Second),
+		senderRules:                    keepRulesSender,
+		receiverRules:                  keepRulesReceiver,
+		retryWait:                      envconst.Duration("ZREPL_PRUNER_RETRY_INTERVAL", 10*time.Second),
 		considerSnapAtCursorReplicated: considerSnapAtCursorReplicated,
-		promPruneSecs: promPruneSecs,
+		promPruneSecs:                  promPruneSecs,
 	}
 	return f, nil
 }
@@ -127,7 +135,7 @@ func NewPrunerFactory(in config.PruningSenderReceiver, promPruneSecs *prometheus
 func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, receiver History) *Pruner {
 	p := &Pruner{
 		args: args{
-			WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "sender")),
+			context.WithValue(ctx, contextKeyPruneSide, "sender"),
 			target,
 			receiver,
 			f.senderRules,
@@ -143,7 +151,7 @@ func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, re
 func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target, receiver History) *Pruner {
 	p := &Pruner{
 		args: args{
-			WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "receiver")),
+			context.WithValue(ctx, contextKeyPruneSide, "receiver"),
 			target,
 			receiver,
 			f.receiverRules,
@@ -156,82 +164,70 @@ func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target,
 	return p
 }
 
+func (f *LocalPrunerFactory) BuildLocalPruner(ctx context.Context, target Target, receiver History) *Pruner {
+	p := &Pruner{
+		args: args{
+			context.WithValue(ctx, contextKeyPruneSide, "local"),
+			target,
+			receiver,
+			f.keepRules,
+			f.retryWait,
+			false, // considerSnapAtCursorReplicated is not relevant for local pruning
+			f.promPruneSecs.WithLabelValues("local"),
+		},
+		state: Plan,
+	}
+	return p
+}
+
 //go:generate enumer -type=State
 type State int
 
 const (
 	Plan State = 1 << iota
-	PlanWait
+	PlanErr
 	Exec
-	ExecWait
-	ErrPerm
+	ExecErr
 	Done
 )
 
-func (s State) statefunc() state {
-	var statemap = map[State]state{
-		Plan:     statePlan,
-		PlanWait: statePlanWait,
-		Exec:     stateExec,
-		ExecWait: stateExecWait,
-		ErrPerm:  nil,
-		Done:     nil,
-	}
-	return statemap[s]
-}
-
-func (s State) IsTerminal() bool {
-	return s.statefunc() == nil
-}
-
-type updater func(func(*Pruner)) State
-type state func(args *args, u updater) state
+type updater func(func(*Pruner))
 
 func (p *Pruner) Prune() {
 	p.prune(p.args)
 }
 
 func (p *Pruner) prune(args args) {
-	s := p.state.statefunc()
-	for s != nil {
-		pre := p.state
-		s = s(&args, func(f func(*Pruner)) State {
-			p.mtx.Lock()
-			defer p.mtx.Unlock()
-			f(p)
-			return p.state
-		})
-		post := p.state
-		GetLogger(args.ctx).
-			WithField("transition", fmt.Sprintf("%s=>%s", pre, post)).
-			Debug("state transition")
-		if err := p.Error(); err != nil {
-			GetLogger(args.ctx).
-				WithError(p.err).
-				WithField("state", post.String()).
-				Error("entering error state after error")
-		}
+	u := func(f func(*Pruner)) {
+		p.mtx.Lock()
+		defer p.mtx.Unlock()
+		f(p)
 	}
+	// TODO support automatic retries
+	// It is advisable to merge this code with package replication/driver before
+	// That will likely require re-modelling struct fs like replication/driver.attempt,
+	// including figuring out how to resume a plan after being interrupted by network errors
+	// The non-retrying code in this package should move straight to replication/logic.
+	doOneAttempt(&args, u)
 }
 
 type Report struct {
-	State string
-	SleepUntil time.Time
-	Error string
+	State              string
+	Error              string
 	Pending, Completed []FSReport
 }
 
 type FSReport struct {
-	Filesystem string
+	Filesystem                string
 	SnapshotList, DestroyList []SnapshotReport
-	ErrorCount int
-	LastError string
+	SkipReason                FSSkipReason
+	LastError                 string
 }
 
 type SnapshotReport struct {
-	Name string
+	Name       string
 	Replicated bool
-	Date time.Time
+	Date       time.Time
 }
 
 func (p *Pruner) Report() *Report {
@@ -240,13 +236,8 @@ func (p *Pruner) Report() *Report {
 
 	r := Report{State: p.state.String()}
 
-	if p.state & (PlanWait|ExecWait) != 0 {
-		r.SleepUntil = p.sleepUntil
-	}
-	if p.state & (PlanWait|ExecWait|ErrPerm) != 0 {
-		if p.err != nil {
-			r.Error = p.err.Error()
-		}
+	if p.err != nil {
+		r.Error = p.err.Error()
 	}
 
 	if p.execQueue != nil {
@@ -262,17 +253,16 @@ func (p *Pruner) State() State {
 	return p.state
 }
 
-func (p *Pruner) Error() error {
-	p.mtx.Lock()
-	defer p.mtx.Unlock()
-	if p.state & (PlanWait|ExecWait|ErrPerm) != 0 {
-		return p.err
-	}
-	return nil
-}
-
 type fs struct {
-	path  string
+	path string
+
+	// permanent error during planning
+	planErr        error
+	planErrContext string
+
+	// if != "", the fs was skipped for planning and the field
+	// contains the reason
+	skipReason FSSkipReason
 
 	// snapshots presented by target
 	// (type snapshot)
@@ -285,8 +275,18 @@ type fs struct {
 
 	// only during Exec state, also used by execQueue
 	execErrLast error
-	execErrCount int
+}
 
+type FSSkipReason string
+
+const (
+	NotSkipped                   = ""
+	SkipPlaceholder              = "filesystem is placeholder"
+	SkipNoCorrespondenceOnSender = "filesystem has no correspondence on sender"
+)
+
+func (r FSSkipReason) NotSkipped() bool {
+	return r == NotSkipped
 }
 
 func (f *fs) Report() FSReport {
@@ -295,8 +295,14 @@ func (f *fs) Report() FSReport {
 
 	r := FSReport{}
 	r.Filesystem = f.path
-	r.ErrorCount = f.execErrCount
-	if f.execErrLast != nil {
+	r.SkipReason = f.skipReason
+	if !r.SkipReason.NotSkipped() {
+		return r
+	}
+
+	if f.planErr != nil {
+		r.LastError = f.planErr.Error()
+	} else if f.execErrLast != nil {
 		r.LastError = f.execErrLast.Error()
 	}
 
@@ -306,7 +312,7 @@ func (f *fs) Report() FSReport {
 	}
 
 	r.DestroyList = make([]SnapshotReport, len(f.destroyList))
-	for i, snap := range f.destroyList{
+	for i, snap := range f.destroyList {
 		r.DestroyList[i] = snap.(snapshot).Report()
 	}
 
@@ -335,90 +341,85 @@ func (s snapshot) Replicated() bool { return s.replicated }
 
 func (s snapshot) Date() time.Time { return s.date }
 
-type Error interface {
-	error
-	Temporary() bool
-}
-
-var _ Error = net.Error(nil)
-var _ Error = streamrpc.Error(nil)
-
-func shouldRetry(e error) bool {
-	if neterr, ok := e.(net.Error); ok {
-		return neterr.Temporary()
-	}
-	return false
-}
-
-func onErr(u updater, e error) state {
-	return u(func(p *Pruner) {
-		p.err = e
-		if !shouldRetry(e) {
-			p.state = ErrPerm
-			return
-		}
-		switch p.state {
-		case Plan:
-			p.state = PlanWait
-		case Exec:
-			p.state = ExecWait
-		default:
-			panic(p.state)
-		}
-	}).statefunc()
-}
-
-func statePlan(a *args, u updater) state {
+func doOneAttempt(a *args, u updater) {
 
 	ctx, target, receiver := a.ctx, a.target, a.receiver
-	var ka *watchdog.KeepAlive
-	u(func(pruner *Pruner) {
-		ka = &pruner.Progress
-	})
 
-	tfss, err := target.ListFilesystems(ctx)
+	sfssres, err := receiver.ListFilesystems(ctx, &pdu.ListFilesystemReq{})
 	if err != nil {
-		return onErr(u, err)
+		u(func(p *Pruner) {
+			p.state = PlanErr
+			p.err = err
+		})
+		return
+	}
+	sfss := make(map[string]*pdu.Filesystem)
+	for _, sfs := range sfssres.GetFilesystems() {
+		sfss[sfs.GetPath()] = sfs
 	}
 
+	tfssres, err := target.ListFilesystems(ctx, &pdu.ListFilesystemReq{})
+	if err != nil {
+		u(func(p *Pruner) {
+			p.state = PlanErr
+			p.err = err
+		})
+		return
+	}
+	tfss := tfssres.GetFilesystems()
+
 	pfss := make([]*fs, len(tfss))
+tfss_loop:
 	for i, tfs := range tfss {
 
 		l := GetLogger(ctx).WithField("fs", tfs.Path)
 		l.Debug("plan filesystem")
 
-
 		pfs := &fs{
-			path:  tfs.Path,
+			path: tfs.Path,
 		}
 		pfss[i] = pfs
 
-		tfsvs, err := target.ListFilesystemVersions(ctx, tfs.Path)
-		if err != nil {
-			l.WithError(err).Error("cannot list filesystem versions")
-			return onErr(u, err)
+		if tfs.GetIsPlaceholder() {
+			pfs.skipReason = SkipPlaceholder
+			l.WithField("skip_reason", pfs.skipReason).Debug("skipping filesystem")
+			continue
+		} else if sfs := sfss[tfs.GetPath()]; sfs == nil {
+			pfs.skipReason = SkipNoCorrespondenceOnSender
+			l.WithField("skip_reason", pfs.skipReason).WithField("sfs", sfs.GetPath()).Debug("skipping filesystem")
+			continue
 		}
+
+		pfsPlanErrAndLog := func(err error, message string) {
+			t := fmt.Sprintf("%T", err)
+			pfs.planErr = err
+			pfs.planErrContext = message
+			l.WithField("orig_err_type", t).WithError(err).Error(fmt.Sprintf("%s: plan error, skipping filesystem", message))
+		}
+
+		tfsvsres, err := target.ListFilesystemVersions(ctx, &pdu.ListFilesystemVersionsReq{Filesystem: tfs.Path})
+		if err != nil {
+			pfsPlanErrAndLog(err, "cannot list filesystem versions")
+			continue tfss_loop
+		}
+		tfsvs := tfsvsres.GetVersions()
 		// no progress here since we could run in a live-lock (must have used target AND receiver before progress)
 
 		pfs.snaps = make([]pruning.Snapshot, 0, len(tfsvs))
 
 		rcReq := &pdu.ReplicationCursorReq{
 			Filesystem: tfs.Path,
-			Op:         &pdu.ReplicationCursorReq_Get{
-				Get: &pdu.ReplicationCursorReq_GetOp{},
-			},
 		}
 		rc, err := receiver.ReplicationCursor(ctx, rcReq)
 		if err != nil {
-			l.WithError(err).Error("cannot get replication cursor")
-			return onErr(u, err)
+			pfsPlanErrAndLog(err, "cannot get replication cursor bookmark")
+			continue tfss_loop
 		}
-		if rc.GetError() != "" {
-			l.WithField("reqErr", rc.GetError()).Error("cannot get replication cursor")
-			return onErr(u, err)
+		if rc.GetNotexist() {
+			err := errors.New("replication cursor bookmark does not exist (one successful replication is required before pruning works)")
+			pfsPlanErrAndLog(err, "")
+			continue tfss_loop
 		}
-		ka.MadeProgress()
-
 
 		// scan from older to newer, all snapshots older than cursor are interpreted as replicated
 		sort.Slice(tfsvs, func(i, j int) bool {
@@ -441,11 +442,9 @@ func statePlan(a *args, u updater) state {
 			}
 			creation, err := tfsv.CreationAsTime()
 			if err != nil {
-				err := fmt.Errorf("%s%s has invalid creation date: %s", tfs, tfsv.RelName(), err)
-				l.WithError(err).
-					WithField("tfsv", tfsv.RelName()).
-					Error("error with fileesystem version")
-				return onErr(u, err)
+				err := fmt.Errorf("%s: %s", tfsv.RelName(), err)
+				pfsPlanErrAndLog(err, "fs version with invalid creation date")
+				continue tfss_loop
 			}
 			// note that we cannot use CreateTXG because target and receiver could be on different pools
 			atCursor := tfsv.Guid == rc.GetGuid()
@@ -457,43 +456,61 @@ func statePlan(a *args, u updater) state {
 			})
 		}
 		if preCursor {
-			err := fmt.Errorf("replication cursor not found in prune target filesystem versions")
-			l.Error(err.Error())
-			return onErr(u, err)
+			pfsPlanErrAndLog(fmt.Errorf("replication cursor not found in prune target filesystem versions"), "")
+			continue tfss_loop
 		}
 
 		// Apply prune rules
 		pfs.destroyList = pruning.PruneSnapshots(pfs.snaps, a.rules)
-		ka.MadeProgress()
 	}
 
-	return u(func(pruner *Pruner) {
-		pruner.Progress.MadeProgress()
+	u(func(pruner *Pruner) {
 		pruner.execQueue = newExecQueue(len(pfss))
 		for _, pfs := range pfss {
 			pruner.execQueue.Put(pfs, nil, false)
 		}
 		pruner.state = Exec
-	}).statefunc()
-}
+	})
 
-func stateExec(a *args, u updater) state {
-
-	var pfs *fs
-	state := u(func(pruner *Pruner) {
-		pfs = pruner.execQueue.Pop()
+	for {
+		var pfs *fs
+		u(func(pruner *Pruner) {
+			pfs = pruner.execQueue.Pop()
+		})
 		if pfs == nil {
-			nextState := Done
-			if pruner.execQueue.HasCompletedFSWithErrors() {
-				nextState = ErrPerm
-			}
-			pruner.state = nextState
-			return
+			break
+		}
+		doOneAttemptExec(a, u, pfs)
+	}
+
+	var rep *Report
+	{
+		// must not hold lock for report
+		var pruner *Pruner
+		u(func(p *Pruner) {
+			pruner = p
+		})
+		rep = pruner.Report()
+	}
+	u(func(p *Pruner) {
+		if len(rep.Pending) > 0 {
+			panic("queue should not have pending items at this point")
+		}
+		hadErr := false
+		for _, fsr := range rep.Completed {
+			hadErr = hadErr || fsr.SkipReason.NotSkipped() && fsr.LastError != ""
+		}
+		if hadErr {
+			p.state = ExecErr
+		} else {
+			p.state = Done
 		}
 	})
-	if state != Exec {
-		return state.statefunc()
-	}
+
+}
+
+// attempts to exec pfs, puts it back into the queue with the result
+func doOneAttemptExec(a *args, u updater, pfs *fs) {
 
 	destroyList := make([]*pdu.FilesystemVersion, len(pfs.destroyList))
 	for i := range destroyList {
@@ -513,7 +530,7 @@ func stateExec(a *args, u updater) state {
 		u(func(pruner *Pruner) {
 			pruner.execQueue.Put(pfs, err, false)
 		})
-		return onErr(u, err)
+		return
 	}
 	// check if all snapshots were destroyed
 	destroyResults := make(map[string]*pdu.DestroySnapshotRes)
@@ -523,20 +540,20 @@ func stateExec(a *args, u updater) state {
 	err = nil
 	destroyFails := make([]*pdu.DestroySnapshotRes, 0)
 	for _, reqDestroy := range destroyList {
-		 res, ok := destroyResults[reqDestroy.Name]
-		 if !ok {
-		 	err = fmt.Errorf("missing destroy-result for %s", reqDestroy.RelName())
-		 	break
-		 } else if res.Error != "" {
-		 	destroyFails = append(destroyFails, res)
-		 }
+		res, ok := destroyResults[reqDestroy.Name]
+		if !ok {
+			err = fmt.Errorf("missing destroy-result for %s", reqDestroy.RelName())
+			break
+		} else if res.Error != "" {
+			destroyFails = append(destroyFails, res)
+		}
 	}
 	if err == nil && len(destroyFails) > 0 {
 		names := make([]string, len(destroyFails))
 		pairs := make([]string, len(destroyFails))
 		allSame := true
 		lastMsg := destroyFails[0].Error
-		for i := 0; i < len(destroyFails); i++{
+		for i := 0; i < len(destroyFails); i++ {
 			allSame = allSame && destroyFails[i].Error == lastMsg
 			relname := destroyFails[i].Snapshot.RelName()
 			names[i] = relname
@@ -554,31 +571,6 @@ func stateExec(a *args, u updater) state {
 	})
 	if err != nil {
 		GetLogger(a.ctx).WithError(err).Error("target could not destroy snapshots")
-		return onErr(u, err)
-	}
-
-	return u(func(pruner *Pruner) {
-		pruner.Progress.MadeProgress()
-	}).statefunc()
-}
-
-func stateExecWait(a *args, u updater) state {
-	return doWait(Exec, a, u)
-}
-
-func statePlanWait(a *args, u updater) state {
-	return doWait(Plan, a, u)
-}
-
-func doWait(goback State, a *args, u updater) state {
-	timer := time.NewTimer(a.retryWait)
-	defer timer.Stop()
-	select {
-	case <-timer.C:
-		return u(func(pruner *Pruner) {
-			pruner.state = goback
-		}).statefunc()
-	case <-a.ctx.Done():
-		return onErr(u, a.ctx.Err())
+		return
 	}
 }