config: support zrepl's day and week units for snapshotting.interval

Originally, I had a patch that would replace all usages of
time.Duration in package config with the new config.Duration
types, but:
1. these are all timeouts/retry intervals that have default values.
   Most users don't touch them, and if they do, they don't need
   day or week units.
2. go-yaml's error reporting for yaml.Unmarshaler is inferior to
   built-in types (line numbers are missing, so the error would not have
   sufficient context)

fixes https://github.com/zrepl/zrepl/issues/486

.circleci/config.yml

@@ -0,0 +1,379 @@
+version: 2.1
+
+commands:
+  setup-home-local-bin:
+    steps:
+      - run:
+          shell: /bin/bash -euo pipefail
+          command: |
+            mkdir -p "$HOME/.local/bin"
+            line='export PATH="$HOME/.local/bin:$PATH"'
+            if grep "$line" $BASH_ENV >/dev/null; then
+              echo "$line" >> $BASH_ENV
+            fi
+
+  invoke-lazy-sh:
+    parameters:
+      subcommand:
+        type: string
+    steps:
+      - run:
+          environment:
+            TERM: xterm
+          command: ./lazy.sh <<parameters.subcommand>>
+
+  apt-update-and-install-common-deps:
+    steps:
+      - run: sudo apt update && sudo apt install gawk make
+
+  restore-cache-gomod:
+    steps:
+      - restore_cache:
+          key: go-mod-v4-{{ checksum "go.sum" }}
+  save-cache-gomod:
+    steps:
+      - save_cache:
+          key: go-mod-v4-{{ checksum "go.sum" }}
+          paths:
+            - "/go/pkg/mod"
+
+  install-godep:
+    steps:
+      - apt-update-and-install-common-deps
+      - invoke-lazy-sh:
+          subcommand: godep
+
+  install-docdep:
+    steps:
+      - apt-update-and-install-common-deps
+      - run: sudo apt install python3 python3-pip libgirepository1.0-dev
+      - invoke-lazy-sh:
+          subcommand: docdep
+
+  download-and-install-minio-client:
+    steps:
+      - setup-home-local-bin
+      - restore_cache:
+          key: minio-client-v2
+      - run:
+          shell: /bin/bash -eo pipefail
+          command: |
+            if which mc; then exit 0; fi
+            sudo curl -sSL https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2020-08-20T00-23-01Z \
+              -o "$HOME/.local/bin/mc"
+            sudo chmod +x "$HOME/.local/bin/mc"
+      - save_cache:
+          key: minio-client-v2
+          paths:
+            - "$HOME/.local/bin/mc"
+
+  upload-minio:
+    parameters:
+      src:
+        type: string
+      dst:
+        type: string
+    steps:
+      - run:
+          shell: /bin/bash -eo pipefail
+          when: always
+          command: |
+            if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
+              echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
+              exit 0
+            fi
+            set -u # from now on
+
+            mc config host add --api s3v4 zrepl-minio https://minio.cschwarz.com ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
+
+            # keep in sync with set-github-minio-status
+            jobprefix=zrepl-ci-artifacts/${CIRCLE_SHA1}-pipeline-<<pipeline.number>>/${CIRCLE_JOB}
+
+            # Upload artifacts
+            mkdir -p ./artifacts
+            mc cp -r <<parameters.src>> "zrepl-minio/$jobprefix/<<parameters.dst>>"
+
+  set-github-minio-status:
+    parameters:
+      context:
+        type: string
+      description:
+        type: string
+      minio-dst:
+        type: string
+    steps:
+      - run:
+          shell: /bin/bash -eo pipefail
+          command: |
+            if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
+              echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
+              exit 0
+            fi
+            set -u # from now on
+
+            # keep in sync with with upload-minio command
+            jobprefix=zrepl-ci-artifacts/${CIRCLE_SHA1}-pipeline-<<pipeline.number>>/${CIRCLE_JOB}
+            # Push Artifact Link to GitHub
+            REPO="zrepl/zrepl"
+            COMMIT="${CIRCLE_SHA1}"
+            JOB_NAME="${CIRCLE_JOB}"
+            CONTEXT="<<parameters.context>>"
+            DESCRIPTION="<<parameters.description>>"
+            TARGETURL=https://minio.cschwarz.com/minio/"$jobprefix"/"<<parameters.minio-dst>>"
+            curl "https://api.github.com/repos/$REPO/statuses/$COMMIT" \
+              -H "Content-Type: application/json" \
+              -H "Authorization: token $GITHUB_COMMIT_STATUS_TOKEN" \
+              -X POST \
+              -d '{"context":"'"$CONTEXT"'", "state": "success", "description":"'"$DESCRIPTION"'", "target_url":"'"$TARGETURL"'"}'
+
+
+  trigger-pipeline:
+    parameters:
+      body_no_shell_subst:
+        type: string
+    steps:
+      - run: |
+          curl -X POST https://circleci.com/api/v2/project/github/zrepl/zrepl/pipeline \
+            -H 'Content-Type: application/json' \
+            -H 'Accept: application/json' \
+            -H "Circle-Token: $ZREPL_BOT_CIRCLE_TOKEN" \
+            --data '<<parameters.body_no_shell_subst>>'
+
+parameters:
+  do_ci:
+    type: boolean
+    default: true
+
+  do_release:
+    type: boolean
+    default: false
+
+  release_docker_baseimage_tag:
+    type: string
+    default: "1.17"
+
+workflows:
+  version: 2
+
+  ci:
+    when: << pipeline.parameters.do_ci >>
+    jobs:
+      - quickcheck-docs
+      - quickcheck-go: &quickcheck-go-smoketest
+          name: quickcheck-go-amd64-linux-1.17
+          goversion: &latest-go-release "1.17"
+          goos: linux
+          goarch: amd64
+      - test-go-on-latest-go-release:
+          goversion: *latest-go-release
+      - quickcheck-go:
+          requires:
+            - quickcheck-go-amd64-linux-1.17 #quickcheck-go-smoketest.name
+          matrix: &quickcheck-go-matrix
+            alias: quickcheck-go-matrix
+            parameters:
+              goversion: [*latest-go-release, "1.12"]
+              goos: ["linux", "freebsd"]
+              goarch: ["amd64", "arm64"]
+            exclude:
+              # don't re-do quickcheck-go-smoketest
+              - goversion: *latest-go-release
+                goos: linux
+                goarch: amd64
+              # not supported by Go 1.12
+              - goversion: "1.12"
+                goos: freebsd
+                goarch: arm64
+
+  release:
+    when: << pipeline.parameters.do_release >>
+    jobs:
+      - release-build
+      - release-deb:
+          requires:
+            - release-build
+      - release-rpm:
+          requires:
+            - release-build
+      - release-upload:
+          requires:
+            - release-build
+            - release-deb
+            - release-rpm
+
+  periodic:
+    triggers:
+      - schedule:
+          cron: "00 17 * * *"
+          filters:
+            branches:
+              only:
+                - master
+                - stable
+                - problame/circleci-build
+    jobs:
+      - periodic-full-pipeline-run
+
+  zrepl.github.io:
+    jobs:
+      - publish-zrepl-github-io:
+          filters:
+            branches:
+              only:
+                - stable
+
+jobs:
+  quickcheck-docs:
+    docker:
+      - image: cimg/base:2020.08
+    steps:
+      - checkout
+      - install-docdep
+      - run: make docs
+
+      - download-and-install-minio-client
+      - upload-minio:
+          src: artifacts
+          dst: ""
+      - set-github-minio-status:
+          context: artifacts/${CIRCLE_JOB}
+          description: artifacts of CI job ${CIRCLE_JOB}
+          minio-dst: ""
+
+  quickcheck-go:
+    parameters:
+      goversion:
+        type: string
+      goos:
+        type: string
+      goarch:
+        type: string
+    docker:
+      - image: circleci/golang:<<parameters.goversion>>
+    environment:
+      GOOS: <<parameters.goos>>
+      GOARCH: <<parameters.goarch>>
+
+    steps:
+      - checkout
+
+      - restore-cache-gomod
+      - run: go mod download
+      - run: cd build && go mod download
+      - save-cache-gomod
+
+      - install-godep
+      - run: make formatcheck
+      - run: make generate-platform-test-list
+      - run: make zrepl-bin test-platform-bin
+      - run: make vet
+      - run: make lint
+
+      - download-and-install-minio-client
+      - run: rm -f artifacts/generate-platform-test-list
+      - store_artifacts:
+          path: artifacts
+      - upload-minio:
+          src: artifacts
+          dst: ""
+      - set-github-minio-status:
+          context: artifacts/${CIRCLE_JOB}
+          description: artifacts of CI job ${CIRCLE_JOB}
+          minio-dst: ""
+
+  test-go-on-latest-go-release:
+    parameters:
+      goversion:
+        type: string
+    docker:
+      - image: circleci/golang:<<parameters.goversion>>
+    steps:
+      - checkout
+      - restore-cache-gomod
+      - run: make test-go
+      # don't save-cache-gomod here, test-go doesn't pull all the dependencies
+
+  release-build:
+    machine:
+      image: ubuntu-2004:202201-02
+    steps:
+      - checkout
+      - run: make release-docker RELEASE_DOCKER_BASEIMAGE_TAG=<<pipeline.parameters.release_docker_baseimage_tag>>
+      - persist_to_workspace:
+          root: .
+          paths: [.]
+  release-deb:
+    machine:
+      image: ubuntu-2004:202201-02
+    steps:
+      - attach_workspace:
+          at: .
+      - run: make debs-docker
+      - persist_to_workspace:
+          root: .
+          paths:
+            - "artifacts/*.deb"
+
+  release-rpm:
+    machine:
+      image: ubuntu-2004:202201-02
+    steps:
+      - attach_workspace:
+          at: .
+      - run: make rpms-docker
+      - persist_to_workspace:
+          root: .
+          paths:
+            - "artifacts/*.rpm"
+
+  release-upload:
+    docker:
+      - image: cimg/base:2020.08
+    steps:
+      - attach_workspace:
+          at: .
+      - store_artifacts:
+          path: artifacts
+      - download-and-install-minio-client
+      - upload-minio:
+          src: artifacts
+          dst: ""
+      - set-github-minio-status:
+          context: artifacts/release
+          description: CI-generated release artifacts
+          minio-dst: ""
+
+  periodic-full-pipeline-run:
+    docker:
+      - image: cimg/base:2020.08
+    steps:
+      - trigger-pipeline:
+          body_no_shell_subst: '{"branch":"<<pipeline.git.branch>>", "parameters": { "do_ci": true, "do_release": true }}'
+
+  publish-zrepl-github-io:
+    docker:
+      - image: cimg/python:3.7
+    steps:
+      - checkout
+      - invoke-lazy-sh:
+          subcommand: docdep
+      - run:
+          command: |
+            git config --global user.email "zreplbot@cschwarz.com"
+            git config --global user.name "zrepl-github-io-ci"
+
+      # https://circleci.com/docs/2.0/add-ssh-key/#adding-multiple-keys-with-blank-hostnames
+      - run: ssh-add -D
+      # the default circleci ssh config only additional ssh keys for Host !github.com
+      - run:
+          command: |
+            cat > ~/.ssh/config \<<EOF
+            Host *
+              IdentityFile /home/circleci/.ssh/id_rsa_458e62c517f6c480e40452126ce47421
+            EOF
+      - add_ssh_keys:
+          fingerprints:
+            # deploy key for zrepl.github.io
+            - "45:8e:62:c5:17:f6:c4:80:e4:04:52:12:6c:e4:74:21"
+
+      - run: bash -x docs/publish.sh -c -a

.circleci/trigger_debian_binary_packaging_workflow.bash

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+COMMIT="$1"
+GO_VERSION="$2"
+
+curl -v -X POST https://api.github.com/repos/zrepl/debian-binary-packaging/dispatches \
+    -H 'Accept: application/vnd.github.v3+json' \
+    -H "Authorization: token $GITHUB_ACCESS_TOKEN" \
+    --data '{"event_type": "push", "client_payload": { "zrepl_main_repo_commit": "'"$COMMIT"'", "go_version": "'"$GO_VERSION"'" }}'

.github/FUNDING.yml

@@ -0,0 +1,5 @@
+github: problame
+patreon: zrepl
+liberapay: zrepl
+custom: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96
+ko_fi: problame

.gitmodules

@@ -1,4 +0,0 @@
-[submodule "docs/themes/docdock"]
-	path = docs/themes/docdock
-	url = git@github.com:zrepl/hugo-theme-docdock.git
-

.golangci.yml

@@ -0,0 +1,16 @@
+linters:
+  enable:
+  - goimports
+
+issues:
+  exclude-rules:
+    - path: _test\.go
+      linters:
+        - errcheck
+    # Disable staticcheck 'Empty body in an if or else branch' as it's useful
+    # to put a comment into an empty else-clause that explains why whatever
+    # is done in the if-caluse is not necessary if the condition is false.
+    - linters:
+        - staticcheck
+      text: "SA9003:"
+

Gopkg.lock

@@ -1,177 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/fsnotify/fsnotify"
-  packages = ["."]
-  revision = "629574ca2a5df945712d3079857300b5e4da0236"
-  version = "v1.4.2"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/ftrvxmtrx/fd"
-  packages = ["."]
-  revision = "c6d800382fff6dc1412f34269f71b7f83bd059ad"
-
-[[projects]]
-  name = "github.com/go-logfmt/logfmt"
-  packages = ["."]
-  revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5"
-  version = "v0.3.0"
-
-[[projects]]
-  branch = "v2"
-  name = "github.com/go-yaml/yaml"
-  packages = ["."]
-  revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/hcl"
-  packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"]
-  revision = "68e816d1c783414e79bc65b3994d9ab6b0a722ab"
-
-[[projects]]
-  name = "github.com/inconshreveable/mousetrap"
-  packages = ["."]
-  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
-  version = "v1.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/jinzhu/copier"
-  packages = ["."]
-  revision = "db4671f3a9b8df855e993f7c94ec5ef1ffb0a23b"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/kr/logfmt"
-  packages = ["."]
-  revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/kr/pretty"
-  packages = ["."]
-  revision = "cfb55aafdaf3ec08f0db22699ab822c50091b1c4"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/kr/text"
-  packages = ["."]
-  revision = "7cafcd837844e784b526369c9bce262804aebc60"
-
-[[projects]]
-  name = "github.com/magiconair/properties"
-  packages = ["."]
-  revision = "be5ece7dd465ab0765a9682137865547526d1dfb"
-  version = "v1.7.3"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/mitchellh/go-homedir"
-  packages = ["."]
-  revision = "b8bc1bf767474819792c23f32d8286a45736f1c6"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/mitchellh/mapstructure"
-  packages = ["."]
-  revision = "d0303fe809921458f417bcf828397a65db30a7e4"
-
-[[projects]]
-  name = "github.com/pelletier/go-buffruneio"
-  packages = ["."]
-  revision = "c37440a7cf42ac63b919c752ca73a85067e05992"
-  version = "v0.2.0"
-
-[[projects]]
-  name = "github.com/pelletier/go-toml"
-  packages = ["."]
-  revision = "5ccdfb18c776b740aecaf085c4d9a2779199c279"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  name = "github.com/pmezard/go-difflib"
-  packages = ["difflib"]
-  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/spf13/afero"
-  packages = [".","mem"]
-  revision = "ee1bd8ee15a1306d1f9201acc41ef39cd9f99a1b"
-
-[[projects]]
-  name = "github.com/spf13/cast"
-  packages = ["."]
-  revision = "acbeb36b902d72a7a4c18e8f3241075e7ab763e4"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/spf13/cobra"
-  packages = ["."]
-  revision = "b78744579491c1ceeaaa3b40205e56b0591b93a3"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/spf13/jwalterweatherman"
-  packages = ["."]
-  revision = "12bd96e66386c1960ab0f74ced1362f66f552f7b"
-
-[[projects]]
-  name = "github.com/spf13/pflag"
-  packages = ["."]
-  revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/spf13/viper"
-  packages = ["."]
-  revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/stretchr/testify"
-  packages = ["assert"]
-  revision = "69483b4bd14f5845b5a1e55bca19e954e827f1d0"
-  version = "v1.1.4"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sys"
-  packages = ["unix"]
-  revision = "429f518978ab01db8bb6f44b66785088e7fba58b"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/text"
-  packages = ["internal/gen","internal/triegen","internal/ucd","transform","unicode/cldr","unicode/norm"]
-  revision = "1cbadb444a806fd9430d14ad08967ed91da4fa0a"
-
-[[projects]]
-  branch = "v2"
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  inputs-digest = "7ce2ead5225e4bb72a34132538171a649b26de574596e909730658ddbc904cd5"
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,66 +0,0 @@
-
-# Gopkg.toml example
-#
-# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
-# for detailed Gopkg.toml documentation.
-#
-# required = ["github.com/user/thing/cmd/thing"]
-# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
-#
-# [[constraint]]
-#   name = "github.com/user/project"
-#   version = "1.0.0"
-#
-# [[constraint]]
-#   name = "github.com/user/project2"
-#   branch = "dev"
-#   source = "github.com/myfork/project2"
-#
-# [[override]]
-#  name = "github.com/x/y"
-#  version = "2.4.0"
-
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/ftrvxmtrx/fd"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/jinzhu/copier"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/kr/pretty"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/mitchellh/go-homedir"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/mitchellh/mapstructure"
-
-[[constraint]]
-  name = "github.com/pkg/errors"
-  version = "0.8.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/spf13/cobra"
-
-[[constraint]]
-  name = "github.com/spf13/viper"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "github.com/stretchr/testify"
-  version = "1.1.4"
-
-[[constraint]]
-  branch = "v2"
-  name = "github.com/go-yaml/yaml"
-
-[[constraint]]
-  name = "github.com/go-logfmt/logfmt"
-  version = "*"

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2017 Christian Schwarz
+Copyright (c) 2017-2018 Christian Schwarz
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,47 +1,344 @@
-.PHONY: generate build test vet cover release clean
-
-ROOT := github.com/zrepl/zrepl
-SUBPKGS := cmd logger rpc sshbytestream util
-
-_TESTPKGS := $(ROOT) $(foreach p,$(SUBPKGS),$(ROOT)/$(p))
+.PHONY: generate build test vet cover release docs docs-clean clean format lint platformtest
+.PHONY: release bins-all release-noarch
+.DEFAULT_GOAL := zrepl-bin
 
 ARTIFACTDIR := artifacts
 
-build: generate
-		go build -o $(ARTIFACTDIR)/zrepl
+ifdef ZREPL_VERSION
+    _ZREPL_VERSION := $(ZREPL_VERSION)
+endif
+ifndef _ZREPL_VERSION
+    _ZREPL_VERSION := $(shell git describe --always --dirty 2>/dev/null || echo "ZREPL_BUILD_INVALID_VERSION" )
+    ifeq ($(_ZREPL_VERSION),ZREPL_BUILD_INVALID_VERSION) # can't use .SHELLSTATUS because Debian Stretch is still on gmake 4.1
+        $(error cannot infer variable ZREPL_VERSION using git and variable is not overriden by make invocation)
+    endif
+endif
 
-generate:
-	@for pkg in $(_TESTPKGS); do\
-		go generate "$$pkg" || exit 1; \
-	done;
+GO := go
+GOOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOOS"')
+GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"')
+GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"')
+GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"')
+GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
+GO_ENV_VARS := GO111MODULE=on
+GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
+GO_MOD_READONLY := -mod=readonly
+GO_EXTRA_BUILDFLAGS :=
+GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
+GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
+GOLANGCI_LINT := golangci-lint
+GOCOVMERGE := gocovmerge
+RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.17
+RELEASE_DOCKER_BASEIMAGE ?= golang:$(RELEASE_DOCKER_BASEIMAGE_TAG)
 
-test:
-	@for pkg in $(_TESTPKGS); do \
-		echo "Testing $$pkg"; \
-		go test "$$pkg" || exit 1; \
-	done;
+ifneq ($(GOARM),)
+	ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM)
+else
+	ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)
+endif
+
+.PHONY: printvars
+printvars:
+	@echo GOOS=$(GOOS)
+	@echo GOARCH=$(GOARCH)
+	@echo GOARM=$(GOARM)
+
+
+##################### PRODUCING A RELEASE #############
+.PHONY: release wrapup-and-checksum check-git-clean sign clean
+
+release: clean
+	# no cross-platform support for target test
+	$(MAKE) test-go
+	$(MAKE) bins-all
+	$(MAKE) noarch
+	$(MAKE) wrapup-and-checksum
+	$(MAKE) check-git-clean
+ifeq (SIGN, 1)
+	$(MAKE) sign
+endif
+	@echo "ZREPL RELEASE ARTIFACTS AVAILABLE IN artifacts/release"
+
+release-docker: $(ARTIFACTDIR)
+	sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > artifacts/release-docker.Dockerfile
+	docker build -t zrepl_release --pull -f artifacts/release-docker.Dockerfile .
+	docker run --rm -i -v $(CURDIR):/src -u $$(id -u):$$(id -g) \
+		zrepl_release \
+		make release GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+
+debs-docker:
+	$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=deb
+rpms-docker:
+	$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=rpm
+_debs_or_rpms_docker: # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
+	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=amd64
+	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=arm64
+	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=arm GOARM=7
+	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=386
+
+rpm: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
+	$(eval _ZREPL_RPM_VERSION := $(subst -,.,$(_ZREPL_VERSION)))
+	$(eval _ZREPL_RPM_TOPDIR_ABS := $(CURDIR)/$(ARTIFACTDIR)/rpmbuild)
+	rm -rf "$(_ZREPL_RPM_TOPDIR_ABS)"
+	mkdir "$(_ZREPL_RPM_TOPDIR_ABS)"
+	mkdir -p "$(_ZREPL_RPM_TOPDIR_ABS)"/{SPECS,RPMS,BUILD,BUILDROOT}
+	sed "s/^Version:.*/Version:          $(_ZREPL_RPM_VERSION)/g" \
+	    packaging/rpm/zrepl.spec >  $(_ZREPL_RPM_TOPDIR_ABS)/SPECS/zrepl.spec
+
+	# see /usr/lib/rpm/platform
+ifeq ($(GOARCH),amd64)
+	$(eval _ZREPL_RPMBUILD_TARGET := x86_64)
+else ifeq ($(GOARCH), 386)
+	$(eval _ZREPL_RPMBUILD_TARGET := i386)
+else ifeq ($(GOARCH), arm64)
+	$(eval _ZREPL_RPMBUILD_TARGET := aarch64)
+else ifeq ($(GOARCH), arm)
+	$(eval _ZREPL_RPMBUILD_TARGET := armv7hl)
+else
+	$(eval _ZREPL_RPMBUILD_TARGET := $(GOARCH))
+endif
+	rpmbuild \
+		--build-in-place \
+		--define "_sourcedir $(CURDIR)" \
+		--define "_topdir $(_ZREPL_RPM_TOPDIR_ABS)" \
+		--define "_zrepl_binary_filename zrepl-$(ZREPL_TARGET_TUPLE)" \
+		--target $(_ZREPL_RPMBUILD_TARGET) \
+		-bb "$(_ZREPL_RPM_TOPDIR_ABS)"/SPECS/zrepl.spec
+	cp "$(_ZREPL_RPM_TOPDIR_ABS)"/RPMS/$(_ZREPL_RPMBUILD_TARGET)/zrepl-$(_ZREPL_RPM_VERSION)*.rpm $(ARTIFACTDIR)/
+
+rpm-docker:
+	docker build -t zrepl_rpm_pkg --pull -f packaging/rpm/Dockerfile .
+	docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
+		zrepl_rpm_pkg \
+		make rpm GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+
+deb: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
+
+	cp packaging/deb/debian/changelog.template packaging/deb/debian/changelog
+	sed -i 's/DATE_DASH_R_OUTPUT/$(shell date -R)/' packaging/deb/debian/changelog
+	VERSION="$(subst -,.,$(_ZREPL_VERSION))"; \
+		export VERSION="$${VERSION#v}"; \
+		sed -i 's/VERSION/'"$$VERSION"'/' packaging/deb/debian/changelog
+
+ifeq ($(GOARCH), arm)
+	$(eval  DEB_HOST_ARCH := armhf)
+else ifeq ($(GOARCH), 386)
+	$(eval DEB_HOST_ARCH := i386)
+else
+	$(eval DEB_HOST_ARCH := $(GOARCH))
+endif
+
+	export ZREPL_DPKG_ZREPL_BINARY_FILENAME=zrepl-$(ZREPL_TARGET_TUPLE); \
+		dpkg-buildpackage -b --no-sign --host-arch $(DEB_HOST_ARCH)
+	cp ../*.deb artifacts/
+
+deb-docker:
+	docker build -t zrepl_debian_pkg --pull -f packaging/deb/Dockerfile .
+	docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
+		zrepl_debian_pkg \
+		make deb GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+
+# expects `release` target to have run before
+NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
+wrapup-and-checksum:
+	rm -f $(NOARCH_TARBALL)
+	tar --mtime='1970-01-01' --sort=name \
+		--transform 's/$(ARTIFACTDIR)/zrepl-$(_ZREPL_VERSION)-noarch/' \
+		--transform 's#dist#zrepl-$(_ZREPL_VERSION)-noarch/dist#' \
+		--transform 's#config/samples#zrepl-$(_ZREPL_VERSION)-noarch/config#' \
+		-acf $(NOARCH_TARBALL) \
+		$(ARTIFACTDIR)/docs/html \
+		$(ARTIFACTDIR)/bash_completion \
+		$(ARTIFACTDIR)/_zrepl.zsh_completion \
+		$(ARTIFACTDIR)/go_env.txt \
+		dist \
+		config/samples
+	rm -rf "$(ARTIFACTDIR)/release"
+	mkdir -p "$(ARTIFACTDIR)/release"
+	cp -l $(ARTIFACTDIR)/zrepl-* \
+		$(ARTIFACTDIR)/platformtest-* \
+		"$(ARTIFACTDIR)/release"
+	cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
+
+check-git-clean:
+	@# note that we use ZREPL_VERSION and not _ZREPL_VERSION because we want to detect the override
+	@if git describe --always --dirty 2>/dev/null | grep dirty >/dev/null; then \
+        echo '[INFO] either git reports checkout is dirty or git is not installed or this is not a git checkout'; \
+		if [ "$(ZREPL_VERSION)" = "" ]; then \
+			echo '[WARN] git checkout is dirty and make variable ZREPL_VERSION was not used to override'; \
+			git status; \
+			echo "git diff:";  \
+			git diff | cat; \
+			exit 1; \
+		fi; \
+	fi;
+
+tag-release:
+	test -n "$(ZREPL_TAG_VERSION)" || exit 1
+	git tag -u E27CA5FC -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)"
+
+sign:
+	gpg -u "89BC 5D89 C845 568B F578  B306 CDBD 8EC8 E27C A5FC" \
+		--armor \
+		--detach-sign $(ARTIFACTDIR)/release/sha512sum.txt
+
+clean: docs-clean
+	rm -rf "$(ARTIFACTDIR)"
+
+##################### BINARIES #####################
+.PHONY: bins-all lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin generate-platform-test-list
+
+BINS_ALL_TARGETS := zrepl-bin test-platform-bin vet lint
+GO_SUPPORTS_ILLUMOS := $(shell $(GO) version | gawk -F '.' '/^go version /{split($$0, comps, " "); split(comps[3], v, "."); if (v[1] == "go1" && v[2] >= 13) { print "illumos"; } else { print "noillumos"; }}')
+bins-all:
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=amd64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=386
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=arm	  GOARM=7
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=arm64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=amd64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm     GOARM=7
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=386
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=darwin    GOARCH=amd64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=solaris   GOARCH=amd64
+ifeq ($(GO_SUPPORTS_ILLUMOS), illumos)
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=illumos   GOARCH=amd64
+else ifeq ($(GO_SUPPORTS_ILLUMOS), noillumos)
+	@echo "SKIPPING ILLUMOS BUILD BECAUSE GO VERSION DOESN'T SUPPORT IT"
+else
+	@echo "CANNOT DETERMINE WHETHER GO VERSION SUPPORTS GOOS=illumos"; exit 1
+endif
+
+lint:
+	$(GO_ENV_VARS) $(GOLANGCI_LINT) run ./...
 
 vet:
-	@for pkg in $(_TESTPKGS); do \
-		echo "Vetting $$pkg"; \
-		go vet "$$pkg" || exit 1; \
-	done;
+	$(GO_ENV_VARS) $(GO) vet $(GO_BUILDFLAGS) ./...
 
-cover: artifacts
-	@for pkg in $(_TESTPKGS); do \
-		profile="$(ARTIFACTDIR)/cover-$$(basename $$pkg).out"; \
-		go test -coverprofile "$$profile" $$pkg || exit 1; \
-		if [ -f "$$profile" ]; then \
-   			go tool cover -html="$$profile" -o "$${profile}.html" || exit 2; \
-		fi; \
-	done;
+test-go: $(ARTIFACTDIR)
+	rm -f "$(ARTIFACTDIR)/gotest.cover"
+ifeq ($(COVER),1)
+	$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
+		-coverpkg github.com/zrepl/zrepl/... \
+		-covermode atomic \
+		-coverprofile "$(ARTIFACTDIR)/gotest.cover" \
+		./...
+else
+	$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
+		./...
+endif
 
-artifacts:
-	mkdir artifacts
+zrepl-bin:
+	$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(ZREPL_TARGET_TUPLE)"
 
-release: generate artifacts vet test
-	GOOS=linux GOARCH=amd64   go build -o "$(ARTIFACTDIR)/zrepl-linux-amd64"
-	GOOS=freebsd GOARCH=amd64 go build -o "$(ARTIFACTDIR)/zrepl-freebsd-amd64"
+generate-platform-test-list:
+	$(GO_BUILD) -o $(ARTIFACTDIR)/generate-platform-test-list ./platformtest/tests/gen
 
-clean:
-	rm -rf "$(ARTIFACTDIR)"
+COVER_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-cover-$(ZREPL_TARGET_TUPLE)
+cover-platform-bin:
+	$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
+		-c -o "$(COVER_PLATFORM_BIN_PATH)" \
+		-covermode=atomic -cover -coverpkg github.com/zrepl/zrepl/... \
+		./platformtest/harness
+cover-platform:
+	# do not track dependency on cover-platform-bin to allow build of binary outside of test VM
+	export _TEST_PLATFORM_CMD="$(COVER_PLATFORM_BIN_PATH) \
+		-test.coverprofile \"$(ARTIFACTDIR)/platformtest.cover\" \
+		-test.v \
+		__DEVEL--i-heard-you-like-tests"; \
+		$(MAKE) _test-or-cover-platform-impl
+
+TEST_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)
+test-platform-bin:
+	$(GO_BUILD) -o "$(TEST_PLATFORM_BIN_PATH)" ./platformtest/harness
+test-platform:
+	export _TEST_PLATFORM_CMD="\"$(TEST_PLATFORM_BIN_PATH)\""; \
+		$(MAKE) _test-or-cover-platform-impl
+
+ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
+ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
+ZREPL_PLATFORMTEST_MOUNTPOINT := /tmp/zreplplatformtest.pool
+ZREPL_PLATFORMTEST_ZFS_LOG := /tmp/zreplplatformtest.zfs.log
+# ZREPL_PLATFORMTEST_STOP_AND_KEEP := -failure.stop-and-keep-pool
+ZREPL_PLATFORMTEST_ARGS :=
+_test-or-cover-platform-impl: $(ARTIFACTDIR)
+ifndef _TEST_PLATFORM_CMD
+	$(error _TEST_PLATFORM_CMD is undefined, caller 'cover-platform' or 'test-platform' should have defined it)
+endif
+	rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
+	rm -f "$(ARTIFACTDIR)/platformtest.cover"
+	platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
+		$(_TEST_PLATFORM_CMD) \
+		-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
+		-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
+		-mountpoint "$(ZREPL_PLATFORMTEST_MOUNTPOINT)" \
+		$(ZREPL_PLATFORMTEST_STOP_AND_KEEP) \
+		$(ZREPL_PLATFORMTEST_ARGS)
+
+cover-merge: $(ARTIFACTDIR)
+	$(GOCOVMERGE) $(ARTIFACTDIR)/platformtest.cover $(ARTIFACTDIR)/gotest.cover > $(ARTIFACTDIR)/merged.cover
+cover-html: cover-merge
+	$(GO) tool cover -html "$(ARTIFACTDIR)/merged.cover" -o "$(ARTIFACTDIR)/merged.cover.html"
+
+cover-full:
+	test "$$(id -u)" = "0" || echo "MUST RUN AS ROOT" 1>&2
+	$(MAKE) test-go COVER=1
+	$(MAKE) cover-platform-bin
+	$(MAKE) cover-platform
+	$(MAKE) cover-html
+
+##################### DEV TARGETS #####################
+# not part of the build, must do that manually
+.PHONY: generate formatcheck format
+
+generate: generate-platform-test-list
+	protoc -I=replication/logic/pdu --go_out=replication/logic/pdu --go-grpc_out=replication/logic/pdu replication/logic/pdu/pdu.proto
+	protoc -I=rpc/grpcclientidentity/example --go_out=rpc/grpcclientidentity/example/pdu --go-grpc_out=rpc/grpcclientidentity/example/pdu rpc/grpcclientidentity/example/grpcauth.proto
+	$(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./...
+
+GOIMPORTS := goimports -srcdir . -local 'github.com/zrepl/zrepl'
+FINDSRCFILES := find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go'
+
+formatcheck:
+	@# goimports doesn't have a knob to exit with non-zero status code if formatting is needed
+	@# see https://go-review.googlesource.com/c/tools/+/237378
+	@ affectedfiles=$$($(GOIMPORTS) -l $(shell $(FINDSRCFILES)) | tee /dev/stderr | wc -l); test "$$affectedfiles" = 0
+
+format:
+	@ $(GOIMPORTS) -w -d $(shell  $(FINDSRCFILES))
+
+##################### NOARCH #####################
+.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
+
+
+$(ARTIFACTDIR):
+	mkdir -p "$@"
+$(ARTIFACTDIR)/docs: $(ARTIFACTDIR)
+	mkdir -p "$@"
+
+noarch: $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs
+	# pass
+
+$(ARTIFACTDIR)/bash_completion:
+	$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
+	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion bash "$@"
+
+$(ARTIFACTDIR)/_zrepl.zsh_completion:
+	$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
+	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion zsh "$@"
+
+$(ARTIFACTDIR)/go_env.txt:
+	$(GO_ENV_VARS) $(GO) env > $@
+	$(GO) version >> $@
+
+docs: $(ARTIFACTDIR)/docs
+	# https://www.sphinx-doc.org/en/master/man/sphinx-build.html
+	$(MAKE) -C docs \
+		html \
+		BUILDDIR=../artifacts/docs \
+		SPHINXOPTS="-W --keep-going -n"
+
+docs-clean:
+	$(MAKE) -C docs \
+		clean \
+		BUILDDIR=../artifacts/docs

README.md

@@ -1,9 +1,19 @@
+[![GitHub license](https://img.shields.io/github/license/zrepl/zrepl.svg)](https://github.com/zrepl/zrepl/blob/master/LICENSE)
+[![Language: Go](https://img.shields.io/badge/lang-Go-6ad7e5.svg)](https://golang.org/)
+[![User Docs](https://img.shields.io/badge/docs-web-blue.svg)](https://zrepl.github.io)
+[![Support me on Patreon](https://img.shields.io/badge/dynamic/json?color=yellow&label=Patreon&query=data.attributes.patron_count&url=https%3A%2F%2Fwww.patreon.com%2Fapi%2Fcampaigns%2F3095079)](https://patreon.com/zrepl)
+[![Donate via GitHub Sponsors](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&style=flat&color=yellow)](https://github.com/sponsors/problame)
+[![Donate via Liberapay](https://img.shields.io/liberapay/patrons/zrepl.svg?logo=liberapay)](https://liberapay.com/zrepl/donate)
+[![Donate via PayPal](https://img.shields.io/badge/donate-paypal-yellow.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
+[![Twitter](https://img.shields.io/twitter/url/https/github.com/zrepl/zrepl.svg?style=social)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl)
+[![Chat](https://img.shields.io/badge/chat-matrix-blue.svg)](https://matrix.to/#/#zrepl:matrix.org)
+
 # zrepl
-zrepl is a ZFS filesystem backup & replication solution written in Go.
+zrepl is a one-stop ZFS backup & replication solution.
 
 ## User Documentation
 
-**User Documentation** cab be found at [zrepl.github.io](https://zrepl.github.io).
+**User Documentation** can be found at [zrepl.github.io](https://zrepl.github.io).
 
 ## Bug Reports
 
@@ -12,43 +22,114 @@ zrepl is a ZFS filesystem backup & replication solution written in Go.
 
 ## Feature Requests
 
-1. Does you feature request require default values / some kind of configuration?
+1. Does your feature request require default values / some kind of configuration?
    If so, think of an expressive configuration example.
 2. Think of at least one use case that generalizes from your concrete application.
 3. Open an issue on GitHub with example conf & use case attached.
+4. **Optional**: [Post a bounty](https://www.bountysource.com/teams/zrepl) on the issue, or [contact Christian Schwarz](https://cschwarz.com) for contract work.
 
 The above does not apply if you already implemented everything.
 Check out the *Coding Workflow* section below for details.
 
-## Developer Documentation
+## Building, Releasing, Downstream-Packaging
 
-### Overall Architecture
+This section provides an overview of the zrepl build & release process.
+Check out `docs/installation/compile-from-source.rst` for build-from-source instructions.
 
-The application architecture is documented as part of the user docs in the *Implementation* section (`docs/content/impl`).
-Make sure to develop an understanding how zrepl is typically used by studying the user docs first.
+### Overview
 
-### Project Structure
+zrepl is written in [Go](https://golang.org) and uses [Go modules](https://github.com/golang/go/wiki/Modules) to manage dependencies.
+The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
 
-```
-├── cmd
-│   ├── sampleconf          # example configuration
-├── docs
-│   ├── content             # hugo-based documentation -> sources for ./public_git
-│   ├── deploy.sh           # shell script for automated rendering & deploy to zrepl.github.io repo
-│   ├── public_git          # used by above shell script
-│   └── themes
-│       └── docdock         # submodule of our docdock theme fork
-├── jobrun                  # OBSOLETE
-├── logger                  # logger package used by zrepl
-├── rpc                     # rpc protocol implementation
-├── scratchpad              # small example programs demoing some internal packages. probably OBSOLETE
-├── sshbytestream           # io.ReadWriteCloser over SSH
-├── util
-└── zfs                     # ZFS wrappers, filesystemm diffing
-```
+Install **build dependencies** using  `./lazy.sh devsetup`.
+`lazy.sh` uses `python3-pip` to fetch the build dependencies for the docs - you might want to use a [venv](https://docs.python.org/3/library/venv.html).
+If you just want to install the Go dependencies, run `./lazy.sh godep`.
 
-### Coding Workflow
+The **test suite** is split into pure **Go tests** (`make test-go`) and **platform tests** that interact with ZFS and thus generally **require root privileges** (`sudo make test-platform`).
+Platform tests run on their own pool with the name `zreplplatformtest`, which is created using the file vdev in `/tmp`.
+
+For a full **code coverage** profile, run `make test-go COVER=1 && sudo make test-platform && make cover-merge`.
+An HTML report can be generated using `make cover-html`.
+
+**Code generation** is triggered by `make generate`. Generated code is committed to the source tree.
+
+### Build & Release Process
+
+**The `Makefile` is catering to the needs of developers & CI, not distro packagers**.
+It provides phony targets for
+* local development (building, running tests, etc)
+* building a release in Docker (used by the CI & release management)
+* building .deb and .rpm packages out of the release artifacts.
+
+**Build tooling & dependencies** are documented as code in `lazy.sh`.
+Go dependencies are then fetched by the go command and pip dependencies are pinned through a `requirements.txt`.
+
+**We use CircleCI for continuous integration**.
+There are two workflows:
+
+* `ci` runs for every commit / branch / tag pushed to GitHub.
+  It is supposed to run very fast (<5min and provides quick feedback to developers).
+  It runs formatting checks, lints and tests on the most important OSes / architectures.
+  Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
+
+* `release` runs
+  * on manual triggers through the CircleCI API (in order to produce a release)
+  * periodically on `master`
+  Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
+
+**Releases** are issued via Git tags + GitHub Releases feature.
+The procedure to issue a release is as follows:
+* Issue the source release:
+  * Git tag the release on the `master` branch.
+  * Push the tag.
+  * Run `./docs/publish.sh` to re-build & push zrepl.github.io.
+* Issue the official binary release:
+  * Run the `release` pipeline (triggered via CircleCI API)
+  * Download the artifacts to the release manager's machine.
+  * Create a GitHub release, edit the changelog, upload all the release artifacts, including .rpm and .deb files.
+  * Issue the GitHub release.
+  * Add the .rpm and .deb files to the official zrepl repos, publish those.
+
+**Official binary releases are not re-built when Go receives an update. If the Go update is critical to zrepl (e.g. a Go security update that affects zrepl), we'd issue a new source release**.
+The rationale for this is that whereas distros provide a mechanism for this (`$zrepl_source_release-$distro_package_revision`), GitHub Releases doesn't which means we'd need to update the existing GitHub release's assets, which nobody would notice (no RSS feed updates, etc.).
+Downstream packagers can read the changelog to determine whether they want to push that minor release into their distro or simply skip it.
+
+### Additional Notes to Distro Package Maintainers
+
+* Run the platform tests (Docs -> Usage -> Platform Tests) **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
+* Ship a default config that adheres to your distro's `hier` and logging system.
+* Ship a service manager file and _please_ try to upstream it to this repository.
+  * `dist/systemd` contains a Systemd unit template.
+* Ship other material provided in `./dist`, e.g. in `/usr/share/zrepl/`.
+* Have a look at the `Makefile`'s `ZREPL_VERSION` variable and how it passed to Go's `ldFlags`.
+  This is how `zrepl version` knows what version number to show.
+  Your build system should set the `ldFlags` flags appropriately and add a prefix or suffix that indicates that the given zrepl binary is a distro build, not an official one.
+* Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
+
+
+## Contributing Code
 
 * Open an issue when starting to hack on a new feature
 * Commits should reference the issue they are related to
 * Docs improvements not documenting new features do not require an issue.
+
+### Breaking Changes
+
+Backward-incompatible changes must be documented in the git commit message and are listed in `docs/changelog.rst`.
+
+### Glossary & Naming Inconsistencies
+
+In ZFS, *dataset* refers to the objects *filesystem*, *ZVOL* and *snapshot*. <br />
+However, we need a word for *filesystem* & *ZVOL* but not a snapshot, bookmark, etc.
+
+Toward the user, the following terminology is used:
+
+* **filesystem**: a ZFS filesystem or a ZVOL
+* **filesystem version**: a ZFS snapshot or a bookmark
+
+Sadly, the zrepl implementation is inconsistent in its use of these words:
+variables and types are often named *dataset* when they in fact refer to a *filesystem*.
+
+There will not be a big refactoring (an attempt was made, but it's destroying too much history without much gain).
+
+However, new contributions & patches should fix naming without further notice in the commit message.

build.Dockerfile

@@ -0,0 +1,30 @@
+FROM !SUBSTITUTED_BY_MAKEFILE
+
+RUN apt-get update && apt-get install -y \
+    python3-pip \
+    unzip \
+    gawk
+
+ADD build.installprotoc.bash ./
+RUN bash build.installprotoc.bash
+
+ADD lazy.sh /tmp/lazy.sh
+ADD docs/requirements.txt /tmp/requirements.txt
+ENV ZREPL_LAZY_DOCS_REQPATH=/tmp/requirements.txt
+RUN /tmp/lazy.sh docdep
+
+# prepare volume mount of git checkout to /zrepl
+RUN mkdir -p /src/github.com/zrepl/zrepl
+RUN mkdir -p /.cache && chmod -R 0777 /.cache
+
+# $GOPATH is /go
+# Go 1.12 doesn't use modules within GOPATH, but 1.13 and later do
+# => store source outside of GOPATH
+WORKDIR /src
+
+# Install build tools (e.g. protobuf generator, stringer) into $GOPATH/bin
+ADD build/ /tmp/build
+RUN /tmp/lazy.sh godep
+
+RUN chmod -R 0777 /go
+

build.installprotoc.bash

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+
+MACH=$(uname -m)
+MACH="${MACH/aarch64/aarch_64}"
+
+VERSION=3.6.1
+FILENAME=protoc-"$VERSION"-linux-"$MACH".zip
+
+if [ -e "$FILENAME" ]; then
+    echo "$FILENAME" already exists 1>&2
+    exit 1
+fi
+
+wget https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
+
+stat "$FILENAME"
+
+sha256sum -c --ignore-missing <<EOF
+6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807  protoc-3.6.1-linux-x86_64.zip
+af8e5aaaf39ddec62ec8dd2be1b8d9602c6da66564883a16393ade5f71170922  protoc-3.6.1-linux-aarch_64.zip
+EOF
+
+unzip -d /usr "$FILENAME"

build/go.mod

@@ -0,0 +1,16 @@
+module github.com/zrepl/zrepl/build
+
+go 1.12
+
+require (
+	github.com/alvaroloes/enumer v1.1.1
+	github.com/golangci/golangci-lint v1.35.2
+	github.com/golangci/misspell v0.3.4 // indirect
+	github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 // indirect
+	github.com/spf13/afero v1.2.2 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad
+	golang.org/x/tools v0.0.0-20210105210202-9ed45478a130
+	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 // indirect
+	google.golang.org/protobuf v1.25.0
+)

build/go.sum

@@ -0,0 +1,698 @@
+4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a h1:wFEQiK85fRsEVF0CRrPAos5LoAryUsIX1kPW/WrIqFw=
+4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a/go.mod h1:wfdC5ZjKSPr7CybKEcgJhUOgeAQW1+7WcyK8OvUilfo=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/Djarvur/go-err113 v0.0.0-20200511133814-5174e21577d5 h1:XTrzB+F8+SpRmbhAH8HLxhiiG6nYNwaBZjrFps1oWEk=
+github.com/Djarvur/go-err113 v0.0.0-20200511133814-5174e21577d5/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us=
+github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM=
+github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alvaroloes/enumer v1.1.1 h1:v+1scNqEhSFAPb9tLwblQegeTXJhnw8hf9O0amTbOvQ=
+github.com/alvaroloes/enumer v1.1.1/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
+github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/ashanbrown/forbidigo v1.0.0 h1:QdNXBduDUopc3GW+YVYZn8jzmIMklQiCfdN2N5+dQeE=
+github.com/ashanbrown/forbidigo v1.0.0/go.mod h1:PH+zMRWE15yW69fYfe7Kn8nYR6yYyafc3ntEGh2BBAg=
+github.com/ashanbrown/makezero v0.0.0-20201205152432-7b7cdbb3025a h1:/U9tbJzDRof4fOR51vwzWdIBsIH6R2yU0KG1MBRM2Js=
+github.com/ashanbrown/makezero v0.0.0-20201205152432-7b7cdbb3025a/go.mod h1:oG9Dnez7/ESBqc4EdrdNlryeo7d0KcW1ftXHm7nU/UU=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/bombsimon/wsl/v3 v3.1.0 h1:E5SRssoBgtVFPcYWUOFJEcgaySgdtTNYzsSKDOY7ss8=
+github.com/bombsimon/wsl/v3 v3.1.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/daixiang0/gci v0.2.8 h1:1mrIGMBQsBu0P7j7m1M8Lb+ZeZxsZL+jyGX4YoMJJpg=
+github.com/daixiang0/gci v0.2.8/go.mod h1:+4dZ7TISfSmqfAGv59ePaHfNzgGtIkHAhhdKggP1JAc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/denis-tingajkin/go-header v0.4.2 h1:jEeSF4sdv8/3cT/WY8AgDHUoItNSoEZ7qg9dX7pc218=
+github.com/denis-tingajkin/go-header v0.4.2/go.mod h1:eLRHAVXzE5atsKAnNRDB90WHCFFnBUn4RN0nRcs1LJA=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
+github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-critic/go-critic v0.5.3 h1:xQEweNxzBNpSqI3wotXZAixRarETng3PTG4pkcrLCOA=
+github.com/go-critic/go-critic v0.5.3/go.mod h1:2Lrs1m4jtOnnG/EdezbSpAoL0F2pRW+9HWJUZ+QaktY=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=
+github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
+github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8=
+github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
+github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ=
+github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
+github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k=
+github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
+github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
+github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg=
+github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
+github.com/go-toolsmith/pkgload v1.0.0 h1:4DFWWMXVfbcN5So1sBNW9+yeiMqLFGl1wFLTL5R0Tgg=
+github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
+github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4=
+github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
+github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA=
+github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
+github.com/go-toolsmith/typep v1.0.2 h1:8xdsa1+FSIH/RhEkgnD1j2CJOy5mNllW1Q9tRiYwvlk=
+github.com/go-toolsmith/typep v1.0.2/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
+github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b h1:khEcpUM4yFcxg4/FHQWkvVRmgijNXRfzkIDHh23ggEo=
+github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gofrs/flock v0.8.0 h1:MSdYClljsF3PbENUUEx85nkWfJSGfzYI9yEBZOJz6CY=
+github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0=
+github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
+github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
+github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
+github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w=
+github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
+github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw=
+github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
+github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d h1:pXTK/gkVNs7Zyy7WKgLXmpQ5bHTrq5GDsp8R9Qs67g0=
+github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
+github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=
+github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
+github.com/golangci/golangci-lint v1.35.2 h1:hD1999/sq3tCPXhhI4UpunxpAAdH9pK7kDIObqoGuWA=
+github.com/golangci/golangci-lint v1.35.2/go.mod h1:Sg5fFp5oLLI1B8gXfUVUSePju8XF0uWefMkuZuGIHUo=
+github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
+github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
+github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA=
+github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
+github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
+github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
+github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
+github.com/golangci/misspell v0.3.4 h1:kAD5J0611Zo2VirUn9KFP15RjEqkmfEfnFxyIVxZCYg=
+github.com/golangci/misspell v0.3.4/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
+github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us=
+github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
+github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
+github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 h1:XQKc8IYQOeRwVs36tDrEmTgDgP88d5iEURwpmtiAlOM=
+github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
+github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
+github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gookit/color v1.3.1/go.mod h1:R3ogXq2B9rTbXoSHJ1HyUVAZ3poOJHpd9nQmyGZsfvQ=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw=
+github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
+github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
+github.com/gostaticanalysis/analysisutil v0.1.0/go.mod h1:dMhHRU9KTiDcuLGdy87/2gTR8WruwYZrKdRq9m1O6uw=
+github.com/gostaticanalysis/analysisutil v0.4.1 h1:/7clKqrVfiVwiBQLM0Uke4KvXnO6JcCTS7HwF2D6wG8=
+github.com/gostaticanalysis/analysisutil v0.4.1/go.mod h1:18U/DLpRgIUd459wGxVHE0fRgmo1UgHDcbw7F5idXu0=
+github.com/gostaticanalysis/comment v1.3.0/go.mod h1:xMicKDx7XRXYdVwY9f9wQpDJVnqWxw9wCauCMKp+IBI=
+github.com/gostaticanalysis/comment v1.4.1 h1:xHopR5L2lRz6OsjH4R2HG5wRhW9ySl3FsHIvi5pcXwc=
+github.com/gostaticanalysis/comment v1.4.1/go.mod h1:ih6ZxzTHLdadaiSnF5WY3dxUoXfXAlTaRzuaNDlSado=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
+github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jgautheron/goconst v0.0.0-20201117150253-ccae5bf973f3 h1:7nkB9fLPMwtn/R6qfPcHileL/x9ydlhw8XyDrLI1ZXg=
+github.com/jgautheron/goconst v0.0.0-20201117150253-ccae5bf973f3/go.mod h1:aAosetZ5zaeC/2EfMeRswtxUFBpe2Hr7HzkgX4fanO4=
+github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a h1:GmsqmapfzSJkm28dhRoHz2tLRbJmqhU86IPgBtN3mmk=
+github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s=
+github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3 h1:jNYPNLe3d8smommaoQlK7LOA5ESyUJJ+Wf79ZtA7Vp4=
+github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0=
+github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
+github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kulti/thelper v0.2.1 h1:H4rSHiB3ALx//SXr+k9OPqKoOw2cAZpIQwVNH1RL5T4=
+github.com/kulti/thelper v0.2.1/go.mod h1:vMu2Cizjy/grP+jmsvOFDx1kYP6+PD1lqg4Yu5exl2U=
+github.com/kunwardeep/paralleltest v1.0.2 h1:/jJRv0TiqPoEy/Y8dQxCFJhD56uS/pnvtatgTZBHokU=
+github.com/kunwardeep/paralleltest v1.0.2/go.mod h1:ZPqNm1fVHPllh5LPVujzbVz1JN2GhLxSfY+oqUsvG30=
+github.com/kyoh86/exportloopref v0.1.8 h1:5Ry/at+eFdkX9Vsdw3qU4YkvGtzuVfzT4X7S77LoN/M=
+github.com/kyoh86/exportloopref v0.1.8/go.mod h1:1tUcJeiioIs7VWe5gcOObrux3lb66+sBqGZrRkMwPgg=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/maratori/testpackage v1.0.1 h1:QtJ5ZjqapShm0w5DosRjg0PRlSdAdlx+W6cCKoALdbQ=
+github.com/maratori/testpackage v1.0.1/go.mod h1:ddKdw+XG0Phzhx8BFDTKgpWP4i7MpApTE5fXSKAqwDU=
+github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE=
+github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
+github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mbilski/exhaustivestruct v1.1.0 h1:4ykwscnAFeHJruT+EY3M3vdeP8uXMh0VV2E61iR7XD8=
+github.com/mbilski/exhaustivestruct v1.1.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aksxSUOUy+nvtVEfzXc=
+github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
+github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/moricho/tparallel v0.2.1 h1:95FytivzT6rYzdJLdtfn6m1bfFJylOJK41+lgv/EHf4=
+github.com/moricho/tparallel v0.2.1/go.mod h1:fXEIZxG2vdfl0ZF8b42f5a78EhjjD5mX8qUplsoSU4k=
+github.com/mozilla/tls-observatory v0.0.0-20200317151703-4fa42e1c2dee/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nakabonne/nestif v0.3.0 h1:+yOViDGhg8ygGrmII72nV9B/zGxY188TYpfolntsaPw=
+github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c=
+github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E=
+github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nishanths/exhaustive v0.1.0 h1:kVlMw8h2LHPMGUVqUj6230oQjjTMFjwcZrnkhXzFfl8=
+github.com/nishanths/exhaustive v0.1.0/go.mod h1:S1j9110vxV1ECdCudXRkeMnFQ/DQk9ajLT0Uf2MYZQQ=
+github.com/nishanths/predeclared v0.2.1 h1:1TXtjmy4f3YCFjTxRd8zcFHOmoUir+gp0ESzjFzG2sw=
+github.com/nishanths/predeclared v0.2.1/go.mod h1:HvkGJcA3naj4lOwnFXFDkFxVtSqQMB9sbB1usJ+xjQE=
+github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4=
+github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs=
+github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 h1:/I3lTljEEDNYLho3/FUB7iD/oc2cEFgVmbHzV+O0PtU=
+github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ=
+github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
+github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d h1:CdDQnGF8Nq9ocOS/xlSptM1N3BbrA6/kmaep5ggwaIA=
+github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw=
+github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/polyfloyd/go-errorlint v0.0.0-20201127212506-19bd8db6546f h1:xAw10KgJqG5NJDfmRqJ05Z0IFblKumjtMeyiOLxj3+4=
+github.com/polyfloyd/go-errorlint v0.0.0-20201127212506-19bd8db6546f/go.mod h1:wi9BfjxjF/bwiZ701TzmfKu6UKC357IOAtNr0Td0Lvw=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
+github.com/quasilyte/go-ruleguard v0.2.1-0.20201030093329-408e96760278 h1:5gcJ7tORNCNB2QjOJF+MYjzS9aiWpxhP3gntf7RVrOQ=
+github.com/quasilyte/go-ruleguard v0.2.1-0.20201030093329-408e96760278/go.mod h1:2RT/tf0Ce0UDj5y243iWKosQogJd8+1G3Rs2fxmlYnw=
+github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 h1:L8QM9bvf68pVdQ3bCFZMDmnt9yqcMBro1pC7F+IPYMY=
+github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryancurrah/gomodguard v1.2.0 h1:YWfhGOrXwLGiqcC/u5EqG6YeS8nh+1fw0HEc85CVZro=
+github.com/ryancurrah/gomodguard v1.2.0/go.mod h1:rNqbC4TOIdUDcVMSIpNNAzTbzXAZa6W5lnUepvuMMgQ=
+github.com/ryanrolds/sqlclosecheck v0.3.0 h1:AZx+Bixh8zdUBxUA1NxbxVAS78vTPq4rCb8OUZI9xFw=
+github.com/ryanrolds/sqlclosecheck v0.3.0/go.mod h1:1gREqxyTGR3lVtpngyFo3hZAgk0KCtEdgEkHwDbigdA=
+github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/securego/gosec/v2 v2.5.0 h1:kjfXLeKdk98gBe2+eYRFMpC4+mxmQQtbidpiiOQ69Qc=
+github.com/securego/gosec/v2 v2.5.0/go.mod h1:L/CDXVntIff5ypVHIkqPXbtRpJiNCh6c6Amn68jXDjo=
+github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU=
+github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs=
+github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
+github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/sonatard/noctx v0.0.1 h1:VC1Qhl6Oxx9vvWo3UDgrGXYCeKCe3Wbw7qAWL6FrmTY=
+github.com/sonatard/noctx v0.0.1/go.mod h1:9D2D/EoULe8Yy2joDHJj7bv3sZoq9AaSb8B4lqBjiZI=
+github.com/sourcegraph/go-diff v0.6.1 h1:hmA1LzxW0n1c3Q4YbrFgg4P99GSnebYa3x8gr0HZqLQ=
+github.com/sourcegraph/go-diff v0.6.1/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
+github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
+github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
+github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
+github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
+github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
+github.com/ssgreg/nlreturn/v2 v2.1.0 h1:6/s4Rc49L6Uo6RLjhWZGBpWWjfzk2yrf1nIW8m4wgVA=
+github.com/ssgreg/nlreturn/v2 v2.1.0/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
+github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2 h1:Xr9gkxfOP0KQWXKNqmwe8vEeSUiUj4Rlee9CMVX2ZUQ=
+github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
+github.com/tetafro/godot v1.3.2 h1:HzWC3XjadkyeuBZxkfAFNY20UVvle0YD51I6zf6RKlU=
+github.com/tetafro/godot v1.3.2/go.mod h1:ah7jjYmOMnIjS9ku2krapvGQrFNtTLo9Z/qB3dGU1eU=
+github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q=
+github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tomarrell/wrapcheck v0.0.0-20200807122107-df9e8bcb914d h1:3EZyvNUMsGD1QA8cu0STNn1L7I77rvhf2IhOcHYQhSw=
+github.com/tomarrell/wrapcheck v0.0.0-20200807122107-df9e8bcb914d/go.mod h1:yiFB6fFoV7saXirUGfuK+cPtUh4NX/Hf5y2WC2lehu0=
+github.com/tommy-muehle/go-mnd v1.3.1-0.20201008215730-16041ac3fe65 h1:Y0bLA422kvb32uZI4fy/Plop/Tbld0l9pSzl+j1FWok=
+github.com/tommy-muehle/go-mnd v1.3.1-0.20201008215730-16041ac3fe65/go.mod h1:T22e7iRN4LsFPZGyRLRXeF+DWVXFuV9thsyO7NjbbTI=
+github.com/ultraware/funlen v0.0.3 h1:5ylVWm8wsNwH5aWo9438pwvsK0QiqVuUrt9bn7S/iLA=
+github.com/ultraware/funlen v0.0.3/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
+github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg=
+github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
+github.com/uudashr/gocognit v1.0.1 h1:MoG2fZ0b/Eo7NXoIwCVFLG5JED3qgQz5/NEE+rOsjPs=
+github.com/uudashr/gocognit v1.0.1/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
+github.com/valyala/quicktemplate v1.6.3/go.mod h1:fwPzK2fHuYEODzJ9pkw0ipCPNHZ2tD5KW4lOuSdPKzY=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad h1:W0LEBv82YCGEtcmPA3uNZBI33/qF//HAAs3MawDjRa0=
+github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad/go.mod h1:Hy8o65+MXnS6EwGElrSRjUzQDLXreJlzYLlWiHtt8hM=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8=
+golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 h1:bNEHhJCnrwMKNMmOx3yAynp5vs5/gRy+XWFtZFu7NBM=
+golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190307163923-6a08e3108db3/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b h1:iEAPfYPbYbxG/2lNN4cMOHkmgKNsCuUwkxlDCK46UlU=
+golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190916130336-e45ffcd953cc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200117220505-0cba7a3a9ee9/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200324003944-a576cf524670/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200414032229-332987a829c3/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200422022333-3d57cf2e726e/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200622203043-20e05c1c8ffa/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200624225443-88f3c62a19ff/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200625211823-6506e20df31f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200812195022-5ae4c3c160a0/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200820010801-b793a1359eac/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200831203904-5a2aa26beb65/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20201001104356-43ebab892c4c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
+golang.org/x/tools v0.0.0-20201002184944-ecd9fd270d5d/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
+golang.org/x/tools v0.0.0-20201007032633-0806396f153e/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
+golang.org/x/tools v0.0.0-20201011145850-ed2f50202694/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
+golang.org/x/tools v0.0.0-20201030010431-2feb2bb1ff51/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201114224030-61ea331ec02b/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201118003311-bd56c0adb394/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201230224404-63754364767c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210105210202-9ed45478a130 h1:8qSBr5nyKsEgkP918Pu5FFDZpTtLIjXSo6mrtdVOFfk=
+golang.org/x/tools v0.0.0-20210105210202-9ed45478a130/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8=
+google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE=
+google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
+gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.6 h1:W18jzjh8mfPez+AwGLxmOImucz/IFjpNlrKVnaj2YVc=
+honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzEYLhQB2YY=
+mvdan.cc/gofumpt v0.1.0 h1:hsVv+Y9UsZ/mFZTxJZuHVI6shSQCtzZ11h1JEFPAZLw=
+mvdan.cc/gofumpt v0.1.0/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48=
+mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
+mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
+mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=
+mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
+mvdan.cc/unparam v0.0.0-20200501210554-b37ab49443f7 h1:kAREL6MPwpsk1/PQPFD3Eg7WAQR5mPTWZJaBiG5LDbY=
+mvdan.cc/unparam v0.0.0-20200501210554-b37ab49443f7/go.mod h1:HGC5lll35J70Y5v7vCGb9oLhHoScFwkHDJm/05RdSTc=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

build/tools.go

@@ -0,0 +1,15 @@
+//go:build tools
+// +build tools
+
+package main
+
+// the lines are parsed by lazy.sh, do not edit
+import (
+	_ "github.com/alvaroloes/enumer"
+	_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
+	_ "github.com/wadey/gocovmerge"
+	_ "golang.org/x/tools/cmd/goimports"
+	_ "golang.org/x/tools/cmd/stringer"
+	_ "google.golang.org/grpc/cmd/protoc-gen-go-grpc"
+	_ "google.golang.org/protobuf/cmd/protoc-gen-go"
+)

cli/cli.go

@@ -0,0 +1,155 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+
+	"github.com/zrepl/zrepl/config"
+)
+
+var rootArgs struct {
+	configPath string
+}
+
+var rootCmd = &cobra.Command{
+	Use:   "zrepl",
+	Short: "One-stop ZFS replication solution",
+}
+
+func init() {
+	rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
+}
+
+var genCompletionCmd = &cobra.Command{
+	Use:   "gencompletion",
+	Short: "generate shell auto-completions",
+}
+
+type completionCmdInfo struct {
+	genFunc func(outpath string) error
+	help    string
+}
+
+var completionCmdMap = map[string]completionCmdInfo{
+	"zsh": {
+		rootCmd.GenZshCompletionFile,
+		"  save to file `_zrepl` in your zsh's $fpath",
+	},
+	"bash": {
+		rootCmd.GenBashCompletionFile,
+		"  save to a path and source that path in your .bashrc",
+	},
+}
+
+func init() {
+	for sh, info := range completionCmdMap {
+		sh, info := sh, info
+		genCompletionCmd.AddCommand(&cobra.Command{
+			Use:     fmt.Sprintf("%s path/to/out/file", sh),
+			Short:   fmt.Sprintf("generate %s completions", sh),
+			Example: info.help,
+			Run: func(cmd *cobra.Command, args []string) {
+				if len(args) != 1 {
+					fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
+					err := cmd.Usage()
+					if err != nil {
+						panic(err)
+					}
+					os.Exit(1)
+				}
+				if err := info.genFunc(args[0]); err != nil {
+					fmt.Fprintf(os.Stderr, "error generating %s completion: %s", sh, err)
+					os.Exit(1)
+				}
+			},
+		})
+	}
+	rootCmd.AddCommand(genCompletionCmd)
+}
+
+type Subcommand struct {
+	Use              string
+	Short            string
+	Example          string
+	NoRequireConfig  bool
+	Run              func(ctx context.Context, subcommand *Subcommand, args []string) error
+	SetupFlags       func(f *pflag.FlagSet)
+	SetupSubcommands func() []*Subcommand
+
+	config    *config.Config
+	configErr error
+}
+
+func (s *Subcommand) ConfigParsingError() error {
+	return s.configErr
+}
+
+func (s *Subcommand) Config() *config.Config {
+	if !s.NoRequireConfig && s.config == nil {
+		panic("command that requires config is running and has no config set")
+	}
+	return s.config
+}
+
+func (s *Subcommand) run(cmd *cobra.Command, args []string) {
+	s.tryParseConfig()
+	ctx := context.Background()
+	endTask := trace.WithTaskFromStackUpdateCtx(&ctx)
+	defer endTask()
+	err := s.Run(ctx, s, args)
+	endTask()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+		os.Exit(1)
+	}
+}
+
+func (s *Subcommand) tryParseConfig() {
+	config, err := config.ParseConfig(rootArgs.configPath)
+	s.configErr = err
+	if err != nil {
+		if s.NoRequireConfig {
+			// doesn't matter
+			return
+		} else {
+			fmt.Fprintf(os.Stderr, "could not parse config: %s\n", err)
+			os.Exit(1)
+		}
+	}
+	s.config = config
+}
+
+func AddSubcommand(s *Subcommand) {
+	addSubcommandToCobraCmd(rootCmd, s)
+}
+
+func addSubcommandToCobraCmd(c *cobra.Command, s *Subcommand) {
+	cmd := cobra.Command{
+		Use:     s.Use,
+		Short:   s.Short,
+		Example: s.Example,
+	}
+	if s.SetupSubcommands == nil {
+		cmd.Run = s.run
+	} else {
+		for _, sub := range s.SetupSubcommands() {
+			addSubcommandToCobraCmd(&cmd, sub)
+		}
+	}
+	if s.SetupFlags != nil {
+		s.SetupFlags(cmd.Flags())
+	}
+	c.AddCommand(&cmd)
+}
+
+func Run() {
+	if err := rootCmd.Execute(); err != nil {
+		os.Exit(1)
+	}
+}

client/configcheck.go

@@ -0,0 +1,131 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/kr/pretty"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+	"github.com/zrepl/yaml-config"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/logger"
+)
+
+var configcheckArgs struct {
+	format        string
+	what          string
+	skipCertCheck bool
+}
+
+var ConfigcheckCmd = &cli.Subcommand{
+	Use:   "configcheck",
+	Short: "check if config can be parsed without errors",
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]")
+		f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]")
+		f.BoolVar(&configcheckArgs.skipCertCheck, "skip-cert-check", false, "skip checking cert files")
+	},
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		formatMap := map[string]func(interface{}){
+			"": func(i interface{}) {},
+			"pretty": func(i interface{}) {
+				if _, err := pretty.Println(i); err != nil {
+					panic(err)
+				}
+			},
+			"json": func(i interface{}) {
+				if err := json.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
+					panic(err)
+				}
+			},
+			"yaml": func(i interface{}) {
+				if err := yaml.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
+					panic(err)
+				}
+			},
+		}
+
+		formatter, ok := formatMap[configcheckArgs.format]
+		if !ok {
+			return fmt.Errorf("unsupported --format %q", configcheckArgs.format)
+		}
+
+		var hadErr bool
+
+		parseFlags := config.ParseFlagsNone
+
+		if configcheckArgs.skipCertCheck {
+			parseFlags |= config.ParseFlagsNoCertCheck
+		}
+
+		// further: try to build jobs
+		confJobs, err := job.JobsFromConfig(subcommand.Config(), parseFlags)
+
+		if err != nil {
+			err := errors.Wrap(err, "cannot build jobs from config")
+			if configcheckArgs.what == "jobs" {
+				return err
+			} else {
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				confJobs = nil
+				hadErr = true
+			}
+		}
+
+		// further: try to build logging outlets
+		outlets, err := logging.OutletsFromConfig(*subcommand.Config().Global.Logging)
+		if err != nil {
+			err := errors.Wrap(err, "cannot build logging from config")
+			if configcheckArgs.what == "logging" {
+				return err
+			} else {
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				outlets = nil
+				hadErr = true
+			}
+		}
+
+		whatMap := map[string]func(){
+			"all": func() {
+				o := struct {
+					config  *config.Config
+					jobs    []job.Job
+					logging *logger.Outlets
+				}{
+					subcommand.Config(),
+					confJobs,
+					outlets,
+				}
+				formatter(o)
+			},
+			"config": func() {
+				formatter(subcommand.Config())
+			},
+			"jobs": func() {
+				formatter(confJobs)
+			},
+			"logging": func() {
+				formatter(outlets)
+			},
+		}
+
+		wf, ok := whatMap[configcheckArgs.what]
+		if !ok {
+			return fmt.Errorf("unsupported --format %q", configcheckArgs.what)
+		}
+		wf()
+
+		if hadErr {
+			return fmt.Errorf("config parsing failed")
+		} else {
+			return nil
+		}
+	},
+}

client/jsonclient.go

@@ -0,0 +1,49 @@
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net"
+	"net/http"
+
+	"github.com/pkg/errors"
+)
+
+func controlHttpClient(sockpath string) (client http.Client, err error) {
+	return http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", sockpath)
+			},
+		},
+	}, nil
+}
+
+func jsonRequestResponse(c http.Client, endpoint string, req interface{}, res interface{}) error {
+	var buf bytes.Buffer
+	encodeErr := json.NewEncoder(&buf).Encode(req)
+	if encodeErr != nil {
+		return encodeErr
+	}
+
+	resp, err := c.Post("http://unix"+endpoint, "application/json", &buf)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		var msg bytes.Buffer
+		_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
+		return errors.Errorf("%s", msg.String())
+	}
+
+	decodeError := json.NewDecoder(resp.Body).Decode(&res)
+	if decodeError != nil {
+		return decodeError
+	}
+
+	return nil
+}

client/migrate.go

@@ -0,0 +1,268 @@
+package client
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/fatih/color"
+	"github.com/kr/pretty"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+)
+
+var (
+	MigrateCmd = &cli.Subcommand{
+		Use:   "migrate",
+		Short: "perform migration of the on-disk / zfs properties",
+		SetupSubcommands: func() []*cli.Subcommand {
+			return migrations
+		},
+	}
+)
+
+var migrations = []*cli.Subcommand{
+	&cli.Subcommand{
+		Use: "0.0.X:0.1:placeholder",
+		Run: doMigratePlaceholder0_1,
+		SetupFlags: func(f *pflag.FlagSet) {
+			f.BoolVar(&migratePlaceholder0_1Args.dryRun, "dry-run", false, "dry run")
+		},
+	},
+	&cli.Subcommand{
+		Use: "replication-cursor:v1-v2",
+		Run: doMigrateReplicationCursor,
+		SetupFlags: func(f *pflag.FlagSet) {
+			f.BoolVar(&migrateReplicationCursorArgs.dryRun, "dry-run", false, "dry run")
+		},
+	},
+}
+
+var migratePlaceholder0_1Args struct {
+	dryRun bool
+}
+
+func doMigratePlaceholder0_1(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	if len(args) != 0 {
+		return fmt.Errorf("migration does not take arguments, got %v", args)
+	}
+
+	cfg := sc.Config()
+
+	allFSS, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
+	if err != nil {
+		return errors.Wrap(err, "cannot list filesystems")
+	}
+
+	type workItem struct {
+		jobName string
+		rootFS  *zfs.DatasetPath
+		fss     []*zfs.DatasetPath
+	}
+	var wis []workItem
+	for i, j := range cfg.Jobs {
+		var rfsS string
+		switch job := j.Ret.(type) {
+		case *config.SinkJob:
+			rfsS = job.RootFS
+		case *config.PullJob:
+			rfsS = job.RootFS
+		default:
+			fmt.Printf("ignoring job %q (%d/%d, type %T)\n", j.Name(), i, len(cfg.Jobs), j.Ret)
+			continue
+		}
+		rfs, err := zfs.NewDatasetPath(rfsS)
+		if err != nil {
+			return errors.Wrapf(err, "root fs for job %q is not a valid dataset path", j.Name())
+		}
+		var fss []*zfs.DatasetPath
+		for _, fs := range allFSS {
+			if fs.HasPrefix(rfs) {
+				fss = append(fss, fs)
+			}
+		}
+		wis = append(wis, workItem{j.Name(), rfs, fss})
+	}
+
+	for _, wi := range wis {
+		fmt.Printf("job %q => migrate filesystems below root_fs %q\n", wi.jobName, wi.rootFS.ToString())
+		if len(wi.fss) == 0 {
+			fmt.Printf("\tno filesystems\n")
+			continue
+		}
+		for _, fs := range wi.fss {
+			fmt.Printf("\t%q ... ", fs.ToString())
+			r, err := zfs.ZFSMigrateHashBasedPlaceholderToCurrent(ctx, fs, migratePlaceholder0_1Args.dryRun)
+			if err != nil {
+				fmt.Printf("error: %s\n", err)
+			} else if !r.NeedsModification {
+				fmt.Printf("unchanged (placeholder=%v)\n", r.OriginalState.IsPlaceholder)
+			} else {
+				fmt.Printf("migrate (placeholder=%v) (old value = %q)\n",
+					r.OriginalState.IsPlaceholder, r.OriginalState.RawLocalPropertyValue)
+			}
+		}
+	}
+
+	return nil
+}
+
+var migrateReplicationCursorArgs struct {
+	dryRun bool
+}
+
+var bold = color.New(color.Bold)
+var succ = color.New(color.FgGreen)
+var fail = color.New(color.FgRed)
+
+var migrateReplicationCursorSkipSentinel = fmt.Errorf("skipping this filesystem")
+
+func doMigrateReplicationCursor(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	if len(args) != 0 {
+		return fmt.Errorf("migration does not take arguments, got %v", args)
+	}
+
+	cfg := sc.Config()
+	jobs, err := job.JobsFromConfig(cfg, config.ParseFlagsNone)
+	if err != nil {
+		fmt.Printf("cannot parse config:\n%s\n\n", err)
+		fmt.Printf("NOTE: this migration was released together with a change in job name requirements.\n")
+		return fmt.Errorf("exiting migration after error")
+	}
+
+	v1cursorJobs := make([]job.Job, 0, len(cfg.Jobs))
+	for i, j := range cfg.Jobs {
+		if jobs[i].Name() != j.Name() {
+			panic("implementation error")
+		}
+		switch j.Ret.(type) {
+		case *config.PushJob:
+			v1cursorJobs = append(v1cursorJobs, jobs[i])
+		case *config.SourceJob:
+			v1cursorJobs = append(v1cursorJobs, jobs[i])
+		default:
+			fmt.Printf("ignoring job %q (%d/%d, type %T), not supposed to create v1 replication cursors\n", j.Name(), i, len(cfg.Jobs), j.Ret)
+			continue
+		}
+	}
+
+	// scan all filesystems for v1 replication cursors
+
+	fss, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
+	if err != nil {
+		return errors.Wrap(err, "list filesystems")
+	}
+
+	var hadError bool
+	for _, fs := range fss {
+
+		bold.Printf("INSPECT FILESYSTEM %q\n", fs.ToString())
+
+		err := doMigrateReplicationCursorFS(ctx, v1cursorJobs, fs)
+		if err == migrateReplicationCursorSkipSentinel {
+			bold.Printf("FILESYSTEM SKIPPED\n")
+		} else if err != nil {
+			hadError = true
+			fail.Printf("MIGRATION FAILED: %s\n", err)
+		} else {
+			succ.Printf("FILESYSTEM %q COMPLETE\n", fs.ToString())
+		}
+	}
+
+	if hadError {
+		fail.Printf("\n\none or more filesystems could not be migrated, please inspect output and or re-run migration")
+		return errors.Errorf("")
+	}
+	return nil
+}
+
+func doMigrateReplicationCursorFS(ctx context.Context, v1CursorJobs []job.Job, fs *zfs.DatasetPath) error {
+
+	var owningJob job.Job = nil
+	for _, job := range v1CursorJobs {
+		conf := job.SenderConfig()
+		if conf == nil {
+			continue
+		}
+		pass, err := conf.FSF.Filter(fs)
+		if err != nil {
+			return errors.Wrapf(err, "filesystem filter error in job %q for fs %q", job.Name(), fs.ToString())
+		}
+		if !pass {
+			continue
+		}
+		if owningJob != nil {
+			return errors.Errorf("jobs %q and %q both match %q\ncannot attribute replication cursor to either one", owningJob.Name(), job.Name(), fs)
+		}
+		owningJob = job
+	}
+	if owningJob == nil {
+		fmt.Printf("no job's Filesystems filter matches\n")
+		return migrateReplicationCursorSkipSentinel
+	}
+	fmt.Printf("identified owning job %q\n", owningJob.Name())
+
+	bookmarks, err := zfs.ZFSListFilesystemVersions(ctx, fs, zfs.ListFilesystemVersionsOptions{
+		Types: zfs.Bookmarks,
+	})
+	if err != nil {
+		return errors.Wrapf(err, "list filesystem versions of %q", fs.ToString())
+	}
+
+	var oldCursor *zfs.FilesystemVersion
+	for i, fsv := range bookmarks {
+		_, _, err := endpoint.ParseReplicationCursorBookmarkName(fsv.ToAbsPath(fs))
+		if err != endpoint.ErrV1ReplicationCursor {
+			continue
+		}
+
+		if oldCursor != nil {
+			fmt.Printf("unexpected v1 replication cursor candidate: %q", fsv.ToAbsPath(fs))
+			return errors.Wrap(err, "multiple filesystem versions identified as v1 replication cursors")
+		}
+
+		oldCursor = &bookmarks[i]
+
+	}
+
+	if oldCursor == nil {
+		bold.Printf("no v1 replication cursor found for filesystem %q\n", fs.ToString())
+		return migrateReplicationCursorSkipSentinel
+	}
+
+	fmt.Printf("found v1 replication cursor:\n%s\n", pretty.Sprint(oldCursor))
+
+	mostRecentNew, err := endpoint.GetMostRecentReplicationCursorOfJob(ctx, fs.ToString(), owningJob.SenderConfig().JobID)
+	if err != nil {
+		return errors.Wrapf(err, "get most recent v2 replication cursor")
+	}
+
+	if mostRecentNew == nil {
+		return errors.Errorf("no v2 replication cursor found for job %q on filesystem %q", owningJob.SenderConfig().JobID, fs.ToString())
+	}
+
+	fmt.Printf("most recent v2 replication cursor:\n%#v", oldCursor)
+
+	if !(mostRecentNew.CreateTXG >= oldCursor.CreateTXG) {
+		return errors.Errorf("v1 replication cursor createtxg is higher than v2 cursor's, skipping this filesystem")
+	}
+
+	fmt.Printf("determined that v2 cursor is bookmark of same or newer version than v1 cursor\n")
+	fmt.Printf("destroying v1 cursor %q\n", oldCursor.ToAbsPath(fs))
+
+	if migrateReplicationCursorArgs.dryRun {
+		succ.Printf("DRY RUN\n")
+	} else {
+		if err := zfs.ZFSDestroyFilesystemVersion(ctx, fs, oldCursor); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

client/migrate_test.go

@@ -0,0 +1,16 @@
+package client
+
+import "testing"
+
+func TestMigrationsUnambiguousNames(t *testing.T) {
+	names := make(map[string]bool)
+	for _, mig := range migrations {
+		if _, ok := names[mig.Use]; ok {
+			t.Errorf("duplicate migration name %q", mig.Use)
+			t.FailNow()
+			return
+		} else {
+			names[mig.Use] = true
+		}
+	}
+}

client/pprof.go

@@ -0,0 +1,10 @@
+package client
+
+import "github.com/zrepl/zrepl/cli"
+
+var PprofCmd = &cli.Subcommand{
+	Use: "pprof",
+	SetupSubcommands: func() []*cli.Subcommand {
+		return []*cli.Subcommand{PprofListenCmd, pprofActivityTraceCmd}
+	},
+}

client/pprof_activity_trace.go

@@ -0,0 +1,44 @@
+package client
+
+import (
+	"context"
+	"io"
+	"log"
+	"os"
+
+	"golang.org/x/net/websocket"
+
+	"github.com/zrepl/zrepl/cli"
+)
+
+var pprofActivityTraceCmd = &cli.Subcommand{
+	Use:   "activity-trace ZREPL_PPROF_HOST:ZREPL_PPROF_PORT",
+	Short: "attach to zrepl daemon with activated pprof listener and dump an activity-trace to stdout",
+	Run:   runPProfActivityTrace,
+}
+
+func runPProfActivityTrace(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+	log := log.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Printf("exiting after error")
+		os.Exit(1)
+	}
+
+	if len(args) != 1 {
+		log.Printf("exactly one positional argument is required")
+		die()
+	}
+
+	url := "ws://" + args[0] + "/debug/zrepl/activity-trace" // FIXME dont' repeat that
+
+	log.Printf("attaching to activity trace stream %s", url)
+	ws, err := websocket.Dial(url, "", url)
+	if err != nil {
+		log.Printf("error: %s", err)
+		die()
+	}
+
+	_, err = io.Copy(os.Stdout, ws)
+	return err
+}

client/pprof_listen.go

@@ -0,0 +1,68 @@
+package client
+
+import (
+	"context"
+	"errors"
+	"log"
+	"os"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+)
+
+var pprofListenCmd struct {
+	daemon.PprofServerControlMsg
+}
+
+var PprofListenCmd = &cli.Subcommand{
+	Use:   "listen off | [on TCP_LISTEN_ADDRESS]",
+	Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		if len(args) < 1 {
+			goto enargs
+		}
+		switch args[0] {
+		case "on":
+			pprofListenCmd.Run = true
+			if len(args) != 2 {
+				return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
+			}
+			pprofListenCmd.HttpListenAddress = args[1]
+		case "off":
+			if len(args) != 1 {
+				goto enargs
+			}
+			pprofListenCmd.Run = false
+		}
+
+		RunPProf(subcommand.Config())
+		return nil
+	enargs:
+		return errors.New("invalid number of positional arguments")
+
+	},
+}
+
+func RunPProf(conf *config.Config) {
+	log := log.New(os.Stderr, "", 0)
+
+	die := func() {
+		log.Printf("exiting after error")
+		os.Exit(1)
+	}
+
+	log.Printf("connecting to zrepl daemon")
+
+	httpc, err := controlHttpClient(conf.Global.Control.SockPath)
+	if err != nil {
+		log.Printf("error creating http client: %s", err)
+		die()
+	}
+	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofListenCmd.PprofServerControlMsg, struct{}{})
+	if err != nil {
+		log.Printf("error sending control message: %s", err)
+		die()
+	}
+	log.Printf("finished")
+}

client/signal.go

@@ -0,0 +1,42 @@
+package client
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+)
+
+var SignalCmd = &cli.Subcommand{
+	Use:   "signal [wakeup|reset] JOB",
+	Short: "wake up a job from wait state or abort its current invocation",
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		return runSignalCmd(subcommand.Config(), args)
+	},
+}
+
+func runSignalCmd(config *config.Config, args []string) error {
+	if len(args) != 2 {
+		return errors.Errorf("Expected 2 arguments: [wakeup|reset] JOB")
+	}
+
+	httpc, err := controlHttpClient(config.Global.Control.SockPath)
+	if err != nil {
+		return err
+	}
+
+	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointSignal,
+		struct {
+			Name string
+			Op   string
+		}{
+			Name: args[1],
+			Op:   args[0],
+		},
+		struct{}{},
+	)
+	return err
+}

client/status/client/client.go

@@ -0,0 +1,111 @@
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net"
+	"net/http"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/daemon"
+)
+
+type Client struct {
+	h *http.Client
+}
+
+func New(network, addr string) (*Client, error) {
+	httpc, err := makeControlHttpClient(func(_ context.Context) (net.Conn, error) { return net.Dial(network, addr) })
+	if err != nil {
+		return nil, err
+	}
+	return &Client{httpc}, nil
+}
+
+func (c *Client) Status() (s daemon.Status, _ error) {
+	err := jsonRequestResponse(c.h, daemon.ControlJobEndpointStatus,
+		struct{}{},
+		&s,
+	)
+	return s, err
+}
+
+func (c *Client) StatusRaw() ([]byte, error) {
+	var r json.RawMessage
+	err := jsonRequestResponse(c.h, daemon.ControlJobEndpointStatus, struct{}{}, &r)
+	if err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+
+func (c *Client) signal(job, sig string) error {
+	return jsonRequestResponse(c.h, daemon.ControlJobEndpointSignal,
+		struct {
+			Name string
+			Op   string
+		}{
+			Name: job,
+			Op:   sig,
+		},
+		struct{}{},
+	)
+}
+
+func (c *Client) SignalWakeup(job string) error {
+	return c.signal(job, "wakeup")
+}
+
+func (c *Client) SignalReset(job string) error {
+	return c.signal(job, "reset")
+}
+
+func makeControlHttpClient(dialfunc func(context.Context) (net.Conn, error)) (client *http.Client, err error) {
+	return &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+				return dialfunc(ctx)
+			},
+		},
+	}, nil
+}
+
+func jsonRequestResponse(c *http.Client, endpoint string, req interface{}, res interface{}) error {
+	var buf bytes.Buffer
+	encodeErr := json.NewEncoder(&buf).Encode(req)
+	if encodeErr != nil {
+		return encodeErr
+	}
+
+	hreq, err := http.NewRequest("POST", "http://unix"+endpoint, &buf)
+	if err != nil {
+		return err
+	}
+	hreq.Header.Set("Content-Type", "application/json")
+	// Prevent EOF errors when client request frequency and server keepalive close are at the same time.
+	// Found this by watching http.Server.ConnState changes, then found
+	// https://stackoverflow.com/questions/17714494/golang-http-request-results-in-eof-errors-when-making-multiple-requests-successi
+	// Note: The issue seems even more prounounced with local TCP sockets than unix domain sockets. So, I used that for debugging.
+	hreq.Close = true
+	resp, err := c.Do(hreq)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		var msg bytes.Buffer
+		_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
+		return errors.Errorf("%s", msg.String())
+	}
+
+	decodeError := json.NewDecoder(resp.Body).Decode(&res)
+	if decodeError != nil {
+		return decodeError
+	}
+
+	return nil
+}

client/status/status.go

@@ -0,0 +1,97 @@
+package status
+
+import (
+	"context"
+	"os"
+	"time"
+
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/client/status/client"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+	"github.com/zrepl/zrepl/util/choices"
+)
+
+type Client interface {
+	Status() (daemon.Status, error)
+	StatusRaw() ([]byte, error)
+	SignalWakeup(job string) error
+	SignalReset(job string) error
+}
+
+type statusFlags struct {
+	Mode  choices.Choices
+	Job   string
+	Delay time.Duration
+}
+
+var statusv2Flags statusFlags
+
+type statusv2Mode int
+
+const (
+	StatusV2ModeInteractive statusv2Mode = 1 + iota
+	StatusV2ModeDump
+	StatusV2ModeRaw
+	StatusV2ModeLegacy
+)
+
+var Subcommand = &cli.Subcommand{
+	Use:   "status",
+	Short: "retrieve & display daemon status information",
+	SetupFlags: func(f *pflag.FlagSet) {
+		statusv2Flags.Mode.Init(
+			"interactive", StatusV2ModeInteractive,
+			"dump", StatusV2ModeDump,
+			"raw", StatusV2ModeRaw,
+			"legacy", StatusV2ModeLegacy,
+		)
+		statusv2Flags.Mode.SetTypeString("mode")
+		statusv2Flags.Mode.SetDefaultValue(StatusV2ModeInteractive)
+		f.Var(&statusv2Flags.Mode, "mode", statusv2Flags.Mode.Usage())
+		f.StringVar(&statusv2Flags.Job, "job", "", "only show specified job (works in \"dump\" and \"interactive\" mode)")
+		f.DurationVarP(&statusv2Flags.Delay, "delay", "d", 1*time.Second, "use -d 3s for 3 seconds delay (minimum delay is 1s)")
+	},
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		return runStatusV2Command(ctx, subcommand.Config(), args)
+	},
+}
+
+func runStatusV2Command(ctx context.Context, config *config.Config, args []string) error {
+
+	c, err := client.New("unix", config.Global.Control.SockPath)
+	if err != nil {
+		return errors.Wrapf(err, "connect to daemon socket at %q", config.Global.Control.SockPath)
+	}
+
+	mode := statusv2Flags.Mode.Value().(statusv2Mode)
+
+	if !isatty.IsTerminal(os.Stdout.Fd()) && mode != StatusV2ModeDump && mode != StatusV2ModeRaw {
+		dumpmode, err := statusv2Flags.Mode.InputForChoice(StatusV2ModeDump)
+		if err != nil {
+			panic(err)
+		}
+		rawmode, err := statusv2Flags.Mode.InputForChoice(StatusV2ModeRaw)
+		if err != nil {
+			panic(err)
+		}
+		return errors.Errorf("error: stdout is not a tty, please use --mode %s or --mode %s", dumpmode, rawmode)
+	}
+
+	switch mode {
+	case StatusV2ModeInteractive:
+		return interactive(c, statusv2Flags)
+	case StatusV2ModeDump:
+		return dump(c, statusv2Flags.Job)
+	case StatusV2ModeRaw:
+		return raw(c)
+	case StatusV2ModeLegacy:
+		return legacy(c, statusv2Flags)
+	default:
+		panic("unreachable")
+	}
+}

client/status/status_dump.go

@@ -0,0 +1,70 @@
+package status
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/gdamore/tcell"
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/client/status/viewmodel"
+)
+
+func dump(c Client, job string) error {
+	s, err := c.Status()
+	if err != nil {
+		return err
+	}
+
+	if job != "" {
+		if _, ok := s.Jobs[job]; !ok {
+			return errors.Errorf("job %q not found", job)
+		}
+	}
+
+	width := (1 << 31) - 1
+	wrap := false
+	hline := strings.Repeat("-", 80)
+	if isatty.IsTerminal(os.Stdout.Fd()) {
+		wrap = true
+		screen, err := tcell.NewScreen()
+		if err != nil {
+			return errors.Wrap(err, "get terminal dimensions")
+		}
+		if err := screen.Init(); err != nil {
+			return errors.Wrap(err, "init screen")
+		}
+		width, _ = screen.Size()
+		screen.Fini()
+		hline = strings.Repeat("-", width)
+	}
+
+	m := viewmodel.New()
+	params := viewmodel.Params{
+		Report:                  s.Jobs,
+		ReportFetchError:        nil,
+		SelectedJob:             nil,
+		FSFilter:                func(s string) bool { return true },
+		DetailViewWidth:         width,
+		DetailViewWrap:          wrap,
+		ShortKeybindingOverview: "",
+	}
+	m.Update(params)
+	for _, j := range m.Jobs() {
+		if job != "" && j.Name() != job {
+			continue
+		}
+		params.SelectedJob = j
+		m.Update(params)
+		fmt.Println(m.SelectedJob().FullDescription())
+		if job != "" {
+			return nil
+		} else {
+			fmt.Println(hline)
+		}
+	}
+
+	return nil
+}

client/status/status_interactive.go

@@ -0,0 +1,347 @@
+package status
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gdamore/tcell/v2"
+	tview "gitlab.com/tslocum/cview"
+
+	"github.com/zrepl/zrepl/client/status/viewmodel"
+)
+
+func interactive(c Client, flag statusFlags) error {
+
+	// Set this so we don't overwrite the default terminal colors
+	// See https://github.com/rivo/tview/blob/master/styles.go
+	tview.Styles.PrimitiveBackgroundColor = tcell.ColorDefault
+	tview.Styles.ContrastBackgroundColor = tcell.ColorDefault
+	tview.Styles.PrimaryTextColor = tcell.ColorDefault
+	tview.Styles.BorderColor = tcell.ColorDefault
+	app := tview.NewApplication()
+
+	jobDetailSplit := tview.NewFlex()
+	jobMenu := tview.NewTreeView()
+	jobMenuRoot := tview.NewTreeNode("jobs")
+	jobMenuRoot.SetSelectable(true)
+	jobMenu.SetRoot(jobMenuRoot)
+	jobMenu.SetCurrentNode(jobMenuRoot)
+	jobMenu.SetSelectedTextColor(tcell.ColorGreen)
+	jobTextDetail := tview.NewTextView()
+	jobTextDetail.SetWrap(false)
+
+	jobMenu.SetBorder(true)
+	jobTextDetail.SetBorder(true)
+
+	toolbarSplit := tview.NewFlex()
+	toolbarSplit.SetDirection(tview.FlexRow)
+	inputBarContainer := tview.NewFlex()
+	fsFilterInput := tview.NewInputField()
+	fsFilterInput.SetBorder(false)
+	fsFilterInput.SetFieldBackgroundColor(tcell.ColorDefault)
+	inputBarLabel := tview.NewTextView()
+	inputBarLabel.SetText("[::b]FILTER ")
+	inputBarLabel.SetDynamicColors(true)
+	inputBarContainer.AddItem(inputBarLabel, 7, 1, false)
+	inputBarContainer.AddItem(fsFilterInput, 0, 10, false)
+	toolbarSplit.AddItem(inputBarContainer, 1, 0, false)
+	toolbarSplit.AddItem(jobDetailSplit, 0, 10, false)
+
+	bottombar := tview.NewFlex()
+	bottombar.SetDirection(tview.FlexColumn)
+	bottombarDateView := tview.NewTextView()
+	bottombar.AddItem(bottombarDateView, len(time.Now().String()), 0, false)
+	bottomBarStatus := tview.NewTextView()
+	bottomBarStatus.SetDynamicColors(true)
+	bottomBarStatus.SetTextAlign(tview.AlignRight)
+	bottombar.AddItem(bottomBarStatus, 0, 10, false)
+	toolbarSplit.AddItem(bottombar, 1, 0, false)
+
+	tabbableWithJobMenu := []tview.Primitive{jobMenu, jobTextDetail, fsFilterInput}
+	tabbableWithoutJobMenu := []tview.Primitive{jobTextDetail, fsFilterInput}
+	var tabbable []tview.Primitive
+	tabbableActiveIndex := 0
+	tabbableRedraw := func() {
+		if len(tabbable) == 0 {
+			app.SetFocus(nil)
+			return
+		}
+		if tabbableActiveIndex >= len(tabbable) {
+			app.SetFocus(tabbable[0])
+			return
+		}
+		app.SetFocus(tabbable[tabbableActiveIndex])
+	}
+	tabbableCycle := func() {
+		if len(tabbable) == 0 {
+			return
+		}
+		tabbableActiveIndex = (tabbableActiveIndex + 1) % len(tabbable)
+		app.SetFocus(tabbable[tabbableActiveIndex])
+		tabbableRedraw()
+	}
+
+	jobMenuVisisble := false
+	reconfigureJobDetailSplit := func(setJobMenuVisible bool) {
+		if jobMenuVisisble == setJobMenuVisible {
+			return
+		}
+		jobMenuVisisble = setJobMenuVisible
+		if setJobMenuVisible {
+			jobDetailSplit.RemoveItem(jobTextDetail)
+			jobDetailSplit.AddItem(jobMenu, 0, 1, true)
+			jobDetailSplit.AddItem(jobTextDetail, 0, 5, false)
+			tabbable = tabbableWithJobMenu
+		} else {
+			jobDetailSplit.RemoveItem(jobMenu)
+			tabbable = tabbableWithoutJobMenu
+		}
+		tabbableRedraw()
+	}
+
+	showModal := func(m *tview.Modal, modalDoneFunc func(idx int, label string)) {
+		preModalFocus := app.GetFocus()
+		m.SetDoneFunc(func(idx int, label string) {
+			if modalDoneFunc != nil {
+				modalDoneFunc(idx, label)
+			}
+			app.SetRoot(toolbarSplit, true)
+			app.SetFocus(preModalFocus)
+			app.Draw()
+		})
+		app.SetRoot(m, true)
+		app.Draw()
+	}
+
+	app.SetRoot(toolbarSplit, true)
+	// initial focus
+	tabbableActiveIndex = len(tabbable)
+	tabbableCycle()
+	reconfigureJobDetailSplit(true)
+
+	m := viewmodel.New()
+	params := &viewmodel.Params{
+		Report:                  nil,
+		SelectedJob:             nil,
+		FSFilter:                func(_ string) bool { return true },
+		DetailViewWidth:         100,
+		DetailViewWrap:          false,
+		ShortKeybindingOverview: "[::b]Q[::-] quit  [::b]<TAB>[::-] switch panes [::b]W[::-] wrap lines  [::b]Shift+M[::-] toggle navbar  [::b]Shift+S[::-] signal job [::b]</>[::-] filter filesystems",
+	}
+	paramsMtx := &sync.Mutex{}
+	var redraw func()
+	viewmodelupdate := func(cb func(*viewmodel.Params)) {
+		paramsMtx.Lock()
+		defer paramsMtx.Unlock()
+		cb(params)
+		m.Update(*params)
+	}
+	redraw = func() {
+		jobs := m.Jobs()
+		if flag.Job != "" {
+			job_found := false
+			for _, job := range jobs {
+				if strings.Compare(flag.Job, job.Name()) == 0 {
+					jobs = []*viewmodel.Job{job}
+					job_found = true
+					break
+				}
+			}
+			if !job_found {
+				jobs = nil
+			}
+		}
+		redrawJobsList := false
+		var selectedJobN *tview.TreeNode
+		if len(jobMenuRoot.GetChildren()) == len(jobs) {
+			for i, jobN := range jobMenuRoot.GetChildren() {
+				if jobN.GetReference().(*viewmodel.Job) != jobs[i] {
+					redrawJobsList = true
+					break
+				}
+				if jobN.GetReference().(*viewmodel.Job) == m.SelectedJob() {
+					selectedJobN = jobN
+				}
+			}
+		} else {
+			redrawJobsList = true
+		}
+		if redrawJobsList {
+			selectedJobN = nil
+			children := make([]*tview.TreeNode, len(jobs))
+			for i := range jobs {
+				jobN := tview.NewTreeNode(jobs[i].JobTreeTitle())
+				jobN.SetReference(jobs[i])
+				jobN.SetSelectable(true)
+				children[i] = jobN
+				jobN.SetSelectedFunc(func() {
+					viewmodelupdate(func(p *viewmodel.Params) {
+						p.SelectedJob = jobN.GetReference().(*viewmodel.Job)
+					})
+				})
+				if jobs[i] == m.SelectedJob() {
+					selectedJobN = jobN
+				}
+			}
+			jobMenuRoot.SetChildren(children)
+		}
+
+		if selectedJobN != nil && jobMenu.GetCurrentNode() != selectedJobN {
+			jobMenu.SetCurrentNode(selectedJobN)
+		} else if selectedJobN == nil {
+			// select something, otherwise selection breaks (likely bug in tview)
+			jobMenu.SetCurrentNode(jobMenuRoot)
+		}
+
+		if selJ := m.SelectedJob(); selJ != nil {
+			jobTextDetail.SetText(selJ.FullDescription())
+		} else {
+			jobTextDetail.SetText("please select a job")
+		}
+
+		bottombardatestring := m.DateString()
+		bottombarDateView.SetText(bottombardatestring)
+		bottombar.ResizeItem(bottombarDateView, len(bottombardatestring), 0)
+
+		bottomBarStatus.SetText(m.BottomBarStatus())
+
+		app.Draw()
+
+	}
+
+	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				app.Suspend(func() {
+					panic(err)
+				})
+			}
+		}()
+		for {
+			st, err := c.Status()
+			viewmodelupdate(func(p *viewmodel.Params) {
+				p.Report = st.Jobs
+				p.ReportFetchError = err
+			})
+			app.QueueUpdateDraw(redraw)
+
+			time.Sleep(flag.Delay)
+		}
+	}()
+
+	jobMenu.SetChangedFunc(func(jobN *tview.TreeNode) {
+		viewmodelupdate(func(p *viewmodel.Params) {
+			p.SelectedJob, _ = jobN.GetReference().(*viewmodel.Job)
+		})
+		redraw()
+		jobTextDetail.ScrollToBeginning()
+	})
+	jobMenu.SetSelectedFunc(func(jobN *tview.TreeNode) {
+		app.SetFocus(jobTextDetail)
+	})
+
+	app.SetBeforeDrawFunc(func(screen tcell.Screen) bool {
+		viewmodelupdate(func(p *viewmodel.Params) {
+			_, _, p.DetailViewWidth, _ = jobTextDetail.GetInnerRect()
+		})
+		return false
+	})
+
+	app.SetInputCapture(func(e *tcell.EventKey) *tcell.EventKey {
+		if e.Key() == tcell.KeyTab {
+			tabbableCycle()
+			return nil
+		}
+
+		if e.Key() == tcell.KeyRune && app.GetFocus() == fsFilterInput {
+			return e
+		}
+
+		if e.Key() == tcell.KeyRune && e.Rune() == '/' {
+			if app.GetFocus() != fsFilterInput {
+				app.SetFocus(fsFilterInput)
+			}
+			return e
+		}
+
+		if e.Key() == tcell.KeyRune && e.Rune() == 'M' {
+			reconfigureJobDetailSplit(!jobMenuVisisble)
+			return nil
+		}
+
+		if e.Key() == tcell.KeyRune && e.Rune() == 'q' {
+			app.Stop()
+		}
+
+		if e.Key() == tcell.KeyRune && e.Rune() == 'S' {
+			job, ok := jobMenu.GetCurrentNode().GetReference().(*viewmodel.Job)
+			if !ok {
+				return nil
+			}
+			signals := []string{"wakeup", "reset"}
+			clientFuncs := []func(job string) error{c.SignalWakeup, c.SignalReset}
+			sigMod := tview.NewModal()
+			sigMod.SetBackgroundColor(tcell.ColorDefault)
+			sigMod.SetBorder(true)
+			sigMod.GetForm().SetButtonTextColorFocused(tcell.ColorGreen)
+			sigMod.AddButtons(signals)
+			sigMod.SetText(fmt.Sprintf("Send a signal to job %q", job.Name()))
+			showModal(sigMod, func(idx int, _ string) {
+				go func() {
+					if idx == -1 {
+						return
+					}
+					err := clientFuncs[idx](job.Name())
+					if err != nil {
+						app.QueueUpdate(func() {
+							me := tview.NewModal()
+							me.SetText(fmt.Sprintf("signal error: %s", err))
+							me.AddButtons([]string{"Close"})
+							showModal(me, nil)
+						})
+					}
+				}()
+			})
+		}
+
+		return e
+	})
+
+	fsFilterInput.SetChangedFunc(func(searchterm string) {
+		viewmodelupdate(func(p *viewmodel.Params) {
+			p.FSFilter = func(fs string) bool {
+				r, err := regexp.Compile(searchterm)
+				if err != nil {
+					return true
+				}
+				return r.MatchString(fs)
+			}
+		})
+		redraw()
+		jobTextDetail.ScrollToBeginning()
+	})
+	fsFilterInput.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+		if event.Key() == tcell.KeyEnter {
+			app.SetFocus(jobTextDetail)
+			return nil
+		}
+		return event
+	})
+
+	jobTextDetail.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
+		if event.Key() == tcell.KeyRune && event.Rune() == 'w' {
+			// toggle wrapping
+			viewmodelupdate(func(p *viewmodel.Params) {
+				p.DetailViewWrap = !p.DetailViewWrap
+			})
+			redraw()
+			return nil
+		}
+		return event
+	})
+
+	return app.Run()
+}

client/status/status_legacy.go

@@ -0,0 +1,128 @@
+package status
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gdamore/tcell/v2"
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+	tview "gitlab.com/tslocum/cview"
+
+	"github.com/zrepl/zrepl/client/status/viewmodel"
+)
+
+func legacy(c Client, flag statusFlags) error {
+
+	// Set this so we don't overwrite the default terminal colors
+	// See https://github.com/rivo/tview/blob/master/styles.go
+	tview.Styles.PrimitiveBackgroundColor = tcell.ColorDefault
+	tview.Styles.ContrastBackgroundColor = tcell.ColorDefault
+	tview.Styles.PrimaryTextColor = tcell.ColorDefault
+	tview.Styles.BorderColor = tcell.ColorDefault
+	app := tview.NewApplication()
+
+	textView := tview.NewTextView()
+	textView.SetWrap(true)
+	textView.SetScrollable(true) // so that it allows us to set scroll position
+	textView.SetScrollBarVisibility(tview.ScrollBarNever)
+
+	app.SetRoot(textView, true)
+
+	width := (1 << 31) - 1
+	wrap := false
+	if isatty.IsTerminal(os.Stdout.Fd()) {
+		wrap = true
+		screen, err := tcell.NewScreen()
+		if err != nil {
+			return errors.Wrap(err, "get terminal dimensions")
+		}
+		if err := screen.Init(); err != nil {
+			return errors.Wrap(err, "init screen")
+		}
+		width, _ = screen.Size()
+		screen.Fini()
+	}
+
+	paramsMtx := &sync.Mutex{}
+	params := viewmodel.Params{
+		Report:                  nil,
+		ReportFetchError:        nil,
+		SelectedJob:             nil,
+		FSFilter:                func(s string) bool { return true },
+		DetailViewWidth:         width,
+		DetailViewWrap:          wrap,
+		ShortKeybindingOverview: "",
+	}
+
+	redraw := func() {
+		textView.Clear()
+
+		paramsMtx.Lock()
+		defer paramsMtx.Unlock()
+
+		if params.ReportFetchError != nil {
+			fmt.Fprintln(textView, params.ReportFetchError.Error())
+		} else if params.Report != nil {
+			m := viewmodel.New()
+			m.Update(params)
+			for _, j := range m.Jobs() {
+				if flag.Job != "" && j.Name() != flag.Job {
+					continue
+				}
+				params.SelectedJob = j
+				m.Update(params)
+				fmt.Fprintln(textView, m.SelectedJob().FullDescription())
+				if flag.Job != "" {
+					break
+				} else {
+					hline := strings.Repeat("-", params.DetailViewWidth)
+					fmt.Fprintln(textView, hline)
+				}
+			}
+		} else {
+			fmt.Fprintln(textView, "waiting for request results")
+		}
+		textView.ScrollToBeginning()
+	}
+
+	app.SetBeforeDrawFunc(func(screen tcell.Screen) bool {
+		// sync resizes to `params`
+		paramsMtx.Lock()
+		_, _, newWidth, _ := textView.GetInnerRect()
+		if newWidth != params.DetailViewWidth {
+			params.DetailViewWidth = newWidth
+			app.QueueUpdateDraw(redraw)
+		}
+		paramsMtx.Unlock()
+
+		textView.ScrollToBeginning() // has the effect of inhibiting user scrolls
+		return false
+	})
+
+	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				app.Suspend(func() {
+					panic(err)
+				})
+			}
+		}()
+		for {
+			st, err := c.Status()
+			paramsMtx.Lock()
+			params.Report = st.Jobs
+			params.ReportFetchError = err
+			paramsMtx.Unlock()
+
+			app.QueueUpdateDraw(redraw)
+
+			time.Sleep(flag.Delay)
+		}
+	}()
+
+	return app.Run()
+}

client/status/status_raw.go

@@ -0,0 +1,18 @@
+package status
+
+import (
+	"bytes"
+	"io"
+	"os"
+)
+
+func raw(c Client) error {
+	b, err := c.StatusRaw()
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(os.Stdout, bytes.NewReader(b)); err != nil {
+		return err
+	}
+	return nil
+}

client/status/viewmodel/bytecountbinary.go

@@ -0,0 +1,26 @@
+package viewmodel
+
+import (
+	"fmt"
+	"math"
+)
+
+func ByteCountBinaryUint(b uint64) string {
+	if b > math.MaxInt64 {
+		panic(b)
+	}
+	return ByteCountBinary(int64(b))
+}
+
+func ByteCountBinary(b int64) string {
+	const unit = 1024
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := unit, 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
+}

client/status/viewmodel/bytesprogresshistory.go

@@ -0,0 +1,49 @@
+package viewmodel
+
+import "time"
+
+type byteProgressMeasurement struct {
+	time time.Time
+	val  uint64
+}
+
+type bytesProgressHistory struct {
+	last        *byteProgressMeasurement // pointer as poor man's optional
+	changeCount int
+	lastChange  time.Time
+}
+
+func (p *bytesProgressHistory) Update(currentVal uint64) (bytesPerSecondAvg int64, changeCount int) {
+
+	if p.last == nil {
+		p.last = &byteProgressMeasurement{
+			time: time.Now(),
+			val:  currentVal,
+		}
+		return 0, 0
+	}
+
+	if p.last.val != currentVal {
+		p.changeCount++
+		p.lastChange = time.Now()
+	}
+
+	if time.Since(p.lastChange) > 3*time.Second {
+		p.last = nil
+		return 0, 0
+	}
+
+	var deltaV int64
+	if currentVal >= p.last.val {
+		deltaV = int64(currentVal - p.last.val)
+	} else {
+		deltaV = -int64(p.last.val - currentVal)
+	}
+	deltaT := time.Since(p.last.time)
+	rate := float64(deltaV) / deltaT.Seconds()
+
+	p.last.time = time.Now()
+	p.last.val = currentVal
+
+	return int64(rate), p.changeCount
+}

client/status/viewmodel/render.go

@@ -0,0 +1,656 @@
+package viewmodel
+
+import (
+	"fmt"
+	"math"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/go-playground/validator/v10"
+	yaml "github.com/zrepl/yaml-config"
+
+	"github.com/zrepl/zrepl/client/status/viewmodel/stringbuilder"
+	"github.com/zrepl/zrepl/daemon"
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/daemon/pruner"
+	"github.com/zrepl/zrepl/daemon/snapper"
+	"github.com/zrepl/zrepl/replication/report"
+)
+
+type M struct {
+	jobs            map[string]*Job
+	jobsList        []*Job
+	selectedJob     *Job
+	dateString      string
+	bottomBarStatus string
+}
+
+type Job struct {
+	// long-lived
+	name         string
+	byteProgress *bytesProgressHistory
+
+	lastStatus      *job.Status
+	fulldescription string
+}
+
+func New() *M {
+	return &M{
+		jobs:        make(map[string]*Job),
+		jobsList:    make([]*Job, 0),
+		selectedJob: nil,
+	}
+}
+
+type FilterFunc func(string) bool
+
+type Params struct {
+	Report                  map[string]*job.Status
+	ReportFetchError        error
+	SelectedJob             *Job
+	FSFilter                FilterFunc `validate:"required"`
+	DetailViewWidth         int        `validate:"gte=1"`
+	DetailViewWrap          bool
+	ShortKeybindingOverview string
+}
+
+var validate = validator.New()
+
+func (m *M) Update(p Params) {
+
+	if err := validate.Struct(p); err != nil {
+		panic(err)
+	}
+
+	if p.ReportFetchError != nil {
+		m.bottomBarStatus = fmt.Sprintf("[red::]status fetch: %s", p.ReportFetchError)
+	} else {
+		m.bottomBarStatus = p.ShortKeybindingOverview
+		for jobname, st := range p.Report {
+			// TODO handle job renames & deletions
+			j, ok := m.jobs[jobname]
+			if !ok {
+				j = &Job{
+					name:         jobname,
+					byteProgress: &bytesProgressHistory{},
+				}
+				m.jobs[jobname] = j
+				m.jobsList = append(m.jobsList, j)
+			}
+			j.lastStatus = st
+		}
+	}
+
+	// filter out internal jobs
+	var jobsList []*Job
+	for _, j := range m.jobsList {
+		if daemon.IsInternalJobName(j.name) {
+			continue
+		}
+		jobsList = append(jobsList, j)
+	}
+	m.jobsList = jobsList
+
+	// determinism!
+	sort.Slice(m.jobsList, func(i, j int) bool {
+		return strings.Compare(m.jobsList[i].name, m.jobsList[j].name) < 0
+	})
+
+	// try to not lose the selected job
+	m.selectedJob = nil
+	for _, j := range m.jobsList {
+		j.updateFullDescription(p)
+		if j == p.SelectedJob {
+			m.selectedJob = j
+		}
+	}
+
+	m.dateString = time.Now().Format(time.RFC3339)
+
+}
+
+func (m *M) BottomBarStatus() string { return m.bottomBarStatus }
+
+func (m *M) Jobs() []*Job { return m.jobsList }
+
+// may be nil
+func (m *M) SelectedJob() *Job { return m.selectedJob }
+
+func (m *M) DateString() string { return m.dateString }
+
+func (j *Job) updateFullDescription(p Params) {
+	width := p.DetailViewWidth
+	if !p.DetailViewWrap {
+		width = 10000000 // FIXME
+	}
+	b := stringbuilder.New(stringbuilder.Config{
+		IndentMultiplier: 3,
+		Width:            width,
+	})
+	drawJob(b, j.name, j.lastStatus, j.byteProgress, p.FSFilter)
+	j.fulldescription = b.String()
+}
+
+func (j *Job) JobTreeTitle() string {
+	return j.name
+}
+
+func (j *Job) FullDescription() string {
+	return j.fulldescription
+}
+
+func (j *Job) Name() string {
+	return j.name
+}
+
+func drawJob(t *stringbuilder.B, name string, v *job.Status, history *bytesProgressHistory, fsfilter FilterFunc) {
+
+	t.Printf("Job: %s\n", name)
+	t.Printf("Type: %s\n\n", v.Type)
+
+	if v.Type == job.TypePush || v.Type == job.TypePull {
+		activeStatus, ok := v.JobSpecific.(*job.ActiveSideStatus)
+		if !ok || activeStatus == nil {
+			t.Printf("ActiveSideStatus is null")
+			t.Newline()
+			return
+		}
+
+		t.Printf("Replication:")
+		t.AddIndentAndNewline(1)
+		renderReplicationReport(t, activeStatus.Replication, history, fsfilter)
+		t.AddIndentAndNewline(-1)
+
+		t.Printf("Pruning Sender:")
+		t.AddIndentAndNewline(1)
+		renderPrunerReport(t, activeStatus.PruningSender, fsfilter)
+		t.AddIndentAndNewline(-1)
+
+		t.Printf("Pruning Receiver:")
+		t.AddIndentAndNewline(1)
+		renderPrunerReport(t, activeStatus.PruningReceiver, fsfilter)
+		t.AddIndentAndNewline(-1)
+
+		if v.Type == job.TypePush {
+			t.Printf("Snapshotting:")
+			t.AddIndentAndNewline(1)
+			renderSnapperReport(t, activeStatus.Snapshotting, fsfilter)
+			t.AddIndentAndNewline(-1)
+		}
+
+	} else if v.Type == job.TypeSnap {
+		snapStatus, ok := v.JobSpecific.(*job.SnapJobStatus)
+		if !ok || snapStatus == nil {
+			t.Printf("SnapJobStatus is null")
+			t.Newline()
+			return
+		}
+		t.Printf("Pruning snapshots:")
+		t.AddIndentAndNewline(1)
+		renderPrunerReport(t, snapStatus.Pruning, fsfilter)
+		t.AddIndentAndNewline(-1)
+		t.Printf("Snapshotting:")
+		t.AddIndentAndNewline(1)
+		renderSnapperReport(t, snapStatus.Snapshotting, fsfilter)
+		t.AddIndentAndNewline(-1)
+	} else if v.Type == job.TypeSource {
+
+		st := v.JobSpecific.(*job.PassiveStatus)
+		t.Printf("Snapshotting:\n")
+		t.AddIndent(1)
+		renderSnapperReport(t, st.Snapper, fsfilter)
+		t.AddIndentAndNewline(-1)
+
+	} else {
+		t.Printf("No status representation for job type '%s', dumping as YAML", v.Type)
+		t.Newline()
+		asYaml, err := yaml.Marshal(v.JobSpecific)
+		if err != nil {
+			t.Printf("Error marshaling status to YAML: %s", err)
+			t.Newline()
+			return
+		}
+		t.Write(string(asYaml))
+		t.Newline()
+	}
+}
+
+func printFilesystemStatus(t *stringbuilder.B, rep *report.FilesystemReport, maxFS int) {
+
+	expected, replicated, containsInvalidSizeEstimates := rep.BytesSum()
+	sizeEstimationImpreciseNotice := ""
+	if containsInvalidSizeEstimates {
+		sizeEstimationImpreciseNotice = " (some steps lack size estimation)"
+	}
+	if rep.CurrentStep < len(rep.Steps) && rep.Steps[rep.CurrentStep].Info.BytesExpected == 0 {
+		sizeEstimationImpreciseNotice = " (step lacks size estimation)"
+	}
+
+	userVisisbleCurrentStep, userVisibleTotalSteps := rep.CurrentStep, len(rep.Steps)
+	// `.CurrentStep` is == len(rep.Steps) if all steps are done.
+	// Until then, it's an index into .Steps that starts at 0.
+	// For the user, we want it to start at 1.
+	if rep.CurrentStep >= len(rep.Steps) {
+		// rep.CurrentStep is what we want to show.
+		// We check for >= and not == for robustness.
+	} else {
+		// We're not done yet, so, make step count start at 1
+		// (The `.State` is included in the output, indicating we're not done yet)
+		userVisisbleCurrentStep = rep.CurrentStep + 1
+	}
+	status := fmt.Sprintf("%s (step %d/%d, %s/%s)%s",
+		strings.ToUpper(string(rep.State)),
+		userVisisbleCurrentStep, userVisibleTotalSteps,
+		ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected),
+		sizeEstimationImpreciseNotice,
+	)
+
+	activeIndicator := " "
+	if rep.BlockedOn == report.FsBlockedOnNothing &&
+		(rep.State == report.FilesystemPlanning || rep.State == report.FilesystemStepping) {
+		activeIndicator = "*"
+	}
+	t.AddIndent(1)
+	t.Printf("%s %s %s ",
+		activeIndicator,
+		stringbuilder.RightPad(rep.Info.Name, maxFS, " "),
+		status)
+
+	next := ""
+	if err := rep.Error(); err != nil {
+		next = err.Err
+	} else if rep.State != report.FilesystemDone {
+		if nextStep := rep.NextStep(); nextStep != nil {
+			if nextStep.IsIncremental() {
+				next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
+			} else {
+				next = fmt.Sprintf("next: full send %s", nextStep.Info.To)
+			}
+			attribs := []string{}
+
+			if nextStep.Info.Resumed {
+				attribs = append(attribs, "resumed")
+			}
+
+			if len(attribs) > 0 {
+				next += fmt.Sprintf(" (%s)", strings.Join(attribs, ", "))
+			}
+		} else {
+			next = "" // individual FSes may still be in planning state
+		}
+
+	}
+	t.Printf("%s", next)
+
+	t.AddIndent(-1)
+	t.Newline()
+}
+
+func renderReplicationReport(t *stringbuilder.B, rep *report.Report, history *bytesProgressHistory, fsfilter FilterFunc) {
+	if rep == nil {
+		t.Printf("...\n")
+		return
+	}
+
+	if rep.WaitReconnectError != nil {
+		t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: %s", rep.WaitReconnectError)
+		t.Newline()
+	}
+	if !rep.WaitReconnectSince.IsZero() {
+		delta := time.Until(rep.WaitReconnectUntil).Round(time.Second)
+		if rep.WaitReconnectUntil.IsZero() || delta > 0 {
+			var until string
+			if rep.WaitReconnectUntil.IsZero() {
+				until = "waiting indefinitely"
+			} else {
+				until = fmt.Sprintf("hard fail in %s @ %s", delta, rep.WaitReconnectUntil)
+			}
+			t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnecting with exponential backoff (since %s) (%s)",
+				rep.WaitReconnectSince, until)
+		} else {
+			t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnects reached hard-fail timeout @ %s", rep.WaitReconnectUntil)
+		}
+		t.Newline()
+	}
+
+	// TODO visualize more than the latest attempt by folding all attempts into one
+	if len(rep.Attempts) == 0 {
+		t.Printf("no attempts made yet")
+		return
+	} else {
+		t.Printf("Attempt #%d", len(rep.Attempts))
+		if len(rep.Attempts) > 1 {
+			t.Printf(". Previous attempts failed with the following statuses:")
+			t.AddIndentAndNewline(1)
+			for i, a := range rep.Attempts[:len(rep.Attempts)-1] {
+				t.PrintfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)\n", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt))
+			}
+			t.AddIndentAndNewline(-1)
+		} else {
+			t.Newline()
+		}
+	}
+
+	latest := rep.Attempts[len(rep.Attempts)-1]
+	sort.Slice(latest.Filesystems, func(i, j int) bool {
+		return latest.Filesystems[i].Info.Name < latest.Filesystems[j].Info.Name
+	})
+
+	// apply filter
+	filtered := make([]*report.FilesystemReport, 0, len(latest.Filesystems))
+	for _, fs := range latest.Filesystems {
+		if !fsfilter(fs.Info.Name) {
+			continue
+		}
+		filtered = append(filtered, fs)
+	}
+	latest.Filesystems = filtered
+
+	t.Printf("Status: %s", latest.State)
+	t.Newline()
+	if !latest.FinishAt.IsZero() {
+		t.Printf("Last Run: %s (lasted %s)\n", latest.FinishAt.Round(time.Second), latest.FinishAt.Sub(latest.StartAt).Round(time.Second))
+	} else {
+		t.Printf("Started: %s (lasting %s)\n", latest.StartAt.Round(time.Second), time.Since(latest.StartAt).Round(time.Second))
+	}
+
+	if latest.State == report.AttemptPlanningError {
+		t.Printf("Problem: ")
+		t.PrintfDrawIndentedAndWrappedIfMultiline("%s", latest.PlanError)
+		t.Newline()
+	} else if latest.State == report.AttemptFanOutError {
+		t.Printf("Problem: one or more of the filesystems encountered errors")
+		t.Newline()
+	}
+
+	if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
+		// Draw global progress bar
+		// Progress: [---------------]
+		expected, replicated, containsInvalidSizeEstimates := latest.BytesSum()
+		rate, changeCount := history.Update(replicated)
+		eta := time.Duration(0)
+		if rate > 0 {
+			eta = time.Duration((float64(expected)-float64(replicated))/float64(rate)) * time.Second
+		}
+
+		t.Write("Progress: ")
+		t.DrawBar(50, replicated, expected, changeCount)
+		t.Write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected), ByteCountBinary(rate)))
+		if eta != 0 {
+			t.Write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
+		}
+		t.Newline()
+		if containsInvalidSizeEstimates {
+			t.Write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
+			t.Newline()
+		}
+
+		if len(latest.Filesystems) == 0 {
+			t.Write("NOTE: no filesystems were considered for replication!")
+			t.Newline()
+		}
+
+		var maxFSLen int
+		for _, fs := range latest.Filesystems {
+			if len(fs.Info.Name) > maxFSLen {
+				maxFSLen = len(fs.Info.Name)
+			}
+		}
+		for _, fs := range latest.Filesystems {
+			printFilesystemStatus(t, fs, maxFSLen)
+		}
+
+	}
+}
+
+func humanizeDuration(duration time.Duration) string {
+	days := int64(duration.Hours() / 24)
+	hours := int64(math.Mod(duration.Hours(), 24))
+	minutes := int64(math.Mod(duration.Minutes(), 60))
+	seconds := int64(math.Mod(duration.Seconds(), 60))
+
+	var parts []string
+
+	force := false
+	chunks := []int64{days, hours, minutes, seconds}
+	for i, chunk := range chunks {
+		if force || chunk > 0 {
+			padding := 0
+			if force {
+				padding = 2
+			}
+			parts = append(parts, fmt.Sprintf("%*d%c", padding, chunk, "dhms"[i]))
+			force = true
+		}
+	}
+
+	return strings.Join(parts, " ")
+}
+
+func renderPrunerReport(t *stringbuilder.B, r *pruner.Report, fsfilter FilterFunc) {
+	if r == nil {
+		t.Printf("...\n")
+		return
+	}
+
+	state, err := pruner.StateString(r.State)
+	if err != nil {
+		t.Printf("Status: %q (parse error: %q)\n", r.State, err)
+		return
+	}
+
+	t.Printf("Status: %s", state)
+	t.Newline()
+
+	if r.Error != "" {
+		t.Printf("Error: %s\n", r.Error)
+	}
+
+	type commonFS struct {
+		*pruner.FSReport
+		completed bool
+	}
+	all := make([]commonFS, 0, len(r.Pending)+len(r.Completed))
+	for i := range r.Pending {
+		all = append(all, commonFS{&r.Pending[i], false})
+	}
+	for i := range r.Completed {
+		all = append(all, commonFS{&r.Completed[i], true})
+	}
+
+	// filter all
+	filtered := make([]commonFS, 0, len(all))
+	for _, fs := range all {
+		if fsfilter(fs.FSReport.Filesystem) {
+			filtered = append(filtered, fs)
+		}
+	}
+	all = filtered
+
+	switch state {
+	case pruner.Plan:
+		fallthrough
+	case pruner.PlanErr:
+		return
+	}
+
+	if len(all) == 0 {
+		t.Printf("nothing to do\n")
+		return
+	}
+
+	var totalDestroyCount, completedDestroyCount int
+	var maxFSname int
+	for _, fs := range all {
+		totalDestroyCount += len(fs.DestroyList)
+		if fs.completed {
+			completedDestroyCount += len(fs.DestroyList)
+		}
+		if maxFSname < len(fs.Filesystem) {
+			maxFSname = len(fs.Filesystem)
+		}
+	}
+
+	// global progress bar
+	progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
+	t.Write("Progress: ")
+	t.Write("[")
+	t.Write(stringbuilder.Times("=", progress))
+	t.Write(">")
+	t.Write(stringbuilder.Times("-", 80-progress))
+	t.Write("]")
+	t.Printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
+	t.Newline()
+
+	sort.SliceStable(all, func(i, j int) bool {
+		return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
+	})
+
+	// Draw a table-like representation of 'all'
+	for _, fs := range all {
+		t.Write(stringbuilder.RightPad(fs.Filesystem, maxFSname, " "))
+		t.Write(" ")
+		if !fs.SkipReason.NotSkipped() {
+			t.Printf("skipped: %s\n", fs.SkipReason)
+			continue
+		}
+		if fs.LastError != "" {
+			if strings.ContainsAny(fs.LastError, "\r\n") {
+				t.Printf("ERROR:")
+				t.PrintfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError)
+			} else {
+				t.PrintfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError)
+			}
+			t.Newline()
+			continue
+		}
+
+		pruneRuleActionStr := fmt.Sprintf("(destroy %d of %d snapshots)",
+			len(fs.DestroyList), len(fs.SnapshotList))
+
+		if fs.completed {
+			t.Printf("Completed  %s\n", pruneRuleActionStr)
+			continue
+		}
+
+		t.Write("Pending    ") // whitespace is padding 10
+		if len(fs.DestroyList) == 1 {
+			t.Write(fs.DestroyList[0].Name)
+		} else {
+			t.Write(pruneRuleActionStr)
+		}
+		t.Newline()
+	}
+
+}
+
+func renderSnapperReport(t *stringbuilder.B, r *snapper.Report, fsfilter FilterFunc) {
+	if r == nil {
+		t.Printf("<no snapshotting report available>\n")
+		return
+	}
+	t.Printf("Type: %s\n", r.Type)
+	if r.Periodic != nil {
+		renderSnapperReportPeriodic(t, r.Periodic, fsfilter)
+	} else if r.Cron != nil {
+		renderSnapperReportCron(t, r.Cron, fsfilter)
+	} else {
+		t.Printf("<no details available>")
+	}
+}
+
+func renderSnapperReportPeriodic(t *stringbuilder.B, r *snapper.PeriodicReport, fsfilter FilterFunc) {
+	t.Printf("Status: %s", r.State)
+	t.Newline()
+
+	if r.Error != "" {
+		t.Printf("Error: %s\n", r.Error)
+	}
+	if !r.SleepUntil.IsZero() {
+		t.Printf("Sleep until: %s\n", r.SleepUntil)
+	}
+
+	renderSnapperPlanReportFilesystem(t, r.Progress, fsfilter)
+}
+
+func renderSnapperReportCron(t *stringbuilder.B, r *snapper.CronReport, fsfilter FilterFunc) {
+	t.Printf("State: %s\n", r.State)
+
+	now := time.Now()
+	if r.WakeupTime.After(now) {
+		t.Printf("Sleep until: %s (%s remaining)\n", r.WakeupTime, r.WakeupTime.Sub(now).Round(time.Second))
+	} else {
+		t.Printf("Started: %s (lasting %s)\n", r.WakeupTime, now.Sub(r.WakeupTime).Round(time.Second))
+	}
+
+	renderSnapperPlanReportFilesystem(t, r.Progress, fsfilter)
+}
+
+func renderSnapperPlanReportFilesystem(t *stringbuilder.B, fss []*snapper.ReportFilesystem, fsfilter FilterFunc) {
+	sort.Slice(fss, func(i, j int) bool {
+		return strings.Compare(fss[i].Path, fss[j].Path) == -1
+	})
+
+	dur := func(d time.Duration) string {
+		return d.Round(100 * time.Millisecond).String()
+	}
+
+	type row struct {
+		path, state, duration, remainder, hookReport string
+	}
+	var widths struct {
+		path, state, duration int
+	}
+	rows := make([]*row, 0, len(fss))
+	for _, fs := range fss {
+		if !fsfilter(fs.Path) {
+			continue
+		}
+		r := &row{
+			path:  fs.Path,
+			state: fs.State.String(),
+		}
+		if fs.HooksHadError {
+			r.hookReport = fs.Hooks // FIXME render here, not in daemon
+		}
+		switch fs.State {
+		case snapper.SnapPending:
+			r.duration = "..."
+			r.remainder = ""
+		case snapper.SnapStarted:
+			r.duration = dur(time.Since(fs.StartAt))
+			r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
+		case snapper.SnapDone:
+			fallthrough
+		case snapper.SnapError:
+			r.duration = dur(fs.DoneAt.Sub(fs.StartAt))
+			r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
+		}
+		rows = append(rows, r)
+		if len(r.path) > widths.path {
+			widths.path = len(r.path)
+		}
+		if len(r.state) > widths.state {
+			widths.state = len(r.state)
+		}
+		if len(r.duration) > widths.duration {
+			widths.duration = len(r.duration)
+		}
+	}
+
+	for _, r := range rows {
+		path := stringbuilder.RightPad(r.path, widths.path, " ")
+		state := stringbuilder.RightPad(r.state, widths.state, " ")
+		duration := stringbuilder.RightPad(r.duration, widths.duration, " ")
+		t.Printf("%s %s %s", path, state, duration)
+		t.PrintfDrawIndentedAndWrappedIfMultiline(" %s", r.remainder)
+		if r.hookReport != "" {
+			t.AddIndent(1)
+			t.Newline()
+			t.Printf("%s", r.hookReport)
+			t.AddIndent(-1)
+		}
+		t.Newline()
+	}
+}

client/status/viewmodel/stringbuilder/stringbuilder.go

@@ -0,0 +1,119 @@
+package stringbuilder
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/go-playground/validator/v10"
+)
+
+type B struct {
+	// const
+	indentMultiplier int
+
+	// mut
+	sb     *strings.Builder
+	indent int
+	width  int
+	x, y   int
+}
+
+type Config struct {
+	IndentMultiplier int `validate:"gte=1"`
+	Width            int `validate:"gte=1"`
+}
+
+var validate = validator.New()
+
+func New(config Config) *B {
+
+	if err := validate.Struct(config); err != nil {
+		panic(err)
+	}
+
+	return &B{sb: &strings.Builder{}, width: config.Width, indentMultiplier: config.IndentMultiplier}
+}
+
+func (b *B) String() string { return b.sb.String() }
+
+func (w *B) Newline() {
+	w.Write("\n")
+}
+
+func (w *B) PrintfDrawIndentedAndWrappedIfMultiline(format string, args ...interface{}) {
+	whole := fmt.Sprintf(format, args...)
+	if strings.ContainsAny(whole, "\n\r") {
+		w.AddIndent(1)
+		defer w.AddIndent(-1)
+	}
+	w.Write(whole)
+}
+
+func (w *B) Printf(format string, args ...interface{}) {
+	whole := fmt.Sprintf(format, args...)
+	w.Write(whole)
+}
+
+func (t *B) AddIndent(delta int) {
+	t.indent += delta * t.indentMultiplier
+}
+
+func (t *B) AddIndentAndNewline(delta int) {
+	t.indent += delta * t.indentMultiplier
+	t.Write("\n")
+}
+
+func (w *B) Write(s string) {
+	for _, c := range s {
+		if c == '\n' {
+			fmt.Fprint(w.sb, "\n")
+			w.x = 0
+			fmt.Fprint(w.sb, Times(" ", w.indent-w.x))
+			w.x = w.indent
+			w.y++
+			continue
+		}
+		if w.x >= w.width {
+			fmt.Fprint(w.sb, "\n")
+			w.x = 0
+			fmt.Fprint(w.sb, Times(" ", w.indent-w.x))
+			w.x = w.indent
+		}
+		fmt.Fprintf(w.sb, "%c", c)
+		w.x++
+	}
+}
+
+func Times(str string, n int) (out string) {
+	for i := 0; i < n; i++ {
+		out += str
+	}
+	return
+}
+
+func RightPad(str string, length int, pad string) string {
+	if len(str) > length {
+		return str[:length]
+	}
+	return str + strings.Repeat(pad, length-len(str))
+}
+
+// changeCount = 0 indicates stall / no progress
+func (w *B) DrawBar(length int, bytes, totalBytes uint64, changeCount int) {
+	const arrowPositions = `>\|/`
+	var completedLength int
+	if totalBytes > 0 {
+		completedLength = int(uint64(length) * bytes / totalBytes)
+		if completedLength > length {
+			completedLength = length
+		}
+	} else if totalBytes == bytes {
+		completedLength = length
+	}
+
+	w.Write("[")
+	w.Write(Times("=", completedLength))
+	w.Write(string(arrowPositions[changeCount%len(arrowPositions)]))
+	w.Write(Times("-", length-completedLength))
+	w.Write("]")
+}

client/stdinserver.go

@@ -0,0 +1,51 @@
+package client
+
+import (
+	"os"
+
+	"github.com/problame/go-netssh"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+
+	"context"
+	"errors"
+	"log"
+	"path"
+)
+
+var StdinserverCmd = &cli.Subcommand{
+	Use:   "stdinserver CLIENT_IDENTITY",
+	Short: "stdinserver transport mode (started from authorized_keys file as forced command)",
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		return runStdinserver(subcommand.Config(), args)
+	},
+}
+
+func runStdinserver(config *config.Config, args []string) error {
+
+	// NOTE: the netssh proxying protocol requires exiting with non-zero status if anything goes wrong
+	defer os.Exit(1)
+
+	log := log.New(os.Stderr, "", log.LUTC|log.Ldate|log.Ltime)
+
+	if len(args) != 1 || args[0] == "" {
+		err := errors.New("must specify client_identity as positional argument")
+		return err
+	}
+
+	identity := args[0]
+	unixaddr := path.Join(config.Global.Serve.StdinServer.SockDir, identity)
+
+	log.Printf("proxying client identity '%s' to zrepl daemon '%s'", identity, unixaddr)
+
+	ctx := netssh.ContextWithLog(context.TODO(), log)
+
+	err := netssh.Proxy(ctx, unixaddr)
+	if err == nil {
+		log.Print("proxying finished successfully, exiting with status 0")
+		os.Exit(0)
+	}
+	log.Printf("error proxying: %s", err)
+	return nil
+}

client/testcmd.go

@@ -0,0 +1,220 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var TestCmd = &cli.Subcommand{
+	Use: "test",
+	SetupSubcommands: func() []*cli.Subcommand {
+		return []*cli.Subcommand{testFilter, testPlaceholder, testDecodeResumeToken}
+	},
+}
+
+var testFilterArgs struct {
+	job   string
+	all   bool
+	input string
+}
+
+var testFilter = &cli.Subcommand{
+	Use:   "filesystems --job JOB [--all | --input INPUT]",
+	Short: "test filesystems filter specified in push or source job",
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&testFilterArgs.job, "job", "", "the name of the push or source job")
+		f.StringVar(&testFilterArgs.input, "input", "", "a filesystem name to test against the job's filters")
+		f.BoolVar(&testFilterArgs.all, "all", false, "test all local filesystems")
+	},
+	Run: runTestFilterCmd,
+}
+
+func runTestFilterCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+
+	if testFilterArgs.job == "" {
+		return fmt.Errorf("must specify --job flag")
+	}
+	if !(testFilterArgs.all != (testFilterArgs.input != "")) { // xor
+		return fmt.Errorf("must set one: --all or --input")
+	}
+
+	conf := subcommand.Config()
+
+	var confFilter config.FilesystemsFilter
+	job, err := conf.Job(testFilterArgs.job)
+	if err != nil {
+		return err
+	}
+	switch j := job.Ret.(type) {
+	case *config.SourceJob:
+		confFilter = j.Filesystems
+	case *config.PushJob:
+		confFilter = j.Filesystems
+	case *config.SnapJob:
+		confFilter = j.Filesystems
+	default:
+		return fmt.Errorf("job type %T does not have filesystems filter", j)
+	}
+
+	f, err := filters.DatasetMapFilterFromConfig(confFilter)
+	if err != nil {
+		return fmt.Errorf("filter invalid: %s", err)
+	}
+
+	var fsnames []string
+	if testFilterArgs.input != "" {
+		fsnames = []string{testFilterArgs.input}
+	} else {
+		out, err := zfs.ZFSList(ctx, []string{"name"})
+		if err != nil {
+			return fmt.Errorf("could not list ZFS filesystems: %s", err)
+		}
+		for _, row := range out {
+
+			fsnames = append(fsnames, row[0])
+		}
+	}
+
+	fspaths := make([]*zfs.DatasetPath, len(fsnames))
+	for i, fsname := range fsnames {
+		path, err := zfs.NewDatasetPath(fsname)
+		if err != nil {
+			return err
+		}
+		fspaths[i] = path
+	}
+
+	hadFilterErr := false
+	for _, in := range fspaths {
+		var res string
+		var errStr string
+		pass, err := f.Filter(in)
+		if err != nil {
+			res = "ERROR"
+			errStr = err.Error()
+			hadFilterErr = true
+		} else if pass {
+			res = "ACCEPT"
+		} else {
+			res = "REJECT"
+		}
+		fmt.Printf("%s\t%s\t%s\n", res, in.ToString(), errStr)
+	}
+
+	if hadFilterErr {
+		return fmt.Errorf("filter errors occurred")
+	}
+	return nil
+}
+
+var testPlaceholderArgs struct {
+	ds  string
+	all bool
+}
+
+var testPlaceholder = &cli.Subcommand{
+	Use:   "placeholder [--all | --dataset DATASET]",
+	Short: fmt.Sprintf("list received placeholder filesystems (zfs property %q)", zfs.PlaceholderPropertyName),
+	Example: `
+	placeholder --all
+	placeholder --dataset path/to/sink/clientident/fs`,
+	NoRequireConfig: true,
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&testPlaceholderArgs.ds, "dataset", "", "dataset path (not required to exist)")
+		f.BoolVar(&testPlaceholderArgs.all, "all", false, "list tab-separated placeholder status of all filesystems")
+	},
+	Run: runTestPlaceholder,
+}
+
+func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+
+	var checkDPs []*zfs.DatasetPath
+	var datasetWasExplicitArgument bool
+
+	// all actions first
+	if testPlaceholderArgs.all {
+		datasetWasExplicitArgument = false
+		out, err := zfs.ZFSList(ctx, []string{"name"})
+		if err != nil {
+			return errors.Wrap(err, "could not list ZFS filesystems")
+		}
+		for _, row := range out {
+			dp, err := zfs.NewDatasetPath(row[0])
+			if err != nil {
+				panic(err)
+			}
+			checkDPs = append(checkDPs, dp)
+		}
+	} else {
+		datasetWasExplicitArgument = true
+		dp, err := zfs.NewDatasetPath(testPlaceholderArgs.ds)
+		if err != nil {
+			return err
+		}
+		if dp.Empty() {
+			return fmt.Errorf("must specify --dataset DATASET or --all")
+		}
+		checkDPs = append(checkDPs, dp)
+	}
+
+	fmt.Printf("IS_PLACEHOLDER\tDATASET\tzrepl:placeholder\n")
+	for _, dp := range checkDPs {
+		ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, dp)
+		if err != nil {
+			return errors.Wrap(err, "cannot get placeholder state")
+		}
+		if !ph.FSExists {
+			if datasetWasExplicitArgument {
+				return errors.Errorf("filesystem %q does not exist", ph.FS)
+			} else {
+				// got deleted between ZFSList and ZFSGetFilesystemPlaceholderState
+				continue
+			}
+		}
+		is := "yes"
+		if !ph.IsPlaceholder {
+			is = "no"
+		}
+		fmt.Printf("%s\t%s\t%s\n", is, dp.ToString(), ph.RawLocalPropertyValue)
+	}
+	return nil
+}
+
+var testDecodeResumeTokenArgs struct {
+	token string
+}
+
+var testDecodeResumeToken = &cli.Subcommand{
+	Use:   "decoderesumetoken --token TOKEN",
+	Short: "decode resume token",
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&testDecodeResumeTokenArgs.token, "token", "", "the resume token obtained from the receive_resume_token property")
+	},
+	Run: runTestDecodeResumeTokenCmd,
+}
+
+func runTestDecodeResumeTokenCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+	if testDecodeResumeTokenArgs.token == "" {
+		return fmt.Errorf("token argument must be specified")
+	}
+	token, err := zfs.ParseResumeToken(ctx, testDecodeResumeTokenArgs.token)
+	if err != nil {
+		return err
+	}
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", " ")
+	if err := enc.Encode(&token); err != nil {
+		panic(err)
+	}
+	return nil
+}

client/version.go

@@ -0,0 +1,75 @@
+package client
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+	"github.com/zrepl/zrepl/version"
+)
+
+var versionArgs struct {
+	Show      string
+	Config    *config.Config
+	ConfigErr error
+}
+
+var VersionCmd = &cli.Subcommand{
+	Use:             "version",
+	Short:           "print version of zrepl binary and running daemon",
+	NoRequireConfig: true,
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)")
+	},
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		versionArgs.Config = subcommand.Config()
+		versionArgs.ConfigErr = subcommand.ConfigParsingError()
+		return runVersionCmd()
+	},
+}
+
+func runVersionCmd() error {
+	args := versionArgs
+
+	if args.Show != "daemon" && args.Show != "client" && args.Show != "" {
+		return fmt.Errorf("show flag must be 'client' or 'server' or be left empty")
+	}
+
+	var clientVersion, daemonVersion *version.ZreplVersionInformation
+	if args.Show == "client" || args.Show == "" {
+		clientVersion = version.NewZreplVersionInformation()
+		fmt.Printf("client: %s\n", clientVersion.String())
+	}
+	if args.Show == "daemon" || args.Show == "" {
+
+		if args.ConfigErr != nil {
+			return fmt.Errorf("config parsing error: %s", args.ConfigErr)
+		}
+
+		httpc, err := controlHttpClient(args.Config.Global.Control.SockPath)
+		if err != nil {
+			return fmt.Errorf("server: error: %s\n", err)
+		}
+
+		var info version.ZreplVersionInformation
+		err = jsonRequestResponse(httpc, daemon.ControlJobEndpointVersion, "", &info)
+		if err != nil {
+			return fmt.Errorf("server: error: %s\n", err)
+		}
+		daemonVersion = &info
+		fmt.Printf("server: %s\n", daemonVersion.String())
+	}
+
+	if args.Show == "" {
+		if clientVersion.Version != daemonVersion.Version {
+			fmt.Fprintf(os.Stderr, "WARNING: client version != daemon version, restart zrepl daemon\n")
+		}
+	}
+
+	return nil
+}

client/zfsabstractions.go

@@ -0,0 +1,147 @@
+package client
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var (
+	ZFSAbstractionsCmd = &cli.Subcommand{
+		Use:   "zfs-abstraction",
+		Short: "manage abstractions that zrepl builds on top of ZFS",
+		SetupSubcommands: func() []*cli.Subcommand {
+			return []*cli.Subcommand{
+				zabsCmdList,
+				zabsCmdReleaseAll,
+				zabsCmdReleaseStale,
+				zabsCmdCreate,
+			}
+		},
+	}
+)
+
+// a common set of CLI flags that map to the fields of an
+// endpoint.ListZFSHoldsAndBookmarksQuery
+type zabsFilterFlags struct {
+	Filesystems FilesystemsFilterFlag
+	Job         JobIDFlag
+	Types       AbstractionTypesFlag
+	Concurrency int64
+}
+
+// produce a query from the CLI flags
+func (f zabsFilterFlags) Query() (endpoint.ListZFSHoldsAndBookmarksQuery, error) {
+	q := endpoint.ListZFSHoldsAndBookmarksQuery{
+		FS:          f.Filesystems.FlagValue(),
+		What:        f.Types.FlagValue(),
+		JobID:       f.Job.FlagValue(),
+		Concurrency: f.Concurrency,
+	}
+	return q, q.Validate()
+}
+
+func (f *zabsFilterFlags) registerZabsFilterFlags(s *pflag.FlagSet, verb string) {
+	// Note: the default value is defined in the .FlagValue methods
+	s.Var(&f.Filesystems, "fs", fmt.Sprintf("only %s holds on the specified filesystem [default: all filesystems] [comma-separated list of <dataset-pattern>:<ok|!> pairs]", verb))
+	s.Var(&f.Job, "job", fmt.Sprintf("only %s holds created by the specified job [default: any job]", verb))
+
+	variants := make([]string, 0, len(endpoint.AbstractionTypesAll))
+	for v := range endpoint.AbstractionTypesAll {
+		variants = append(variants, string(v))
+	}
+	variants = sort.StringSlice(variants)
+	variantsJoined := strings.Join(variants, "|")
+	s.Var(&f.Types, "type", fmt.Sprintf("only %s holds of the specified type [default: all] [comma-separated list of %s]", verb, variantsJoined))
+
+	s.Int64VarP(&f.Concurrency, "concurrency", "p", 1, "number of concurrently queried filesystems")
+}
+
+type JobIDFlag struct{ J *endpoint.JobID }
+
+func (f *JobIDFlag) Set(s string) error {
+	if len(s) == 0 {
+		*f = JobIDFlag{J: nil}
+		return nil
+	}
+
+	jobID, err := endpoint.MakeJobID(s)
+	if err != nil {
+		return err
+	}
+	*f = JobIDFlag{J: &jobID}
+	return nil
+}
+func (f JobIDFlag) Type() string               { return "job-ID" }
+func (f JobIDFlag) String() string             { return fmt.Sprint(f.J) }
+func (f JobIDFlag) FlagValue() *endpoint.JobID { return f.J }
+
+type AbstractionTypesFlag map[endpoint.AbstractionType]bool
+
+func (f *AbstractionTypesFlag) Set(s string) error {
+	ats, err := endpoint.AbstractionTypeSetFromStrings(strings.Split(s, ","))
+	if err != nil {
+		return err
+	}
+	*f = AbstractionTypesFlag(ats)
+	return nil
+}
+func (f AbstractionTypesFlag) Type() string { return "abstraction-type" }
+func (f AbstractionTypesFlag) String() string {
+	return endpoint.AbstractionTypeSet(f).String()
+}
+func (f AbstractionTypesFlag) FlagValue() map[endpoint.AbstractionType]bool {
+	if len(f) > 0 {
+		return f
+	}
+	return endpoint.AbstractionTypesAll
+}
+
+type FilesystemsFilterFlag struct {
+	F endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter
+}
+
+func (flag *FilesystemsFilterFlag) Set(s string) error {
+	mappings := strings.Split(s, ",")
+	if len(mappings) == 1 && !strings.Contains(mappings[0], ":") {
+		flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &mappings[0],
+		}
+		return nil
+	}
+
+	f := filters.NewDatasetMapFilter(len(mappings), true)
+	for _, m := range mappings {
+		thisMappingErr := fmt.Errorf("expecting comma-separated list of <dataset-pattern>:<ok|!> pairs, got %q", m)
+		lhsrhs := strings.SplitN(m, ":", 2)
+		if len(lhsrhs) != 2 {
+			return thisMappingErr
+		}
+		err := f.Add(lhsrhs[0], lhsrhs[1])
+		if err != nil {
+			return fmt.Errorf("%s: %s", thisMappingErr, err)
+		}
+	}
+	flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+		Filter: f,
+	}
+	return nil
+}
+func (flag FilesystemsFilterFlag) Type() string { return "filesystem filter spec" }
+func (flag FilesystemsFilterFlag) String() string {
+	return fmt.Sprintf("%v", flag.F)
+}
+func (flag FilesystemsFilterFlag) FlagValue() endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter {
+	var z FilesystemsFilterFlag
+	if flag == z {
+		return endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{Filter: zfs.NoFilter()}
+	}
+	return flag.F
+}

client/zfsabstractions_create.go

@@ -0,0 +1,14 @@
+package client
+
+import "github.com/zrepl/zrepl/cli"
+
+var zabsCmdCreate = &cli.Subcommand{
+	Use:             "create",
+	NoRequireConfig: true,
+	Short:           `create zrepl ZFS abstractions (mostly useful for debugging & development, users should not need to use this command)`,
+	SetupSubcommands: func() []*cli.Subcommand {
+		return []*cli.Subcommand{
+			zabsCmdCreateStepHold,
+		}
+	},
+}

client/zfsabstractions_create_step_hold.go

@@ -0,0 +1,58 @@
+package client
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var zabsCreateStepHoldFlags struct {
+	target string
+	jobid  JobIDFlag
+}
+
+var zabsCmdCreateStepHold = &cli.Subcommand{
+	Use:             "step",
+	Run:             doZabsCreateStep,
+	NoRequireConfig: true,
+	Short:           `create a step hold or bookmark`,
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVarP(&zabsCreateStepHoldFlags.target, "target", "t", "", "snapshot to be held / bookmark to be held")
+		f.VarP(&zabsCreateStepHoldFlags.jobid, "jobid", "j", "jobid for which the hold is installed")
+	},
+}
+
+func doZabsCreateStep(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	if len(args) > 0 {
+		return errors.New("subcommand takes no arguments")
+	}
+
+	f := &zabsCreateStepHoldFlags
+
+	fs, _, _, err := zfs.DecomposeVersionString(f.target)
+	if err != nil {
+		return errors.Wrapf(err, "%q invalid target", f.target)
+	}
+
+	if f.jobid.FlagValue() == nil {
+		return errors.Errorf("jobid must be set")
+	}
+
+	v, err := zfs.ZFSGetFilesystemVersion(ctx, f.target)
+	if err != nil {
+		return errors.Wrapf(err, "get info about target %q", f.target)
+	}
+
+	step, err := endpoint.HoldStep(ctx, fs, v, *f.jobid.FlagValue())
+	if err != nil {
+		return errors.Wrap(err, "create step hold")
+	}
+	fmt.Println(step.String())
+	return nil
+}

client/zfsabstractions_list.go

@@ -0,0 +1,100 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/util/chainlock"
+)
+
+var zabsListFlags struct {
+	Filter zabsFilterFlags
+	Json   bool
+}
+
+var zabsCmdList = &cli.Subcommand{
+	Use:             "list",
+	Short:           `list zrepl ZFS abstractions`,
+	Run:             doZabsList,
+	NoRequireConfig: true,
+	SetupFlags: func(f *pflag.FlagSet) {
+		zabsListFlags.Filter.registerZabsFilterFlags(f, "list")
+		f.BoolVar(&zabsListFlags.Json, "json", false, "emit JSON")
+	},
+}
+
+func doZabsList(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsListFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	abstractions, errors, drainDone, err := endpoint.ListAbstractionsStreamed(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+	defer drainDone()
+
+	var line chainlock.L
+	var wg sync.WaitGroup
+	defer wg.Wait()
+
+	// print results
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		enc := json.NewEncoder(os.Stdout)
+		for a := range abstractions {
+			func() {
+				defer line.Lock().Unlock()
+				if zabsListFlags.Json {
+					enc.SetIndent("", "  ")
+					if err := enc.Encode(a); err != nil {
+						panic(err)
+					}
+					fmt.Println()
+				} else {
+					fmt.Println(a)
+				}
+			}()
+		}
+	}()
+
+	// print errors to stderr
+	errorColor := color.New(color.FgRed)
+	var errorsSlice []endpoint.ListAbstractionsError
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for err := range errors {
+			func() {
+				defer line.Lock().Unlock()
+				errorsSlice = append(errorsSlice, err)
+				errorColor.Fprintf(os.Stderr, "%s\n", err)
+			}()
+		}
+	}()
+	wg.Wait()
+	if len(errorsSlice) > 0 {
+		errorColor.Add(color.Bold).Fprintf(os.Stderr, "there were errors in listing the abstractions")
+		return fmt.Errorf("")
+	} else {
+		return nil
+	}
+
+}

client/zfsabstractions_release.go

@@ -0,0 +1,143 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+)
+
+// shared between release-all and release-step
+var zabsReleaseFlags struct {
+	Filter zabsFilterFlags
+	Json   bool
+	DryRun bool
+}
+
+func registerZabsReleaseFlags(s *pflag.FlagSet) {
+	zabsReleaseFlags.Filter.registerZabsFilterFlags(s, "release")
+	s.BoolVar(&zabsReleaseFlags.Json, "json", false, "emit json instead of pretty-printed")
+	s.BoolVar(&zabsReleaseFlags.DryRun, "dry-run", false, "do a dry-run")
+}
+
+var zabsCmdReleaseAll = &cli.Subcommand{
+	Use:             "release-all",
+	Run:             doZabsReleaseAll,
+	NoRequireConfig: true,
+	Short:           `(DANGEROUS) release ALL zrepl ZFS abstractions (mostly useful for uninstalling zrepl completely or for "de-zrepl-ing" a filesystem)`,
+	SetupFlags:      registerZabsReleaseFlags,
+}
+
+var zabsCmdReleaseStale = &cli.Subcommand{
+	Use:             "release-stale",
+	Run:             doZabsReleaseStale,
+	NoRequireConfig: true,
+	Short:           `release stale zrepl ZFS abstractions (useful if zrepl has a bug and does not do it by itself)`,
+	SetupFlags:      registerZabsReleaseFlags,
+}
+
+func doZabsReleaseAll(ctx context.Context, sc *cli.Subcommand, args []string) error {
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsReleaseFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	abstractions, listErrors, err := endpoint.ListAbstractions(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+	if len(listErrors) > 0 {
+		color.New(color.FgRed).Fprintf(os.Stderr, "there were errors in listing the abstractions:\n%s\n", listErrors)
+		// proceed anyways with rest of abstractions
+	}
+
+	return doZabsRelease_Common(ctx, abstractions)
+}
+
+func doZabsReleaseStale(ctx context.Context, sc *cli.Subcommand, args []string) error {
+
+	var err error
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsReleaseFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	stalenessInfo, err := endpoint.ListStale(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+
+	return doZabsRelease_Common(ctx, stalenessInfo.Stale)
+}
+
+func doZabsRelease_Common(ctx context.Context, destroy []endpoint.Abstraction) error {
+
+	if zabsReleaseFlags.DryRun {
+		if zabsReleaseFlags.Json {
+			m, err := json.MarshalIndent(destroy, "", "  ")
+			if err != nil {
+				panic(err)
+			}
+			if _, err := os.Stdout.Write(m); err != nil {
+				panic(err)
+			}
+			fmt.Println()
+		} else {
+			for _, a := range destroy {
+				fmt.Printf("would destroy %s\n", a)
+			}
+		}
+		return nil
+	}
+
+	outcome := endpoint.BatchDestroy(ctx, destroy)
+	hadErr := false
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	colorErr := color.New(color.FgRed)
+	printfSuccess := color.New(color.FgGreen).FprintfFunc()
+	printfSection := color.New(color.Bold).FprintfFunc()
+
+	for res := range outcome {
+		hadErr = hadErr || res.DestroyErr != nil
+		if zabsReleaseFlags.Json {
+			err := enc.Encode(res)
+			if err != nil {
+				colorErr.Fprintf(os.Stderr, "cannot marshal there were errors in destroying the abstractions")
+			}
+		} else {
+			printfSection(os.Stdout, "destroy %s ...", res.Abstraction)
+			if res.DestroyErr != nil {
+				colorErr.Fprintf(os.Stdout, " failed:\n%s\n", res.DestroyErr)
+			} else {
+				printfSuccess(os.Stdout, " OK\n")
+			}
+		}
+	}
+
+	if hadErr {
+		colorErr.Add(color.Bold).Fprintf(os.Stderr, "there were errors in destroying the abstractions")
+		return fmt.Errorf("")
+	} else {
+		return nil
+	}
+}

cmd/autosnap.go

@@ -1,148 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"github.com/zrepl/zrepl/zfs"
-	"sort"
-	"time"
-)
-
-type IntervalAutosnap struct {
-	DatasetFilter    zfs.DatasetFilter
-	Prefix           string
-	SnapshotInterval time.Duration
-
-	log       Logger
-	snaptimes []snapTime
-}
-
-type snapTime struct {
-	ds   *zfs.DatasetPath
-	time time.Time
-}
-
-func (a *IntervalAutosnap) Run(ctx context.Context, didSnaps chan struct{}) {
-
-	a.log = ctx.Value(contextKeyLog).(Logger)
-
-	const LOG_TIME_FMT string = time.ANSIC
-
-	ds, err := zfs.ZFSListMapping(a.DatasetFilter)
-	if err != nil {
-		a.log.WithError(err).Error("cannot list datasets")
-		return
-	}
-	if len(ds) == 0 {
-		a.log.WithError(err).Error("no datasets matching dataset filter")
-		return
-	}
-
-	a.snaptimes = make([]snapTime, len(ds))
-
-	now := time.Now()
-
-	a.log.Debug("examine filesystem state")
-	for i, d := range ds {
-
-		l := a.log.WithField(logFSField, d.ToString())
-
-		fsvs, err := zfs.ZFSListFilesystemVersions(d, &PrefixSnapshotFilter{a.Prefix})
-		if err != nil {
-			l.WithError(err).Error("cannot list filesystem versions")
-			continue
-		}
-		if len(fsvs) <= 0 {
-			l.WithField("prefix", a.Prefix).Info("no filesystem versions with prefix")
-			a.snaptimes[i] = snapTime{d, now}
-			continue
-		}
-
-		// Sort versions by creation
-		sort.SliceStable(fsvs, func(i, j int) bool {
-			return fsvs[i].CreateTXG < fsvs[j].CreateTXG
-		})
-
-		latest := fsvs[len(fsvs)-1]
-		l.WithField("creation", latest.Creation).
-			Debug("found latest snapshot")
-
-		since := now.Sub(latest.Creation)
-		if since < 0 {
-			l.WithField("snapshot", latest.Name).
-				WithField("creation", latest.Creation).
-				Error("snapshot is from the future")
-			continue
-		}
-		next := now
-		if since < a.SnapshotInterval {
-			next = latest.Creation.Add(a.SnapshotInterval)
-		}
-		a.snaptimes[i] = snapTime{d, next}
-	}
-
-	sort.Slice(a.snaptimes, func(i, j int) bool {
-		return a.snaptimes[i].time.Before(a.snaptimes[j].time)
-	})
-
-	syncPoint := a.snaptimes[0]
-	a.log.WithField("sync_point", syncPoint.time.Format(LOG_TIME_FMT)).
-		Info("wait for sync point")
-
-	select {
-	case <-ctx.Done():
-		a.log.WithError(ctx.Err()).Info("context done")
-		return
-
-	case <-time.After(syncPoint.time.Sub(now)):
-		a.log.Debug("snapshot all filesystems to enable further snaps in lockstep")
-		a.doSnapshots(didSnaps)
-	}
-
-	ticker := time.NewTicker(a.SnapshotInterval)
-
-	for {
-		select {
-		case <-ctx.Done():
-			ticker.Stop()
-			a.log.WithError(ctx.Err()).Info("context done")
-			return
-
-		case <-ticker.C:
-			a.doSnapshots(didSnaps)
-		}
-	}
-
-}
-
-func (a *IntervalAutosnap) doSnapshots(didSnaps chan struct{}) {
-
-	// fetch new dataset list in case user added new dataset
-	ds, err := zfs.ZFSListMapping(a.DatasetFilter)
-	if err != nil {
-		a.log.WithError(err).Error("cannot list datasets")
-		return
-	}
-
-	// TODO channel programs -> allow a little jitter?
-	for _, d := range ds {
-		suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
-		snapname := fmt.Sprintf("%s%s", a.Prefix, suffix)
-
-		a.log.WithField(logFSField, d.ToString()).
-			WithField("snapname", snapname).
-			Info("create snapshot")
-
-		err := zfs.ZFSSnapshot(d, snapname, false)
-		if err != nil {
-			a.log.WithError(err).Error("cannot create snapshot")
-		}
-	}
-
-	select {
-	case didSnaps <- struct{}{}:
-	default:
-		a.log.Error("warning: callback channel is full, discarding")
-	}
-
-}

cmd/config.go

@@ -1,100 +0,0 @@
-package cmd
-
-import (
-	"io"
-
-	"fmt"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type Config struct {
-	Global Global
-	Jobs   map[string]Job
-}
-
-func (c *Config) LookupJob(name string) (j Job, err error) {
-	j, ok := c.Jobs[name]
-	if !ok {
-		return nil, errors.Errorf("job '%s' is not defined", name)
-	}
-	return j, nil
-}
-
-type Global struct {
-	Serve struct {
-		Stdinserver struct {
-			SockDir string
-		}
-	}
-	Control struct {
-		Sockpath string
-	}
-	logging *LoggingConfig
-}
-
-type JobDebugSettings struct {
-	Conn struct {
-		ReadDump  string `mapstructure:"read_dump"`
-		WriteDump string `mapstructure:"write_dump"`
-	}
-	RPC struct {
-		Log bool
-	}
-}
-
-type RWCConnecter interface {
-	Connect() (io.ReadWriteCloser, error)
-}
-type AuthenticatedChannelListenerFactory interface {
-	Listen() (AuthenticatedChannelListener, error)
-}
-
-type AuthenticatedChannelListener interface {
-	Accept() (ch io.ReadWriteCloser, err error)
-	Close() (err error)
-}
-
-type SSHStdinServerConnectDescr struct {
-}
-
-type PrunePolicy interface {
-	Prune(fs *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error)
-}
-
-type PruningJob interface {
-	Pruner(side PrunePolicySide, dryRun bool) (Pruner, error)
-}
-
-// A type for constants describing different prune policies of a PruningJob
-// This is mostly a special-case for LocalJob, which is the only job that has two prune policies
-// instead of one.
-// It implements github.com/spf13/pflag.Value to be used as CLI flag for the test subcommand
-type PrunePolicySide string
-
-const (
-	PrunePolicySideDefault PrunePolicySide = ""
-	PrunePolicySideLeft    PrunePolicySide = "left"
-	PrunePolicySideRight   PrunePolicySide = "right"
-)
-
-func (s *PrunePolicySide) String() string {
-	return string(*s)
-}
-
-func (s *PrunePolicySide) Set(news string) error {
-	p := PrunePolicySide(news)
-	switch p {
-	case PrunePolicySideRight:
-		fallthrough
-	case PrunePolicySideLeft:
-		*s = p
-	default:
-		return errors.Errorf("must be either %s or %s", PrunePolicySideLeft, PrunePolicySideRight)
-	}
-	return nil
-}
-
-func (s *PrunePolicySide) Type() string {
-	return fmt.Sprintf("%s | %s", PrunePolicySideLeft, PrunePolicySideRight)
-}

cmd/config_connect.go

@@ -1,46 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"io"
-
-	"github.com/jinzhu/copier"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/sshbytestream"
-)
-
-type SSHStdinserverConnecter struct {
-	Host                 string
-	User                 string
-	Port                 uint16
-	IdentityFile         string   `mapstructure:"identity_file"`
-	TransportOpenCommand []string `mapstructure:"transport_open_command"`
-	SSHCommand           string   `mapstructure:"ssh_command"`
-	Options              []string
-}
-
-func parseSSHStdinserverConnecter(i map[string]interface{}) (c *SSHStdinserverConnecter, err error) {
-
-	c = &SSHStdinserverConnecter{}
-	if err = mapstructure.Decode(i, c); err != nil {
-		err = errors.New(fmt.Sprintf("could not parse ssh transport: %s", err))
-		return nil, err
-	}
-
-	// TODO assert fields are filled
-	return
-
-}
-
-func (c *SSHStdinserverConnecter) Connect() (rwc io.ReadWriteCloser, err error) {
-	var rpcTransport sshbytestream.SSHTransport
-	if err = copier.Copy(&rpcTransport, c); err != nil {
-		return
-	}
-	if rwc, err = sshbytestream.Outgoing(rpcTransport); err != nil {
-		err = errors.WithStack(err)
-		return
-	}
-	return
-}

cmd/config_fsvfilter.go

@@ -1,33 +0,0 @@
-package cmd
-
-import (
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/zfs"
-	"strings"
-)
-
-type PrefixSnapshotFilter struct {
-	Prefix string
-}
-
-func parseSnapshotPrefix(i string) (p string, err error) {
-	if len(i) <= 0 {
-		err = errors.Errorf("snapshot prefix must not be empty string")
-		return
-	}
-	p = i
-	return
-}
-
-func parsePrefixSnapshotFilter(i string) (f *PrefixSnapshotFilter, err error) {
-	if !(len(i) > 0) {
-		err = errors.Errorf("snapshot prefix must be longer than 0 characters")
-		return
-	}
-	f = &PrefixSnapshotFilter{i}
-	return
-}
-
-func (f *PrefixSnapshotFilter) Filter(fsv zfs.FilesystemVersion) (accept bool, err error) {
-	return fsv.Type == zfs.Snapshot && strings.HasPrefix(fsv.Name, f.Prefix), nil
-}

cmd/config_job_control.go

@@ -1,86 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"github.com/pkg/errors"
-	"net"
-	"net/http"
-	"net/http/pprof"
-)
-
-type ControlJob struct {
-	Name     string
-	sockaddr *net.UnixAddr
-}
-
-func NewControlJob(name, sockpath string) (j *ControlJob, err error) {
-	j = &ControlJob{Name: name}
-
-	j.sockaddr, err = net.ResolveUnixAddr("unix", sockpath)
-	if err != nil {
-		err = errors.Wrap(err, "cannot resolve unix address")
-		return
-	}
-
-	return
-}
-
-func (j *ControlJob) JobName() string {
-	return j.Name
-}
-
-const (
-	ControlJobEndpointProfile string = "/debug/pprof/profile"
-)
-
-func (j *ControlJob) JobStart(ctx context.Context) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-	defer log.Info("control job finished")
-
-	l, err := ListenUnixPrivate(j.sockaddr)
-	if err != nil {
-		log.WithError(err).Error("error listening")
-		return
-	}
-
-	mux := http.NewServeMux()
-	mux.Handle(ControlJobEndpointProfile, requestLogger{log, pprof.Profile})
-	server := http.Server{Handler: mux}
-
-outer:
-	for {
-
-		served := make(chan error)
-		go func() {
-			served <- server.Serve(l)
-			close(served)
-		}()
-
-		select {
-		case <-ctx.Done():
-			log.WithError(err).Info("contex done")
-			server.Shutdown(context.Background())
-			break outer
-		case err = <-served:
-			if err != nil {
-				log.WithError(err).Error("error serving")
-				break outer
-			}
-		}
-
-	}
-
-}
-
-type requestLogger struct {
-	log         Logger
-	handlerFunc func(w http.ResponseWriter, r *http.Request)
-}
-
-func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	log := l.log.WithField("method", r.Method).WithField("url", r.URL)
-	log.Info("start")
-	l.handlerFunc(w, r)
-	log.Info("finish")
-}

cmd/config_job_local.go

@@ -1,205 +0,0 @@
-package cmd
-
-import (
-	"time"
-
-	"context"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/zfs"
-	"sync"
-)
-
-type LocalJob struct {
-	Name              string
-	Mapping           *DatasetMapFilter
-	SnapshotPrefix    string
-	Interval          time.Duration
-	InitialReplPolicy InitialReplPolicy
-	PruneLHS          PrunePolicy
-	PruneRHS          PrunePolicy
-	Debug             JobDebugSettings
-}
-
-func parseLocalJob(c JobParsingContext, name string, i map[string]interface{}) (j *LocalJob, err error) {
-
-	var asMap struct {
-		Mapping           map[string]string
-		SnapshotPrefix    string `mapstructure:"snapshot_prefix"`
-		Interval          string
-		InitialReplPolicy string                 `mapstructure:"initial_repl_policy"`
-		PruneLHS          map[string]interface{} `mapstructure:"prune_lhs"`
-		PruneRHS          map[string]interface{} `mapstructure:"prune_rhs"`
-		Debug             map[string]interface{}
-	}
-
-	if err = mapstructure.Decode(i, &asMap); err != nil {
-		err = errors.Wrap(err, "mapstructure error")
-		return nil, err
-	}
-
-	j = &LocalJob{Name: name}
-
-	if j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false); err != nil {
-		return
-	}
-
-	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
-		return
-	}
-
-	if j.Interval, err = time.ParseDuration(asMap.Interval); err != nil {
-		err = errors.Wrap(err, "cannot parse interval")
-		return
-	}
-
-	if j.InitialReplPolicy, err = parseInitialReplPolicy(asMap.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY); err != nil {
-		return
-	}
-
-	if j.PruneLHS, err = parsePrunePolicy(asMap.PruneLHS); err != nil {
-		err = errors.Wrap(err, "cannot parse 'prune_lhs'")
-		return
-	}
-	if j.PruneRHS, err = parsePrunePolicy(asMap.PruneRHS); err != nil {
-		err = errors.Wrap(err, "cannot parse 'prune_rhs'")
-		return
-	}
-
-	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
-		err = errors.Wrap(err, "cannot parse 'debug'")
-		return
-	}
-
-	return
-}
-
-func (j *LocalJob) JobName() string {
-	return j.Name
-}
-
-func (j *LocalJob) JobStart(ctx context.Context) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-	defer log.Info("exiting")
-
-	local := rpc.NewLocalRPC()
-	// Allow access to any dataset since we control what mapping
-	// is passed to the pull routine.
-	// All local datasets will be passed to its Map() function,
-	// but only those for which a mapping exists will actually be pulled.
-	// We can pay this small performance penalty for now.
-	handler := NewHandler(log.WithField(logTaskField, "handler"), localPullACL{}, &PrefixSnapshotFilter{j.SnapshotPrefix})
-
-	registerEndpoints(local, handler)
-
-	snapper := IntervalAutosnap{
-		DatasetFilter:    j.Mapping.AsFilter(),
-		Prefix:           j.SnapshotPrefix,
-		SnapshotInterval: j.Interval,
-	}
-
-	plhs, err := j.Pruner(PrunePolicySideLeft, false)
-	if err != nil {
-		log.WithError(err).Error("error creating lhs pruner")
-		return
-	}
-	prhs, err := j.Pruner(PrunePolicySideRight, false)
-	if err != nil {
-		log.WithError(err).Error("error creating rhs pruner")
-		return
-	}
-
-	makeCtx := func(parent context.Context, taskName string) (ctx context.Context) {
-		return context.WithValue(parent, contextKeyLog, log.WithField(logTaskField, taskName))
-	}
-	var snapCtx, plCtx, prCtx, pullCtx context.Context
-	snapCtx = makeCtx(ctx, "autosnap")
-	plCtx = makeCtx(ctx, "prune_lhs")
-	prCtx = makeCtx(ctx, "prune_rhs")
-	pullCtx = makeCtx(ctx, "repl")
-
-	didSnaps := make(chan struct{})
-	go snapper.Run(snapCtx, didSnaps)
-
-outer:
-	for {
-
-		select {
-		case <-ctx.Done():
-			break outer
-		case <-didSnaps:
-			log.Debug("finished taking snapshots")
-			log.Info("starting replication procedure")
-		}
-
-		{
-			log := pullCtx.Value(contextKeyLog).(Logger)
-			log.Debug("replicating from lhs to rhs")
-			err := doPull(PullContext{local, log, j.Mapping, j.InitialReplPolicy})
-			if err != nil {
-				log.WithError(err).Error("error replicating lhs to rhs")
-			}
-			// use a ctx as soon as doPull gains ctx support
-			select {
-			case <-ctx.Done():
-				break outer
-			default:
-			}
-		}
-
-		var wg sync.WaitGroup
-
-		log.Info("pruning lhs")
-		wg.Add(1)
-		go func() {
-			plhs.Run(plCtx)
-			wg.Done()
-		}()
-
-		log.Info("pruning rhs")
-		wg.Add(1)
-		go func() {
-			prhs.Run(prCtx)
-			wg.Done()
-		}()
-
-		wg.Wait()
-
-	}
-
-	log.WithError(ctx.Err()).Info("context")
-
-}
-
-func (j *LocalJob) Pruner(side PrunePolicySide, dryRun bool) (p Pruner, err error) {
-
-	var dsfilter zfs.DatasetFilter
-	var pp PrunePolicy
-	switch side {
-	case PrunePolicySideLeft:
-		pp = j.PruneLHS
-		dsfilter = j.Mapping.AsFilter()
-	case PrunePolicySideRight:
-		pp = j.PruneRHS
-		dsfilter, err = j.Mapping.InvertedFilter()
-		if err != nil {
-			err = errors.Wrap(err, "cannot invert mapping for prune_rhs")
-			return
-		}
-	default:
-		err = errors.Errorf("must be either left or right side")
-		return
-	}
-
-	p = Pruner{
-		time.Now(),
-		dryRun,
-		dsfilter,
-		j.SnapshotPrefix,
-		pp,
-	}
-
-	return
-}

cmd/config_job_pull.go

@@ -1,165 +0,0 @@
-package cmd
-
-import (
-	"time"
-
-	"context"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/util"
-)
-
-type PullJob struct {
-	Name     string
-	Connect  RWCConnecter
-	Interval time.Duration
-	Mapping  *DatasetMapFilter
-	// constructed from mapping during parsing
-	pruneFilter       *DatasetMapFilter
-	SnapshotPrefix    string
-	InitialReplPolicy InitialReplPolicy
-	Prune             PrunePolicy
-	Debug             JobDebugSettings
-}
-
-func parsePullJob(c JobParsingContext, name string, i map[string]interface{}) (j *PullJob, err error) {
-
-	var asMap struct {
-		Connect           map[string]interface{}
-		Interval          string
-		Mapping           map[string]string
-		InitialReplPolicy string `mapstructure:"initial_repl_policy"`
-		Prune             map[string]interface{}
-		SnapshotPrefix    string `mapstructure:"snapshot_prefix"`
-		Debug             map[string]interface{}
-	}
-
-	if err = mapstructure.Decode(i, &asMap); err != nil {
-		err = errors.Wrap(err, "mapstructure error")
-		return nil, err
-	}
-
-	j = &PullJob{Name: name}
-
-	j.Connect, err = parseSSHStdinserverConnecter(asMap.Connect)
-	if err != nil {
-		err = errors.Wrap(err, "cannot parse 'connect'")
-		return nil, err
-	}
-
-	if j.Interval, err = time.ParseDuration(asMap.Interval); err != nil {
-		err = errors.Wrap(err, "cannot parse 'interval'")
-		return nil, err
-	}
-
-	j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false)
-	if err != nil {
-		err = errors.Wrap(err, "cannot parse 'mapping'")
-		return nil, err
-	}
-
-	if j.pruneFilter, err = j.Mapping.InvertedFilter(); err != nil {
-		err = errors.Wrap(err, "cannot automatically invert 'mapping' for prune job")
-		return nil, err
-	}
-
-	j.InitialReplPolicy, err = parseInitialReplPolicy(asMap.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY)
-	if err != nil {
-		err = errors.Wrap(err, "cannot parse 'initial_repl_policy'")
-		return
-	}
-
-	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
-		return
-	}
-
-	if j.Prune, err = parsePrunePolicy(asMap.Prune); err != nil {
-		err = errors.Wrap(err, "cannot parse prune policy")
-		return
-	}
-
-	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
-		err = errors.Wrap(err, "cannot parse 'debug'")
-		return
-	}
-
-	return
-}
-
-func (j *PullJob) JobName() string {
-	return j.Name
-}
-
-func (j *PullJob) JobStart(ctx context.Context) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-	defer log.Info("exiting")
-
-	ticker := time.NewTicker(j.Interval)
-
-start:
-
-	log.Info("connecting")
-	rwc, err := j.Connect.Connect()
-	if err != nil {
-		log.WithError(err).Error("error connecting")
-		return
-	}
-
-	rwc, err = util.NewReadWriteCloserLogger(rwc, j.Debug.Conn.ReadDump, j.Debug.Conn.WriteDump)
-	if err != nil {
-		return
-	}
-
-	client := rpc.NewClient(rwc)
-	if j.Debug.RPC.Log {
-		client.SetLogger(log, true)
-	}
-
-	log.Info("starting pull")
-
-	pullLog := log.WithField(logTaskField, "pull")
-	err = doPull(PullContext{client, pullLog, j.Mapping, j.InitialReplPolicy})
-	if err != nil {
-		log.WithError(err).Error("error doing pull")
-	}
-
-	closeRPCWithTimeout(log, client, time.Second*10, "")
-
-	log.Info("starting prune")
-	prunectx := context.WithValue(ctx, contextKeyLog, log.WithField(logTaskField, "prune"))
-	pruner, err := j.Pruner(PrunePolicySideDefault, false)
-	if err != nil {
-		log.WithError(err).Error("error creating pruner")
-		return
-	}
-
-	pruner.Run(prunectx)
-	log.Info("finish prune")
-
-	log.Info("wait for next interval")
-	select {
-	case <-ctx.Done():
-		log.WithError(ctx.Err()).Info("context")
-		return
-	case <-ticker.C:
-		goto start
-	}
-
-}
-
-func (j *PullJob) Pruner(side PrunePolicySide, dryRun bool) (p Pruner, err error) {
-	p = Pruner{
-		time.Now(),
-		dryRun,
-		j.pruneFilter,
-		j.SnapshotPrefix,
-		j.Prune,
-	}
-	return
-}
-
-func (j *PullJob) doRun(ctx context.Context) {
-
-}

cmd/config_job_source.go

@@ -1,191 +0,0 @@
-package cmd
-
-import (
-	"context"
-	mapstructure "github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/util"
-	"io"
-	"time"
-)
-
-type SourceJob struct {
-	Name           string
-	Serve          AuthenticatedChannelListenerFactory
-	Datasets       *DatasetMapFilter
-	SnapshotPrefix string
-	Interval       time.Duration
-	Prune          PrunePolicy
-	Debug          JobDebugSettings
-}
-
-func parseSourceJob(c JobParsingContext, name string, i map[string]interface{}) (j *SourceJob, err error) {
-
-	var asMap struct {
-		Serve          map[string]interface{}
-		Datasets       map[string]string
-		SnapshotPrefix string `mapstructure:"snapshot_prefix"`
-		Interval       string
-		Prune          map[string]interface{}
-		Debug          map[string]interface{}
-	}
-
-	if err = mapstructure.Decode(i, &asMap); err != nil {
-		err = errors.Wrap(err, "mapstructure error")
-		return nil, err
-	}
-
-	j = &SourceJob{Name: name}
-
-	if j.Serve, err = parseAuthenticatedChannelListenerFactory(c, asMap.Serve); err != nil {
-		return
-	}
-
-	if j.Datasets, err = parseDatasetMapFilter(asMap.Datasets, true); err != nil {
-		return
-	}
-
-	if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
-		return
-	}
-
-	if j.Interval, err = time.ParseDuration(asMap.Interval); err != nil {
-		err = errors.Wrap(err, "cannot parse 'interval'")
-		return
-	}
-
-	if j.Prune, err = parsePrunePolicy(asMap.Prune); err != nil {
-		err = errors.Wrap(err, "cannot parse 'prune'")
-		return
-	}
-
-	if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
-		err = errors.Wrap(err, "cannot parse 'debug'")
-		return
-	}
-
-	return
-}
-
-func (j *SourceJob) JobName() string {
-	return j.Name
-}
-
-func (j *SourceJob) JobStart(ctx context.Context) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-	defer log.Info("exiting")
-
-	a := IntervalAutosnap{DatasetFilter: j.Datasets, Prefix: j.SnapshotPrefix, SnapshotInterval: j.Interval}
-	p, err := j.Pruner(PrunePolicySideDefault, false)
-	if err != nil {
-		log.WithError(err).Error("error creating pruner")
-		return
-	}
-
-	snapContext := context.WithValue(ctx, contextKeyLog, log.WithField(logTaskField, "autosnap"))
-	prunerContext := context.WithValue(ctx, contextKeyLog, log.WithField(logTaskField, "prune"))
-	serveContext := context.WithValue(ctx, contextKeyLog, log.WithField(logTaskField, "serve"))
-	didSnaps := make(chan struct{})
-
-	go j.serve(serveContext)
-	go a.Run(snapContext, didSnaps)
-
-outer:
-	for {
-		select {
-		case <-ctx.Done():
-			break outer
-		case <-didSnaps:
-			log.Info("starting pruner")
-			p.Run(prunerContext)
-			log.Info("pruner done")
-		}
-	}
-	log.WithError(prunerContext.Err()).Info("context")
-
-}
-
-func (j *SourceJob) Pruner(side PrunePolicySide, dryRun bool) (p Pruner, err error) {
-	p = Pruner{
-		time.Now(),
-		dryRun,
-		j.Datasets,
-		j.SnapshotPrefix,
-		j.Prune,
-	}
-	return
-}
-
-func (j *SourceJob) serve(ctx context.Context) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-
-	listener, err := j.Serve.Listen()
-	if err != nil {
-		log.WithError(err).Error("error listening")
-		return
-	}
-
-	rwcChan := make(chan io.ReadWriteCloser)
-
-	// Serve connections until interrupted or error
-outer:
-	for {
-
-		go func() {
-			rwc, err := listener.Accept()
-			if err != nil {
-				log.WithError(err).Error("error accepting connection")
-				close(rwcChan)
-				return
-			}
-			rwcChan <- rwc
-		}()
-
-		select {
-
-		case rwc, notClosed := <-rwcChan:
-
-			if !notClosed {
-				break outer // closed because of accept error
-			}
-
-			rwc, err := util.NewReadWriteCloserLogger(rwc, j.Debug.Conn.ReadDump, j.Debug.Conn.WriteDump)
-			if err != nil {
-				panic(err)
-			}
-
-			// construct connection handler
-			handler := NewHandler(log, j.Datasets, &PrefixSnapshotFilter{j.SnapshotPrefix})
-
-			// handle connection
-			rpcServer := rpc.NewServer(rwc)
-			if j.Debug.RPC.Log {
-				rpclog := log.WithField("subsystem", "rpc")
-				rpcServer.SetLogger(rpclog, true)
-			}
-			registerEndpoints(rpcServer, handler)
-			if err = rpcServer.Serve(); err != nil {
-				log.WithError(err).Error("error serving connection")
-			}
-			rwc.Close()
-
-		case <-ctx.Done():
-			log.WithError(ctx.Err()).Info("context")
-			break outer
-
-		}
-
-	}
-
-	log.Info("closing listener")
-	err = listener.Close()
-	if err != nil {
-		log.WithError(err).Error("error closing listener")
-	}
-
-	return
-
-}

cmd/config_logging.go

@@ -1,191 +0,0 @@
-package cmd
-
-import (
-	"crypto/tls"
-	"crypto/x509"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/logger"
-	"io/ioutil"
-	"os"
-	"time"
-)
-
-type LoggingConfig struct {
-	Outlets logger.Outlets
-}
-
-type SetNoMetadataFormatter interface {
-	SetNoMetadata(noMetadata bool)
-}
-
-func parseLogging(i interface{}) (c *LoggingConfig, err error) {
-
-	c = &LoggingConfig{}
-	if i == nil {
-		return c, nil
-	}
-
-	var asMap struct {
-		Stdout struct {
-			Level  string
-			Format string
-		}
-		TCP struct {
-			Level         string
-			Format        string
-			Net           string
-			Address       string
-			RetryInterval string `mapstructure:"retry_interval"`
-			TLS           *struct {
-				CA   string
-				Cert string
-				Key  string
-			}
-		}
-		Syslog struct {
-			Enable        bool
-			Format        string
-			RetryInterval string `mapstructure:"retry_interval"`
-		}
-	}
-	if err = mapstructure.Decode(i, &asMap); err != nil {
-		return nil, errors.Wrap(err, "mapstructure error")
-	}
-
-	c.Outlets = logger.NewOutlets()
-
-	if asMap.Stdout.Level != "" {
-
-		out := WriterOutlet{
-			&HumanFormatter{},
-			os.Stdout,
-		}
-
-		level, err := logger.ParseLevel(asMap.Stdout.Level)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot parse 'level'")
-		}
-
-		if asMap.Stdout.Format != "" {
-			out.Formatter, err = parseLogFormat(asMap.Stdout.Format)
-			if err != nil {
-				return nil, errors.Wrap(err, "cannot parse 'format'")
-			}
-		}
-
-		c.Outlets.Add(out, level)
-
-	}
-
-	if asMap.TCP.Address != "" {
-
-		out := &TCPOutlet{}
-
-		out.Formatter, err = parseLogFormat(asMap.TCP.Format)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot parse 'format'")
-		}
-
-		lvl, err := logger.ParseLevel(asMap.TCP.Level)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot parse 'level'")
-		}
-
-		out.RetryInterval, err = time.ParseDuration(asMap.TCP.RetryInterval)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
-		}
-
-		out.Net, out.Address = asMap.TCP.Net, asMap.TCP.Address
-
-		if asMap.TCP.TLS != nil {
-
-			cert, err := tls.LoadX509KeyPair(asMap.TCP.TLS.Cert, asMap.TCP.TLS.Key)
-			if err != nil {
-				return nil, errors.Wrap(err, "cannot load client cert")
-			}
-
-			var rootCAs *x509.CertPool
-			if asMap.TCP.TLS.CA == "" {
-				if rootCAs, err = x509.SystemCertPool(); err != nil {
-					return nil, errors.Wrap(err, "cannot open system cert pool")
-				}
-			} else {
-				rootCAs = x509.NewCertPool()
-				rootCAPEM, err := ioutil.ReadFile(asMap.TCP.TLS.CA)
-				if err != nil {
-					return nil, errors.Wrap(err, "cannot load CA cert")
-				}
-				if !rootCAs.AppendCertsFromPEM(rootCAPEM) {
-					return nil, errors.New("cannot parse CA cert")
-				}
-			}
-			if err != nil && asMap.TCP.TLS.CA == "" {
-				return nil, errors.Wrap(err, "cannot load root ca pool")
-			}
-
-			out.TLS = &tls.Config{
-				Certificates: []tls.Certificate{cert},
-				RootCAs:      rootCAs,
-			}
-
-			out.TLS.BuildNameToCertificate()
-		}
-
-		c.Outlets.Add(out, lvl)
-	}
-
-	if asMap.Syslog.Enable {
-
-		out := &SyslogOutlet{}
-
-		out.Formatter = &HumanFormatter{}
-		if asMap.Syslog.Format != "" {
-			out.Formatter, err = parseLogFormat(asMap.Syslog.Format)
-			if err != nil {
-				return nil, errors.Wrap(err, "cannot parse 'format'")
-			}
-		}
-
-		if f, ok := out.Formatter.(SetNoMetadataFormatter); ok {
-			f.SetNoMetadata(true)
-		}
-
-		out.RetryInterval = 0 // default to 0 as we assume local syslog will just work
-		if asMap.Syslog.RetryInterval != "" {
-			out.RetryInterval, err = time.ParseDuration(asMap.Syslog.RetryInterval)
-			if err != nil {
-				return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
-			}
-		}
-
-		c.Outlets.Add(out, logger.Debug)
-
-	}
-
-	return c, nil
-
-}
-
-func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
-	var is string
-	switch j := i.(type) {
-	case string:
-		is = j
-	default:
-		return nil, errors.Errorf("invalid log format: wrong type: %T", i)
-	}
-
-	switch is {
-	case "human":
-		return &HumanFormatter{}, nil
-	case "logfmt":
-		return &LogfmtFormatter{}, nil
-	case "json":
-		return &JSONFormatter{}, nil
-	default:
-		return nil, errors.Errorf("invalid log format: '%s'", is)
-	}
-
-}

cmd/config_parse.go

@@ -1,254 +0,0 @@
-package cmd
-
-import (
-	"io/ioutil"
-
-	"fmt"
-	yaml "github.com/go-yaml/yaml"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"os"
-)
-
-var ConfigFileDefaultLocations []string = []string{
-	"/etc/zrepl/zrepl.yml",
-	"/usr/local/etc/zrepl/zrepl.yml",
-}
-
-const (
-	JobNameControl string = "control"
-)
-
-var ReservedJobNames []string = []string{
-	JobNameControl,
-}
-
-type ConfigParsingContext struct {
-	Global *Global
-}
-
-func ParseConfig(path string) (config *Config, err error) {
-
-	if path == "" {
-		// Try default locations
-		for _, l := range ConfigFileDefaultLocations {
-			stat, err := os.Stat(l)
-			if err != nil {
-				continue
-			}
-			if !stat.Mode().IsRegular() {
-				err = errors.Errorf("file at default location is not a regular file: %s", l)
-				continue
-			}
-			path = l
-			break
-		}
-	}
-
-	var i interface{}
-
-	var bytes []byte
-
-	if bytes, err = ioutil.ReadFile(path); err != nil {
-		err = errors.WithStack(err)
-		return
-	}
-
-	if err = yaml.Unmarshal(bytes, &i); err != nil {
-		err = errors.WithStack(err)
-		return
-	}
-
-	return parseConfig(i)
-}
-
-func parseConfig(i interface{}) (c *Config, err error) {
-
-	var asMap struct {
-		Global map[string]interface{}
-		Jobs   []map[string]interface{}
-	}
-	if err := mapstructure.Decode(i, &asMap); err != nil {
-		return nil, errors.Wrap(err, "config root must be a dict")
-	}
-
-	c = &Config{}
-
-	// Parse global with defaults
-	c.Global.Serve.Stdinserver.SockDir = "/var/run/zrepl/stdinserver"
-	c.Global.Control.Sockpath = "/var/run/zrepl/control"
-
-	err = mapstructure.Decode(asMap.Global, &c.Global)
-	if err != nil {
-		err = errors.Wrap(err, "mapstructure error on 'global' section: %s")
-		return
-	}
-
-	if c.Global.logging, err = parseLogging(asMap.Global["logging"]); err != nil {
-		return nil, errors.Wrap(err, "cannot parse logging section")
-	}
-
-	cpc := ConfigParsingContext{&c.Global}
-	jpc := JobParsingContext{cpc}
-
-	// Jobs
-	c.Jobs = make(map[string]Job, len(asMap.Jobs))
-	for i := range asMap.Jobs {
-		job, err := parseJob(jpc, asMap.Jobs[i])
-		if err != nil {
-			// Try to find its name
-			namei, ok := asMap.Jobs[i]["name"]
-			if !ok {
-				namei = fmt.Sprintf("<no name, entry #%d in list>", i)
-			}
-			err = errors.Wrapf(err, "cannot parse job '%v'", namei)
-			return nil, err
-		}
-		jn := job.JobName()
-		if _, ok := c.Jobs[jn]; ok {
-			err = errors.Errorf("duplicate job name: %s", jn)
-			return nil, err
-		}
-		c.Jobs[job.JobName()] = job
-	}
-
-	cj, err := NewControlJob(JobNameControl, jpc.Global.Control.Sockpath)
-	if err != nil {
-		err = errors.Wrap(err, "cannot create control job")
-		return
-	}
-	c.Jobs[JobNameControl] = cj
-
-	return c, nil
-
-}
-
-func extractStringField(i map[string]interface{}, key string, notempty bool) (field string, err error) {
-	vi, ok := i[key]
-	if !ok {
-		err = errors.Errorf("must have field '%s'", key)
-		return "", err
-	}
-	field, ok = vi.(string)
-	if !ok {
-		err = errors.Errorf("'%s' field must have type string", key)
-		return "", err
-	}
-	if notempty && len(field) <= 0 {
-		err = errors.Errorf("'%s' field must not be empty", key)
-		return "", err
-	}
-	return
-}
-
-type JobParsingContext struct {
-	ConfigParsingContext
-}
-
-func parseJob(c JobParsingContext, i map[string]interface{}) (j Job, err error) {
-
-	name, err := extractStringField(i, "name", true)
-	if err != nil {
-		return
-	}
-
-	for _, r := range ReservedJobNames {
-		if name == r {
-			err = errors.Errorf("job name '%s' is reserved", name)
-			return
-		}
-	}
-
-	jobtype, err := extractStringField(i, "type", true)
-	if err != nil {
-		return
-	}
-
-	switch jobtype {
-	case "pull":
-		return parsePullJob(c, name, i)
-	case "source":
-		return parseSourceJob(c, name, i)
-	case "local":
-		return parseLocalJob(c, name, i)
-	default:
-		return nil, errors.Errorf("unknown job type '%s'", jobtype)
-	}
-
-}
-
-func parseConnect(i map[string]interface{}) (c RWCConnecter, err error) {
-
-	t, err := extractStringField(i, "type", true)
-	if err != nil {
-		return nil, err
-	}
-
-	switch t {
-	case "ssh+stdinserver":
-		return parseSSHStdinserverConnecter(i)
-	default:
-		return nil, errors.Errorf("unknown connection type '%s'", t)
-	}
-
-}
-
-func parseInitialReplPolicy(v interface{}, defaultPolicy InitialReplPolicy) (p InitialReplPolicy, err error) {
-	s, ok := v.(string)
-	if !ok {
-		goto err
-	}
-
-	switch {
-	case s == "":
-		p = defaultPolicy
-	case s == "most_recent":
-		p = InitialReplPolicyMostRecent
-	case s == "all":
-		p = InitialReplPolicyAll
-	default:
-		goto err
-	}
-
-	return
-
-err:
-	err = errors.New(fmt.Sprintf("expected InitialReplPolicy, got %#v", v))
-	return
-}
-
-func parsePrunePolicy(v map[string]interface{}) (p PrunePolicy, err error) {
-
-	policyName, err := extractStringField(v, "policy", true)
-	if err != nil {
-		return
-	}
-
-	switch policyName {
-	case "grid":
-		return parseGridPrunePolicy(v)
-	case "noprune":
-		return NoPrunePolicy{}, nil
-	default:
-		err = errors.Errorf("unknown policy '%s'", policyName)
-		return
-	}
-
-}
-
-func parseAuthenticatedChannelListenerFactory(c JobParsingContext, v map[string]interface{}) (p AuthenticatedChannelListenerFactory, err error) {
-
-	t, err := extractStringField(v, "type", true)
-	if err != nil {
-		return nil, err
-	}
-
-	switch t {
-	case "stdinserver":
-		return parseStdinserverListenerFactory(c, v)
-	default:
-		err = errors.Errorf("unknown type '%s'", t)
-		return
-	}
-
-}

cmd/config_prune_grid.go

@@ -1,201 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/util"
-	"github.com/zrepl/zrepl/zfs"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-	"time"
-)
-
-type GridPrunePolicy struct {
-	RetentionGrid *util.RetentionGrid
-}
-
-type retentionGridAdaptor struct {
-	zfs.FilesystemVersion
-}
-
-func (a retentionGridAdaptor) Date() time.Time {
-	return a.Creation
-}
-
-func (a retentionGridAdaptor) LessThan(b util.RetentionGridEntry) bool {
-	return a.CreateTXG < b.(retentionGridAdaptor).CreateTXG
-}
-
-func (p *GridPrunePolicy) Prune(_ *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error) {
-
-	// Build adaptors for retention grid
-	adaptors := make([]util.RetentionGridEntry, len(versions))
-	for fsv := range versions {
-		adaptors[fsv] = retentionGridAdaptor{versions[fsv]}
-	}
-
-	sort.SliceStable(adaptors, func(i, j int) bool {
-		return adaptors[i].LessThan(adaptors[j])
-	})
-	now := adaptors[len(adaptors)-1].Date()
-
-	// Evaluate retention grid
-	keepa, removea := p.RetentionGrid.FitEntries(now, adaptors)
-
-	// Revert adaptors
-	keep = make([]zfs.FilesystemVersion, len(keepa))
-	for i := range keepa {
-		keep[i] = keepa[i].(retentionGridAdaptor).FilesystemVersion
-	}
-	remove = make([]zfs.FilesystemVersion, len(removea))
-	for i := range removea {
-		remove[i] = removea[i].(retentionGridAdaptor).FilesystemVersion
-	}
-	return
-
-}
-
-func parseGridPrunePolicy(e map[string]interface{}) (p *GridPrunePolicy, err error) {
-
-	var i struct {
-		Grid string
-	}
-
-	if err = mapstructure.Decode(e, &i); err != nil {
-		err = errors.Wrapf(err, "mapstructure error")
-		return
-	}
-
-	p = &GridPrunePolicy{}
-
-	// Parse grid policy
-	intervals, err := parseRetentionGridIntervalsString(i.Grid)
-	if err != nil {
-		err = fmt.Errorf("cannot parse retention grid: %s", err)
-		return
-	}
-	// Assert intervals are of increasing length (not necessarily required, but indicates config mistake)
-	lastDuration := time.Duration(0)
-	for i := range intervals {
-		if intervals[i].Length < lastDuration {
-			err = fmt.Errorf("retention grid interval length must be monotonically increasing:"+
-				"interval %d is shorter than %d", i+1, i)
-			return
-		} else {
-			lastDuration = intervals[i].Length
-		}
-	}
-	p.RetentionGrid = util.NewRetentionGrid(intervals)
-
-	return
-}
-
-var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
-
-func parseDuration(e string) (d time.Duration, err error) {
-	comps := durationStringRegex.FindStringSubmatch(e)
-	if len(comps) != 3 {
-		err = fmt.Errorf("does not match regex: %s %#v", e, comps)
-		return
-	}
-
-	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
-	if err != nil {
-		return
-	}
-
-	var durationUnit time.Duration
-	switch comps[2] {
-	case "s":
-		durationUnit = time.Second
-	case "m":
-		durationUnit = time.Minute
-	case "h":
-		durationUnit = time.Hour
-	case "d":
-		durationUnit = 24 * time.Hour
-	case "w":
-		durationUnit = 24 * 7 * time.Hour
-	default:
-		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
-		return
-	}
-
-	d = time.Duration(durationFactor) * durationUnit
-	return
-
-}
-
-var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
-
-func parseRetentionGridIntervalString(e string) (intervals []util.RetentionInterval, err error) {
-
-	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
-	if comps == nil {
-		err = fmt.Errorf("retention string does not match expected format")
-		return
-	}
-
-	times, err := strconv.Atoi(comps[1])
-	if err != nil {
-		return nil, err
-	} else if times <= 0 {
-		return nil, fmt.Errorf("contains factor <= 0")
-	}
-
-	duration, err := parseDuration(comps[2])
-	if err != nil {
-		return nil, err
-	}
-
-	keepCount := 1
-	if comps[3] != "" {
-		// Decompose key=value, comma separated
-		// For now, only keep_count is supported
-		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
-		res := re.FindStringSubmatch(comps[4])
-		if res == nil || len(res) != 2 {
-			err = fmt.Errorf("interval parameter contains unknown parameters")
-			return
-		}
-		if res[1] == "all" {
-			keepCount = util.RetentionGridKeepCountAll
-		} else {
-			keepCount, err = strconv.Atoi(res[1])
-			if err != nil {
-				err = fmt.Errorf("cannot parse keep_count value")
-				return
-			}
-		}
-	}
-
-	intervals = make([]util.RetentionInterval, times)
-	for i := range intervals {
-		intervals[i] = util.RetentionInterval{
-			Length:    duration,
-			KeepCount: keepCount,
-		}
-	}
-
-	return
-
-}
-
-func parseRetentionGridIntervalsString(s string) (intervals []util.RetentionInterval, err error) {
-
-	ges := strings.Split(s, "|")
-	intervals = make([]util.RetentionInterval, 0, 7*len(ges))
-
-	for intervalIdx, e := range ges {
-		parsed, err := parseRetentionGridIntervalString(e)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
-		}
-		intervals = append(intervals, parsed...)
-	}
-
-	return
-}

cmd/config_prune_none.go

@@ -1,11 +0,0 @@
-package cmd
-
-import "github.com/zrepl/zrepl/zfs"
-
-type NoPrunePolicy struct{}
-
-func (p NoPrunePolicy) Prune(fs *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error) {
-	keep = versions
-	remove = []zfs.FilesystemVersion{}
-	return
-}

cmd/config_serve_stdinserver.go

@@ -1,104 +0,0 @@
-package cmd
-
-import (
-	"github.com/ftrvxmtrx/fd"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-	"io"
-	"net"
-	"os"
-	"path"
-)
-
-type StdinserverListenerFactory struct {
-	ClientIdentity string `mapstructure:"client_identity"`
-	sockaddr       *net.UnixAddr
-}
-
-func parseStdinserverListenerFactory(c JobParsingContext, i map[string]interface{}) (f *StdinserverListenerFactory, err error) {
-
-	f = &StdinserverListenerFactory{}
-
-	if err = mapstructure.Decode(i, f); err != nil {
-		return nil, errors.Wrap(err, "mapstructure error")
-	}
-	if !(len(f.ClientIdentity) > 0) {
-		err = errors.Errorf("must specify 'client_identity'")
-		return
-	}
-
-	f.sockaddr, err = stdinserverListenerSocket(c.Global.Serve.Stdinserver.SockDir, f.ClientIdentity)
-	if err != nil {
-		return
-	}
-
-	return
-}
-
-func stdinserverListenerSocket(sockdir, clientIdentity string) (addr *net.UnixAddr, err error) {
-	sockpath := path.Join(sockdir, clientIdentity)
-	addr, err = net.ResolveUnixAddr("unix", sockpath)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot resolve unix address")
-	}
-	return addr, nil
-}
-
-func (f *StdinserverListenerFactory) Listen() (al AuthenticatedChannelListener, err error) {
-
-	ul, err := ListenUnixPrivate(f.sockaddr)
-	if err != nil {
-		return nil, errors.Wrapf(err, "cannot listen on unix socket %s", f.sockaddr)
-	}
-
-	l := &StdinserverListener{ul}
-
-	return l, nil
-}
-
-type StdinserverListener struct {
-	l *net.UnixListener
-}
-
-type fdRWC struct {
-	stdin, stdout *os.File
-	control       *net.UnixConn
-}
-
-func (f fdRWC) Read(p []byte) (n int, err error) {
-	return f.stdin.Read(p)
-}
-
-func (f fdRWC) Write(p []byte) (n int, err error) {
-	return f.stdout.Write(p)
-}
-
-func (f fdRWC) Close() (err error) {
-	f.stdin.Close()
-	f.stdout.Close()
-	return f.control.Close()
-}
-
-func (l *StdinserverListener) Accept() (ch io.ReadWriteCloser, err error) {
-	c, err := l.l.Accept()
-	if err != nil {
-		err = errors.Wrap(err, "error accepting on unix listener")
-		return
-	}
-
-	// Read the stdin and stdout of the stdinserver command
-	files, err := fd.Get(c.(*net.UnixConn), 2, []string{"stdin", "stdout"})
-	if err != nil {
-		err = errors.Wrap(err, "error receiving fds from stdinserver command")
-		c.Close()
-	}
-
-	rwc := fdRWC{files[0], files[1], c.(*net.UnixConn)}
-
-	return rwc, nil
-
-}
-
-func (l *StdinserverListener) Close() (err error) {
-	return l.l.Close() // removes socket file automatically
-}

cmd/config_test.go

@@ -1,222 +0,0 @@
-package cmd
-
-import (
-	"testing"
-	"time"
-
-	"github.com/kr/pretty"
-	"github.com/stretchr/testify/assert"
-	"github.com/zrepl/zrepl/util"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
-
-	paths := []string{
-		"./sampleconf/localbackup/host1.yml",
-		"./sampleconf/pullbackup/backuphost.yml",
-		"./sampleconf/pullbackup/productionhost.yml",
-		"./sampleconf/random/debugging.yml",
-		"./sampleconf/random/logging.yml",
-	}
-
-	for _, p := range paths {
-
-		c, err := ParseConfig(p)
-		if err != nil {
-			t.Errorf("error parsing %s:\n%+v", p, err)
-		}
-
-		t.Logf("file: %s", p)
-		t.Log(pretty.Sprint(c))
-
-	}
-
-}
-
-func TestParseRetentionGridStringParsing(t *testing.T) {
-
-	intervals, err := parseRetentionGridIntervalsString("2x10m(keep=2) | 1x1h | 3x1w")
-
-	assert.Nil(t, err)
-	assert.Len(t, intervals, 6)
-	proto := util.RetentionInterval{
-		KeepCount: 2,
-		Length:    10 * time.Minute,
-	}
-	assert.EqualValues(t, proto, intervals[0])
-	assert.EqualValues(t, proto, intervals[1])
-
-	proto.KeepCount = 1
-	proto.Length = 1 * time.Hour
-	assert.EqualValues(t, proto, intervals[2])
-
-	proto.Length = 7 * 24 * time.Hour
-	assert.EqualValues(t, proto, intervals[3])
-	assert.EqualValues(t, proto, intervals[4])
-	assert.EqualValues(t, proto, intervals[5])
-
-	intervals, err = parseRetentionGridIntervalsString("|")
-	assert.Error(t, err)
-	intervals, err = parseRetentionGridIntervalsString("2x10m")
-	assert.NoError(t, err)
-
-	intervals, err = parseRetentionGridIntervalsString("1x10m(keep=all)")
-	assert.NoError(t, err)
-	assert.Len(t, intervals, 1)
-	assert.EqualValues(t, util.RetentionGridKeepCountAll, intervals[0].KeepCount)
-
-}
-
-func TestDatasetMapFilter(t *testing.T) {
-
-	expectMapping := func(m map[string]string, from, to string) {
-		dmf, err := parseDatasetMapFilter(m, false)
-		if err != nil {
-			t.Logf("expect test map to be valid: %s", err)
-			t.FailNow()
-		}
-		fromPath, err := zfs.NewDatasetPath(from)
-		if err != nil {
-			t.Logf("expect test from path to be valid: %s", err)
-			t.FailNow()
-		}
-
-		res, err := dmf.Map(fromPath)
-		if to == "" {
-			assert.Nil(t, res)
-			assert.Nil(t, err)
-			t.Logf("%s => NOT MAPPED", fromPath.ToString())
-			return
-		}
-
-		assert.Nil(t, err)
-		toPath, err := zfs.NewDatasetPath(to)
-		if err != nil {
-			t.Logf("expect test to path to be valid: %s", err)
-			t.FailNow()
-		}
-		assert.True(t, res.Equal(toPath))
-	}
-
-	expectFilter := func(m map[string]string, path string, pass bool) {
-		dmf, err := parseDatasetMapFilter(m, true)
-		if err != nil {
-			t.Logf("expect test filter to be valid: %s", err)
-			t.FailNow()
-		}
-		p, err := zfs.NewDatasetPath(path)
-		if err != nil {
-			t.Logf("expect test path to be valid: %s", err)
-			t.FailNow()
-		}
-		res, err := dmf.Filter(p)
-		assert.Nil(t, err)
-		assert.Equal(t, pass, res)
-	}
-
-	map1 := map[string]string{
-		"a/b/c<":     "root1",
-		"a/b<":       "root2",
-		"<":          "root3/b/c",
-		"b":          "!",
-		"a/b/c/d/e<": "!",
-		"q<":         "root4/1/2",
-	}
-
-	expectMapping(map1, "a/b/c", "root1")
-	expectMapping(map1, "a/b/c/d", "root1/d")
-	expectMapping(map1, "a/b/c/d/e", "")
-	expectMapping(map1, "a/b/e", "root2/e")
-	expectMapping(map1, "a/b", "root2")
-	expectMapping(map1, "x", "root3/b/c")
-	expectMapping(map1, "x/y", "root3/b/c/y")
-	expectMapping(map1, "q", "root4/1/2")
-	expectMapping(map1, "b", "")
-	expectMapping(map1, "q/r", "root4/1/2/r")
-
-	filter1 := map[string]string{
-		"<":    "!",
-		"a<":   "ok",
-		"a/b<": "!",
-	}
-
-	expectFilter(filter1, "b", false)
-	expectFilter(filter1, "a", true)
-	expectFilter(filter1, "a/d", true)
-	expectFilter(filter1, "a/b", false)
-	expectFilter(filter1, "a/b/c", false)
-
-	filter2 := map[string]string{}
-	expectFilter(filter2, "foo", false) // default to omit
-
-}
-
-func TestDatasetMapFilter_AsFilter(t *testing.T) {
-
-	mapspec := map[string]string{
-		"a/b/c<":     "root1",
-		"a/b<":       "root2",
-		"<":          "root3/b/c",
-		"b":          "!",
-		"a/b/c/d/e<": "!",
-		"q<":         "root4/1/2",
-	}
-
-	m, err := parseDatasetMapFilter(mapspec, false)
-	assert.Nil(t, err)
-
-	f := m.AsFilter()
-
-	t.Logf("Mapping:\n%s\nFilter:\n%s", pretty.Sprint(m), pretty.Sprint(f))
-
-	tf := func(f zfs.DatasetFilter, path string, pass bool) {
-		p, err := zfs.NewDatasetPath(path)
-		assert.Nil(t, err)
-		r, err := f.Filter(p)
-		assert.Nil(t, err)
-		assert.Equal(t, pass, r)
-	}
-
-	tf(f, "a/b/c", true)
-	tf(f, "a/b", true)
-	tf(f, "b", false)
-	tf(f, "a/b/c/d/e", false)
-	tf(f, "a/b/c/d/e/f", false)
-	tf(f, "a", true)
-
-}
-
-func TestDatasetMapFilter_InvertedFilter(t *testing.T) {
-	mapspec := map[string]string{
-		"a/b":      "1/2",
-		"a/b/c<":   "3",
-		"a/b/c/d<": "1/2/a",
-		"a/b/d":    "!",
-	}
-
-	m, err := parseDatasetMapFilter(mapspec, false)
-	assert.Nil(t, err)
-
-	inv, err := m.InvertedFilter()
-	assert.Nil(t, err)
-
-	t.Log(pretty.Sprint(inv))
-
-	expectMapping := func(m *DatasetMapFilter, ps string, expRes bool) {
-		p, err := zfs.NewDatasetPath(ps)
-		assert.Nil(t, err)
-		r, err := m.Filter(p)
-		assert.Nil(t, err)
-		assert.Equal(t, expRes, r)
-	}
-
-	expectMapping(inv, "4", false)
-	expectMapping(inv, "3", true)
-	expectMapping(inv, "3/x", true)
-	expectMapping(inv, "1", false)
-	expectMapping(inv, "1/2", true)
-	expectMapping(inv, "1/2/3", false)
-	expectMapping(inv, "1/2/a/b", true)
-
-}

cmd/control.go

@@ -1,100 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"github.com/spf13/cobra"
-	"io"
-	golog "log"
-	"net"
-	"net/http"
-	"net/url"
-	"os"
-)
-
-var controlCmd = &cobra.Command{
-	Use:   "control",
-	Short: "control zrepl daemon",
-}
-
-var pprofCmd = &cobra.Command{
-	Use:   "pprof cpu OUTFILE",
-	Short: "pprof CPU of daemon to OUTFILE (- for stdout)",
-	Run:   doControlPProf,
-}
-var pprofCmdArgs struct {
-	seconds int64
-}
-
-func init() {
-	RootCmd.AddCommand(controlCmd)
-	controlCmd.AddCommand(pprofCmd)
-	pprofCmd.Flags().Int64Var(&pprofCmdArgs.seconds, "seconds", 30, "seconds to profile")
-}
-
-func doControlPProf(cmd *cobra.Command, args []string) {
-
-	log := golog.New(os.Stderr, "", 0)
-
-	die := func() {
-		log.Printf("exiting after error")
-		os.Exit(1)
-	}
-
-	conf, err := ParseConfig(rootArgs.configFile)
-	if err != nil {
-		log.Printf("error parsing config: %s", err)
-		die()
-	}
-
-	if cmd.Flags().Arg(0) != "cpu" {
-		log.Printf("only CPU profiles are supported")
-		log.Printf("%s", cmd.UsageString())
-		die()
-	}
-
-	outfn := cmd.Flags().Arg(1)
-	if outfn == "" {
-		log.Printf("must specify output filename")
-		log.Printf("%s", cmd.UsageString())
-		die()
-	}
-	var out io.Writer
-	if outfn == "-" {
-		out = os.Stdout
-	} else {
-		out, err = os.Create(outfn)
-		if err != nil {
-			log.Printf("error creating output file: %s", err)
-			die()
-		}
-	}
-
-	log.Printf("connecting to daemon")
-	httpc := http.Client{
-		Transport: &http.Transport{
-			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
-				return net.Dial("unix", conf.Global.Control.Sockpath)
-			},
-		},
-	}
-
-	log.Printf("profiling...")
-	v := url.Values{}
-	v.Set("seconds", fmt.Sprintf("%d", pprofCmdArgs.seconds))
-	v.Encode()
-	resp, err := httpc.Get("http://unix" + ControlJobEndpointProfile + "?" + v.Encode())
-	if err != nil {
-		log.Printf("error: %s", err)
-		die()
-	}
-
-	_, err = io.Copy(out, resp.Body)
-	if err != nil {
-		log.Printf("error writing profile: %s", err)
-		die()
-	}
-
-	log.Printf("finished")
-
-}

cmd/daemon.go

@@ -1,110 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/logger"
-	"os"
-	"os/signal"
-	"syscall"
-	"time"
-)
-
-// daemonCmd represents the daemon command
-var daemonCmd = &cobra.Command{
-	Use:   "daemon",
-	Short: "start daemon",
-	Run:   doDaemon,
-}
-
-func init() {
-	RootCmd.AddCommand(daemonCmd)
-}
-
-type Job interface {
-	JobName() string
-	JobStart(ctxt context.Context)
-}
-
-func doDaemon(cmd *cobra.Command, args []string) {
-
-	conf, err := ParseConfig(rootArgs.configFile)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "error parsing config: %s", err)
-		os.Exit(1)
-	}
-
-	log := logger.NewLogger(conf.Global.logging.Outlets, 1*time.Second)
-
-	log.Debug("starting daemon")
-	ctx := context.WithValue(context.Background(), contextKeyLog, log)
-	ctx = context.WithValue(ctx, contextKeyLog, log)
-
-	d := NewDaemon(conf)
-	d.Loop(ctx)
-
-}
-
-type contextKey string
-
-const (
-	contextKeyLog contextKey = contextKey("log")
-)
-
-type Daemon struct {
-	conf *Config
-}
-
-func NewDaemon(initialConf *Config) *Daemon {
-	return &Daemon{initialConf}
-}
-
-func (d *Daemon) Loop(ctx context.Context) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-
-	ctx, cancel := context.WithCancel(ctx)
-
-	sigChan := make(chan os.Signal, 1)
-	finishs := make(chan Job)
-
-	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
-
-	log.Info("starting jobs from config")
-	i := 0
-	for _, job := range d.conf.Jobs {
-		logger := log.WithField(logJobField, job.JobName())
-		logger.Info("starting")
-		i++
-		jobCtx := context.WithValue(ctx, contextKeyLog, logger)
-		go func(j Job) {
-			j.JobStart(jobCtx)
-			finishs <- j
-		}(job)
-	}
-
-	finishCount := 0
-outer:
-	for {
-		select {
-		case <-finishs:
-			finishCount++
-			if finishCount == len(d.conf.Jobs) {
-				log.Info("all jobs finished")
-				break outer
-			}
-
-		case sig := <-sigChan:
-			log.WithField("signal", sig).Info("received signal")
-			log.Info("cancelling all jobs")
-			cancel()
-		}
-	}
-
-	signal.Stop(sigChan)
-	cancel() // make go vet happy
-
-	log.Info("exiting")
-
-}

cmd/handler.go

@@ -1,181 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"io"
-
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type DatasetMapping interface {
-	Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error)
-}
-
-type FilesystemRequest struct {
-	Roots []string // may be nil, indicating interest in all filesystems
-}
-
-type FilesystemVersionsRequest struct {
-	Filesystem *zfs.DatasetPath
-}
-
-type InitialTransferRequest struct {
-	Filesystem        *zfs.DatasetPath
-	FilesystemVersion zfs.FilesystemVersion
-}
-
-type IncrementalTransferRequest struct {
-	Filesystem *zfs.DatasetPath
-	From       zfs.FilesystemVersion
-	To         zfs.FilesystemVersion
-}
-
-type Handler struct {
-	logger Logger
-	dsf    zfs.DatasetFilter
-	fsvf   zfs.FilesystemVersionFilter
-}
-
-func NewHandler(logger Logger, dsfilter zfs.DatasetFilter, snapfilter zfs.FilesystemVersionFilter) (h Handler) {
-	return Handler{logger, dsfilter, snapfilter}
-}
-
-func registerEndpoints(server rpc.RPCServer, handler Handler) (err error) {
-	err = server.RegisterEndpoint("FilesystemRequest", handler.HandleFilesystemRequest)
-	if err != nil {
-		panic(err)
-	}
-	err = server.RegisterEndpoint("FilesystemVersionsRequest", handler.HandleFilesystemVersionsRequest)
-	if err != nil {
-		panic(err)
-	}
-	err = server.RegisterEndpoint("InitialTransferRequest", handler.HandleInitialTransferRequest)
-	if err != nil {
-		panic(err)
-	}
-	err = server.RegisterEndpoint("IncrementalTransferRequest", handler.HandleIncrementalTransferRequest)
-	if err != nil {
-		panic(err)
-	}
-	return nil
-}
-
-func (h Handler) HandleFilesystemRequest(r *FilesystemRequest, roots *[]*zfs.DatasetPath) (err error) {
-
-	log := h.logger.WithField("endpoint", "FilesystemRequest")
-
-	log.WithField("request", r).Debug("request")
-	log.WithField("dataset_filter", h.dsf).Debug("dsf")
-
-	allowed, err := zfs.ZFSListMapping(h.dsf)
-	if err != nil {
-		log.WithError(err).Error("error listing filesystems")
-		return
-	}
-
-	log.WithField("response", allowed).Debug("response")
-	*roots = allowed
-	return
-}
-
-func (h Handler) HandleFilesystemVersionsRequest(r *FilesystemVersionsRequest, versions *[]zfs.FilesystemVersion) (err error) {
-
-	log := h.logger.WithField("endpoint", "FilesystemVersionsRequest")
-
-	log.WithField("request", r).Debug("request")
-
-	// allowed to request that?
-	if h.pullACLCheck(r.Filesystem, nil); err != nil {
-		log.WithError(err).Warn("pull ACL check failed")
-		return
-	}
-
-	// find our versions
-	vs, err := zfs.ZFSListFilesystemVersions(r.Filesystem, h.fsvf)
-	if err != nil {
-		log.WithError(err).Error("cannot list filesystem versions")
-		return
-	}
-
-	log.WithField("resposne", vs).Debug("response")
-
-	*versions = vs
-	return
-
-}
-
-func (h Handler) HandleInitialTransferRequest(r *InitialTransferRequest, stream *io.Reader) (err error) {
-
-	log := h.logger.WithField("endpoint", "InitialTransferRequest")
-
-	log.WithField("request", r).Debug("request")
-	if err = h.pullACLCheck(r.Filesystem, &r.FilesystemVersion); err != nil {
-		log.WithError(err).Warn("pull ACL check failed")
-		return
-	}
-
-	log.Debug("invoking zfs send")
-
-	s, err := zfs.ZFSSend(r.Filesystem, &r.FilesystemVersion, nil)
-	if err != nil {
-		log.WithError(err).Error("cannot send filesystem")
-	}
-	*stream = s
-
-	return
-
-}
-
-func (h Handler) HandleIncrementalTransferRequest(r *IncrementalTransferRequest, stream *io.Reader) (err error) {
-
-	log := h.logger.WithField("endpoint", "IncrementalTransferRequest")
-	log.WithField("request", r).Debug("request")
-	if err = h.pullACLCheck(r.Filesystem, &r.From); err != nil {
-		log.WithError(err).Warn("pull ACL check failed")
-		return
-	}
-	if err = h.pullACLCheck(r.Filesystem, &r.To); err != nil {
-		log.WithError(err).Warn("pull ACL check failed")
-		return
-	}
-
-	log.Debug("invoking zfs send")
-
-	s, err := zfs.ZFSSend(r.Filesystem, &r.From, &r.To)
-	if err != nil {
-		log.WithError(err).Error("cannot send filesystem")
-	}
-
-	*stream = s
-	return
-
-}
-
-func (h Handler) pullACLCheck(p *zfs.DatasetPath, v *zfs.FilesystemVersion) (err error) {
-	var fsAllowed, vAllowed bool
-	fsAllowed, err = h.dsf.Filter(p)
-	if err != nil {
-		err = fmt.Errorf("error evaluating ACL: %s", err)
-		return
-	}
-	if !fsAllowed {
-		err = fmt.Errorf("ACL prohibits access to %s", p.ToString())
-		return
-	}
-	if v == nil {
-		return
-	}
-
-	vAllowed, err = h.fsvf.Filter(*v)
-	if err != nil {
-		err = errors.Wrap(err, "error evaluating version filter")
-		return
-	}
-	if !vAllowed {
-		err = fmt.Errorf("ACL prohibits access to %s", v.ToAbsPath(p))
-		return
-	}
-	return
-}

cmd/helpers.go

@@ -1,35 +0,0 @@
-package cmd
-
-import (
-	"github.com/pkg/errors"
-	"net"
-	"os"
-	"path/filepath"
-)
-
-func ListenUnixPrivate(sockaddr *net.UnixAddr) (*net.UnixListener, error) {
-
-	sockdir := filepath.Dir(sockaddr.Name)
-	sdstat, err := os.Stat(sockdir)
-	if err != nil {
-		return nil, errors.Wrapf(err, "cannot stat(2) '%s'", sockdir)
-	}
-	if !sdstat.IsDir() {
-		return nil, errors.Errorf("not a directory: %s", sockdir)
-	}
-	p := sdstat.Mode().Perm()
-	if p&0007 != 0 {
-		return nil, errors.Errorf("socket directory not be world-accessible: %s (permissions are %#o)", sockdir, p)
-	}
-
-	// Maybe things have not been cleaned up before
-	s, err := os.Stat(sockaddr.Name)
-	if err == nil {
-		if s.Mode()&os.ModeSocket != 0 {
-			// opportunistically try to remove it, but if this fails, it is not an error
-			os.Remove(sockaddr.Name)
-		}
-	}
-
-	return net.ListenUnix("unix", sockaddr)
-}

cmd/logging_formatters.go

@@ -1,174 +0,0 @@
-package cmd
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"github.com/go-logfmt/logfmt"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/logger"
-	"strings"
-	"time"
-)
-
-type EntryFormatter interface {
-	Format(e *logger.Entry) ([]byte, error)
-}
-
-const (
-	FieldLevel   = "level"
-	FieldMessage = "msg"
-	FieldTime    = "time"
-)
-
-const (
-	logJobField     string = "job"
-	logTaskField    string = "task"
-	logFSField      string = "filesystem"
-	logMapFromField string = "map_from"
-	logMapToField   string = "map_to"
-	logIncFromField string = "inc_from"
-	logIncToField   string = "inc_to"
-)
-
-type NoFormatter struct{}
-
-func (f NoFormatter) Format(e *logger.Entry) ([]byte, error) {
-	return []byte(e.Message), nil
-}
-
-type HumanFormatter struct {
-	NoMetadata bool
-}
-
-var _ SetNoMetadataFormatter = &HumanFormatter{}
-
-func (f *HumanFormatter) SetNoMetadata(noMetadata bool) {
-	f.NoMetadata = noMetadata
-}
-
-func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
-
-	var line bytes.Buffer
-
-	if !f.NoMetadata {
-		fmt.Fprintf(&line, "[%s]", e.Level.Short())
-	}
-
-	prefixFields := []string{logJobField, logTaskField, logFSField}
-	prefixed := make(map[string]bool, len(prefixFields)+2)
-	for _, field := range prefixFields {
-		val, ok := e.Fields[field].(string)
-		if ok {
-			fmt.Fprintf(&line, "[%s]", val)
-			prefixed[field] = true
-		} else {
-			break
-		}
-	}
-	// even more prefix fields
-	mapFrom, mapFromOk := e.Fields[logMapFromField].(string)
-	mapTo, mapToOk := e.Fields[logMapToField].(string)
-	if mapFromOk && mapToOk {
-		fmt.Fprintf(&line, "[%s => %s]", mapFrom, mapTo)
-		prefixed[logMapFromField], prefixed[logMapToField] = true, true
-	}
-	incFrom, incFromOk := e.Fields[logIncFromField].(string)
-	incTo, incToOk := e.Fields[logIncToField].(string)
-	if incFromOk && incToOk {
-		fmt.Fprintf(&line, "[%s => %s]", incFrom, incTo)
-		prefixed[logIncFromField], prefixed[logIncToField] = true, true
-	}
-
-	if line.Len() > 0 {
-		fmt.Fprint(&line, ": ")
-	}
-	fmt.Fprint(&line, e.Message)
-
-	for field, value := range e.Fields {
-
-		if prefixed[field] {
-			continue
-		}
-
-		if strings.ContainsAny(field, " \t") {
-			return nil, errors.Errorf("field must not contain whitespace: '%s'", field)
-		}
-		fmt.Fprintf(&line, " %s=\"%s\"", field, value)
-	}
-
-	return line.Bytes(), nil
-}
-
-type JSONFormatter struct{}
-
-func (f *JSONFormatter) Format(e *logger.Entry) ([]byte, error) {
-	data := make(logger.Fields, len(e.Fields)+3)
-	for k, v := range e.Fields {
-		switch v := v.(type) {
-		case error:
-			// Otherwise errors are ignored by `encoding/json`
-			// https://github.com/sirupsen/logrus/issues/137
-			data[k] = v.Error()
-		default:
-			_, err := json.Marshal(v)
-			if err != nil {
-				return nil, errors.Errorf("field is not JSON encodable: %s", k)
-			}
-			data[k] = v
-		}
-	}
-
-	data[FieldMessage] = e.Message
-	data[FieldTime] = e.Time.Format(time.RFC3339)
-	data[FieldLevel] = e.Level
-
-	return json.Marshal(data)
-
-}
-
-type LogfmtFormatter struct {
-	NoMetadata bool
-}
-
-var _ SetNoMetadataFormatter = &LogfmtFormatter{}
-
-func (f *LogfmtFormatter) SetNoMetadata(noMetadata bool) {
-	f.NoMetadata = noMetadata
-}
-
-func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
-	var buf bytes.Buffer
-	enc := logfmt.NewEncoder(&buf)
-
-	if !f.NoMetadata {
-		enc.EncodeKeyval(FieldTime, e.Time)
-		enc.EncodeKeyval(FieldLevel, e.Level)
-	}
-
-	// at least try and put job and task in front
-	prefixed := make(map[string]bool, 2)
-	prefix := []string{logJobField, logTaskField}
-	for _, pf := range prefix {
-		v, ok := e.Fields[pf]
-		if !ok {
-			break
-		}
-		enc.EncodeKeyval(pf, v)
-		prefixed[pf] = true
-	}
-
-	enc.EncodeKeyval(FieldMessage, e.Message)
-
-	for k, v := range e.Fields {
-		if !prefixed[k] {
-			enc.EncodeKeyval(k, v)
-		}
-	}
-
-	if err := enc.EndRecord(); err != nil {
-		return nil, err
-	}
-
-	return buf.Bytes(), nil
-}

cmd/logging_outlets.go

@@ -1,124 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"crypto/tls"
-	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/logger"
-	"io"
-	"log/syslog"
-	"net"
-	"time"
-)
-
-type WriterOutlet struct {
-	Formatter EntryFormatter
-	Writer    io.Writer
-}
-
-func (h WriterOutlet) WriteEntry(ctx context.Context, entry logger.Entry) error {
-	bytes, err := h.Formatter.Format(&entry)
-	if err != nil {
-		return err
-	}
-	_, err = h.Writer.Write(bytes)
-	h.Writer.Write([]byte("\n"))
-	return err
-}
-
-type TCPOutlet struct {
-	Formatter    EntryFormatter
-	Net, Address string
-	Dialer       net.Dialer
-	TLS          *tls.Config
-	// Specifies how much time must pass between a connection error and a reconnection attempt
-	// Log entries written to the outlet during this time interval are silently dropped.
-	RetryInterval time.Duration
-	// nil if there was an error sending / connecting to remote server
-	conn net.Conn
-	// Last time an error occurred when sending / connecting to remote server
-	retry time.Time
-}
-
-func (h *TCPOutlet) WriteEntry(ctx context.Context, e logger.Entry) error {
-
-	b, err := h.Formatter.Format(&e)
-	if err != nil {
-		return err
-	}
-
-	if h.conn == nil {
-		if time.Now().Sub(h.retry) < h.RetryInterval {
-			// cool-down phase, drop the log entry
-			return nil
-		}
-
-		if h.TLS != nil {
-			h.conn, err = tls.DialWithDialer(&h.Dialer, h.Net, h.Address, h.TLS)
-		} else {
-			h.conn, err = h.Dialer.DialContext(ctx, h.Net, h.Address)
-		}
-		if err != nil {
-			h.conn = nil
-			h.retry = time.Now()
-			return errors.Wrap(err, "cannot dial")
-		}
-	}
-
-	_, err = h.conn.Write(b)
-	if err == nil {
-		_, err = h.conn.Write([]byte("\n"))
-	}
-	if err != nil {
-		h.conn.Close()
-		h.conn = nil
-		h.retry = time.Now()
-		return errors.Wrap(err, "cannot write")
-	}
-
-	return nil
-}
-
-type SyslogOutlet struct {
-	Formatter          EntryFormatter
-	RetryInterval      time.Duration
-	writer             *syslog.Writer
-	lastConnectAttempt time.Time
-}
-
-func (o *SyslogOutlet) WriteEntry(ctx context.Context, entry logger.Entry) error {
-
-	bytes, err := o.Formatter.Format(&entry)
-	if err != nil {
-		return err
-	}
-
-	s := string(bytes)
-
-	if o.writer == nil {
-		now := time.Now()
-		if now.Sub(o.lastConnectAttempt) < o.RetryInterval {
-			return nil // not an error toward logger
-		}
-		o.writer, err = syslog.New(syslog.LOG_LOCAL0, "zrepl")
-		o.lastConnectAttempt = time.Now()
-		if err != nil {
-			o.writer = nil
-			return err
-		}
-	}
-
-	switch entry.Level {
-	case logger.Debug:
-		return o.writer.Debug(s)
-	case logger.Info:
-		return o.writer.Info(s)
-	case logger.Warn:
-		return o.writer.Warning(s)
-	case logger.Error:
-		return o.writer.Err(s)
-	default:
-		return o.writer.Err(s) // write as error as reaching this case is in fact an error
-	}
-
-}

cmd/main.go

@@ -1,43 +0,0 @@
-// zrepl replicates ZFS filesystems & volumes between pools
-//
-// Code Organization
-//
-// The cmd package uses github.com/spf13/cobra for its CLI.
-//
-// It combines the other packages in the zrepl project to implement zrepl functionality.
-//
-// Each subcommand's code is in the corresponding *.go file.
-// All other *.go files contain code shared by the subcommands.
-package cmd
-
-import (
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/logger"
-)
-
-//
-//type Logger interface {
-//	Printf(format string, v ...interface{})
-//}
-
-type Logger = *logger.Logger
-
-var RootCmd = &cobra.Command{
-	Use:   "zrepl",
-	Short: "ZFS dataset replication",
-	Long: `Replicate ZFS filesystems & volumes between pools:
-
-  - push & pull mode
-  - automatic snapshot creation & pruning
-  - local / over the network
-  - ACLs instead of blank SSH access`,
-}
-
-var rootArgs struct {
-	configFile string
-}
-
-func init() {
-	//cobra.OnInitialize(initConfig)
-	RootCmd.PersistentFlags().StringVar(&rootArgs.configFile, "config", "", "config file path")
-}

cmd/prune.go

@@ -1,103 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"github.com/zrepl/zrepl/zfs"
-	"time"
-)
-
-type Pruner struct {
-	Now            time.Time
-	DryRun         bool
-	DatasetFilter  zfs.DatasetFilter
-	SnapshotPrefix string
-	PrunePolicy    PrunePolicy
-}
-
-type PruneResult struct {
-	Filesystem *zfs.DatasetPath
-	All        []zfs.FilesystemVersion
-	Keep       []zfs.FilesystemVersion
-	Remove     []zfs.FilesystemVersion
-}
-
-func (p *Pruner) Run(ctx context.Context) (r []PruneResult, err error) {
-
-	log := ctx.Value(contextKeyLog).(Logger)
-
-	if p.DryRun {
-		log.Info("doing dry run")
-	}
-
-	filesystems, err := zfs.ZFSListMapping(p.DatasetFilter)
-	if err != nil {
-		log.WithError(err).Error("error applying filesystem filter")
-		return nil, err
-	}
-	if len(filesystems) <= 0 {
-		log.Info("no filesystems matching filter")
-		return nil, err
-	}
-
-	r = make([]PruneResult, 0, len(filesystems))
-
-	for _, fs := range filesystems {
-
-		log := log.WithField(logFSField, fs.ToString())
-
-		fsversions, err := zfs.ZFSListFilesystemVersions(fs, &PrefixSnapshotFilter{p.SnapshotPrefix})
-		if err != nil {
-			log.WithError(err).Error("error listing filesytem versions")
-			continue
-		}
-		if len(fsversions) == 0 {
-			log.WithField("prefix", p.SnapshotPrefix).Info("no filesystem versions matching prefix")
-			continue
-		}
-
-		keep, remove, err := p.PrunePolicy.Prune(fs, fsversions)
-		if err != nil {
-			log.WithError(err).Error("error evaluating prune policy")
-			continue
-		}
-
-		log.WithField("fsversions", fsversions).
-			WithField("keep", keep).
-			WithField("remove", remove).
-			Debug("prune policy debug dump")
-
-		r = append(r, PruneResult{fs, fsversions, keep, remove})
-
-		makeFields := func(v zfs.FilesystemVersion) (fields map[string]interface{}) {
-			fields = make(map[string]interface{})
-			fields["version"] = v.ToAbsPath(fs)
-			timeSince := v.Creation.Sub(p.Now)
-			fields["age_ns"] = timeSince
-			const day time.Duration = 24 * time.Hour
-			days := timeSince / day
-			remainder := timeSince % day
-			fields["age_str"] = fmt.Sprintf("%dd%s", days, remainder)
-			return
-		}
-
-		for _, v := range remove {
-			fields := makeFields(v)
-			log.WithFields(fields).Info("destroying version")
-			// echo what we'll do and exec zfs destroy if not dry run
-			// TODO special handling for EBUSY (zfs hold)
-			// TODO error handling for clones? just echo to cli, skip over, and exit with non-zero status code (we're idempotent)
-			if !p.DryRun {
-				err := zfs.ZFSDestroyFilesystemVersion(fs, v)
-				if err != nil {
-					// handle
-					log.WithFields(fields).WithError(err).Error("error destroying version")
-				}
-			}
-		}
-
-	}
-
-	return
-
-}

cmd/replication.go

@@ -1,319 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"io"
-	"time"
-
-	"bytes"
-	"encoding/json"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/util"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type localPullACL struct{}
-
-func (a localPullACL) Filter(p *zfs.DatasetPath) (pass bool, err error) {
-	return true, nil
-}
-
-const LOCAL_TRANSPORT_IDENTITY string = "local"
-
-const DEFAULT_INITIAL_REPL_POLICY = InitialReplPolicyMostRecent
-
-type InitialReplPolicy string
-
-const (
-	InitialReplPolicyMostRecent InitialReplPolicy = "most_recent"
-	InitialReplPolicyAll        InitialReplPolicy = "all"
-)
-
-func closeRPCWithTimeout(log Logger, remote rpc.RPCClient, timeout time.Duration, goodbye string) {
-	log.Info("closing rpc connection")
-
-	ch := make(chan error)
-	go func() {
-		ch <- remote.Close()
-		close(ch)
-	}()
-
-	var err error
-	select {
-	case <-time.After(timeout):
-		err = fmt.Errorf("timeout exceeded (%s)", timeout)
-	case closeRequestErr := <-ch:
-		err = closeRequestErr
-	}
-
-	if err != nil {
-		log.WithError(err).Error("error closing connection")
-	}
-	return
-}
-
-type PullContext struct {
-	Remote            rpc.RPCClient
-	Log               Logger
-	Mapping           DatasetMapping
-	InitialReplPolicy InitialReplPolicy
-}
-
-func doPull(pull PullContext) (err error) {
-
-	remote := pull.Remote
-	log := pull.Log
-
-	log.Info("request remote filesystem list")
-	fsr := FilesystemRequest{}
-	var remoteFilesystems []*zfs.DatasetPath
-	if err = remote.Call("FilesystemRequest", &fsr, &remoteFilesystems); err != nil {
-		return
-	}
-
-	log.Debug("map remote filesystems to local paths and determine order for per-filesystem sync")
-	type RemoteLocalMapping struct {
-		Remote *zfs.DatasetPath
-		Local  *zfs.DatasetPath
-	}
-	replMapping := make(map[string]RemoteLocalMapping, len(remoteFilesystems))
-	localTraversal := zfs.NewDatasetPathForest()
-	for fs := range remoteFilesystems {
-		var err error
-		var localFs *zfs.DatasetPath
-		localFs, err = pull.Mapping.Map(remoteFilesystems[fs])
-		if err != nil {
-			err := fmt.Errorf("error mapping %s: %s", remoteFilesystems[fs], err)
-			log.WithError(err).WithField(logMapFromField, remoteFilesystems[fs]).Error("cannot map")
-			return err
-		}
-		if localFs == nil {
-			continue
-		}
-		log.WithField(logMapFromField, remoteFilesystems[fs].ToString()).
-			WithField(logMapToField, localFs.ToString()).Debug("mapping")
-		m := RemoteLocalMapping{remoteFilesystems[fs], localFs}
-		replMapping[m.Local.ToString()] = m
-		localTraversal.Add(m.Local)
-	}
-
-	log.Debug("build cache for already present local filesystem state")
-	localFilesystemState, err := zfs.ZFSListFilesystemState()
-	if err != nil {
-		log.WithError(err).Error("cannot request local filesystem state")
-		return err
-	}
-
-	log.Info("start per-filesystem sync")
-	localTraversal.WalkTopDown(func(v zfs.DatasetPathVisit) bool {
-
-		log := log.WithField(logFSField, v.Path.ToString())
-
-		if v.FilledIn {
-			if _, exists := localFilesystemState[v.Path.ToString()]; exists {
-				// No need to verify if this is a placeholder or not. It is sufficient
-				// to know we can add child filesystems to it
-				return true
-			}
-			log.Debug("create placeholder filesystem")
-			err = zfs.ZFSCreatePlaceholderFilesystem(v.Path)
-			if err != nil {
-				log.Error("cannot create placeholder filesystem")
-				return false
-			}
-			return true
-		}
-
-		m, ok := replMapping[v.Path.ToString()]
-		if !ok {
-			panic("internal inconsistency: replMapping should contain mapping for any path that was not filled in by WalkTopDown()")
-		}
-
-		log = log.WithField(logMapToField, m.Remote.ToString()).
-			WithField(logMapFromField, m.Local.ToString())
-
-		log.Debug("examing local filesystem state")
-		localState, localExists := localFilesystemState[m.Local.ToString()]
-		var versions []zfs.FilesystemVersion
-		switch {
-		case !localExists:
-			log.Info("local filesystem does not exist")
-		case localState.Placeholder:
-			log.Info("local filesystem is marked as placeholder")
-		default:
-			log.Debug("local filesystem exists")
-			log.Debug("requesting local filesystem versions")
-			if versions, err = zfs.ZFSListFilesystemVersions(m.Local, nil); err != nil {
-				log.WithError(err).Error("cannot get local filesystem versions")
-				return false
-			}
-		}
-
-		log.Info("requesting remote filesystem versions")
-		r := FilesystemVersionsRequest{
-			Filesystem: m.Remote,
-		}
-		var theirVersions []zfs.FilesystemVersion
-		if err = remote.Call("FilesystemVersionsRequest", &r, &theirVersions); err != nil {
-			log.WithError(err).Error("cannot get remote filesystem versions")
-			log.Warn("stopping replication for all filesystems mapped as children of receiving filesystem")
-			return false
-		}
-
-		log.Debug("computing diff between remote and local filesystem versions")
-		diff := zfs.MakeFilesystemDiff(versions, theirVersions)
-		log.WithField("diff", diff).Debug("diff between local and remote filesystem")
-
-		if localState.Placeholder && diff.Conflict != zfs.ConflictAllRight {
-			panic("internal inconsistency: local placeholder implies ConflictAllRight")
-		}
-
-		switch diff.Conflict {
-		case zfs.ConflictAllRight:
-
-			log.WithField("replication_policy", pull.InitialReplPolicy).Info("performing initial sync, following policy")
-
-			if pull.InitialReplPolicy != InitialReplPolicyMostRecent {
-				panic(fmt.Sprintf("policy '%s' not implemented", pull.InitialReplPolicy))
-			}
-
-			snapsOnly := make([]zfs.FilesystemVersion, 0, len(diff.MRCAPathRight))
-			for s := range diff.MRCAPathRight {
-				if diff.MRCAPathRight[s].Type == zfs.Snapshot {
-					snapsOnly = append(snapsOnly, diff.MRCAPathRight[s])
-				}
-			}
-
-			if len(snapsOnly) < 1 {
-				log.Warn("cannot perform initial sync: no remote snapshots")
-				return false
-			}
-
-			r := InitialTransferRequest{
-				Filesystem:        m.Remote,
-				FilesystemVersion: snapsOnly[len(snapsOnly)-1],
-			}
-
-			log.WithField("version", r.FilesystemVersion).Debug("requesting snapshot stream")
-
-			var stream io.Reader
-
-			if err = remote.Call("InitialTransferRequest", &r, &stream); err != nil {
-				log.WithError(err).Error("cannot request initial transfer")
-				return false
-			}
-			log.Debug("received initial transfer request response")
-
-			log.Debug("invoke zfs receive")
-			watcher := util.IOProgressWatcher{Reader: stream}
-			watcher.KickOff(1*time.Second, func(p util.IOProgress) {
-				log.WithField("total_rx", p.TotalRX).Info("progress on receive operation")
-			})
-
-			recvArgs := []string{"-u"}
-			if localState.Placeholder {
-				log.Info("receive with forced rollback to replace placeholder filesystem")
-				recvArgs = append(recvArgs, "-F")
-			}
-
-			if err = zfs.ZFSRecv(m.Local, &watcher, recvArgs...); err != nil {
-				log.WithError(err).Error("canot receive stream")
-				return false
-			}
-			log.WithField("total_rx", watcher.Progress().TotalRX).
-				Info("finished receiving stream")
-
-			log.Debug("configuring properties of received filesystem")
-			if err = zfs.ZFSSet(m.Local, "readonly", "on"); err != nil {
-				log.WithError(err).Error("cannot set readonly property")
-			}
-
-			log.Info("finished initial transfer")
-			return true
-
-		case zfs.ConflictIncremental:
-
-			if len(diff.IncrementalPath) < 2 {
-				log.Info("remote and local are in sync")
-				return true
-			}
-
-			log.Info("following incremental path from diff")
-			var pathRx uint64
-
-			for i := 0; i < len(diff.IncrementalPath)-1; i++ {
-
-				from, to := diff.IncrementalPath[i], diff.IncrementalPath[i+1]
-
-				log, _ := log.WithField(logIncFromField, from.Name).WithField(logIncToField, to.Name), 0
-
-				log.Debug("requesting incremental snapshot stream")
-				r := IncrementalTransferRequest{
-					Filesystem: m.Remote,
-					From:       from,
-					To:         to,
-				}
-				var stream io.Reader
-				if err = remote.Call("IncrementalTransferRequest", &r, &stream); err != nil {
-					log.WithError(err).Error("cannot request incremental snapshot stream")
-					return false
-				}
-
-				log.Debug("invoking zfs receive")
-				watcher := util.IOProgressWatcher{Reader: stream}
-				watcher.KickOff(1*time.Second, func(p util.IOProgress) {
-					log.WithField("total_rx", p.TotalRX).Info("progress on receive operation")
-				})
-
-				if err = zfs.ZFSRecv(m.Local, &watcher); err != nil {
-					log.WithError(err).Error("cannot receive stream")
-					return false
-				}
-
-				totalRx := watcher.Progress().TotalRX
-				pathRx += totalRx
-				log.WithField("total_rx", totalRx).Info("finished incremental transfer")
-
-			}
-
-			log.WithField("total_rx", pathRx).Info("finished following incremental path")
-			return true
-
-		case zfs.ConflictNoCommonAncestor:
-			fallthrough
-		case zfs.ConflictDiverged:
-
-			var jsonDiff bytes.Buffer
-			if err := json.NewEncoder(&jsonDiff).Encode(diff); err != nil {
-				log.WithError(err).Error("cannot JSON-encode diff")
-				return false
-			}
-
-			var problem, resolution string
-
-			switch diff.Conflict {
-			case zfs.ConflictNoCommonAncestor:
-				problem = "remote and local filesystem have snapshots, but no common one"
-				resolution = "perform manual establish a common snapshot history"
-			case zfs.ConflictDiverged:
-				problem = "remote and local filesystem share a history but have diverged"
-				resolution = "perform manual replication or delete snapshots on the receiving" +
-					"side  to establish an incremental replication parse"
-			}
-
-			log.WithField("diff", jsonDiff.String()).
-				WithField("problem", problem).
-				WithField("resolution", resolution).
-				Error("manual conflict resolution required")
-
-			return false
-
-		}
-
-		panic("should not be reached")
-
-	})
-
-	return
-
-}

cmd/sampleconf/localbackup/host1.yml

@@ -1,27 +0,0 @@
-jobs:
-- name: mirror_local
-  type: local
-
-  # snapshot the filesystems matched by the left-hand-side of the mapping
-  # every 10m with zrepl_ as prefix
-  mapping: {
-    "zroot/var/db<":    "storage/backups/local/zroot/var/db",
-    "zroot/usr/home<":  "storage/backups/local/zroot/usr/home",
-    "zroot/var/tmp": "!", #don't backup /tmp
-  }
-  snapshot_prefix: zrepl_
-  interval: 10m
-  initial_repl_policy: most_recent
-
-  # keep one hour of 10m interval snapshots of filesystems matched by
-  # the left-hand-side of the mapping
-  prune_lhs:
-    policy: grid
-    grid: 1x1h(keep=all)
-
-  # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
-  prune_rhs:
-    policy: grid
-    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-
-

cmd/sampleconf/pullbackup/backuphost.yml

@@ -1,27 +0,0 @@
-jobs:
-- name: fullbackup_prod1
-  type: pull
-  # connect to remote using ssh / stdinserver command
-  connect:
-    type: ssh+stdinserver
-    host: prod1.example.com
-    user: root
-    port: 22
-    identity_file: /root/.ssh/id_ed25519
-
-  # pull (=ask for new snapshots) every 10m, prune afterwards
-  # this will leave us at most 10m behind production
-  interval: 10m
-
-  # pull all offered filesystems to storage/backups/zrepl/pull/prod1.example.com
-  mapping: {
-    "<":"storage/backups/zrepl/pull/prod1.example.com"
-  }
-  initial_repl_policy: most_recent
-
-  # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
-  snapshot_prefix: zrepl_
-  prune:
-    policy: grid
-    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-

cmd/sampleconf/pullbackup/productionhost.yml

@@ -1,45 +0,0 @@
-global:
-  serve:
-    stdinserver:
-      # Directory where AF_UNIX sockets for stdinserver command are placed.
-      #
-      # `zrepl stdinserver CLIENT_IDENTITY`
-      #   * connects to the socket in $sockdir/CLIENT_IDENTITY
-      #   * sends its stdin / stdout file descriptors to the `zrepl daemon` process (see cmsg(3))
-      #   * does nothing more
-      #
-      # This enables a setup where `zrepl daemon` is not directly exposed to the internet
-      # but instead all traffic is tunnelled through SSH.
-      # The server with the source job has an authorized_keys file entry for the public key
-      # used by the corresponding pull job
-      #
-      #   command="/mnt/zrepl stdinserver CLIENT_IDENTITY" ssh-ed25519 AAAAC3NzaC1E... zrepl@pullingserver
-      #
-      # Below is the default value.
-      sockdir: /var/run/zrepl/stdinserver
-
-jobs:
-
-- name: fullbackup_prod1
-  # expect remote to connect via ssh+stdinserver with fullbackup_prod1 as client_identity
-  type: source
-  serve:
-    type: stdinserver # see global.serve.stdinserver for explanation
-    client_identity: fullbackup_prod1
-
-  # snapshot these filesystems every 10m with zrepl_ as prefix
-  datasets: {
-    "zroot/var/db<": "ok",
-    "zroot/usr/home<": "ok",
-    "zroot/var/tmp": "!", #don't backup /tmp
-  }
-  snapshot_prefix: zrepl_
-  interval: 10m
-
-
-  # keep a one day window 10m interval snapshots in case pull doesn't work (link down, etc)
-  # (we cannot keep more than one day because this host will run out of disk space)
-  prune:
-    policy: grid
-    grid: 1x1d(keep=all)
-

cmd/sampleconf/pushbackup/backuphost.yml

@@ -1,20 +0,0 @@
-jobs:
-- name: fullbackup_prod1
-
-  # expect remote to connect via ssh+stdinserver with fullbackup_prod1 as client_identity
-  type: push-sink
-  serve:
-    type: stdinserver
-    client_identity: fullbackup_prod1
-
-  # map all pushed datasets to storage/backups/zrepl/sink/prod1.example.com
-  mapping: {
-    "<":"storage/backups/zrepl/sink/prod1.example.com"
-  }
-
-  # follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
-  prune:
-    policy: grid
-    grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-
-

cmd/sampleconf/pushbackup/productionhost.yml

@@ -1,26 +0,0 @@
-jobs:
-- name: fullbackup_prod1
-
-  # connect to remote using ssh / stdinserver command
-  type: push
-  connect:
-      type: ssh+stdinserver
-      host: prod1.example.com
-      user: root
-      port: 22
-      identity_file: /root/.ssh/id_ed25519
-
-  # snapshot these datsets every 10m with zrepl_ as prefix
-  datasets: {
-    "zroot/var/db<": "ok",
-    "zroot/usr/home<": "!",
-  }
-  snapshot_prefix: zrepl_
-  interval: 10m
-
-  # keep a one day window 10m interval snapshots in case push doesn't work (link down, etc)
-  # (we cannot keep more than one day because this host will run out of disk space)
-  prune:
-    policy: grid
-    grid: 1x1d(keep=all)
-

cmd/sampleconf/random/debugging.yml

@@ -1,33 +0,0 @@
-global:
-  serve:
-    stdinserver:
-      sockdir: /var/run/zrepl/stdinserver
-
-jobs:
-
-- name: debian2_pull
-  # JOB DEBUGGING OPTIONS
-  # should be equal for all job types, but each job implements the debugging itself
-  # => consult job documentation for supported options
-  debug:
-    conn: # debug the io.ReadWriteCloser connection
-      read_dump: /tmp/connlog_read   # dump results of Read() invocations to this file
-      write_dump: /tmp/connlog_write # dump results of Write() invocations to this file
-    rpc: # debug the RPC protocol implementation
-      log: true # log output from rpc layer to the job log
-
-  # ... just to make the unit tests pass.
-  # check other examples, e.g. localbackup or pullbackup for what the sutff below means
-  type: source
-  serve:
-    type: stdinserver
-    client_identity: debian2
-  datasets: {
-    "pool1/db<": ok
-  }
-  snapshot_prefix: zrepl_
-  interval: 1s
-  prune:
-    policy: grid
-    grid: 1x10s(keep=all)
-

cmd/sampleconf/random/logging.yml

@@ -1,20 +0,0 @@
-global:
-  logging:
-    stdout:
-      level: warn
-      format: human
-    tcp:
-      level: debug
-      format: json
-      net: tcp
-      address: 127.0.0.1:8080
-      retry_interval: 1s
-      tls: # if not specified, use plain TCP
-        ca: sampleconf/random/logging/logserver.crt
-        cert: sampleconf/random/logging/client.crt
-        key: sampleconf/random/logging/client.key
-    syslog:
-      enable: true
-      format: logfmt
-
-jobs: []

cmd/sampleconf/random/logging/client.crt

@@ -1,19 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDIzCCAgsCAQEwDQYJKoZIhvcNAQELBQAwWTELMAkGA1UEBhMCQVUxEzARBgNV
-BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0
-ZDESMBAGA1UEAwwJbG9nc2VydmVyMB4XDTE3MDkyNDEyMzAzNloXDTE3MTAyNDEy
-MzAzNlowVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNV
-BAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xpZW50MIIB
-IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/xJTUlqApeJGzRD+w2J8sZS
-Bo+s+04T987L/M6gaCo8aDSTEb/ZH3XSoU5JEmO6kPpwNNapOsaEhTCjndZQdm5F
-uqiUtAg1uW0HCkBEIDkGr9bFHDKzpewGmmMgfQ2+hfiBR/4ZCrc/vd9P0W9BiWQS
-Dtc7p22XraWPVL8HlSz5K/Ih+V6i8O+kBltZkusiJh2bWPoRp/netiTZuc6du+Wp
-kpWp1OBaTU4GXIAlLj5afF14BBphRQK983Yhaz53BkA7OQ76XxowynMjmuLQVGmK
-f1R9zEJuohTX9XIr1tp/ueRHcS4Awk6LcNZUMCV6270FNSIw2f4hbOZvep+t2wID
-AQABMA0GCSqGSIb3DQEBCwUAA4IBAQACK3OeNzScpiNwz/jpg/usQzvXbZ/wDvml
-YLjtzn/A65ox8a8BhxvH1ydyoCM2YAGYX7+y7qXJnMgRO/v8565CQIVcznHhg9ST
-3828/WqZ3bXf2DV5GxKKQf7hPmBnyVUUhn/Ny91MECED27lZucWiX/bczN8ffDeh
-M3+ngezcJxsOBd4x0gLrqIJCoaFRSeepOaFEW6GHQ8loxE9GmA7FQd2phIpJHFSd
-Z7nQl7X5C1iN2OboEApJHwtmNVC45UlOpg53vo2sDTLhSfdogstiWi8x1HmvhIGM
-j3XHs0Illvo9OwVrmgUph8zQ7pvr/AFrTOIbhgzl/9uVUk5ApwFM
------END CERTIFICATE-----

cmd/sampleconf/random/logging/client.csr

@@ -1,16 +0,0 @@
------BEGIN CERTIFICATE REQUEST-----
-MIICmzCCAYMCAQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx
-ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xp
-ZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/xJTUlqApeJGzRD
-+w2J8sZSBo+s+04T987L/M6gaCo8aDSTEb/ZH3XSoU5JEmO6kPpwNNapOsaEhTCj
-ndZQdm5FuqiUtAg1uW0HCkBEIDkGr9bFHDKzpewGmmMgfQ2+hfiBR/4ZCrc/vd9P
-0W9BiWQSDtc7p22XraWPVL8HlSz5K/Ih+V6i8O+kBltZkusiJh2bWPoRp/netiTZ
-uc6du+WpkpWp1OBaTU4GXIAlLj5afF14BBphRQK983Yhaz53BkA7OQ76XxowynMj
-muLQVGmKf1R9zEJuohTX9XIr1tp/ueRHcS4Awk6LcNZUMCV6270FNSIw2f4hbOZv
-ep+t2wIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAKnlr0Qs5KYF85u2YA7DJ5pL
-HwAx+qNoNbox5CS1aynrDBpDTWLaErviUJ+4WxRlRyTMEscMOIOKajbYhqqFmtGZ
-mu3SshZnFihErw8TOQMyU1LGGG+l6r+6ve5TciwJRLla2Y75z7izr6cyvQNRWdLr
-PvxL1/Yqr8LKha12+7o28R4SLf6/GY0GcedqoebRmtuwA/jES0PuGauEUD5lH4cj
-Me8sqRrB+IMHQ5j8hlJX4DbA8UQRUBL64sHkQzeQfWu+qkWmS5I19CFfLNrcH+OV
-yhyjGfN0q0jHyHdpckBhgzS7IIdo6P66AIlm4qpHM7Scra3JaGM7oaZPamJ6f8U=
------END CERTIFICATE REQUEST-----

cmd/sampleconf/random/logging/client.key

@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC3/ElNSWoCl4kb
-NEP7DYnyxlIGj6z7ThP3zsv8zqBoKjxoNJMRv9kfddKhTkkSY7qQ+nA01qk6xoSF
-MKOd1lB2bkW6qJS0CDW5bQcKQEQgOQav1sUcMrOl7AaaYyB9Db6F+IFH/hkKtz+9
-30/Rb0GJZBIO1zunbZetpY9UvweVLPkr8iH5XqLw76QGW1mS6yImHZtY+hGn+d62
-JNm5zp275amSlanU4FpNTgZcgCUuPlp8XXgEGmFFAr3zdiFrPncGQDs5DvpfGjDK
-cyOa4tBUaYp/VH3MQm6iFNf1civW2n+55EdxLgDCTotw1lQwJXrbvQU1IjDZ/iFs
-5m96n63bAgMBAAECggEAF4om0sWe06ARwbJJNFjCGpa3LfG5/xk5Qs5pmPnS2iD1
-Q5veaTnzjKvlfA/pF3o9B4mTS59fXY7Cq8vSU0J1XwGy2DPzeqlGPmgtq2kXjkvd
-iCfhZj8ybvsoyR3/rSBSDRADcnOXPqC9fgyRSMmESBDOoql1D3HdIzF4ii46ySIU
-/XQvExS6NWifbP+Ue6DETV8NhreO5PqjeXLITQhhndtc8MDL/8eCNOyN8XjYIWKX
-smlBYtRQYOOY9BHOQgUn6yvPHrtKJNKci+qcQNvWir66mBhY1o40MH5wTIV+8yP2
-Vbm/VzoNKIYgeROsilBW7QTwGvkDn3R11zeTqfUNSQKBgQD0eFzhJAEZi4uBw6Tg
-NKmBC5Y1IHPOsb5gKPNz9Z9j4qYRDySgYl6ISk+2EdhgUCo1NmTk8EIPQjIerUVf
-S+EogFnpsj8U9LR3OM79DaGkNULxrHqhd209/g8DtVgk7yjkxL4vmVOv8qpHMp/7
-eWsylN7AOxj2RB/eXYQBPrw+jQKBgQDAqae9HasLmvpJ9ktTv30yZSKXC+LP4A0D
-RBBmx410VpPd4CvcpCJxXmjer6B7+9L1xHYP2pvsnMBid5i0knuvyK28dYy7fldl
-CzWvb+lqNA5YYPFXQED4oEdihlQczoI1Bm06SFizeAKD1Q9e2c+lgbR/51j8xuXi
-twvhMj/YBwKBgQCZw97/iQrcC2Zq7yiUEOuQjD4lGk1c83U/vGIsTJC9XcCAOFsc
-OeMlrD/oz96d7a4unBDn4qpaOJOXsfpRT0PGmrxy/jcpMiUUW/ntNpa11v5NTeQw
-DRL8DAFbnsNbL8Yz5f+Nps35fBNYBuKTZLJlNTfKByHTO9QjpAQ0WEZEvQKBgQCi
-Ovm83EuYVSKmvxcE6Tyx/8lVqTOO2Vn7wweQlD4/lVujvE0S2L8L+XSS9w5K+GzW
-eFz10p3zarbw80YJ30L5bSEmjVE43BUZR4woMzM4M6dUsiTm1HshIE2b4ALZ0uZ/
-Ye794ceXL9nmSrVLqFsaQZLNFPCwwYb4FiyRry9lZwKBgAO9VbWcN8SEeBDKo3z8
-yRbRTc6sI+AdKY44Dfx0tqOPmTjO3mE4X1GU4sbfD2Bvg3DdjwTuxxC/jHaKu0GG
-dTM0CbrZGbDAj7E87SOcN/PWEeBckSvuQq5H3DQfwIpTmlS1l5oZn9CxRGbLqC2G
-ifnel8XWUG0ROybsr1tk4mzW
------END PRIVATE KEY-----

cmd/sampleconf/random/logging/logserver.crt

@@ -1,21 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDiDCCAnCgAwIBAgIJALhp/WvTQeg/MA0GCSqGSIb3DQEBCwUAMFkxCzAJBgNV
-BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
-aWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMMCWxvZ3NlcnZlcjAeFw0xNzA5MjQxMjI3
-MDRaFw0yNzA5MjIxMjI3MDRaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21l
-LVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNV
-BAMMCWxvZ3NlcnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKs3
-TLYfXhV3hap71tOkhPQlM+m0EKRAo8Nua50Cci5UhDo4JkVpyYok1h+NFkqmjU2b
-IiIuGvsZZPOWYjbWWnSJE4+n5pBFBzcfNQ4d8xVxjANImFn6Tcehhj0WkbDIv/Ge
-364XUgywS7u3EGQj/FO7vZ8KHlUxBHNuPIOPHftwIVRyleh5K32UyBaSpSmnqGos
-rvI1byMuznavcZpOs4vlebZ+Jy6a20iKf9fj/0f0t0O+F5x3JIk07D3zSywhJ4RM
-M0mGIUmYXbh2SMh+f61KDZLDANpz/pMAPbUJe0mxEtBf0tnwK1gEqc3SLwA0EwiM
-8Hnn2iaH5Ln20UE3LOkCAwEAAaNTMFEwHQYDVR0OBBYEFDXoDcwx9SngzZcRYCeP
-BplBecfiMB8GA1UdIwQYMBaAFDXoDcwx9SngzZcRYCePBplBecfiMA8GA1UdEwEB
-/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBADyNvs4AA91x3gurQb1pcPVhK6nR
-mkYSTN1AsDKSRi/X2iCUmR7G7FlF7XW8mntTpHvVzcs+gr94WckH5wqEOA5iZnaw
-PXUWexmdXUge4hmC2q6kBQ5e2ykhSJMRVZXvOLZOZV9qitceamHESV1cKZSNMvZM
-aCSVA1RK61/nUzs04pVp5PFPv9gFxJp9ki39FYFdsgZmM5RZ5I/FqxxvTJzu4RnH
-VPjsMopzARYwJw6dV2bKdFSYOE8B/Vs3Yv0GxjrABw2ko4PkBPTjLIz22x6+Hd9r
-K9BQi4pVmQfvppF5+SORSftlHSS+N47b0DD1rW1f5R6QGi71dFuJGikOwvY=
------END CERTIFICATE-----

cmd/sampleconf/random/logging/logserver.key

@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCrN0y2H14Vd4Wq
-e9bTpIT0JTPptBCkQKPDbmudAnIuVIQ6OCZFacmKJNYfjRZKpo1NmyIiLhr7GWTz
-lmI21lp0iROPp+aQRQc3HzUOHfMVcYwDSJhZ+k3HoYY9FpGwyL/xnt+uF1IMsEu7
-txBkI/xTu72fCh5VMQRzbjyDjx37cCFUcpXoeSt9lMgWkqUpp6hqLK7yNW8jLs52
-r3GaTrOL5Xm2ficumttIin/X4/9H9LdDvhecdySJNOw980ssISeETDNJhiFJmF24
-dkjIfn+tSg2SwwDac/6TAD21CXtJsRLQX9LZ8CtYBKnN0i8ANBMIjPB559omh+S5
-9tFBNyzpAgMBAAECggEBAIY8ZwJq+WKvQLb3POjWFf8so9TY/ispGrwAeJKy9j5o
-uPrERw0o8YBDfTVjclS43BQ6Srqtly3DLSjlgL8ps+WmCxYYN2ZpGE0ZRIl65bis
-O2/fnML+wbiAZTTD2xnVatfPDeP6GLQmDFpyHoHEzPIBQZvNXRbBxZGSnhMvQ/x7
-FhqSBQG4kf3b1XDCENIbFEVOBOCg7WtMiIgjEGS7QnW3I65/Zt+Ts1LXRZbz+6na
-Gmi0PGHA/oLUh1NRzsF4zuZn6fFzja5zw4mkt+JvCWEoxg1QhRAxRp6QQwmZ6MIc
-1rw1D4Z+c5UEKyqHeIwZj4M6UNPhCfTXVm47c9eSiGECgYEA4U8pB+7eRo2fqX0C
-nWsWMcmsULJvwplQnUSFenUayPn3E8ammS/ZBHksoKhj82vwIdDbtS1hQZn8Bzsi
-atc8au0wz0YRDcVDzHX4HknXVQayHtP/FTPeSr5hwpoY8vhEbySuxBTBkXCrp4dx
-u5ErfOiYEP3Q1ZvPRywelrATu20CgYEAwonV5dgOcen/4oAirlnvufc2NfqhAQwJ
-FJ/JSVMAcXxPYu3sZMv0dGWrX8mLc+P1+XMCuV/7eBM/vU2LbDzmpeUV8sJfB2jw
-wyKqKXZwBgeq60btriA4f+0ElwRGgU2KSiniUuuTX2JmyftFQx4cVAQRCFk27NY0
-09psSsYyre0CgYBo6unabdtH029EB5iOIW3GZXk+Yrk0TxyA/4WAjsOYTv5FUT4H
-G4bdVGf5sDBLDDpYJOAKsEUXvVLlMx5FzlCuIiGWg7QxS2jU7yJJSG1jhKixPlsM
-Toj3GUyAyC1SB1Ymw1g2qsuwpFzquGG3zFQJ6G3Xi7oRnmqZY+wik3+8yQKBgB11
-SdKYOPe++2SNCrNkIw0CBk9+OEs0S1u4Jn7X9sU4kbzlUlqhF89YZe8HUfqmlmTD
-qbHwet/f6lL8HxSw1Cxi2EP+cu1oUqz53tKQgL4pAxTFlNA9SND2Ty+fEh4aY8p/
-NSphSduzxuTnC8HyGVAPnZSqDcsnVLCP7r4T7TCxAoGAbJygkkk/gZ9pT4fZoIaq
-8CMR8FTfxtkwCuZsWccSMUOWtx9nqet3gbCpKHfyoYZiKB4ke+lnUz4uFS16Y3hG
-kN0hFfvfoNa8eB2Ox7vs60cMMfWJac0H7KSaDDy+EvbhE2KtQADT0eWxMyhzGR8p
-5CbIivB0QCjeQIA8dOQpE8E=
------END PRIVATE KEY-----

cmd/sampleconf/zrepl.yml

@@ -1,111 +0,0 @@
-remotes:
-  offsite_backups:
-    transport:
-      ssh:
-        host: 192.168.122.6
-        user: root
-        port: 22
-        identity_file: /etc/zrepl/identities/offsite_backups
-
-push:
-
-  offsite:
-    to: offsite_backups
-    filter: {
-      # like in pull_acls
-      "tank/var/db<":   ok,
-      "tank/usr/home<": ok,
-    }
-
-pull:
-
-  offsite:
-    from: offsite_backups
-    mapping: {
-      # like in sinks
-    }
-
-  # local replication, only allowed in pull mode
-  # the from name 'local' is reserved for this purpose
-  homemirror:
-    from: local
-    repeat:
-      interval: 15m
-    mapping: {
-      "tank/usr/home":"mirrorpool/foo/bar"
-    }
-
-sink:
-
-  db1:
-    mapping: {
-      # direct mapping
-      "ssdpool/var/db/postgresql9.6":"zroot/backups/db1/pg_data"
-    }
-
-  mirror1:
-    mapping: {
-      # "<" subtree wildcard matches the dataset left of < and all its children
-      "tank/foo/bar<":"zroot/backups/mirror1" 
-    }
-
-  mirror2:
-    # more specific path patterns win over less specific ones
-    # direct mappings win over subtree wildcards
-    # detailed rule precedence: check unit tests & docs for exact behavior
-    # TODO subcommand to test a mapping & filter
-    mapping: {
-      "tank<":                       "zroot/backups/mirror1/tank1",
-      "tank/cdn/root<":              "storage/cdn/root",
-      "tank/legacydb":               "legacypool/backups/legacydb",
-    }
-
-
-pull_acl:
-
-  office_backup:
-    filter: {
-      # valid filter results (right hand side): ok, omit
-      # default is to omit
-      # rule precedence is same as for mappings
-      "tank<":         omit,
-      "tank/usr/home": ok,
-    }
-
-
-prune:
-
-  clean_backups:
-    policy: grid
-    grid: 6x10m | 24x1h | 7x1d | 5 x 1w | 4 x 5w
-    dataset_filter: {
-      "tank/backups/legacyscript<": omit,
-      "tank/backups<":              ok,
-    }
-    snapshot_filter: {
-      prefix: zrepl_
-    }
-
-  hfbak_prune: # cleans up after hfbak autosnap job
-    policy: grid
-    grid: 1x1m(keep=all)
-    dataset_filter: {
-      "pool1<": ok
-    }
-    snapshot_filter: {
-      prefix: zrepl_hfbak_
-    }
-    repeat:
-      interval: 10s
-
-autosnap:
-
-  hfbak:
-    prefix: zrepl_hfbak_
-    interval: 1s
-    dataset_filter: {
-      "pool1<": ok
-    }
-    # prune: hfbak_prune
-    # future versions may inline the retention policy here, but for now,
-    # pruning has to be triggered manually (it's safe to run autosnap + prune in parallel)

cmd/stdinserver.go

@@ -1,93 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/ftrvxmtrx/fd"
-	"github.com/spf13/cobra"
-	"io"
-	"log"
-	"net"
-)
-
-var StdinserverCmd = &cobra.Command{
-	Use:   "stdinserver CLIENT_IDENTITY",
-	Short: "start in stdinserver mode (from authorized_keys file)",
-	Run:   cmdStdinServer,
-}
-
-func init() {
-	RootCmd.AddCommand(StdinserverCmd)
-}
-
-func cmdStdinServer(cmd *cobra.Command, args []string) {
-
-	log := log.New(os.Stderr, "", log.LUTC|log.Ldate|log.Ltime)
-
-	die := func() {
-		log.Printf("stdinserver exiting after fatal error")
-		os.Exit(1)
-	}
-
-	conf, err := ParseConfig(rootArgs.configFile)
-	if err != nil {
-		log.Printf("error parsing config: %s", err)
-		die()
-	}
-
-	if len(args) != 1 || args[0] == "" {
-		err = fmt.Errorf("must specify client_identity as positional argument")
-		die()
-	}
-	identity := args[0]
-
-	unixaddr, err := stdinserverListenerSocket(conf.Global.Serve.Stdinserver.SockDir, identity)
-	if err != nil {
-		log.Printf("%s", err)
-		os.Exit(1)
-	}
-
-	log.Printf("opening control connection to zrepld via %s", unixaddr)
-	conn, err := net.DialUnix("unix", nil, unixaddr)
-	if err != nil {
-		log.Printf("error connecting to zrepld: %s", err)
-		die()
-	}
-
-	log.Printf("sending stdin and stdout fds to zrepld")
-	err = fd.Put(conn, os.Stdin, os.Stdout)
-	if err != nil {
-		log.Printf("error: %s", err)
-		die()
-	}
-
-	log.Printf("waiting for zrepld to close control connection")
-	for {
-
-		var buf [64]byte
-		n, err := conn.Read(buf[:])
-		if err == nil && n != 0 {
-			log.Printf("protocol error: read expected to timeout or EOF returned bytes")
-		}
-
-		if err == io.EOF {
-			log.Printf("zrepld closed control connection, terminating")
-			break
-		}
-
-		neterr, ok := err.(net.Error)
-		if !ok {
-			log.Printf("received unexpected error type: %T %s", err, err)
-			die()
-		}
-		if !neterr.Timeout() {
-			log.Printf("receivd unexpected net.Error (not a timeout): %s", neterr)
-			die()
-		}
-		// Read timed out, as expected
-	}
-
-	return
-
-}

cmd/test.go

@@ -1,214 +0,0 @@
-package cmd
-
-import (
-	"os"
-
-	"bytes"
-	"context"
-	"fmt"
-	"sort"
-	"strings"
-
-	"github.com/kr/pretty"
-	"github.com/spf13/cobra"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/zfs"
-	"time"
-)
-
-var testCmd = &cobra.Command{
-	Use:              "test",
-	Short:            "test configuration",
-	PersistentPreRun: testCmdGlobalInit,
-}
-
-var testCmdGlobal struct {
-	log  Logger
-	conf *Config
-}
-
-var testConfigSyntaxCmd = &cobra.Command{
-	Use:   "config",
-	Short: "parse config file and dump parsed datastructure",
-	Run:   doTestConfig,
-}
-
-var testDatasetMapFilter = &cobra.Command{
-	Use:     "pattern jobname test/zfs/dataset/path",
-	Short:   "test dataset mapping / filter specified in config",
-	Example: ` zrepl test pattern prune.clean_backups tank/backups/legacyscript/foo`,
-	Run:     doTestDatasetMapFilter,
-}
-
-var testPrunePolicyArgs struct {
-	side        PrunePolicySide
-	showKept    bool
-	showRemoved bool
-}
-
-var testPrunePolicyCmd = &cobra.Command{
-	Use:   "prune jobname",
-	Short: "do a dry-run of the pruning part of a job",
-	Run:   doTestPrunePolicy,
-}
-
-func init() {
-	RootCmd.AddCommand(testCmd)
-	testCmd.AddCommand(testConfigSyntaxCmd)
-	testCmd.AddCommand(testDatasetMapFilter)
-
-	testPrunePolicyCmd.Flags().VarP(&testPrunePolicyArgs.side, "side", "s", "prune_lhs (left) or prune_rhs (right)")
-	testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showKept, "kept", false, "show kept snapshots")
-	testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showRemoved, "removed", true, "show removed snapshots")
-	testCmd.AddCommand(testPrunePolicyCmd)
-}
-
-func testCmdGlobalInit(cmd *cobra.Command, args []string) {
-
-	out := logger.NewOutlets()
-	out.Add(WriterOutlet{&NoFormatter{}, os.Stdout}, logger.Info)
-	log := logger.NewLogger(out, 1*time.Second)
-	testCmdGlobal.log = log
-
-	var err error
-	if testCmdGlobal.conf, err = ParseConfig(rootArgs.configFile); err != nil {
-		testCmdGlobal.log.Printf("error parsing config file: %s", err)
-		os.Exit(1)
-	}
-
-}
-
-func doTestConfig(cmd *cobra.Command, args []string) {
-
-	log, conf := testCmdGlobal.log, testCmdGlobal.conf
-
-	log.Printf("config ok")
-	log.Printf("%# v", pretty.Formatter(conf))
-	return
-}
-
-func doTestDatasetMapFilter(cmd *cobra.Command, args []string) {
-
-	log, conf := testCmdGlobal.log, testCmdGlobal.conf
-
-	if len(args) != 2 {
-		log.Printf("specify job name as first postitional argument, test input as second")
-		log.Printf(cmd.UsageString())
-		os.Exit(1)
-	}
-	n, i := args[0], args[1]
-
-	jobi, err := conf.LookupJob(n)
-	if err != nil {
-		log.Printf("%s", err)
-		os.Exit(1)
-	}
-
-	var mf *DatasetMapFilter
-	switch j := jobi.(type) {
-	case *PullJob:
-		mf = j.Mapping
-	case *SourceJob:
-		mf = j.Datasets
-	case *LocalJob:
-		mf = j.Mapping
-	default:
-		panic("incomplete implementation")
-	}
-
-	ip, err := zfs.NewDatasetPath(i)
-	if err != nil {
-		log.Printf("cannot parse test input as ZFS dataset path: %s", err)
-		os.Exit(1)
-	}
-
-	if mf.filterMode {
-		pass, err := mf.Filter(ip)
-		if err != nil {
-			log.Printf("error evaluating filter: %s", err)
-			os.Exit(1)
-		}
-		log.Printf("filter result: %v", pass)
-	} else {
-		res, err := mf.Map(ip)
-		if err != nil {
-			log.Printf("error evaluating mapping: %s", err)
-			os.Exit(1)
-		}
-		toStr := "NO MAPPING"
-		if res != nil {
-			toStr = res.ToString()
-		}
-		log.Printf("%s => %s", ip.ToString(), toStr)
-
-	}
-
-}
-
-func doTestPrunePolicy(cmd *cobra.Command, args []string) {
-
-	log, conf := testCmdGlobal.log, testCmdGlobal.conf
-
-	if cmd.Flags().NArg() != 1 {
-		log.Printf("specify job name as first positional argument")
-		log.Printf(cmd.UsageString())
-		os.Exit(1)
-	}
-
-	jobname := cmd.Flags().Arg(0)
-	jobi, err := conf.LookupJob(jobname)
-	if err != nil {
-		log.Printf("%s", err)
-		os.Exit(1)
-	}
-
-	jobp, ok := jobi.(PruningJob)
-	if !ok {
-		log.Printf("job doesn't do any prunes")
-		os.Exit(0)
-	}
-
-	log.Printf("job dump:\n%s", pretty.Sprint(jobp))
-
-	pruner, err := jobp.Pruner(testPrunePolicyArgs.side, true)
-	if err != nil {
-		log.Printf("cannot create test pruner: %s", err)
-		os.Exit(1)
-	}
-
-	log.Printf("start pruning")
-
-	ctx := context.WithValue(context.Background(), contextKeyLog, log)
-	result, err := pruner.Run(ctx)
-	if err != nil {
-		log.Printf("error running pruner: %s", err)
-		os.Exit(1)
-	}
-
-	sort.Slice(result, func(i, j int) bool {
-		return strings.Compare(result[i].Filesystem.ToString(), result[j].Filesystem.ToString()) == -1
-	})
-
-	var b bytes.Buffer
-	for _, r := range result {
-		fmt.Fprintf(&b, "%s\n", r.Filesystem.ToString())
-
-		if testPrunePolicyArgs.showKept {
-			fmt.Fprintf(&b, "\tkept:\n")
-			for _, v := range r.Keep {
-				fmt.Fprintf(&b, "\t- %s\n", v.Name)
-			}
-		}
-
-		if testPrunePolicyArgs.showRemoved {
-			fmt.Fprintf(&b, "\tremoved:\n")
-			for _, v := range r.Remove {
-				fmt.Fprintf(&b, "\t- %s\n", v.Name)
-			}
-		}
-
-	}
-
-	log.Printf("pruning result:\n%s", b.String())
-
-}

config/config.go

@@ -0,0 +1,712 @@
+package config
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log/syslog"
+	"os"
+	"reflect"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/robfig/cron/v3"
+	"github.com/zrepl/yaml-config"
+
+	"github.com/zrepl/zrepl/util/datasizeunit"
+	zfsprop "github.com/zrepl/zrepl/zfs/property"
+)
+
+type ParseFlags uint
+
+const (
+	ParseFlagsNone        ParseFlags = 0
+	ParseFlagsNoCertCheck ParseFlags = 1 << iota
+)
+
+type Config struct {
+	Jobs   []JobEnum `yaml:"jobs"`
+	Global *Global   `yaml:"global,optional,fromdefaults"`
+}
+
+func (c *Config) Job(name string) (*JobEnum, error) {
+	for _, j := range c.Jobs {
+		if j.Name() == name {
+			return &j, nil
+		}
+	}
+	return nil, fmt.Errorf("job %q not defined in config", name)
+}
+
+type JobEnum struct {
+	Ret interface{}
+}
+
+func (j JobEnum) Name() string {
+	var name string
+	switch v := j.Ret.(type) {
+	case *SnapJob:
+		name = v.Name
+	case *PushJob:
+		name = v.Name
+	case *SinkJob:
+		name = v.Name
+	case *PullJob:
+		name = v.Name
+	case *SourceJob:
+		name = v.Name
+	default:
+		panic(fmt.Sprintf("unknown job type %T", v))
+	}
+	return name
+}
+
+type ActiveJob struct {
+	Type               string                `yaml:"type"`
+	Name               string                `yaml:"name"`
+	Connect            ConnectEnum           `yaml:"connect"`
+	Pruning            PruningSenderReceiver `yaml:"pruning"`
+	Debug              JobDebugSettings      `yaml:"debug,optional"`
+	Replication        *Replication          `yaml:"replication,optional,fromdefaults"`
+	ConflictResolution *ConflictResolution   `yaml:"conflict_resolution,optional,fromdefaults"`
+}
+
+type ConflictResolution struct {
+	InitialReplication string `yaml:"initial_replication,optional,default=most_recent"`
+}
+
+type PassiveJob struct {
+	Type  string           `yaml:"type"`
+	Name  string           `yaml:"name"`
+	Serve ServeEnum        `yaml:"serve"`
+	Debug JobDebugSettings `yaml:"debug,optional"`
+}
+
+type SnapJob struct {
+	Type         string            `yaml:"type"`
+	Name         string            `yaml:"name"`
+	Pruning      PruningLocal      `yaml:"pruning"`
+	Debug        JobDebugSettings  `yaml:"debug,optional"`
+	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
+	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+}
+
+type SendOptions struct {
+	Encrypted        bool `yaml:"encrypted,optional,default=false"`
+	Raw              bool `yaml:"raw,optional,default=false"`
+	SendProperties   bool `yaml:"send_properties,optional,default=false"`
+	BackupProperties bool `yaml:"backup_properties,optional,default=false"`
+	LargeBlocks      bool `yaml:"large_blocks,optional,default=false"`
+	Compressed       bool `yaml:"compressed,optional,default=false"`
+	EmbeddedData     bool `yaml:"embedded_data,optional,default=false"`
+	Saved            bool `yaml:"saved,optional,default=false"`
+
+	BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
+}
+
+type RecvOptions struct {
+	// Note: we cannot enforce encrypted recv as the ZFS cli doesn't provide a mechanism for it
+	// Encrypted bool `yaml:"may_encrypted"`
+	// Future:
+	// Reencrypt bool `yaml:"reencrypt"`
+
+	Properties *PropertyRecvOptions `yaml:"properties,fromdefaults"`
+
+	BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
+
+	Placeholder *PlaceholderRecvOptions `yaml:"placeholder,fromdefaults"`
+}
+
+var _ yaml.Unmarshaler = &datasizeunit.Bits{}
+
+type BandwidthLimit struct {
+	Max            datasizeunit.Bits `yaml:"max,default=-1 B"`
+	BucketCapacity datasizeunit.Bits `yaml:"bucket_capacity,default=128 KiB"`
+}
+
+type Replication struct {
+	Protection  *ReplicationOptionsProtection  `yaml:"protection,optional,fromdefaults"`
+	Concurrency *ReplicationOptionsConcurrency `yaml:"concurrency,optional,fromdefaults"`
+}
+
+type ReplicationOptionsProtection struct {
+	Initial     string `yaml:"initial,optional,default=guarantee_resumability"`
+	Incremental string `yaml:"incremental,optional,default=guarantee_resumability"`
+}
+
+type ReplicationOptionsConcurrency struct {
+	Steps         int `yaml:"steps,optional,default=1"`
+	SizeEstimates int `yaml:"size_estimates,optional,default=4"`
+}
+
+type PropertyRecvOptions struct {
+	Inherit  []zfsprop.Property          `yaml:"inherit,optional"`
+	Override map[zfsprop.Property]string `yaml:"override,optional"`
+}
+
+type PlaceholderRecvOptions struct {
+	Encryption string `yaml:"encryption,default=unspecified"`
+}
+
+type PushJob struct {
+	ActiveJob    `yaml:",inline"`
+	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
+	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+	Send         *SendOptions      `yaml:"send,fromdefaults,optional"`
+}
+
+func (j *PushJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
+func (j *PushJob) GetSendOptions() *SendOptions      { return j.Send }
+
+type PullJob struct {
+	ActiveJob `yaml:",inline"`
+	RootFS    string                   `yaml:"root_fs"`
+	Interval  PositiveDurationOrManual `yaml:"interval"`
+	Recv      *RecvOptions             `yaml:"recv,fromdefaults,optional"`
+}
+
+func (j *PullJob) GetRootFS() string             { return j.RootFS }
+func (j *PullJob) GetAppendClientIdentity() bool { return false }
+func (j *PullJob) GetRecvOptions() *RecvOptions  { return j.Recv }
+
+type PositiveDurationOrManual struct {
+	Interval time.Duration
+	Manual   bool
+}
+
+var _ yaml.Unmarshaler = (*PositiveDurationOrManual)(nil)
+
+func (i *PositiveDurationOrManual) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	var s string
+	if err := u(&s, true); err != nil {
+		return err
+	}
+	switch s {
+	case "manual":
+		i.Manual = true
+		i.Interval = 0
+	case "":
+		return fmt.Errorf("value must not be empty")
+	default:
+		i.Manual = false
+		i.Interval, err = parsePositiveDuration(s)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type SinkJob struct {
+	PassiveJob `yaml:",inline"`
+	RootFS     string       `yaml:"root_fs"`
+	Recv       *RecvOptions `yaml:"recv,optional,fromdefaults"`
+}
+
+func (j *SinkJob) GetRootFS() string             { return j.RootFS }
+func (j *SinkJob) GetAppendClientIdentity() bool { return true }
+func (j *SinkJob) GetRecvOptions() *RecvOptions  { return j.Recv }
+
+type SourceJob struct {
+	PassiveJob   `yaml:",inline"`
+	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
+	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+	Send         *SendOptions      `yaml:"send,optional,fromdefaults"`
+}
+
+func (j *SourceJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
+func (j *SourceJob) GetSendOptions() *SendOptions      { return j.Send }
+
+type FilesystemsFilter map[string]bool
+
+type SnapshottingEnum struct {
+	Ret interface{}
+}
+
+type SnapshottingPeriodic struct {
+	Type     string            `yaml:"type"`
+	Prefix   string            `yaml:"prefix"`
+	Interval *PositiveDuration `yaml:"interval"`
+	Hooks    HookList          `yaml:"hooks,optional"`
+}
+
+type CronSpec struct {
+	Schedule cron.Schedule
+}
+
+var _ yaml.Unmarshaler = &CronSpec{}
+
+func (s *CronSpec) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
+	var specString string
+	if err := unmarshal(&specString, false); err != nil {
+		return err
+	}
+
+	// Use standard cron format.
+	// Disable the various "descriptors" (@daily, etc)
+	// They are just aliases to "top of hour", "midnight", etc.
+	parser := cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.SecondOptional)
+
+	sched, err := parser.Parse(specString)
+	if err != nil {
+		return errors.Wrap(err, "cron syntax invalid")
+	}
+	s.Schedule = sched
+	return nil
+}
+
+type SnapshottingCron struct {
+	Type   string   `yaml:"type"`
+	Prefix string   `yaml:"prefix"`
+	Cron   CronSpec `yaml:"cron"`
+	Hooks  HookList `yaml:"hooks,optional"`
+}
+
+type SnapshottingManual struct {
+	Type string `yaml:"type"`
+}
+
+type PruningSenderReceiver struct {
+	KeepSender   []PruningEnum `yaml:"keep_sender"`
+	KeepReceiver []PruningEnum `yaml:"keep_receiver"`
+}
+
+type PruningLocal struct {
+	Keep []PruningEnum `yaml:"keep"`
+}
+
+type LoggingOutletEnumList []LoggingOutletEnum
+
+func (l *LoggingOutletEnumList) SetDefault() {
+	def := `
+type: "stdout"
+time: true
+level: "warn"
+format: "human"
+`
+	s := &StdoutLoggingOutlet{}
+	err := yaml.UnmarshalStrict([]byte(def), &s)
+	if err != nil {
+		panic(err)
+	}
+	*l = []LoggingOutletEnum{LoggingOutletEnum{Ret: s}}
+}
+
+var _ yaml.Defaulter = &LoggingOutletEnumList{}
+
+type Global struct {
+	Logging    *LoggingOutletEnumList `yaml:"logging,optional,fromdefaults"`
+	Monitoring []MonitoringEnum       `yaml:"monitoring,optional"`
+	Control    *GlobalControl         `yaml:"control,optional,fromdefaults"`
+	Serve      *GlobalServe           `yaml:"serve,optional,fromdefaults"`
+}
+
+func Default(i interface{}) {
+	v := reflect.ValueOf(i)
+	if v.Kind() != reflect.Ptr {
+		panic(v)
+	}
+	y := `{}`
+	err := yaml.Unmarshal([]byte(y), v.Interface())
+	if err != nil {
+		panic(err)
+	}
+}
+
+type ConnectEnum struct {
+	Ret interface{}
+}
+
+type ConnectCommon struct {
+	Type string `yaml:"type"`
+}
+
+type TCPConnect struct {
+	ConnectCommon `yaml:",inline"`
+	Address       string        `yaml:"address,hostport"`
+	DialTimeout   time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
+}
+
+type TLSConnect struct {
+	ConnectCommon `yaml:",inline"`
+	Address       string        `yaml:"address,hostport"`
+	Ca            string        `yaml:"ca"`
+	Cert          string        `yaml:"cert"`
+	Key           string        `yaml:"key"`
+	ServerCN      string        `yaml:"server_cn"`
+	DialTimeout   time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
+}
+
+type SSHStdinserverConnect struct {
+	ConnectCommon        `yaml:",inline"`
+	Host                 string        `yaml:"host"`
+	User                 string        `yaml:"user"`
+	Port                 uint16        `yaml:"port"`
+	IdentityFile         string        `yaml:"identity_file"`
+	TransportOpenCommand []string      `yaml:"transport_open_command,optional"` //TODO unused
+	SSHCommand           string        `yaml:"ssh_command,optional"`            //TODO unused
+	Options              []string      `yaml:"options,optional"`
+	DialTimeout          time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
+}
+
+type LocalConnect struct {
+	ConnectCommon  `yaml:",inline"`
+	ListenerName   string        `yaml:"listener_name"`
+	ClientIdentity string        `yaml:"client_identity"`
+	DialTimeout    time.Duration `yaml:"dial_timeout,zeropositive,default=2s"`
+}
+
+type ServeEnum struct {
+	Ret interface{}
+}
+
+type ServeCommon struct {
+	Type string `yaml:"type"`
+}
+
+type TCPServe struct {
+	ServeCommon    `yaml:",inline"`
+	Listen         string            `yaml:"listen,hostport"`
+	ListenFreeBind bool              `yaml:"listen_freebind,default=false"`
+	Clients        map[string]string `yaml:"clients"`
+}
+
+type TLSServe struct {
+	ServeCommon      `yaml:",inline"`
+	Listen           string        `yaml:"listen,hostport"`
+	ListenFreeBind   bool          `yaml:"listen_freebind,default=false"`
+	Ca               string        `yaml:"ca"`
+	Cert             string        `yaml:"cert"`
+	Key              string        `yaml:"key"`
+	ClientCNs        []string      `yaml:"client_cns"`
+	HandshakeTimeout time.Duration `yaml:"handshake_timeout,zeropositive,default=10s"`
+}
+
+type StdinserverServer struct {
+	ServeCommon      `yaml:",inline"`
+	ClientIdentities []string `yaml:"client_identities"`
+}
+
+type LocalServe struct {
+	ServeCommon  `yaml:",inline"`
+	ListenerName string `yaml:"listener_name"`
+}
+
+type PruningEnum struct {
+	Ret interface{}
+}
+
+type PruneKeepNotReplicated struct {
+	Type                 string `yaml:"type"`
+	KeepSnapshotAtCursor bool   `yaml:"keep_snapshot_at_cursor,optional,default=true"`
+}
+
+type PruneKeepLastN struct {
+	Type  string `yaml:"type"`
+	Count int    `yaml:"count"`
+	Regex string `yaml:"regex,optional"`
+}
+
+type PruneKeepRegex struct { // FIXME rename to KeepRegex
+	Type   string `yaml:"type"`
+	Regex  string `yaml:"regex"`
+	Negate bool   `yaml:"negate,optional,default=false"`
+}
+
+type LoggingOutletEnum struct {
+	Ret interface{}
+}
+
+type LoggingOutletCommon struct {
+	Type   string `yaml:"type"`
+	Level  string `yaml:"level"`
+	Format string `yaml:"format"`
+}
+
+type StdoutLoggingOutlet struct {
+	LoggingOutletCommon `yaml:",inline"`
+	Time                bool `yaml:"time,default=true"`
+	Color               bool `yaml:"color,default=true"`
+}
+
+type SyslogLoggingOutlet struct {
+	LoggingOutletCommon `yaml:",inline"`
+	Facility            *SyslogFacility `yaml:"facility,optional,fromdefaults"`
+	RetryInterval       time.Duration   `yaml:"retry_interval,positive,default=10s"`
+}
+
+type TCPLoggingOutlet struct {
+	LoggingOutletCommon `yaml:",inline"`
+	Address             string               `yaml:"address,hostport"`
+	Net                 string               `yaml:"net,default=tcp"`
+	RetryInterval       time.Duration        `yaml:"retry_interval,positive,default=10s"`
+	TLS                 *TCPLoggingOutletTLS `yaml:"tls,optional"`
+}
+
+type TCPLoggingOutletTLS struct {
+	CA   string `yaml:"ca"`
+	Cert string `yaml:"cert"`
+	Key  string `yaml:"key"`
+}
+
+type MonitoringEnum struct {
+	Ret interface{}
+}
+
+type PrometheusMonitoring struct {
+	Type           string `yaml:"type"`
+	Listen         string `yaml:"listen,hostport"`
+	ListenFreeBind bool   `yaml:"listen_freebind,default=false"`
+}
+
+type SyslogFacility syslog.Priority
+
+func (f *SyslogFacility) SetDefault() {
+	*f = SyslogFacility(syslog.LOG_LOCAL0)
+}
+
+var _ yaml.Defaulter = (*SyslogFacility)(nil)
+
+type GlobalControl struct {
+	SockPath string `yaml:"sockpath,default=/var/run/zrepl/control"`
+}
+
+type GlobalServe struct {
+	StdinServer *GlobalStdinServer `yaml:"stdinserver,optional,fromdefaults"`
+}
+
+type GlobalStdinServer struct {
+	SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
+}
+
+type JobDebugSettings struct {
+	Conn *struct {
+		ReadDump  string `yaml:"read_dump"`
+		WriteDump string `yaml:"write_dump"`
+	} `yaml:"conn,optional"`
+	RPCLog bool `yaml:"rpc_log,optional,default=false"`
+}
+
+type HookList []HookEnum
+
+type HookEnum struct {
+	Ret interface{}
+}
+
+type HookCommand struct {
+	Path               string            `yaml:"path"`
+	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
+	Filesystems        FilesystemsFilter `yaml:"filesystems,optional,default={'<': true}"`
+	HookSettingsCommon `yaml:",inline"`
+}
+
+type HookPostgresCheckpoint struct {
+	HookSettingsCommon `yaml:",inline"`
+	DSN                string            `yaml:"dsn"`
+	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
+	Filesystems        FilesystemsFilter `yaml:"filesystems"` // required, user should not CHECKPOINT for every FS
+}
+
+type HookMySQLLockTables struct {
+	HookSettingsCommon `yaml:",inline"`
+	DSN                string            `yaml:"dsn"`
+	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
+	Filesystems        FilesystemsFilter `yaml:"filesystems"`
+}
+
+type HookSettingsCommon struct {
+	Type       string `yaml:"type"`
+	ErrIsFatal bool   `yaml:"err_is_fatal,optional,default=false"`
+}
+
+func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) {
+	var in struct {
+		Type string
+	}
+	if err := u(&in, true); err != nil {
+		return nil, err
+	}
+	if in.Type == "" {
+		return nil, &yaml.TypeError{Errors: []string{"must specify type"}}
+	}
+
+	v, ok := types[in.Type]
+	if !ok {
+		return nil, &yaml.TypeError{Errors: []string{fmt.Sprintf("invalid type name %q", in.Type)}}
+	}
+	if err := u(v, false); err != nil {
+		return nil, err
+	}
+	return v, nil
+}
+
+func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"snap":   &SnapJob{},
+		"push":   &PushJob{},
+		"sink":   &SinkJob{},
+		"pull":   &PullJob{},
+		"source": &SourceJob{},
+	})
+	return
+}
+
+func (t *ConnectEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"tcp":             &TCPConnect{},
+		"tls":             &TLSConnect{},
+		"ssh+stdinserver": &SSHStdinserverConnect{},
+		"local":           &LocalConnect{},
+	})
+	return
+}
+
+func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"tcp":         &TCPServe{},
+		"tls":         &TLSServe{},
+		"stdinserver": &StdinserverServer{},
+		"local":       &LocalServe{},
+	})
+	return
+}
+
+func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"not_replicated": &PruneKeepNotReplicated{},
+		"last_n":         &PruneKeepLastN{},
+		"grid":           &PruneGrid{},
+		"regex":          &PruneKeepRegex{},
+	})
+	return
+}
+
+func (t *SnapshottingEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"periodic": &SnapshottingPeriodic{},
+		"manual":   &SnapshottingManual{},
+		"cron":     &SnapshottingCron{},
+	})
+	return
+}
+
+func (t *LoggingOutletEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"stdout": &StdoutLoggingOutlet{},
+		"syslog": &SyslogLoggingOutlet{},
+		"tcp":    &TCPLoggingOutlet{},
+	})
+	return
+}
+
+func (t *MonitoringEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"prometheus": &PrometheusMonitoring{},
+	})
+	return
+}
+
+func (t *SyslogFacility) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	var s string
+	if err := u(&s, true); err != nil {
+		return err
+	}
+	var level syslog.Priority
+	switch s {
+	case "kern":
+		level = syslog.LOG_KERN
+	case "user":
+		level = syslog.LOG_USER
+	case "mail":
+		level = syslog.LOG_MAIL
+	case "daemon":
+		level = syslog.LOG_DAEMON
+	case "auth":
+		level = syslog.LOG_AUTH
+	case "syslog":
+		level = syslog.LOG_SYSLOG
+	case "lpr":
+		level = syslog.LOG_LPR
+	case "news":
+		level = syslog.LOG_NEWS
+	case "uucp":
+		level = syslog.LOG_UUCP
+	case "cron":
+		level = syslog.LOG_CRON
+	case "authpriv":
+		level = syslog.LOG_AUTHPRIV
+	case "ftp":
+		level = syslog.LOG_FTP
+	case "local0":
+		level = syslog.LOG_LOCAL0
+	case "local1":
+		level = syslog.LOG_LOCAL1
+	case "local2":
+		level = syslog.LOG_LOCAL2
+	case "local3":
+		level = syslog.LOG_LOCAL3
+	case "local4":
+		level = syslog.LOG_LOCAL4
+	case "local5":
+		level = syslog.LOG_LOCAL5
+	case "local6":
+		level = syslog.LOG_LOCAL6
+	case "local7":
+		level = syslog.LOG_LOCAL7
+	default:
+		return fmt.Errorf("invalid syslog level: %q", s)
+	}
+	*t = SyslogFacility(level)
+	return nil
+}
+
+func (t *HookEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
+		"command":             &HookCommand{},
+		"postgres-checkpoint": &HookPostgresCheckpoint{},
+		"mysql-lock-tables":   &HookMySQLLockTables{},
+	})
+	return
+}
+
+var ConfigFileDefaultLocations = []string{
+	"/etc/zrepl/zrepl.yml",
+	"/usr/local/etc/zrepl/zrepl.yml",
+}
+
+func ParseConfig(path string) (i *Config, err error) {
+
+	if path == "" {
+		// Try default locations
+		for _, l := range ConfigFileDefaultLocations {
+			stat, statErr := os.Stat(l)
+			if statErr != nil {
+				continue
+			}
+			if !stat.Mode().IsRegular() {
+				err = errors.Errorf("file at default location is not a regular file: %s", l)
+				return
+			}
+			path = l
+			break
+		}
+	}
+
+	var bytes []byte
+
+	if bytes, err = ioutil.ReadFile(path); err != nil {
+		return
+	}
+
+	return ParseConfigBytes(bytes)
+}
+
+func ParseConfigBytes(bytes []byte) (*Config, error) {
+	var c *Config
+	if err := yaml.UnmarshalStrict(bytes, &c); err != nil {
+		return nil, err
+	}
+	if c == nil {
+		return nil, fmt.Errorf("config is empty or only consists of comments")
+	}
+	return c, nil
+}

config/config_dep_test.go

@@ -0,0 +1,90 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/zrepl/yaml-config"
+)
+
+type A struct {
+	B  *B     `yaml:"b,optional,fromdefaults"`
+	A1 string `yaml:"a1,optional"`
+}
+
+type B struct {
+	C *C     `yaml:"c,optional,fromdefaults"`
+	D string `yaml:"d,default=ddd"`
+	E string `yaml:"e,optional"`
+}
+
+type C struct {
+	Q string `yaml:"q,optional"`
+	R string `yaml:"r,default=r"`
+}
+
+func TestDepFromDefaults(t *testing.T) {
+
+	type testcase struct {
+		name   string
+		yaml   string
+		expect *A
+	}
+
+	tcs := []testcase{
+		{
+			name: "empty",
+			yaml: `{}`,
+			expect: &A{
+				B: &B{
+					C: &C{
+						R: "r",
+					},
+					D: "ddd",
+				},
+			},
+		},
+		{
+			name: "a1 set",
+			yaml: `{"a1":"blah"}`,
+			expect: &A{
+				A1: "blah",
+				B: &B{
+					C: &C{
+						R: "r",
+					},
+					D: "ddd",
+				},
+			},
+		},
+		{
+			name: "D set",
+			yaml: `
+b:
+  d: 4d
+`,
+			expect: &A{
+				B: &B{
+					D: "4d",
+					C: &C{
+						R: "r",
+					},
+				},
+			},
+		},
+	}
+
+	for tci := range tcs {
+		t.Run(fmt.Sprintf("%d-%s", tci, tcs[tci].name), func(t *testing.T) {
+			tc := tcs[tci]
+
+			var a A
+			err := yaml.UnmarshalStrict([]byte(tc.yaml), &a)
+			require.NoError(t, err)
+
+			require.Equal(t, tc.expect, &a)
+		})
+	}
+
+}

config/config_duration.go

@@ -0,0 +1,106 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
+	"time"
+
+	"github.com/kr/pretty"
+	"github.com/zrepl/yaml-config"
+)
+
+type Duration struct{ d time.Duration }
+
+func (d Duration) Duration() time.Duration { return d.d }
+
+var _ yaml.Unmarshaler = &Duration{}
+
+func (d *Duration) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
+	var s string
+	err := unmarshal(&s, false)
+	if err != nil {
+		return err
+	}
+	d.d, err = parseDuration(s)
+	if err != nil {
+		d.d = 0
+		return &yaml.TypeError{Errors: []string{fmt.Sprintf("cannot parse value %q: %s", s, err)}}
+	}
+	return nil
+}
+
+type PositiveDuration struct{ d Duration }
+
+var _ yaml.Unmarshaler = &PositiveDuration{}
+
+func (d PositiveDuration) Duration() time.Duration { return d.d.Duration() }
+
+func (d *PositiveDuration) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
+	err := d.d.UnmarshalYAML(unmarshal)
+	if err != nil {
+		return err
+	}
+	if d.d.Duration() <= 0 {
+		return fmt.Errorf("duration must be positive, got %s", d.d.Duration())
+	}
+	return nil
+}
+
+func parsePositiveDuration(e string) (time.Duration, error) {
+	d, err := parseDuration(e)
+	if err != nil {
+		return d, err
+	}
+	if d <= 0 {
+		return 0, errors.New("duration must be positive integer")
+	}
+	return d, err
+}
+
+var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*([\+-]?\d+)\s*(|s|m|h|d|w)\s*$`)
+
+func parseDuration(e string) (d time.Duration, err error) {
+	comps := durationStringRegex.FindStringSubmatch(e)
+	if comps == nil {
+		err = fmt.Errorf("must match %s", durationStringRegex)
+		return
+	}
+	if len(comps) != 3 {
+		panic(pretty.Sprint(comps))
+	}
+
+	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+
+	var durationUnit time.Duration
+	switch comps[2] {
+	case "":
+		if durationFactor != 0 {
+			err = fmt.Errorf("missing time unit")
+			return
+		} else {
+			// It's the case where user specified '0'.
+			// We want to allow this, just like time.ParseDuration.
+		}
+	case "s":
+		durationUnit = time.Second
+	case "m":
+		durationUnit = time.Minute
+	case "h":
+		durationUnit = time.Hour
+	case "d":
+		durationUnit = 24 * time.Hour
+	case "w":
+		durationUnit = 24 * 7 * time.Hour
+	default:
+		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
+		return
+	}
+
+	d = time.Duration(durationFactor) * durationUnit
+	return
+}

config/config_global_test.go

@@ -0,0 +1,115 @@
+package config
+
+import (
+	"fmt"
+	"log/syslog"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/zrepl/yaml-config"
+)
+
+func testValidGlobalSection(t *testing.T, s string) *Config {
+	jobdef := `
+jobs:
+- name: dummyjob
+  type: sink
+  serve:
+    type: tcp
+    listen: ":2342"
+    clients: {
+      "10.0.0.1":"foo"
+    }
+  root_fs: zroot/foo
+`
+	_, err := ParseConfigBytes([]byte(jobdef))
+	require.NoError(t, err)
+	return testValidConfig(t, s+jobdef)
+}
+
+func TestOutletTypes(t *testing.T) {
+	conf := testValidGlobalSection(t, `
+global:
+  logging:
+  - type: stdout
+    level: debug
+    format: human
+  - type: syslog
+    level: info
+    retry_interval: 20s
+    format: human
+  - type: tcp
+    level: debug
+    format: json
+    address: logserver.example.com:1234
+  - type: tcp
+    level: debug
+    format: json
+    address: encryptedlogserver.example.com:1234
+    retry_interval: 20s 
+    tls:
+      ca: /etc/zrepl/log/ca.crt
+      cert: /etc/zrepl/log/key.pem
+      key: /etc/zrepl/log/cert.pem
+`)
+	assert.Equal(t, 4, len(*conf.Global.Logging))
+	assert.NotNil(t, (*conf.Global.Logging)[3].Ret.(*TCPLoggingOutlet).TLS)
+}
+
+func TestDefaultLoggingOutlet(t *testing.T) {
+	conf := testValidGlobalSection(t, "")
+	assert.Equal(t, 1, len(*conf.Global.Logging))
+	o := (*conf.Global.Logging)[0].Ret.(*StdoutLoggingOutlet)
+	assert.Equal(t, "warn", o.Level)
+	assert.Equal(t, "human", o.Format)
+}
+
+func TestPrometheusMonitoring(t *testing.T) {
+	conf := testValidGlobalSection(t, `
+global:
+  monitoring:
+    - type: prometheus
+      listen: ':9811'
+`)
+	assert.Equal(t, ":9811", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
+}
+
+func TestSyslogLoggingOutletFacility(t *testing.T) {
+	type SyslogFacilityPriority struct {
+		Facility string
+		Priority syslog.Priority
+	}
+	syslogFacilitiesPriorities := []SyslogFacilityPriority{
+		{"", syslog.LOG_LOCAL0}, // default
+		{"kern", syslog.LOG_KERN}, {"daemon", syslog.LOG_DAEMON}, {"auth", syslog.LOG_AUTH},
+		{"syslog", syslog.LOG_SYSLOG}, {"lpr", syslog.LOG_LPR}, {"news", syslog.LOG_NEWS},
+		{"uucp", syslog.LOG_UUCP}, {"cron", syslog.LOG_CRON}, {"authpriv", syslog.LOG_AUTHPRIV},
+		{"ftp", syslog.LOG_FTP}, {"local0", syslog.LOG_LOCAL0}, {"local1", syslog.LOG_LOCAL1},
+		{"local2", syslog.LOG_LOCAL2}, {"local3", syslog.LOG_LOCAL3}, {"local4", syslog.LOG_LOCAL4},
+		{"local5", syslog.LOG_LOCAL5}, {"local6", syslog.LOG_LOCAL6}, {"local7", syslog.LOG_LOCAL7},
+	}
+
+	for _, sFP := range syslogFacilitiesPriorities {
+		logcfg := fmt.Sprintf(`
+global:
+  logging:
+  - type: syslog
+    level: info
+    format: human
+    facility: %s
+`, sFP.Facility)
+		conf := testValidGlobalSection(t, logcfg)
+		assert.Equal(t, 1, len(*conf.Global.Logging))
+		assert.True(t, SyslogFacility(sFP.Priority) == *(*conf.Global.Logging)[0].Ret.(*SyslogLoggingOutlet).Facility)
+	}
+}
+
+func TestLoggingOutletEnumList_SetDefaults(t *testing.T) {
+	e := &LoggingOutletEnumList{}
+	var i yaml.Defaulter = e
+	require.NotPanics(t, func() {
+		i.SetDefault()
+		assert.Equal(t, "warn", (*e)[0].Ret.(*StdoutLoggingOutlet).Level)
+	})
+}

config/config_jobid_test.go

@@ -0,0 +1 @@
+package config

config/config_minimal_test.go

@@ -0,0 +1,40 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfigEmptyFails(t *testing.T) {
+	conf, err := testConfig(t, "\n")
+	assert.Nil(t, conf)
+	assert.Error(t, err)
+}
+
+func TestJobsOnlyWorks(t *testing.T) {
+	testValidConfig(t, `
+jobs:
+- name: push
+  type: push
+  # snapshot the filesystems matched by the left-hand-side of the mapping
+  # every 10m with zrepl_ as prefix
+  connect:
+    type: tcp
+    address: localhost:2342
+  filesystems: {
+    "pool1/var/db<": true,
+    "pool1/usr/home<": true,
+    "pool1/usr/home/paranoid": false, #don't backup paranoid user
+    "pool1/poudriere/ports<": false #don't backup the ports trees
+  }
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    keep_receiver:
+    - type: last_n 
+      count: 1
+`)
+}

config/config_positiveintervalormanual_test.go

@@ -0,0 +1,41 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/zrepl/yaml-config"
+)
+
+func TestPositiveDurationOrManual(t *testing.T) {
+	cases := []struct {
+		Comment, Input string
+		Result         *PositiveDurationOrManual
+	}{
+		{"empty is error", "", nil},
+		{"negative is error", "-1s", nil},
+		{"zero seconds is error", "0s", nil},
+		{"zero is error", "0", nil},
+		{"non-manual is error", "something", nil},
+		{"positive seconds works", "1s", &PositiveDurationOrManual{Manual: false, Interval: 1 * time.Second}},
+		{"manual works", "manual", &PositiveDurationOrManual{Manual: true, Interval: 0}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.Comment, func(t *testing.T) {
+			var out struct {
+				FieldName PositiveDurationOrManual `yaml:"fieldname"`
+			}
+			input := fmt.Sprintf("\nfieldname: %s\n", tc.Input)
+			err := yaml.UnmarshalStrict([]byte(input), &out)
+			if tc.Result == nil {
+				assert.Error(t, err)
+				t.Logf("%#v", out)
+			} else {
+				assert.Equal(t, *tc.Result, out.FieldName)
+			}
+		})
+	}
+
+}

config/config_recv_test.go

@@ -0,0 +1,134 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	zfsprop "github.com/zrepl/zrepl/zfs/property"
+)
+
+func TestRecvOptions(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: pull
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  root_fs: "zreplplatformtest"
+  %s
+  interval: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+
+`
+
+	recv_properties_empty := `
+  recv:
+    properties:
+`
+
+	recv_inherit_empty := `
+  recv:
+    properties:
+      inherit:
+`
+
+	recv_inherit := `
+  recv:
+    properties:
+      inherit:
+        - testprop
+`
+
+	recv_override_empty := `
+  recv:
+    properties:
+      override:
+`
+
+	recv_override := `
+  recv:
+    properties:
+      override:
+        testprop2: "test123"
+`
+
+	recv_override_and_inherit := `
+  recv:
+    properties:
+      inherit:
+        - testprop
+      override:
+        testprop2: "test123"
+`
+
+	recv_empty := `
+  recv: {}
+`
+
+	recv_not_specified := `
+`
+
+	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
+
+	t.Run("recv_inherit_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_inherit_empty))
+		assert.NotNil(t, c)
+	})
+
+	t.Run("recv_inherit", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_inherit))
+		inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
+		assert.NotEmpty(t, inherit)
+		assert.Contains(t, inherit, zfsprop.Property("testprop"))
+	})
+
+	t.Run("recv_override_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_override_empty))
+		assert.NotNil(t, c)
+	})
+
+	t.Run("recv_override", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_override))
+		override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
+		require.Len(t, override, 1)
+		require.Equal(t, "test123", override["testprop2"])
+
+	})
+
+	t.Run("recv_override_and_inherit", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_override_and_inherit))
+		inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
+		override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
+		assert.NotEmpty(t, inherit)
+		assert.Contains(t, inherit, zfsprop.Property("testprop"))
+		assert.NotEmpty(t, override)
+		assert.Equal(t, "test123", override["testprop2"])
+	})
+
+	t.Run("recv_properties_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_properties_empty))
+		assert.NotNil(t, c)
+	})
+
+	t.Run("recv_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_empty))
+		assert.NotNil(t, c)
+	})
+
+	t.Run("send_not_specified", func(t *testing.T) {
+		c := testValidConfig(t, fill(recv_not_specified))
+		assert.NotNil(t, c)
+	})
+
+}

config/config_send_test.go

@@ -0,0 +1,146 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSendOptions(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  %s
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+	encrypted_false := `
+  send:
+    encrypted: false
+`
+
+	encrypted_true := `
+  send:
+    encrypted: true
+`
+
+	raw_true := `
+  send:
+    raw: true
+
+`
+
+	raw_false := `
+  send:
+    raw: false
+
+`
+
+	raw_and_encrypted := `
+  send:
+    encrypted: true
+    raw: true
+`
+
+	properties_and_encrypted := `
+  send:
+    encrypted: true
+    send_properties: true
+`
+
+	properties_true := `
+  send:
+    send_properties: true
+`
+
+	properties_false := `
+  send:
+    send_properties: false
+`
+
+	send_empty := `
+  send: {}
+`
+
+	send_not_specified := `
+`
+
+	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
+
+	t.Run("encrypted_false", func(t *testing.T) {
+		c := testValidConfig(t, fill(encrypted_false))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, false, encrypted)
+	})
+	t.Run("encrypted_true", func(t *testing.T) {
+		c := testValidConfig(t, fill(encrypted_true))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, true, encrypted)
+	})
+
+	t.Run("send_empty", func(t *testing.T) {
+		c := testValidConfig(t, fill(send_empty))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, false, encrypted)
+	})
+
+	t.Run("properties_and_encrypted", func(t *testing.T) {
+		c := testValidConfig(t, fill(properties_and_encrypted))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
+		assert.Equal(t, true, encrypted)
+		assert.Equal(t, true, properties)
+	})
+
+	t.Run("properties_false", func(t *testing.T) {
+		c := testValidConfig(t, fill(properties_false))
+		properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
+		assert.Equal(t, false, properties)
+	})
+
+	t.Run("properties_true", func(t *testing.T) {
+		c := testValidConfig(t, fill(properties_true))
+		properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
+		assert.Equal(t, true, properties)
+	})
+
+	t.Run("raw_true", func(t *testing.T) {
+		c := testValidConfig(t, fill(raw_true))
+		raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
+		assert.Equal(t, true, raw)
+	})
+
+	t.Run("raw_false", func(t *testing.T) {
+		c := testValidConfig(t, fill(raw_false))
+		raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
+		assert.Equal(t, false, raw)
+	})
+
+	t.Run("raw_and_encrypted", func(t *testing.T) {
+		c := testValidConfig(t, fill(raw_and_encrypted))
+		raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, true, raw)
+		assert.Equal(t, true, encrypted)
+	})
+
+	t.Run("send_not_specified", func(t *testing.T) {
+		c := testValidConfig(t, fill(send_not_specified))
+		assert.NotNil(t, c)
+	})
+
+}

config/config_snapshotting_test.go

@@ -0,0 +1,105 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSnapshotting(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  %s
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+	manual := `
+  snapshotting:
+    type: manual
+`
+	periodic := `
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 10m
+`
+
+	periodicDaily := `
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 1d
+`
+
+	hooks := `
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 10m
+    hooks:
+    - type: command
+      path: /tmp/path/to/command
+    - type: command
+      path: /tmp/path/to/command
+      filesystems: { "zroot<": true, "<": false }
+    - type: postgres-checkpoint
+      dsn: "host=localhost port=5432 user=postgres sslmode=disable"
+      filesystems: {
+          "tank/postgres/data11": true
+      }
+    - type: mysql-lock-tables
+      dsn: "root@tcp(localhost)/"
+      filesystems: {
+        "tank/mysql": true
+      }
+`
+
+	fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
+	var c *Config
+
+	t.Run("manual", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(manual))
+		snm := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingManual)
+		assert.Equal(t, "manual", snm.Type)
+	})
+
+	t.Run("periodic", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(periodic))
+		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
+		assert.Equal(t, "periodic", snp.Type)
+		assert.Equal(t, 10*time.Minute, snp.Interval.Duration())
+		assert.Equal(t, "zrepl_", snp.Prefix)
+	})
+
+	t.Run("periodicDaily", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(periodicDaily))
+		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
+		assert.Equal(t, "periodic", snp.Type)
+		assert.Equal(t, 24*time.Hour, snp.Interval.Duration())
+		assert.Equal(t, "zrepl_", snp.Prefix)
+	})
+
+	t.Run("hooks", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(hooks))
+		hs := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic).Hooks
+		assert.Equal(t, hs[0].Ret.(*HookCommand).Filesystems["<"], true)
+		assert.Equal(t, hs[1].Ret.(*HookCommand).Filesystems["zroot<"], true)
+		assert.Equal(t, hs[2].Ret.(*HookPostgresCheckpoint).Filesystems["tank/postgres/data11"], true)
+		assert.Equal(t, hs[3].Ret.(*HookMySQLLockTables).Filesystems["tank/mysql"], true)
+	})
+
+}

config/config_test.go

@@ -0,0 +1,139 @@
+package config
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+	"text/template"
+
+	"github.com/kr/pretty"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/zrepl/yaml-config"
+)
+
+func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
+	paths, err := filepath.Glob("./samples/*")
+	if err != nil {
+		t.Errorf("glob failed: %+v", err)
+	}
+
+	for _, p := range paths {
+
+		if path.Ext(p) != ".yml" {
+			t.Logf("skipping file %s", p)
+			continue
+		}
+
+		t.Run(p, func(t *testing.T) {
+			c, err := ParseConfig(p)
+			if err != nil {
+				t.Errorf("error parsing %s:\n%+v", p, err)
+			}
+
+			t.Logf("file: %s", p)
+			t.Log(pretty.Sprint(c))
+		})
+
+	}
+
+}
+
+// template must be a template/text template with a single '{{ . }}' as placeholder for val
+//nolint[:deadcode,unused]
+func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
+	tmp, err := template.New("master").Parse(tmpl)
+	if err != nil {
+		panic(err)
+	}
+	var buf bytes.Buffer
+	err = tmp.Execute(&buf, val)
+	if err != nil {
+		panic(err)
+	}
+	return testValidConfig(t, buf.String())
+}
+
+func testValidConfig(t *testing.T, input string) *Config {
+	t.Helper()
+	conf, err := testConfig(t, input)
+	require.NoError(t, err)
+	require.NotNil(t, conf)
+	return conf
+}
+
+func testConfig(t *testing.T, input string) (*Config, error) {
+	t.Helper()
+	return ParseConfigBytes([]byte(input))
+}
+
+func trimSpaceEachLineAndPad(s, pad string) string {
+	var out strings.Builder
+	scan := bufio.NewScanner(strings.NewReader(s))
+	for scan.Scan() {
+		fmt.Fprintf(&out, "%s%s\n", pad, bytes.TrimSpace(scan.Bytes()))
+	}
+	return out.String()
+}
+
+func TestTrimSpaceEachLineAndPad(t *testing.T) {
+	foo := `
+	foo
+	bar baz 
+	`
+	assert.Equal(t, "  \n  foo\n  bar baz\n  \n", trimSpaceEachLineAndPad(foo, "  "))
+}
+
+func TestCronSpec(t *testing.T) {
+
+	expectAccept := []string{
+		`"* * * * *"`,
+		`"0-10 * * * *"`,
+		`"* 0-5,8,12 * * *"`,
+	}
+
+	expectFail := []string{
+		`* * * *`,
+		``,
+		`23`,
+		`"@reboot"`,
+		`"@every 1h30m"`,
+		`"@daily"`,
+		`* * * * * *`,
+	}
+
+	for _, input := range expectAccept {
+		t.Run(input, func(t *testing.T) {
+			s := fmt.Sprintf("spec: %s\n", input)
+			var v struct {
+				Spec CronSpec
+			}
+			v.Spec.Schedule = nil
+			t.Logf("input:\n%s", s)
+			err := yaml.UnmarshalStrict([]byte(s), &v)
+			t.Logf("error: %T %s", err, err)
+			require.NoError(t, err)
+			require.NotNil(t, v.Spec.Schedule)
+		})
+	}
+
+	for _, input := range expectFail {
+		t.Run(input, func(t *testing.T) {
+			s := fmt.Sprintf("spec: %s\n", input)
+			var v struct {
+				Spec CronSpec
+			}
+			v.Spec.Schedule = nil
+			t.Logf("input: %q", s)
+			err := yaml.UnmarshalStrict([]byte(s), &v)
+			t.Logf("error: %T %s", err, err)
+			require.Error(t, err)
+			require.Nil(t, v.Spec.Schedule)
+		})
+	}
+
+}

config/config_transport_test.go

@@ -0,0 +1,99 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestTransportConnect(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+%s
+  filesystems: {"<": true}
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+
+	mconf := func(s string) string { return fmt.Sprintf(tmpl, s) }
+
+	type test struct {
+		Name        string
+		ExpectError bool
+		Connect     string
+	}
+
+	testTable := []test{
+		{
+			Name:        "tcp_with_address_and_port",
+			ExpectError: false,
+			Connect: `
+			type: tcp
+			address: 10.0.0.23:42
+			`,
+		},
+		{
+			Name:        "tls_with_host_and_port",
+			ExpectError: false,
+			Connect: `
+			type: tls
+			address: "server1.foo.bar:8888"
+			ca:   /etc/zrepl/ca.crt
+			cert: /etc/zrepl/backupserver.fullchain
+			key:  /etc/zrepl/backupserver.key
+			server_cn: "server1"
+			`,
+		},
+		{
+			Name:        "tcp_without_port",
+			ExpectError: true,
+			Connect: `
+			type: tcp
+			address: 10.0.0.23
+			`,
+		},
+		{
+			Name:        "tls_without_port",
+			ExpectError: true,
+			Connect: `
+			type: tls
+			address: 10.0.0.23
+			ca:   /etc/zrepl/ca.crt
+			cert: /etc/zrepl/backupserver.fullchain
+			key:  /etc/zrepl/backupserver.key
+			server_cn: "server1"
+			`,
+		},
+	}
+
+	for _, tc := range testTable {
+		t.Run(tc.Name, func(t *testing.T) {
+			require.NotEmpty(t, tc.Connect)
+			connect := trimSpaceEachLineAndPad(tc.Connect, "    ")
+			conf := mconf(connect)
+			config, err := testConfig(t, conf)
+			if tc.ExpectError && err == nil {
+				t.Errorf("expected test failure, but got valid config %v", config)
+				return
+			}
+			if !tc.ExpectError && err != nil {
+				t.Errorf("not expecint test failure but got error: %s", err)
+				return
+			}
+			t.Logf("error=%v", err)
+			t.Logf("config=%v", config)
+		})
+	}
+
+}

config/retentiongrid.go

@@ -0,0 +1,119 @@
+package config
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type RetentionIntervalList []RetentionInterval
+
+type PruneGrid struct {
+	Type  string                `yaml:"type"`
+	Grid  RetentionIntervalList `yaml:"grid"`
+	Regex string                `yaml:"regex"`
+}
+
+type RetentionInterval struct {
+	length    time.Duration
+	keepCount int
+}
+
+func (i *RetentionInterval) Length() time.Duration {
+	return i.length
+}
+
+func (i *RetentionInterval) KeepCount() int {
+	return i.keepCount
+}
+
+const RetentionGridKeepCountAll int = -1
+
+func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
+	var in string
+	if err := u(&in, true); err != nil {
+		return err
+	}
+
+	intervals, err := ParseRetentionIntervalSpec(in)
+	if err != nil {
+		return err
+	}
+
+	*t = intervals
+
+	return nil
+}
+
+var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
+
+func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {
+
+	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
+	if comps == nil {
+		err = fmt.Errorf("retention string does not match expected format")
+		return
+	}
+
+	times, err := strconv.Atoi(comps[1])
+	if err != nil {
+		return nil, err
+	} else if times <= 0 {
+		return nil, fmt.Errorf("contains factor <= 0")
+	}
+
+	duration, err := parsePositiveDuration(comps[2])
+	if err != nil {
+		return nil, err
+	}
+
+	keepCount := 1
+	if comps[3] != "" {
+		// Decompose key=value, comma separated
+		// For now, only keep_count is supported
+		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
+		res := re.FindStringSubmatch(comps[4])
+		if res == nil || len(res) != 2 {
+			err = fmt.Errorf("interval parameter contains unknown parameters")
+			return
+		}
+		if res[1] == "all" {
+			keepCount = RetentionGridKeepCountAll
+		} else {
+			keepCount, err = strconv.Atoi(res[1])
+			if err != nil {
+				err = fmt.Errorf("cannot parse keep_count value")
+				return
+			}
+		}
+	}
+
+	intervals = make([]RetentionInterval, times)
+	for i := range intervals {
+		intervals[i] = RetentionInterval{
+			length:    duration,
+			keepCount: keepCount,
+		}
+	}
+
+	return
+
+}
+
+func ParseRetentionIntervalSpec(s string) (intervals []RetentionInterval, err error) {
+
+	ges := strings.Split(s, "|")
+	intervals = make([]RetentionInterval, 0, 7*len(ges))
+
+	for intervalIdx, e := range ges {
+		parsed, err := parseRetentionGridIntervalString(e)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
+		}
+		intervals = append(intervals, parsed...)
+	}
+
+	return
+}

config/samples/bandwidth_limit.yml

@@ -0,0 +1,41 @@
+
+jobs:
+  - type: sink
+    name: "limited_sink"
+    root_fs: "fs0"
+    recv:
+      bandwidth_limit:
+        max: 12345 B
+    serve:
+      type: local
+      listener_name: localsink
+
+  - type: push
+    name: "limited_push"
+    connect:
+      type: local
+      listener_name: localsink
+      client_identity: local_backup
+    filesystems: {
+      "root<": true,
+    }
+    send:
+      bandwidth_limit:
+        max: 54321 B
+        bucket_capacity: 1024 B
+    snapshotting:
+      type: manual
+    pruning:
+      keep_sender:
+        - type: last_n
+          count: 1
+      keep_receiver:
+        - type: last_n
+          count: 1
+
+  - type: sink
+    name: "nolimit_sink"
+    root_fs: "fs1"
+    serve:
+      type: local
+      listener_name: localsink

config/samples/hooks/template.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -e
+
+[ "$ZREPL_DRYRUN" = "true" ] && DRYRUN="echo DRYRUN: "
+
+pre_snapshot() {
+    $DRYRUN date
+}
+
+post_snapshot() {
+    $DRYRUN date
+}
+
+case "$ZREPL_HOOKTYPE" in
+    pre_snapshot|post_snapshot)
+        "$ZREPL_HOOKTYPE"
+        ;;
+    *)
+        printf 'Unrecognized hook type: %s\n' "$ZREPL_HOOKTYPE"
+        exit 255
+        ;;
+esac

config/samples/local.yml

@@ -0,0 +1,31 @@
+
+jobs:
+  - type: sink
+    name: "local_sink"
+    root_fs: "storage/zrepl/sink"
+    serve:
+      type: local
+      listener_name: localsink
+
+  - type: push
+    name: "backup_system"
+    connect:
+      type: local
+      listener_name: localsink
+      client_identity: local_backup
+    filesystems: {
+      "system<": true,
+    }
+    snapshotting:
+      type: periodic
+      interval: 10m
+      prefix: zrepl_
+    pruning:
+      keep_sender:
+      - type: not_replicated
+      - type: last_n
+        count: 10
+      keep_receiver:
+      - type: grid
+        grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+        regex: "zrepl_.*"

config/samples/pull.yml

@@ -0,0 +1,24 @@
+jobs:
+- name: pull_servers
+  type: pull
+  connect:
+    type: tls
+    address: "server1.foo.bar:8888"
+    ca: "/certs/ca.crt"
+    cert: "/certs/cert.crt"
+    key: "/certs/key.pem"
+    server_cn: "server1"
+  root_fs: "pool2/backup_servers"
+  interval: 10m
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    - type: last_n
+      count: 10
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 14x1d
+      regex: "zrepl_.*"
+    keep_receiver:
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+      regex: "zrepl_.*"

config/samples/pull_ssh.yml

@@ -0,0 +1,28 @@
+jobs:
+
+- name: pull_servers
+  type: pull
+  connect:
+    type: ssh+stdinserver
+    host: app-srv.example.com
+    user: root
+    port: 22
+    identity_file: /etc/zrepl/ssh/identity
+    options: # optional, default [], `-o` arguments passed to ssh
+    - "Compression=yes"
+  root_fs: "pool2/backup_servers"
+  interval: 10m
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    - type: last_n
+      count: 10
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 14x1d
+      regex: "^zrepl_.*"
+    keep_receiver:
+    - type: regex
+      regex: keep_
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+      regex: "^zrepl_.*"

config/samples/push.yml

@@ -0,0 +1,26 @@
+jobs:
+  - type: push
+    name: "push"
+    filesystems: {
+      "<": true,
+      "tmp": false
+    }
+    connect:
+      type: tcp
+      address: "backup-server.foo.bar:8888"
+    snapshotting:
+      type: manual
+    send:
+      encrypted: false
+    pruning:
+      keep_sender:
+        - type: not_replicated
+        - type: last_n
+          count: 10
+        - type: grid
+          grid: 1x1h(keep=all) | 24x1h | 14x1d
+          regex: "^zrepl_.*"
+      keep_receiver:
+        - type: grid
+          grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
+          regex: "^zrepl_.*"

config/samples/quickstart_backup_to_external_disk.yml

@@ -0,0 +1,88 @@
+# This config serves as an example for a local zrepl installation that
+# backups the entire zpool `system` to `backuppool/zrepl/sink`
+#
+# The requirements covered by this setup are described in the zrepl documentation's
+# quick start section which inlines this example.
+#
+# CUSTOMIZATIONS YOU WILL LIKELY WANT TO APPLY:
+# - adjust the name of the production pool `system` in the `filesystems` filter of jobs `snapjob` and `push_to_drive`
+# - adjust the name of the backup pool `backuppool` in the `backuppool_sink` job
+# - adjust the occurences of `myhostname` to the name of the system you are backing up (cannot be easily changed once you start replicating)
+# - make sure the `zrepl_` prefix is not being used by any other zfs tools you might have installed (it likely isn't)
+
+jobs:
+
+# this job takes care of snapshot creation + pruning
+- name: snapjob
+  type: snap
+  filesystems: {
+      "system<": true,
+  }
+  # create snapshots with prefix `zrepl_` every 15 minutes
+  snapshotting:
+    type: periodic
+    interval: 15m
+    prefix: zrepl_
+  pruning:
+    keep:
+    # fade-out scheme for snapshots starting with `zrepl_`
+    # - keep all created in the last hour
+    # - then destroy snapshots such that we keep 24 each 1 hour apart
+    # - then destroy snapshots such that we keep 14 each 1 day apart
+    # - then destroy all older snapshots
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 14x1d
+      regex: "^zrepl_.*"
+    # keep all snapshots that don't have the `zrepl_` prefix
+    - type: regex
+      negate: true
+      regex: "^zrepl_.*"
+
+# This job pushes to the local sink defined in job `backuppool_sink`.
+# We trigger replication manually from the command line / udev rules using
+#  `zrepl signal wakeup push_to_drive`
+- type: push
+  name: push_to_drive
+  connect:
+    type: local
+    listener_name: backuppool_sink
+    client_identity: myhostname
+  filesystems: {
+      "system<": true
+  }
+  send:
+    encrypted: true
+  replication:
+    protection:
+      initial: guarantee_resumability
+      # Downgrade protection to guarantee_incremental which uses zfs bookmarks instead of zfs holds.
+      # Thus, when we yank out the backup drive during replication
+      # - we might not be able to resume the interrupted replication step because the partially received `to` snapshot of a `from`->`to` step may be pruned any time
+      # - but in exchange we get back the disk space allocated by `to` when we prune it
+      # - and because we still have the bookmarks created by `guarantee_incremental`, we can still do incremental replication of `from`->`to2` in the future
+      incremental: guarantee_incremental
+  snapshotting:
+    type: manual
+  pruning:
+    # no-op prune rule on sender (keep all snapshots), job `snapshot` takes care of this
+    keep_sender:
+    - type: regex
+      regex: ".*"
+    # retain 
+    keep_receiver:
+    # longer retention on the backup drive, we have more space there
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 360x1d
+      regex: "^zrepl_.*"
+    # retain all non-zrepl snapshots on the backup drive
+    - type: regex
+      negate: true
+      regex: "^zrepl_.*"
+
+# This job receives from job `push_to_drive` into `backuppool/zrepl/sink/myhostname`
+- type: sink
+  name: backuppool_sink
+  root_fs: "backuppool/zrepl/sink"
+  serve:
+    type: local
+    listener_name: backuppool_sink