prototype resumable send & recv

.circleci/config.yml

@@ -1,139 +0,0 @@
-version: 2.0
-workflows:
-  version: 2
-  build:
-    jobs:
-      - build-1.11
-      - build-1.12
-      - build-1.13
-      - build-latest
-      - test-build-in-docker
-jobs:
-
-  # build-latest serves as the template
-  # we use YAML anchors & aliases to exchange the docker image (and hence Go version used for the build)
-  build-latest: &build-latest
-    description: Builds zrepl
-    parameters:
-      image:
-        description: "the docker image that the job should use"
-        type: string
-    docker:
-      - image: circleci/golang:latest
-    environment:
-      # required by lazy.sh
-      TERM: xterm
-    working_directory: /go/src/github.com/zrepl/zrepl
-    steps:
-      - run:
-          name: Setup environment variables
-          command: |
-            # used by pip (for docs)
-            echo 'export PATH="$HOME/.local/bin:$PATH"' >> $BASH_ENV
-            # we use modules
-            echo 'export GO111MODULE=on' >> $BASH_ENV
-
-      - restore_cache:
-          keys:
-            - source
-            - protobuf
-
-      - checkout
-
-      - save_cache:
-          key: source
-          paths:
-            - ".git"
-      
-      # install deps
-      - run: wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
-      - run: echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807  protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
-      - run: sudo unzip -d /usr protoc-3.6.1-linux-x86_64.zip
-      - save_cache:
-          key: protobuf
-          paths:
-            - "/usr/include/google/protobuf"
-            
-      - run: sudo apt update && sudo apt install python3 python3-pip libgirepository1.0-dev gawk
-      - run: ./lazy.sh devsetup
-
-      - run: make zrepl-bin
-      - run: make vet
-      - run: make test
-      - run: make lint
-      - run: make release
-
-
-      - store_artifacts:
-          path: ./artifacts/release
-          when: always
-
-      - run:
-          shell: /bin/bash -eo pipefail
-          when: always
-          command: |
-            if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
-              echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
-              exit 0
-            fi
-            set -u # from now on
-            
-            # Download and install minio
-            curl -sSL https://dl.minio.io/client/mc/release/linux-amd64/mc -o ${GOPATH}/bin/mc
-            chmod +x ${GOPATH}/bin/mc
-            mc config host add --api s3v4 zrepl-minio https://minio.cschwarz.com ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
-            
-            # Upload artifacts
-            echo "$CIRCLE_BUILD_URL" > ./artifacts/release/cirlceci_build_url
-            mc cp -r artifacts/release "zrepl-minio/zrepl-ci-artifacts/${CIRCLE_SHA1}/${CIRCLE_JOB}/"
-
-            # Push Artifact Link to GitHub
-            REPO="zrepl/zrepl"
-            COMMIT="${CIRCLE_SHA1}"
-            JOB_NAME="${CIRCLE_JOB}"
-            curl "https://api.github.com/repos/$REPO/statuses/$COMMIT?access_token=$GITHUB_COMMIT_STATUS_TOKEN" \
-              -H "Content-Type: application/json" \
-              -X POST \
-              -d '{"context":"zrepl/publish-ci-artifacts", "state": "success", "description":"CI Build Artifacts for '"$JOB_NAME"'", "target_url":"https://minio.cschwarz.com/minio/zrepl-ci-artifacts/'"$COMMIT"'/"}'
-
-  build-1.11:
-    <<: *build-latest
-    docker:
-    - image: circleci/golang:1.11
-
-  build-1.12:
-    <<: *build-latest
-    docker:
-    - image: circleci/golang:1.12
-
-  build-1.13:
-    <<: *build-latest
-    docker:
-    - image: circleci/golang:1.13
-
-  # this job tries to mimic the build-in-docker instructions
-  # given in docs/installation.rst
-  #
-  # However, CircleCI doesn't support volume mounts, so we have to copy
-  # the source into the build-container by modifying the Dockerfile
-  test-build-in-docker:
-    description: Check that build-in-docker works
-    docker:
-      - image: circleci/golang:latest
-    environment:
-    working_directory: /go/src/github.com/zrepl/zrepl
-    steps:
-    - checkout
-    - setup_remote_docker
-    - run:
-        name: (hacky) circleci doesn't allow volume mounts, so copy src to container
-        command: echo "ADD . /src" >> build.Dockerfile
-    - run:
-        name: (hacky) commit modified Dockerfile to avoid failing git clean check in Makefile
-        command: git -c user.name='circleci' -c user.email='circleci@localhost' commit -m 'CIRCLECI modified Dockerfile with zrepl src' --author 'autoauthor <circleci@localhost>' -- build.Dockerfile
-    - run:
-        name: build the build image (build deps)
-        command: docker build -t zrepl_build -f build.Dockerfile .
-    - run:
-        name: try compiling
-        command: docker run -it zrepl_build make release

.github/FUNDING.yml

@@ -1,4 +0,0 @@
-patreon: zrepl
-liberapay: zrepl
-custom: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96
-ko_fi: problame

.gitignore

@@ -1,13 +1,3 @@
-# Build
-artifacts/
-
-# Golang
-vendor/
-
-# IDEs
-.idea/
-.vscode/
-
 # Compiled Object files, Static and Dynamic libs (Shared Objects)
 *.o
 *.a

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "docs/themes/docdock"]
+	path = docs/themes/docdock
+	url = git@github.com:zrepl/hugo-theme-docdock.git
+

.golangci.yml

@@ -1,10 +0,0 @@
-linters:
-  enable:
-  - goimports
-
-issues:
-  exclude-rules:
-    - path: _test\.go
-      linters:
-        - errcheck
-

.travis.yml

@@ -1,70 +0,0 @@
-dist: xenial
-services:
-- docker
-
-env: # for allow_failures: https://docs.travis-ci.com/user/customizing-the-build/
-matrix:
-  include:
-
-  - language: go
-    name: "Build in Docker (docs/installation.rst)"
-    script:
-    - sudo docker build -t zrepl_build -f build.Dockerfile .
-    - |
-      sudo docker run -it --rm \
-        -v "${PWD}:/go/src/github.com/zrepl/zrepl" \
-        --user "$(id -u):$(id -g)" \
-        zrepl_build make vendordeps release
-
-  - &zrepl_build_template
-    language: go
-    go_import_path: github.com/zrepl/zrepl
-    before_install:
-        - wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
-        - echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807  protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
-        - sudo unzip -d /usr protoc-3.6.1-linux-x86_64.zip
-        - ./lazy.sh godep
-        - make vendordeps
-    script:
-      - make
-      - make vet
-      - make test
-      - make lint
-      - make artifacts/zrepl-freebsd-amd64
-      - make artifacts/zrepl-linux-amd64
-      - make artifacts/zrepl-darwin-amd64
-    go:
-    - "1.11"
-
-  - <<: *zrepl_build_template
-    go:
-    - "1.12"
-
-  - <<: *zrepl_build_template
-    go:
-    - "master"
-
-  - &zrepl_docs_template 
-    language: python
-    python:
-    - "3.4"
-    install:
-    - sudo apt-get install libgirepository1.0-dev
-    - pip install -r docs/requirements.txt
-    script:
-    - make docs
-  - <<: *zrepl_docs_template
-    python:
-    - "3.5"
-  - <<: *zrepl_docs_template
-    python:
-    - "3.6"
-  - <<: *zrepl_docs_template
-    python:
-    - "3.7"
-  
-
-  allow_failures:
-  - <<: *zrepl_build_template
-    go:
-    - "master"

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2017-2018 Christian Schwarz
+Copyright (c) 2017 Christian Schwarz
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,188 +0,0 @@
-.PHONY: generate build test vet cover release docs docs-clean clean format lint platformtest
-.PHONY: release bins-all release-noarch
-.DEFAULT_GOAL := zrepl-bin
-
-ARTIFACTDIR := artifacts
-
-ifdef ZREPL_VERSION
-    _ZREPL_VERSION := $(ZREPL_VERSION)
-endif
-ifndef _ZREPL_VERSION
-    _ZREPL_VERSION := $(shell git describe --always --dirty 2>/dev/null || echo "ZREPL_BUILD_INVALID_VERSION" )
-    ifeq ($(_ZREPL_VERSION),ZREPL_BUILD_INVALID_VERSION) # can't use .SHELLSTATUS because Debian Stretch is still on gmake 4.1
-        $(error cannot infer variable ZREPL_VERSION using git and variable is not overriden by make invocation)
-    endif
-endif
-
-GO := go
-GOOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOOS"')
-GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"')
-GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"')
-GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"')
-GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
-GO_ENV_VARS := GO111MODULE=on
-GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
-GO_MOD_READONLY := -mod=readonly
-GO_BUILDFLAGS := $(GO_MOD_READONLY)
-GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
-GOLANGCI_LINT := golangci-lint
-ifneq ($(GOARM),)
-	ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM)
-else
-	ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)
-endif
-
-.PHONY: printvars
-printvars:
-	@echo GOOS=$(GOOS)
-	@echo GOARCH=$(GOARCH)
-	@echo GOARM=$(GOARM)
-
-
-##################### PRODUCING A RELEASE #############
-.PHONY: release wrapup-and-checksum check-git-clean sign clean
-
-release: clean
-	# no cross-platform support for target test
-	$(MAKE) test
-	$(MAKE) bins-all
-	$(MAKE) noarch
-	$(MAKE) wrapup-and-checksum
-	$(MAKE) check-git-clean
-ifeq (SIGN, 1)
-	$(make) sign
-endif
-	@echo "ZREPL RELEASE ARTIFACTS AVAILABLE IN artifacts/release"
-
-# expects `release` target to have run before
-NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
-wrapup-and-checksum:
-	rm -f $(NOARCH_TARBALL)
-	tar --mtime='1970-01-01' --sort=name \
-		--transform 's/$(ARTIFACTDIR)/zrepl-$(_ZREPL_VERSION)-noarch/' \
-		--transform 's#dist#zrepl-$(_ZREPL_VERSION)-noarch/dist#' \
-		--transform 's#config/samples#zrepl-$(_ZREPL_VERSION)-noarch/config#' \
-		-acf $(NOARCH_TARBALL) \
-		$(ARTIFACTDIR)/docs/html \
-		$(ARTIFACTDIR)/bash_completion \
-		$(ARTIFACTDIR)/go_env.txt \
-		dist \
-		config/samples
-	rm -rf "$(ARTIFACTDIR)/release"
-	mkdir -p "$(ARTIFACTDIR)/release"
-	cp -l $(ARTIFACTDIR)/zrepl-* \
-		$(ARTIFACTDIR)/platformtest-* \
-		"$(ARTIFACTDIR)/release"
-	cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
-
-check-git-clean:
-	@# note that we use ZREPL_VERSION and not _ZREPL_VERSION because we want to detect the override
-	@if git describe --always --dirty 2>/dev/null | grep dirty >/dev/null; then \
-        echo '[INFO] either git reports checkout is dirty or git is not installed or this is not a git checkout'; \
-		if [ "$(ZREPL_VERSION)" = "" ]; then \
-			echo '[WARN] git checkout is dirty and make variable ZREPL_VERSION was not used to override'; \
-			git status; \
-			echo "git diff:";  \
-			git diff | cat; \
-			exit 1; \
-		fi; \
-	fi;
-
-sign:
-	gpg -u "89BC 5D89 C845 568B F578  B306 CDBD 8EC8 E27C A5FC" \
-		--armor \
-		--detach-sign $(ARTIFACTDIR)/release/sha512sum.txt
-
-clean: docs-clean
-	rm -rf "$(ARTIFACTDIR)"
-
-##################### BINARIES #####################
-.PHONY: bins-all lint test vet zrepl-bin platformtest-bin
-
-BINS_ALL_TARGETS := zrepl-bin platformtest-bin vet lint
-GO_SUPPORTS_ILLUMOS := $(shell $(GO) version | gawk -F '.' '/^go version /{split($$0, comps, " "); split(comps[3], v, "."); if (v[1] == "go1" && v[2] >= 13) { print "illumos"; } else { print "noillumos"; }}')
-bins-all:
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=amd64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=386
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=amd64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm     GOARM=7
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=386
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=darwin    GOARCH=amd64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=solaris   GOARCH=amd64
-ifeq ($(GO_SUPPORTS_ILLUMOS), illumos)
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=illumos   GOARCH=amd64
-else ifeq ($(GO_SUPPORTS_ILLUMOS), noillumos)
-	@echo "SKIPPING ILLUMOS BUILD BECAUSE GO VERSION DOESN'T SUPPORT IT"
-else
-	@echo "CANNOT DETERMINE WHETHER GO VERSION SUPPORTS GOOS=illumos"; exit 1
-endif
-
-lint:
-	$(GO_ENV_VARS) $(GOLANGCI_LINT) run ./...
-test:
-	$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) ./...
-vet:
-	$(GO_ENV_VARS) $(GO) vet $(GO_BUILDFLAGS) ./...
-
-zrepl-bin:
-	$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(ZREPL_TARGET_TUPLE)"
-
-platformtest-bin:
-	$(GO_BUILD) -o "$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" ./platformtest/harness
-
-##################### DEV TARGETS #####################
-# not part of the build, must do that manually
-.PHONY: generate format platformtest
-
-generate:
-	protoc -I=replication/logic/pdu --go_out=plugins=grpc:replication/logic/pdu replication/logic/pdu/pdu.proto
-	$(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./...
-
-format:
-	goimports -srcdir . -local 'github.com/zrepl/zrepl' -w $(shell find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go')
-
-ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
-ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
-ZREPL_PLATFORMTEST_MOUNTPOINT := /tmp/zreplplatformtest.pool
-ZREPL_PLATFORMTEST_ZFS_LOG := /tmp/zreplplatformtest.zfs.log
-# ZREPL_PLATFORMTEST_STOP_AND_KEEP := -failure.stop-and-keep-pool
-ZREPL_PLATFORMTEST_ARGS := 
-platformtest: # do not track dependency on platformtest-bin to allow build of platformtest outside of test VM
-	rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
-	platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
-	"$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" \
-		-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
-		-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
-		-mountpoint "$(ZREPL_PLATFORMTEST_MOUNTPOINT)" \
-		$(ZREPL_PLATFORMTEST_STOP_AND_KEEP) \
-		$(ZREPL_PLATFORMTEST_ARGS)
-
-##################### NOARCH #####################
-.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
-
-
-$(ARTIFACTDIR):
-	mkdir -p "$@"
-$(ARTIFACTDIR)/docs: $(ARTIFACTDIR)
-	mkdir -p "$@"
-
-noarch: $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_env.txt docs
-	# pass
-
-$(ARTIFACTDIR)/bash_completion:
-	$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
-	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) bashcomp "$@"
-
-$(ARTIFACTDIR)/go_env.txt:
-	$(GO_ENV_VARS) $(GO) env > $@
-
-docs: $(ARTIFACTDIR)/docs
-	make -C docs \
-		html \
-		BUILDDIR=../artifacts/docs \
-
-docs-clean:
-	make -C docs \
-		clean \
-		BUILDDIR=../artifacts/docs

README.md

@@ -1,136 +1,2 @@
-[![GitHub license](https://img.shields.io/github/license/zrepl/zrepl.svg)](https://github.com/zrepl/zrepl/blob/master/LICENSE)
-[![Language: Go](https://img.shields.io/badge/language-Go-6ad7e5.svg)](https://golang.org/)
-[![User Docs](https://img.shields.io/badge/docs-web-blue.svg)](https://zrepl.github.io)
-[![Donate via PayPal](https://img.shields.io/badge/donate-paypal-yellow.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
-[![Donate via Liberapay](https://img.shields.io/liberapay/receives/zrepl.svg?logo=liberapay)](https://liberapay.com/zrepl/donate)
-[![Twitter](https://img.shields.io/twitter/url/https/github.com/zrepl/zrepl.svg?style=social)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl)
-
 # zrepl
-zrepl is a one-stop ZFS backup & replication solution.
+ZFS dataset replication. Not a shell script.
-
-## User Documentation
-
-**User Documentation** can be found at [zrepl.github.io](https://zrepl.github.io).
-
-## Bug Reports
-
-1. If the issue is reproducible, enable debug logging, reproduce and capture the log.
-2. Open an issue on GitHub, with logs pasted as GitHub gists / inline.
-
-## Feature Requests
-
-1. Does you feature request require default values / some kind of configuration?
-   If so, think of an expressive configuration example.
-2. Think of at least one use case that generalizes from your concrete application.
-3. Open an issue on GitHub with example conf & use case attached.
-
-The above does not apply if you already implemented everything.
-Check out the *Coding Workflow* section below for details.
-
-## Package Maintainer Information
-
-* Follow the steps in `docs/installation.rst -> Compiling from Source` and read the Makefile / shell scripts used in this process.
-* Make sure your distro is compatible with the paths in `docs/installation.rst`.
-* Ship a default config that adheres to your distro's `hier` and logging system.
-* Ship a service manager file and _please_ try to upstream it to this repository.
-  * `dist/systemd` contains a Systemd unit template.
-* Ship other material provided in `./dist`, e.g. in `/usr/share/zrepl/`.
-* Use `make release ZREPL_VERSION='mydistro-1.2.3_1'`
-    * Your distro's name and any versioning supplemental to zrepl's (e.g. package revision) should be in this string
-* Use `make platformtest` **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
-* Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
-
-## Developer Documentation
-
-zrepl is written in [Go](https://golang.org) and uses [Go modules](https://github.com/golang/go/wiki/Modules) to manage dependencies.
-The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
-
-To get started, run `./lazy.sh devsetup` to easily install build dependencies and read `docs/installation.rst -> Compiling from Source`.
-
-### Overall Architecture
-
-The application architecture is documented as part of the user docs in the *Implementation* section (`docs/content/impl`).
-Make sure to develop an understanding how zrepl is typically used by studying the user docs first.
-
-### Project Structure
-
-```
-├── artifacts               # build artifcats generate by make
-├── cli                     # wrapper around CLI package cobra
-├── client                  # all subcommands that are not `daemon`
-├── config                  # config data types (=> package yaml-config)
-│   └── samples
-├── daemon                  # the implementation of `zrepl daemon` subcommand
-│   ├── filters
-│   ├── job                 # job implementations
-│   ├── logging             # logging outlets + formatters
-│   ├── nethelpers
-│   ├── prometheus
-│   ├── pruner              # pruner implementation
-│   ├── snapper             # snapshotter implementation
-├── docs                    # sphinx-based documentation
-├── dist                    # supplemental material for users & package maintainers
-│   ├── **/*.rst            # documentation in reStructuredText
-│   ├── sphinxconf
-│   │   └── conf.py         # sphinx config (see commit 445a280 why its not in docs/)
-│   ├── requirements.txt    # pip3 requirements to build documentation
-│   ├── publish.sh          # shell script for automated rendering & deploy to zrepl.github.io repo
-│   └── public_git          # checkout of zrepl.github.io managed by above shell script
-├── endpoint                # implementation of replication endpoints (=> package replication)
-├── logger                  # our own logger package
-├── pruning                 # pruning rules (the logic, not the actual execution)
-│   └── retentiongrid
-├── replication
-│   ├── driver              # the driver of the replication logic (status reporting, error handling)
-│   ├── logic               # planning & executing replication steps via rpc
-|   |   └── pdu             # the generated gRPC & protobuf code used in replication (and endpoints)
-│   └── report              # the JSON-serializable report datastructures exposed to the client
-├── rpc                     # the hybrid gRPC + ./dataconn RPC client: connects to a remote replication.Endpoint
-│   ├── dataconn            # Bulk data-transfer RPC protocol
-│   ├── grpcclientidentity  # adaptor to inject package transport's 'client identity' concept into gRPC contexts
-│   ├── netadaptor          # adaptor to convert a package transport's Connecter and Listener into net.* primitives
-│   ├── transportmux        # TCP connecter and listener used to split control & data traffic
-│   └── versionhandshake    # replication protocol version handshake perfomed on newly established connections
-├── tlsconf                 # abstraction for Go TLS server + client config
-├── transport               # transports implementation
-│   ├── fromconfig
-│   ├── local
-│   ├── ssh
-│   ├── tcp
-│   └── tls
-├── util
-├── vendor                  # managed by dep
-├── version                 # abstraction for versions (filled during build by Makefile)
-└── zfs                     # zfs(8) wrappers
-```
-
-### Coding Workflow
-
-* Open an issue when starting to hack on a new feature
-* Commits should reference the issue they are related to
-* Docs improvements not documenting new features do not require an issue.
-
-### Breaking Changes
-
-Backward-incompatible changes must be documented in the git commit message and are listed in `docs/changelog.rst`.
-
-* Config-breaking changes must contain a line `BREAK CONFIG` in the commit message
-* Other breaking changes must contain a line `BREAK` in the commit message
-
-### Glossary & Naming Inconsistencies
-
-In ZFS, *dataset* refers to the objects *filesystem*, *ZVOL* and *snapshot*. <br />
-However, we need a word for *filesystem* & *ZVOL* but not a snapshot, bookmark, etc.
-
-Toward the user, the following terminology is used:
-
-* **filesystem**: a ZFS filesystem or a ZVOL
-* **filesystem version**: a ZFS snapshot or a bookmark
-
-Sadly, the zrepl implementation is inconsistent in its use of these words:
-variables and types are often named *dataset* when they in fact refer to a *filesystem*.
-
-There will not be a big refactoring (an attempt was made, but it's destroying too much history without much gain).
-
-However, new contributions & patches should fix naming without further notice in the commit message.
-

build.Dockerfile

@@ -1,30 +0,0 @@
-FROM golang:latest
-
-RUN apt-get update && apt-get install -y \
-    python3-pip \
-    unzip \
-    gawk
-
-ADD build.installprotoc.bash ./
-RUN bash build.installprotoc.bash
-
-ADD lazy.sh /tmp/lazy.sh
-ADD docs/requirements.txt /tmp/requirements.txt
-ENV ZREPL_LAZY_DOCS_REQPATH=/tmp/requirements.txt
-RUN /tmp/lazy.sh docdep
-
-# prepare volume mount of git checkout to /zrepl
-RUN mkdir -p /src/github.com/zrepl/zrepl
-RUN mkdir -p /.cache && chmod -R 0777 /.cache
-
-# $GOPATH is /go
-# Go 1.12 doesn't use modules within GOPATH, but 1.13 and later do
-# => store source outside of GOPATH
-WORKDIR /src
-
-# Install build tools (e.g. protobuf generator, stringer) into $GOPATH/bin
-ADD build/ /tmp/build
-RUN /tmp/lazy.sh godep
-
-RUN chmod -R 0777 /go
-

build.installprotoc.bash

@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-MACH=$(uname -m)
-MACH="${MACH/aarch64/aarch_64}"
-
-VERSION=3.6.1
-FILENAME=protoc-"$VERSION"-linux-"$MACH".zip
-
-if [ -e "$FILENAME" ]; then
-    echo "$FILENAME" already exists 1>&2
-    exit 1
-fi
-
-wget https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
-
-stat "$FILENAME"
-
-sha256sum -c --ignore-missing <<EOF
-6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807  protoc-3.6.1-linux-x86_64.zip
-af8e5aaaf39ddec62ec8dd2be1b8d9602c6da66564883a16393ade5f71170922  protoc-3.6.1-linux-aarch_64.zip
-EOF
-
-unzip -d /usr "$FILENAME"

build/go.mod

@@ -1,51 +0,0 @@
-module github.com/zrepl/zrepl/build
-
-go 1.12
-
-require (
-	github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 // indirect
-	github.com/alvaroloes/enumer v1.1.1
-	github.com/fatih/color v1.7.0 // indirect
-	github.com/go-critic/go-critic v0.3.4 // indirect
-	github.com/gogo/protobuf v1.2.1 // indirect
-	github.com/golang/mock v1.2.0 // indirect
-	github.com/golang/protobuf v1.2.0
-	github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 // indirect
-	github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 // indirect
-	github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d // indirect
-	github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 // indirect
-	github.com/golangci/golangci-lint v1.17.1
-	github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb // indirect
-	github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 // indirect
-	github.com/golangci/misspell v0.3.4 // indirect
-	github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 // indirect
-	github.com/google/go-cmp v0.3.0 // indirect
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/pkg/errors v0.8.1 // indirect
-	github.com/sirupsen/logrus v1.4.2 // indirect
-	github.com/spf13/afero v1.2.2 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/viper v1.3.2 // indirect
-	github.com/stretchr/testify v1.4.0 // indirect
-	golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 // indirect
-	golang.org/x/sys v0.0.0-20191010194322-b09406accb47 // indirect
-	golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b
-	mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe // indirect
-	sourcegraph.com/sqs/pbtypes v1.0.0 // indirect
-)
-
-// invalid dates in transitive dependencies (first validated in Go 1.13, didn't fail in earlier Go versions)
-replace (
-	github.com/go-critic/go-critic v0.0.0-20181204210945-1df300866540 => github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540
-	github.com/go-critic/go-critic v0.0.0-20181204210945-ee9bf5809ead => github.com/go-critic/go-critic v0.3.5-0.20190210220443-ee9bf5809ead
-	github.com/golangci/errcheck v0.0.0-20181003203344-ef45e06d44b6 => github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6
-	github.com/golangci/go-tools v0.0.0-20180109140146-35a9f45a5db0 => github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0
-	github.com/golangci/go-tools v0.0.0-20180109140146-af6baa5dc196 => github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196
-	github.com/golangci/gofmt v0.0.0-20181105071733-0b8337e80d98 => github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98
-	github.com/golangci/gosec v0.0.0-20180901114220-66fb7fc33547 => github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547
-	github.com/golangci/ineffassign v0.0.0-20180808204949-42439a7714cc => github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc
-	github.com/golangci/lint-1 v0.0.0-20180610141402-ee948d087217 => github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217
-	golang.org/x/tools v0.0.0-20190125232054-379209517ffe => golang.org/x/tools v0.0.0-20190205201329-379209517ffe
-	mvdan.cc/unparam v0.0.0-20190124213536-fbb59629db34 => mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34
-)

build/go.sum

@@ -1,280 +0,0 @@
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/OpenPeeDeeP/depguard v0.0.0-20180806142446-a69c782687b2/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
-github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 h1:pFks+oaqVWDFq0KsLQZFB2pQGB5Us0HfMSBENtieXOs=
-github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
-github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
-github.com/alvaroloes/enumer v1.1.1 h1:v+1scNqEhSFAPb9tLwblQegeTXJhnw8hf9O0amTbOvQ=
-github.com/alvaroloes/enumer v1.1.1/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
-github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
-github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
-github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/go-critic/go-critic v0.3.4 h1:FYaiaLjX0Nqei80KPhm4CyFQUBbmJwSrHxQ73taaGBc=
-github.com/go-critic/go-critic v0.3.4/go.mod h1:AHR42Lk/E/aOznsrYdMYeIQS5RH10HZHSqP+rD6AJrc=
-github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
-github.com/go-lintpack/lintpack v0.5.1/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
-github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0=
-github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
-github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
-github.com/go-toolsmith/astcast v0.0.0-20181028201508-b7a89ed70af1/go.mod h1:TEo3Ghaj7PsZawQHxT/oBvo4HK/sl1RcuUHDKTTju+o=
-github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=
-github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
-github.com/go-toolsmith/astcopy v0.0.0-20180903214859-79b422d080c4/go.mod h1:c9CPdq2AzM8oPomdlPniEfPAC6g1s7NqZzODt8y6ib8=
-github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8=
-github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
-github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
-github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ=
-github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
-github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
-github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k=
-github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
-github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
-github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
-github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg=
-github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
-github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8 h1:vVouagbdmqTVlCIAxpyYsNNTbkKZ3V66VpKOLU/s6W4=
-github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
-github.com/go-toolsmith/pkgload v1.0.0 h1:4DFWWMXVfbcN5So1sBNW9+yeiMqLFGl1wFLTL5R0Tgg=
-github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
-github.com/go-toolsmith/strparse v0.0.0-20180903215201-830b6daa1241/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
-github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4=
-github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
-github.com/go-toolsmith/typep v0.0.0-20181030061450-d63dc7650676/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
-github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA=
-github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
-github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
-github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
-github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
-github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0=
-github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
-github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
-github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
-github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w=
-github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
-github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw=
-github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
-github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 h1:MRhC9XbUjE6XDOInSJ8pwHuPagqsyO89QDU9IdVhe3o=
-github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
-github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
-github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8=
-github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
-github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
-github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d h1:pXTK/gkVNs7Zyy7WKgLXmpQ5bHTrq5GDsp8R9Qs67g0=
-github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
-github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU=
-github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
-github.com/golangci/golangci-lint v1.17.1 h1:lc8Hf9GPCjIr0hg3S/xhvFT1+Hydass8F1xchr8jkME=
-github.com/golangci/golangci-lint v1.17.1/go.mod h1:+5sJSl2h3aly+fpmL2meSP8CaSKua2E4Twi9LPy7b1g=
-github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb h1:Bi7BYmZVg4C+mKGi8LeohcP2GGUl2XJD4xCkJoZSaYc=
-github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb/go.mod h1:ON/c2UR0VAAv6ZEAFKhjCLplESSmRFfZcDLASbI1GWo=
-github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
-github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
-github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
-github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 h1:utua3L2IbQJmauC5IXdEA547bcoU5dozgQAfc8Onsg4=
-github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
-github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
-github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
-github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
-github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
-github.com/golangci/misspell v0.3.4 h1:kAD5J0611Zo2VirUn9KFP15RjEqkmfEfnFxyIVxZCYg=
-github.com/golangci/misspell v0.3.4/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
-github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us=
-github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
-github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
-github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 h1:XQKc8IYQOeRwVs36tDrEmTgDgP88d5iEURwpmtiAlOM=
-github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
-github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
-github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
-github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw=
-github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
-github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
-github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
-github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
-github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM=
-github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
-github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
-github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
-github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
-github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
-github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
-github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
-github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
-github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
-github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
-github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663 h1:Ri1EhipkbhWsffPJ3IPlrb4SkTOPa2PfRXp3jchBczw=
-github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
-github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
-github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 h1:/I3lTljEEDNYLho3/FUB7iD/oc2cEFgVmbHzV+O0PtU=
-github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ=
-github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
-github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
-github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rogpeppe/go-internal v1.2.1/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
-github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
-github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
-github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM=
-github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
-github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc=
-github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
-github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
-github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
-github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs=
-github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
-github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
-github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.2 h1:NfkwRbgViGoyjBKsLI0QMDcuMnhM+SBg3T0cGfpvKDE=
-github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
-github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
-github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
-github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
-github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
-github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb h1:lI9ufgFfvuqRctP9Ny8lDDLbSWCMxBPletcSqrnyFYM=
-github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
-github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
-github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
-github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
-github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
-github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190313220215-9f648a60d977 h1:actzWV6iWn3GLqN8dZjzsB+CLt+gaV2+wsxroxiQI8I=
-golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
-golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181205014116-22934f0fdb62/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8 h1:b0PLhFjEMqrIqsD4rS5sp0YxBYgdqKzX0KkkGGKLV8I=
-golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b h1:iEAPfYPbYbxG/2lNN4cMOHkmgKNsCuUwkxlDCK46UlU=
-golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
-mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
-mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=
-mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
-mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
-mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe h1:Ekmnp+NcP2joadI9pbK4Bva87QKZSeY7le//oiMrc9g=
-mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe/go.mod h1:BnhuWBAqxH3+J5bDybdxgw5ZfS+DsVd4iylsKQePN8o=
-sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
-sourcegraph.com/sqs/pbtypes v1.0.0 h1:f7lAwqviDEGvON4kRv0o5V7FT/IQK+tbkF664XMbP3o=
-sourcegraph.com/sqs/pbtypes v1.0.0/go.mod h1:3AciMUv4qUuRHRHhOG4TZOB+72GdPVz5k+c648qsFS4=

build/tools.go

@@ -1,11 +0,0 @@
-// +build tools
-
-package main
-
-import (
-	_ "github.com/alvaroloes/enumer"
-	_ "github.com/golang/protobuf/protoc-gen-go"
-	_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
-	_ "golang.org/x/tools/cmd/goimports"
-	_ "golang.org/x/tools/cmd/stringer"
-)

cli/cli.go

@@ -1,122 +0,0 @@
-package cli
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
-
-	"github.com/zrepl/zrepl/config"
-)
-
-var rootArgs struct {
-	configPath string
-}
-
-var rootCmd = &cobra.Command{
-	Use:   "zrepl",
-	Short: "One-stop ZFS replication solution",
-}
-
-var bashcompCmd = &cobra.Command{
-	Use:   "bashcomp path/to/out/file",
-	Short: "generate bash completions",
-	Run: func(cmd *cobra.Command, args []string) {
-		if len(args) != 1 {
-			fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
-			err := cmd.Usage()
-			if err != nil {
-				panic(err)
-			}
-			os.Exit(1)
-		}
-		if err := rootCmd.GenBashCompletionFile(args[0]); err != nil {
-			fmt.Fprintf(os.Stderr, "error generating bash completion: %s", err)
-			os.Exit(1)
-		}
-	},
-	Hidden: true,
-}
-
-func init() {
-	rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
-	rootCmd.AddCommand(bashcompCmd)
-}
-
-type Subcommand struct {
-	Use              string
-	Short            string
-	Example          string
-	NoRequireConfig  bool
-	Run              func(subcommand *Subcommand, args []string) error
-	SetupFlags       func(f *pflag.FlagSet)
-	SetupSubcommands func() []*Subcommand
-
-	config    *config.Config
-	configErr error
-}
-
-func (s *Subcommand) ConfigParsingError() error {
-	return s.configErr
-}
-
-func (s *Subcommand) Config() *config.Config {
-	if !s.NoRequireConfig && s.config == nil {
-		panic("command that requires config is running and has no config set")
-	}
-	return s.config
-}
-
-func (s *Subcommand) run(cmd *cobra.Command, args []string) {
-	s.tryParseConfig()
-	err := s.Run(s, args)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s\n", err)
-		os.Exit(1)
-	}
-}
-
-func (s *Subcommand) tryParseConfig() {
-	config, err := config.ParseConfig(rootArgs.configPath)
-	s.configErr = err
-	if err != nil {
-		if s.NoRequireConfig {
-			// doesn't matter
-			return
-		} else {
-			fmt.Fprintf(os.Stderr, "could not parse config: %s\n", err)
-			os.Exit(1)
-		}
-	}
-	s.config = config
-}
-
-func AddSubcommand(s *Subcommand) {
-	addSubcommandToCobraCmd(rootCmd, s)
-}
-
-func addSubcommandToCobraCmd(c *cobra.Command, s *Subcommand) {
-	cmd := cobra.Command{
-		Use:     s.Use,
-		Short:   s.Short,
-		Example: s.Example,
-	}
-	if s.SetupSubcommands == nil {
-		cmd.Run = s.run
-	} else {
-		for _, sub := range s.SetupSubcommands() {
-			addSubcommandToCobraCmd(&cmd, sub)
-		}
-	}
-	if s.SetupFlags != nil {
-		s.SetupFlags(cmd.Flags())
-	}
-	c.AddCommand(&cmd)
-}
-
-func Run() {
-	if err := rootCmd.Execute(); err != nil {
-		os.Exit(1)
-	}
-}

client/configcheck.go

@@ -1,120 +0,0 @@
-package client
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-
-	"github.com/kr/pretty"
-	"github.com/pkg/errors"
-	"github.com/spf13/pflag"
-	"github.com/zrepl/yaml-config"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/daemon/logging"
-	"github.com/zrepl/zrepl/logger"
-)
-
-var configcheckArgs struct {
-	format string
-	what   string
-}
-
-var ConfigcheckCmd = &cli.Subcommand{
-	Use:   "configcheck",
-	Short: "check if config can be parsed without errors",
-	SetupFlags: func(f *pflag.FlagSet) {
-		f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]")
-		f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]")
-	},
-	Run: func(subcommand *cli.Subcommand, args []string) error {
-		formatMap := map[string]func(interface{}){
-			"": func(i interface{}) {},
-			"pretty": func(i interface{}) {
-				if _, err := pretty.Println(i); err != nil {
-					panic(err)
-				}
-			},
-			"json": func(i interface{}) {
-				if err := json.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
-					panic(err)
-				}
-			},
-			"yaml": func(i interface{}) {
-				if err := yaml.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
-					panic(err)
-				}
-			},
-		}
-
-		formatter, ok := formatMap[configcheckArgs.format]
-		if !ok {
-			return fmt.Errorf("unsupported --format %q", configcheckArgs.format)
-		}
-
-		var hadErr bool
-		// further: try to build jobs
-		confJobs, err := job.JobsFromConfig(subcommand.Config())
-		if err != nil {
-			err := errors.Wrap(err, "cannot build jobs from config")
-			if configcheckArgs.what == "jobs" {
-				return err
-			} else {
-				fmt.Fprintf(os.Stderr, "%s\n", err)
-				confJobs = nil
-				hadErr = true
-			}
-		}
-
-		// further: try to build logging outlets
-		outlets, err := logging.OutletsFromConfig(*subcommand.Config().Global.Logging)
-		if err != nil {
-			err := errors.Wrap(err, "cannot build logging from config")
-			if configcheckArgs.what == "logging" {
-				return err
-			} else {
-				fmt.Fprintf(os.Stderr, "%s\n", err)
-				outlets = nil
-				hadErr = true
-			}
-		}
-
-		whatMap := map[string]func(){
-			"all": func() {
-				o := struct {
-					config  *config.Config
-					jobs    []job.Job
-					logging *logger.Outlets
-				}{
-					subcommand.Config(),
-					confJobs,
-					outlets,
-				}
-				formatter(o)
-			},
-			"config": func() {
-				formatter(subcommand.Config())
-			},
-			"jobs": func() {
-				formatter(confJobs)
-			},
-			"logging": func() {
-				formatter(outlets)
-			},
-		}
-
-		wf, ok := whatMap[configcheckArgs.what]
-		if !ok {
-			return fmt.Errorf("unsupported --format %q", configcheckArgs.what)
-		}
-		wf()
-
-		if hadErr {
-			return fmt.Errorf("config parsing failed")
-		} else {
-			return nil
-		}
-	},
-}

client/holds.go

@@ -1,87 +0,0 @@
-package client
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"os"
-	"strings"
-
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-var (
-	HoldsCmd = &cli.Subcommand{
-		Use:   "holds",
-		Short: "manage holds & step bookmarks",
-		SetupSubcommands: func() []*cli.Subcommand {
-			return holdsList
-		},
-	}
-)
-
-var holdsList = []*cli.Subcommand{
-	&cli.Subcommand{
-		Use:             "list [FSFILTER]",
-		Run:             doHoldsList,
-		NoRequireConfig: true,
-		Short: `
-FSFILTER SYNTAX:
-representation of a 'filesystems' filter statement on the command line
-		`,
-	},
-}
-
-func fsfilterFromCliArg(arg string) (zfs.DatasetFilter, error) {
-	mappings := strings.Split(arg, ",")
-	f := filters.NewDatasetMapFilter(len(mappings), true)
-	for _, m := range mappings {
-		thisMappingErr := fmt.Errorf("expecting comma-separated list of <dataset-pattern>:<ok|!> pairs, got %q", m)
-		lhsrhs := strings.SplitN(m, ":", 2)
-		if len(lhsrhs) != 2 {
-			return nil, thisMappingErr
-		}
-		err := f.Add(lhsrhs[0], lhsrhs[1])
-		if err != nil {
-			return nil, fmt.Errorf("%s: %s", thisMappingErr, err)
-		}
-	}
-	return f.AsFilter(), nil
-}
-
-func doHoldsList(sc *cli.Subcommand, args []string) error {
-	var err error
-	ctx := context.Background()
-
-	if len(args) > 1 {
-		return errors.New("this subcommand takes at most one argument")
-	}
-
-	var filter zfs.DatasetFilter
-	if len(args) == 0 {
-		filter = zfs.NoFilter()
-	} else {
-		filter, err = fsfilterFromCliArg(args[0])
-		if err != nil {
-			return errors.Wrap(err, "cannot parse filesystem filter args")
-		}
-	}
-
-	listing, err := endpoint.ListZFSHoldsAndBookmarks(ctx, filter)
-	if err != nil {
-		return err // context clear by invocation of command
-	}
-
-	enc := json.NewEncoder(os.Stdout)
-	enc.SetIndent("  ", "  ")
-	if err := enc.Encode(listing); err != nil {
-		panic(err)
-	}
-
-	return nil
-}

client/jsonclient.go

@@ -1,49 +0,0 @@
-package client
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"io"
-	"net"
-	"net/http"
-
-	"github.com/pkg/errors"
-)
-
-func controlHttpClient(sockpath string) (client http.Client, err error) {
-	return http.Client{
-		Transport: &http.Transport{
-			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
-				return net.Dial("unix", sockpath)
-			},
-		},
-	}, nil
-}
-
-func jsonRequestResponse(c http.Client, endpoint string, req interface{}, res interface{}) error {
-	var buf bytes.Buffer
-	encodeErr := json.NewEncoder(&buf).Encode(req)
-	if encodeErr != nil {
-		return encodeErr
-	}
-
-	resp, err := c.Post("http://unix"+endpoint, "application/json", &buf)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		var msg bytes.Buffer
-		_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
-		return errors.Errorf("%s", msg.String())
-	}
-
-	decodeError := json.NewDecoder(resp.Body).Decode(&res)
-	if decodeError != nil {
-		return decodeError
-	}
-
-	return nil
-}

client/migrate.go

@@ -1,273 +0,0 @@
-package client
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/fatih/color"
-	"github.com/kr/pretty"
-	"github.com/pkg/errors"
-	"github.com/spf13/pflag"
-
-	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/zfs"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-)
-
-var (
-	MigrateCmd = &cli.Subcommand{
-		Use:   "migrate",
-		Short: "perform migration of the on-disk / zfs properties",
-		SetupSubcommands: func() []*cli.Subcommand {
-			return migrations
-		},
-	}
-)
-
-var migrations = []*cli.Subcommand{
-	&cli.Subcommand{
-		Use: "0.0.X:0.1:placeholder",
-		Run: doMigratePlaceholder0_1,
-		SetupFlags: func(f *pflag.FlagSet) {
-			f.BoolVar(&migratePlaceholder0_1Args.dryRun, "dry-run", false, "dry run")
-		},
-	},
-	&cli.Subcommand{
-		Use: "replication-cursor:v1-v2",
-		Run: doMigrateReplicationCursor,
-		SetupFlags: func(f *pflag.FlagSet) {
-			f.BoolVar(&migrateReplicationCursorArgs.dryRun, "dry-run", false, "dry run")
-		},
-	},
-}
-
-var migratePlaceholder0_1Args struct {
-	dryRun bool
-}
-
-func doMigratePlaceholder0_1(sc *cli.Subcommand, args []string) error {
-	if len(args) != 0 {
-		return fmt.Errorf("migration does not take arguments, got %v", args)
-	}
-
-	cfg := sc.Config()
-
-	ctx := context.Background()
-	allFSS, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
-	if err != nil {
-		return errors.Wrap(err, "cannot list filesystems")
-	}
-
-	type workItem struct {
-		jobName string
-		rootFS  *zfs.DatasetPath
-		fss     []*zfs.DatasetPath
-	}
-	var wis []workItem
-	for i, j := range cfg.Jobs {
-		var rfsS string
-		switch job := j.Ret.(type) {
-		case *config.SinkJob:
-			rfsS = job.RootFS
-		case *config.PullJob:
-			rfsS = job.RootFS
-		default:
-			fmt.Printf("ignoring job %q (%d/%d, type %T)\n", j.Name(), i, len(cfg.Jobs), j.Ret)
-			continue
-		}
-		rfs, err := zfs.NewDatasetPath(rfsS)
-		if err != nil {
-			return errors.Wrapf(err, "root fs for job %q is not a valid dataset path", j.Name())
-		}
-		var fss []*zfs.DatasetPath
-		for _, fs := range allFSS {
-			if fs.HasPrefix(rfs) {
-				fss = append(fss, fs)
-			}
-		}
-		wis = append(wis, workItem{j.Name(), rfs, fss})
-	}
-
-	for _, wi := range wis {
-		fmt.Printf("job %q => migrate filesystems below root_fs %q\n", wi.jobName, wi.rootFS.ToString())
-		if len(wi.fss) == 0 {
-			fmt.Printf("\tno filesystems\n")
-			continue
-		}
-		for _, fs := range wi.fss {
-			fmt.Printf("\t%q ... ", fs.ToString())
-			r, err := zfs.ZFSMigrateHashBasedPlaceholderToCurrent(fs, migratePlaceholder0_1Args.dryRun)
-			if err != nil {
-				fmt.Printf("error: %s\n", err)
-			} else if !r.NeedsModification {
-				fmt.Printf("unchanged (placeholder=%v)\n", r.OriginalState.IsPlaceholder)
-			} else {
-				fmt.Printf("migrate (placeholder=%v) (old value = %q)\n",
-					r.OriginalState.IsPlaceholder, r.OriginalState.RawLocalPropertyValue)
-			}
-		}
-	}
-
-	return nil
-}
-
-var migrateReplicationCursorArgs struct {
-	dryRun bool
-}
-
-var bold = color.New(color.Bold)
-var succ = color.New(color.FgGreen)
-var fail = color.New(color.FgRed)
-
-var migrateReplicationCursorSkipSentinel = fmt.Errorf("skipping this filesystem")
-
-func doMigrateReplicationCursor(sc *cli.Subcommand, args []string) error {
-	if len(args) != 0 {
-		return fmt.Errorf("migration does not take arguments, got %v", args)
-	}
-
-	cfg := sc.Config()
-	jobs, err := job.JobsFromConfig(cfg)
-	if err != nil {
-		fmt.Printf("cannot parse config:\n%s\n\n", err)
-		fmt.Printf("NOTE: this migration was released together with a change in job name requirements.\n")
-		return fmt.Errorf("exiting migration after error")
-	}
-
-	ctx := context.Background()
-
-	v1cursorJobs := make([]job.Job, 0, len(cfg.Jobs))
-	for i, j := range cfg.Jobs {
-		if jobs[i].Name() != j.Name() {
-			panic("implementation error")
-		}
-		switch j.Ret.(type) {
-		case *config.PushJob:
-			v1cursorJobs = append(v1cursorJobs, jobs[i])
-		case *config.SourceJob:
-			v1cursorJobs = append(v1cursorJobs, jobs[i])
-		default:
-			fmt.Printf("ignoring job %q (%d/%d, type %T), not supposed to create v1 replication cursors\n", j.Name(), i, len(cfg.Jobs), j.Ret)
-			continue
-		}
-	}
-
-	// scan all filesystems for v1 replication cursors
-
-	fss, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
-	if err != nil {
-		return errors.Wrap(err, "list filesystems")
-	}
-
-	var hadError bool
-	for _, fs := range fss {
-
-		bold.Printf("INSPECT FILESYTEM %q\n", fs.ToString())
-
-		err := doMigrateReplicationCursorFS(ctx, v1cursorJobs, fs)
-		if err == migrateReplicationCursorSkipSentinel {
-			bold.Printf("FILESYSTEM SKIPPED\n")
-		} else if err != nil {
-			hadError = true
-			fail.Printf("MIGRATION FAILED: %s\n", err)
-		} else {
-			succ.Printf("FILESYSTEM %q COMPLETE\n", fs.ToString())
-		}
-	}
-
-	if hadError {
-		fail.Printf("\n\none or more filesystems could not be migrated, please inspect output and or re-run migration")
-		return errors.Errorf("")
-	}
-	return nil
-}
-
-func doMigrateReplicationCursorFS(ctx context.Context, v1CursorJobs []job.Job, fs *zfs.DatasetPath) error {
-
-	var owningJob job.Job = nil
-	for _, job := range v1CursorJobs {
-		conf := job.SenderConfig()
-		if conf == nil {
-			continue
-		}
-		pass, err := conf.FSF.Filter(fs)
-		if err != nil {
-			return errors.Wrapf(err, "filesystem filter error in job %q for fs %q", job.Name(), fs.ToString())
-		}
-		if !pass {
-			continue
-		}
-		if owningJob != nil {
-			return errors.Errorf("jobs %q and %q both match %q\ncannot attribute replication cursor to either one", owningJob.Name(), job.Name(), fs)
-		}
-		owningJob = job
-	}
-	if owningJob == nil {
-		fmt.Printf("no job's Filesystems filter matches\n")
-		return migrateReplicationCursorSkipSentinel
-	}
-	fmt.Printf("identified owning job %q\n", owningJob.Name())
-
-	versions, err := zfs.ZFSListFilesystemVersions(fs, nil)
-	if err != nil {
-		return errors.Wrapf(err, "list filesystem versions of %q", fs.ToString())
-	}
-
-	var oldCursor *zfs.FilesystemVersion
-	for i, fsv := range versions {
-		if fsv.Type != zfs.Bookmark {
-			continue
-		}
-
-		_, _, err := endpoint.ParseReplicationCursorBookmarkName(fsv.ToAbsPath(fs))
-		if err != endpoint.ErrV1ReplicationCursor {
-			continue
-		}
-
-		if oldCursor != nil {
-			fmt.Printf("unexpected v1 replication cursor candidate: %q", fsv.ToAbsPath(fs))
-			return errors.Wrap(err, "multiple filesystem versions identified as v1 replication cursors")
-		}
-
-		oldCursor = &versions[i]
-
-	}
-
-	if oldCursor == nil {
-		bold.Printf("no v1 replication cursor found for filesystem %q\n", fs.ToString())
-		return migrateReplicationCursorSkipSentinel
-	}
-
-	fmt.Printf("found v1 replication cursor:\n%s\n", pretty.Sprint(oldCursor))
-
-	mostRecentNew, err := endpoint.GetMostRecentReplicationCursorOfJob(ctx, fs.ToString(), owningJob.SenderConfig().JobID)
-	if err != nil {
-		return errors.Wrapf(err, "get most recent v2 replication cursor")
-	}
-
-	if mostRecentNew == nil {
-		return errors.Errorf("no v2 replication cursor found for job %q on filesystem %q", owningJob.SenderConfig().JobID, fs.ToString())
-	}
-
-	fmt.Printf("most recent v2 replication cursor:\n%#v", oldCursor)
-
-	if !(mostRecentNew.CreateTXG >= oldCursor.CreateTXG) {
-		return errors.Errorf("v1 replication cursor createtxg is higher than v2 cursor's, skipping this filesystem")
-	}
-
-	fmt.Printf("determined that v2 cursor is bookmark of same or newer version than v1 cursor\n")
-	fmt.Printf("destroying v1 cursor %q\n", oldCursor.ToAbsPath(fs))
-
-	if migrateReplicationCursorArgs.dryRun {
-		succ.Printf("DRY RUN\n")
-	} else {
-		if err := zfs.ZFSDestroyFilesystemVersion(fs, oldCursor); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

client/migrate_test.go

@@ -1,16 +0,0 @@
-package client
-
-import "testing"
-
-func TestMigrationsUnambiguousNames(t *testing.T) {
-	names := make(map[string]bool)
-	for _, mig := range migrations {
-		if _, ok := names[mig.Use]; ok {
-			t.Errorf("duplicate migration name %q", mig.Use)
-			t.FailNow()
-			return
-		} else {
-			names[mig.Use] = true
-		}
-	}
-}

client/pprof.go

@@ -1,67 +0,0 @@
-package client
-
-import (
-	"errors"
-	"log"
-	"os"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon"
-)
-
-var pprofArgs struct {
-	daemon.PprofServerControlMsg
-}
-
-var PprofCmd = &cli.Subcommand{
-	Use:   "pprof off | [on TCP_LISTEN_ADDRESS]",
-	Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
-	Run: func(subcommand *cli.Subcommand, args []string) error {
-		if len(args) < 1 {
-			goto enargs
-		}
-		switch args[0] {
-		case "on":
-			pprofArgs.Run = true
-			if len(args) != 2 {
-				return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
-			}
-			pprofArgs.HttpListenAddress = args[1]
-		case "off":
-			if len(args) != 1 {
-				goto enargs
-			}
-			pprofArgs.Run = false
-		}
-
-		RunPProf(subcommand.Config())
-		return nil
-	enargs:
-		return errors.New("invalid number of positional arguments")
-
-	},
-}
-
-func RunPProf(conf *config.Config) {
-	log := log.New(os.Stderr, "", 0)
-
-	die := func() {
-		log.Printf("exiting after error")
-		os.Exit(1)
-	}
-
-	log.Printf("connecting to zrepl daemon")
-
-	httpc, err := controlHttpClient(conf.Global.Control.SockPath)
-	if err != nil {
-		log.Printf("error creating http client: %s", err)
-		die()
-	}
-	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofArgs.PprofServerControlMsg, struct{}{})
-	if err != nil {
-		log.Printf("error sending control message: %s", err)
-		die()
-	}
-	log.Printf("finished")
-}

client/signal.go

@@ -1,40 +0,0 @@
-package client
-
-import (
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon"
-)
-
-var SignalCmd = &cli.Subcommand{
-	Use:   "signal [wakeup|reset] JOB",
-	Short: "wake up a job from wait state or abort its current invocation",
-	Run: func(subcommand *cli.Subcommand, args []string) error {
-		return runSignalCmd(subcommand.Config(), args)
-	},
-}
-
-func runSignalCmd(config *config.Config, args []string) error {
-	if len(args) != 2 {
-		return errors.Errorf("Expected 2 arguments: [wakeup|reset] JOB")
-	}
-
-	httpc, err := controlHttpClient(config.Global.Control.SockPath)
-	if err != nil {
-		return err
-	}
-
-	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointSignal,
-		struct {
-			Name string
-			Op   string
-		}{
-			Name: args[1],
-			Op:   args[0],
-		},
-		struct{}{},
-	)
-	return err
-}

client/status.go

@@ -1,810 +0,0 @@
-package client
-
-import (
-	"fmt"
-	"io"
-	"math"
-	"net/http"
-	"os"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	// tcell is the termbox-compatbile library for abstracting away escape sequences, etc.
-	// as of tcell#252, the number of default distributed terminals is relatively limited
-	// additional terminal definitions can be included via side-effect import
-	// See https://github.com/gdamore/tcell/blob/master/terminfo/base/base.go
-	// See https://github.com/gdamore/tcell/issues/252#issuecomment-533836078
-	"github.com/gdamore/tcell/termbox"
-	_ "github.com/gdamore/tcell/terminfo/s/screen" // tmux on FreeBSD 11 & 12 without ncurses
-
-	"github.com/pkg/errors"
-	"github.com/spf13/pflag"
-	"github.com/zrepl/yaml-config"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/daemon"
-	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/daemon/pruner"
-	"github.com/zrepl/zrepl/daemon/snapper"
-	"github.com/zrepl/zrepl/replication/report"
-)
-
-type byteProgressMeasurement struct {
-	time time.Time
-	val  int64
-}
-
-type bytesProgressHistory struct {
-	last        *byteProgressMeasurement // pointer as poor man's optional
-	changeCount int
-	lastChange  time.Time
-	bpsAvg      float64
-}
-
-func (p *bytesProgressHistory) Update(currentVal int64) (bytesPerSecondAvg int64, changeCount int) {
-
-	if p.last == nil {
-		p.last = &byteProgressMeasurement{
-			time: time.Now(),
-			val:  currentVal,
-		}
-		return 0, 0
-	}
-
-	if p.last.val != currentVal {
-		p.changeCount++
-		p.lastChange = time.Now()
-	}
-
-	if time.Since(p.lastChange) > 3*time.Second {
-		p.last = nil
-		return 0, 0
-	}
-
-	deltaV := currentVal - p.last.val
-	deltaT := time.Since(p.last.time)
-	rate := float64(deltaV) / deltaT.Seconds()
-
-	factor := 0.3
-	p.bpsAvg = (1-factor)*p.bpsAvg + factor*rate
-
-	p.last.time = time.Now()
-	p.last.val = currentVal
-
-	return int64(p.bpsAvg), p.changeCount
-}
-
-type tui struct {
-	x, y   int
-	indent int
-
-	lock   sync.Mutex //For report and error
-	report map[string]job.Status
-	err    error
-
-	jobFilter string
-
-	replicationProgress map[string]*bytesProgressHistory // by job name
-}
-
-func newTui() tui {
-	return tui{
-		replicationProgress: make(map[string]*bytesProgressHistory),
-	}
-}
-
-const INDENT_MULTIPLIER = 4
-
-func (t *tui) moveLine(dl int, col int) {
-	t.y += dl
-	t.x = t.indent*INDENT_MULTIPLIER + col
-}
-
-func (t *tui) write(text string) {
-	for _, c := range text {
-		if c == '\n' {
-			t.newline()
-			continue
-		}
-		termbox.SetCell(t.x, t.y, c, termbox.ColorDefault, termbox.ColorDefault)
-		t.x += 1
-	}
-}
-
-func (t *tui) printf(text string, a ...interface{}) {
-	t.write(fmt.Sprintf(text, a...))
-}
-
-func wrap(s string, width int) string {
-	var b strings.Builder
-	for len(s) > 0 {
-		rem := width
-		if rem > len(s) {
-			rem = len(s)
-		}
-		if idx := strings.IndexAny(s, "\n\r"); idx != -1 && idx < rem {
-			rem = idx + 1
-		}
-		untilNewline := strings.TrimRight(s[:rem], "\n\r")
-		s = s[rem:]
-		if len(untilNewline) == 0 {
-			continue
-		}
-		b.WriteString(untilNewline)
-		b.WriteString("\n")
-	}
-	return strings.TrimRight(b.String(), "\n\r")
-}
-
-func (t *tui) printfDrawIndentedAndWrappedIfMultiline(format string, a ...interface{}) {
-	whole := fmt.Sprintf(format, a...)
-	width, _ := termbox.Size()
-	if !strings.ContainsAny(whole, "\n\r") && t.x+len(whole) <= width {
-		t.printf(format, a...)
-	} else {
-		t.addIndent(1)
-		t.newline()
-		t.write(wrap(whole, width-INDENT_MULTIPLIER*t.indent))
-		t.addIndent(-1)
-	}
-}
-
-func (t *tui) newline() {
-	t.moveLine(1, 0)
-}
-
-func (t *tui) setIndent(indent int) {
-	t.indent = indent
-	t.moveLine(0, 0)
-}
-
-func (t *tui) addIndent(indent int) {
-	t.indent += indent
-	t.moveLine(0, 0)
-}
-
-var statusFlags struct {
-	Raw bool
-	Job string
-}
-
-var StatusCmd = &cli.Subcommand{
-	Use:   "status",
-	Short: "show job activity or dump as JSON for monitoring",
-	SetupFlags: func(f *pflag.FlagSet) {
-		f.BoolVar(&statusFlags.Raw, "raw", false, "dump raw status description from zrepl daemon")
-		f.StringVar(&statusFlags.Job, "job", "", "only dump specified job")
-	},
-	Run: runStatus,
-}
-
-func runStatus(s *cli.Subcommand, args []string) error {
-	httpc, err := controlHttpClient(s.Config().Global.Control.SockPath)
-	if err != nil {
-		return err
-	}
-
-	if statusFlags.Raw {
-		resp, err := httpc.Get("http://unix" + daemon.ControlJobEndpointStatus)
-		if err != nil {
-			return err
-		}
-		defer resp.Body.Close()
-		if resp.StatusCode != http.StatusOK {
-			fmt.Fprintf(os.Stderr, "Received error response:\n")
-			_, err := io.CopyN(os.Stderr, resp.Body, 4096)
-			if err != nil {
-				return err
-			}
-			return errors.Errorf("exit")
-		}
-		if _, err := io.Copy(os.Stdout, resp.Body); err != nil {
-			return err
-		}
-		return nil
-	}
-
-	t := newTui()
-	t.lock.Lock()
-	t.err = errors.New("Got no report yet")
-	t.lock.Unlock()
-	t.jobFilter = statusFlags.Job
-
-	err = termbox.Init()
-	if err != nil {
-		return err
-	}
-	defer termbox.Close()
-
-	update := func() {
-		m := make(map[string]job.Status)
-
-		err2 := jsonRequestResponse(httpc, daemon.ControlJobEndpointStatus,
-			struct{}{},
-			&m,
-		)
-
-		t.lock.Lock()
-		t.err = err2
-		t.report = m
-		t.lock.Unlock()
-		t.draw()
-	}
-	update()
-
-	ticker := time.NewTicker(500 * time.Millisecond)
-	defer ticker.Stop()
-	go func() {
-		for range ticker.C {
-			update()
-		}
-	}()
-
-	termbox.HideCursor()
-	termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
-
-loop:
-	for {
-		switch ev := termbox.PollEvent(); ev.Type {
-		case termbox.EventKey:
-			switch ev.Key {
-			case termbox.KeyEsc:
-				break loop
-			case termbox.KeyCtrlC:
-				break loop
-			}
-		case termbox.EventResize:
-			t.draw()
-		}
-	}
-
-	return nil
-
-}
-
-func (t *tui) getReplicationProgresHistory(jobName string) *bytesProgressHistory {
-	p, ok := t.replicationProgress[jobName]
-	if !ok {
-		p = &bytesProgressHistory{}
-		t.replicationProgress[jobName] = p
-	}
-	return p
-}
-
-func (t *tui) draw() {
-	t.lock.Lock()
-	defer t.lock.Unlock()
-
-	termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
-	t.x = 0
-	t.y = 0
-	t.indent = 0
-
-	if t.err != nil {
-		t.write(t.err.Error())
-	} else {
-		//Iterate over map in alphabetical order
-		keys := make([]string, 0, len(t.report))
-		for k := range t.report {
-			if len(k) == 0 || daemon.IsInternalJobName(k) { //Internal job
-				continue
-			}
-			if t.jobFilter != "" && k != t.jobFilter {
-				continue
-			}
-			keys = append(keys, k)
-		}
-		sort.Strings(keys)
-
-		if len(keys) == 0 {
-			t.setIndent(0)
-			t.printf("no jobs to display")
-			t.newline()
-			termbox.Flush()
-			return
-		}
-
-		for _, k := range keys {
-			v := t.report[k]
-
-			t.setIndent(0)
-
-			t.printf("Job: %s", k)
-			t.setIndent(1)
-			t.newline()
-			t.printf("Type: %s", v.Type)
-			t.setIndent(1)
-			t.newline()
-
-			if v.Type == job.TypePush || v.Type == job.TypePull {
-				activeStatus, ok := v.JobSpecific.(*job.ActiveSideStatus)
-				if !ok || activeStatus == nil {
-					t.printf("ActiveSideStatus is null")
-					t.newline()
-					continue
-				}
-
-				t.printf("Replication:")
-				t.newline()
-				t.addIndent(1)
-				t.renderReplicationReport(activeStatus.Replication, t.getReplicationProgresHistory(k))
-				t.addIndent(-1)
-
-				t.printf("Pruning Sender:")
-				t.newline()
-				t.addIndent(1)
-				t.renderPrunerReport(activeStatus.PruningSender)
-				t.addIndent(-1)
-
-				t.printf("Pruning Receiver:")
-				t.newline()
-				t.addIndent(1)
-				t.renderPrunerReport(activeStatus.PruningReceiver)
-				t.addIndent(-1)
-
-				if v.Type == job.TypePush {
-					t.printf("Snapshotting:")
-					t.newline()
-					t.addIndent(1)
-					t.renderSnapperReport(activeStatus.Snapshotting)
-					t.addIndent(-1)
-				}
-
-			} else if v.Type == job.TypeSnap {
-				snapStatus, ok := v.JobSpecific.(*job.SnapJobStatus)
-				if !ok || snapStatus == nil {
-					t.printf("SnapJobStatus is null")
-					t.newline()
-					continue
-				}
-				t.printf("Pruning snapshots:")
-				t.newline()
-				t.addIndent(1)
-				t.renderPrunerReport(snapStatus.Pruning)
-				t.addIndent(-1)
-				t.printf("Snapshotting:")
-				t.newline()
-				t.addIndent(1)
-				t.renderSnapperReport(snapStatus.Snapshotting)
-				t.addIndent(-1)
-			} else if v.Type == job.TypeSource {
-
-				st := v.JobSpecific.(*job.PassiveStatus)
-				t.printf("Snapshotting:\n")
-				t.addIndent(1)
-				t.renderSnapperReport(st.Snapper)
-				t.addIndent(-1)
-
-			} else {
-				t.printf("No status representation for job type '%s', dumping as YAML", v.Type)
-				t.newline()
-				asYaml, err := yaml.Marshal(v.JobSpecific)
-				if err != nil {
-					t.printf("Error marshaling status to YAML: %s", err)
-					t.newline()
-					continue
-				}
-				t.write(string(asYaml))
-				t.newline()
-				continue
-			}
-		}
-	}
-	termbox.Flush()
-}
-
-func (t *tui) renderReplicationReport(rep *report.Report, history *bytesProgressHistory) {
-	if rep == nil {
-		t.printf("...\n")
-		return
-	}
-
-	if rep.WaitReconnectError != nil {
-		t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: %s", rep.WaitReconnectError)
-		t.newline()
-	}
-	if !rep.WaitReconnectSince.IsZero() {
-		delta := time.Until(rep.WaitReconnectUntil).Round(time.Second)
-		if rep.WaitReconnectUntil.IsZero() || delta > 0 {
-			var until string
-			if rep.WaitReconnectUntil.IsZero() {
-				until = "waiting indefinitely"
-			} else {
-				until = fmt.Sprintf("hard fail in %s @ %s", delta, rep.WaitReconnectUntil)
-			}
-			t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnecting with exponential backoff (since %s) (%s)",
-				rep.WaitReconnectSince, until)
-		} else {
-			t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnects reached hard-fail timeout @ %s", rep.WaitReconnectUntil)
-		}
-		t.newline()
-	}
-
-	// TODO visualize more than the latest attempt by folding all attempts into one
-	if len(rep.Attempts) == 0 {
-		t.printf("no attempts made yet")
-		return
-	} else {
-		t.printf("Attempt #%d", len(rep.Attempts))
-		if len(rep.Attempts) > 1 {
-			t.printf(". Previous attempts failed with the following statuses:")
-			t.newline()
-			t.addIndent(1)
-			for i, a := range rep.Attempts[:len(rep.Attempts)-1] {
-				t.printfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt))
-				t.newline()
-			}
-			t.addIndent(-1)
-		} else {
-			t.newline()
-		}
-	}
-
-	latest := rep.Attempts[len(rep.Attempts)-1]
-	sort.Slice(latest.Filesystems, func(i, j int) bool {
-		return latest.Filesystems[i].Info.Name < latest.Filesystems[j].Info.Name
-	})
-
-	t.printf("Status: %s", latest.State)
-	t.newline()
-	if latest.State == report.AttemptPlanningError {
-		t.printf("Problem: ")
-		t.printfDrawIndentedAndWrappedIfMultiline("%s", latest.PlanError)
-		t.newline()
-	} else if latest.State == report.AttemptFanOutError {
-		t.printf("Problem: one or more of the filesystems encountered errors")
-		t.newline()
-	}
-
-	if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
-		// Draw global progress bar
-		// Progress: [---------------]
-		expected, replicated, containsInvalidSizeEstimates := latest.BytesSum()
-		rate, changeCount := history.Update(replicated)
-		eta := time.Duration(0)
-		if rate > 0 {
-			eta = time.Duration((expected-replicated)/rate) * time.Second
-		}
-		t.write("Progress: ")
-		t.drawBar(50, replicated, expected, changeCount)
-		t.write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinary(replicated), ByteCountBinary(expected), ByteCountBinary(rate)))
-		if eta != 0 {
-			t.write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
-		}
-		t.newline()
-		if containsInvalidSizeEstimates {
-			t.write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
-			t.newline()
-		}
-
-		var maxFSLen int
-		for _, fs := range latest.Filesystems {
-			if len(fs.Info.Name) > maxFSLen {
-				maxFSLen = len(fs.Info.Name)
-			}
-		}
-		for _, fs := range latest.Filesystems {
-			t.printFilesystemStatus(fs, false, maxFSLen) // FIXME bring 'active' flag back
-		}
-
-	}
-
-}
-
-func (t *tui) renderPrunerReport(r *pruner.Report) {
-	if r == nil {
-		t.printf("...\n")
-		return
-	}
-
-	state, err := pruner.StateString(r.State)
-	if err != nil {
-		t.printf("Status: %q (parse error: %q)\n", r.State, err)
-		return
-	}
-
-	t.printf("Status: %s", state)
-	t.newline()
-
-	if r.Error != "" {
-		t.printf("Error: %s\n", r.Error)
-	}
-
-	type commonFS struct {
-		*pruner.FSReport
-		completed bool
-	}
-	all := make([]commonFS, 0, len(r.Pending)+len(r.Completed))
-	for i := range r.Pending {
-		all = append(all, commonFS{&r.Pending[i], false})
-	}
-	for i := range r.Completed {
-		all = append(all, commonFS{&r.Completed[i], true})
-	}
-
-	switch state {
-	case pruner.Plan:
-		fallthrough
-	case pruner.PlanErr:
-		return
-	}
-
-	if len(all) == 0 {
-		t.printf("nothing to do\n")
-		return
-	}
-
-	var totalDestroyCount, completedDestroyCount int
-	var maxFSname int
-	for _, fs := range all {
-		totalDestroyCount += len(fs.DestroyList)
-		if fs.completed {
-			completedDestroyCount += len(fs.DestroyList)
-		}
-		if maxFSname < len(fs.Filesystem) {
-			maxFSname = len(fs.Filesystem)
-		}
-	}
-
-	// global progress bar
-	progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
-	t.write("Progress: ")
-	t.write("[")
-	t.write(times("=", progress))
-	t.write(">")
-	t.write(times("-", 80-progress))
-	t.write("]")
-	t.printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
-	t.newline()
-
-	sort.SliceStable(all, func(i, j int) bool {
-		return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
-	})
-
-	// Draw a table-like representation of 'all'
-	for _, fs := range all {
-		t.write(rightPad(fs.Filesystem, maxFSname, " "))
-		t.write(" ")
-		if !fs.SkipReason.NotSkipped() {
-			t.printf("skipped: %s\n", fs.SkipReason)
-			continue
-		}
-		if fs.LastError != "" {
-			if strings.ContainsAny(fs.LastError, "\r\n") {
-				t.printf("ERROR:")
-				t.printfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError)
-			} else {
-				t.printfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError)
-			}
-			t.newline()
-			continue
-		}
-
-		pruneRuleActionStr := fmt.Sprintf("(destroy %d of %d snapshots)",
-			len(fs.DestroyList), len(fs.SnapshotList))
-
-		if fs.completed {
-			t.printf("Completed  %s\n", pruneRuleActionStr)
-			continue
-		}
-
-		t.write("Pending    ") // whitespace is padding 10
-		if len(fs.DestroyList) == 1 {
-			t.write(fs.DestroyList[0].Name)
-		} else {
-			t.write(pruneRuleActionStr)
-		}
-		t.newline()
-	}
-
-}
-
-func (t *tui) renderSnapperReport(r *snapper.Report) {
-	if r == nil {
-		t.printf("<snapshot type does not have a report>\n")
-		return
-	}
-
-	t.printf("Status: %s", r.State)
-	t.newline()
-
-	if r.Error != "" {
-		t.printf("Error: %s\n", r.Error)
-	}
-	if !r.SleepUntil.IsZero() {
-		t.printf("Sleep until: %s\n", r.SleepUntil)
-	}
-
-	sort.Slice(r.Progress, func(i, j int) bool {
-		return strings.Compare(r.Progress[i].Path, r.Progress[j].Path) == -1
-	})
-
-	t.addIndent(1)
-	defer t.addIndent(-1)
-	dur := func(d time.Duration) string {
-		return d.Round(100 * time.Millisecond).String()
-	}
-
-	type row struct {
-		path, state, duration, remainder, hookReport string
-	}
-	var widths struct {
-		path, state, duration int
-	}
-	rows := make([]*row, len(r.Progress))
-	for i, fs := range r.Progress {
-		r := &row{
-			path:  fs.Path,
-			state: fs.State.String(),
-		}
-		if fs.HooksHadError {
-			r.hookReport = fs.Hooks // FIXME render here, not in daemon
-		}
-		switch fs.State {
-		case snapper.SnapPending:
-			r.duration = "..."
-			r.remainder = ""
-		case snapper.SnapStarted:
-			r.duration = dur(time.Since(fs.StartAt))
-			r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
-		case snapper.SnapDone:
-			fallthrough
-		case snapper.SnapError:
-			r.duration = dur(fs.DoneAt.Sub(fs.StartAt))
-			r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
-		}
-		rows[i] = r
-		if len(r.path) > widths.path {
-			widths.path = len(r.path)
-		}
-		if len(r.state) > widths.state {
-			widths.state = len(r.state)
-		}
-		if len(r.duration) > widths.duration {
-			widths.duration = len(r.duration)
-		}
-	}
-
-	for _, r := range rows {
-		path := rightPad(r.path, widths.path, " ")
-		state := rightPad(r.state, widths.state, " ")
-		duration := rightPad(r.duration, widths.duration, " ")
-		t.printf("%s %s %s", path, state, duration)
-		t.printfDrawIndentedAndWrappedIfMultiline(" %s", r.remainder)
-		if r.hookReport != "" {
-			t.printfDrawIndentedAndWrappedIfMultiline("%s", r.hookReport)
-		}
-		t.newline()
-	}
-
-}
-
-func times(str string, n int) (out string) {
-	for i := 0; i < n; i++ {
-		out += str
-	}
-	return
-}
-
-func rightPad(str string, length int, pad string) string {
-	if len(str) > length {
-		return str[:length]
-	}
-	return str + strings.Repeat(pad, length-len(str))
-}
-
-var arrowPositions = `>\|/`
-
-// changeCount = 0 indicates stall / no progresss
-func (t *tui) drawBar(length int, bytes, totalBytes int64, changeCount int) {
-	var completedLength int
-	if totalBytes > 0 {
-		completedLength = int(int64(length) * bytes / totalBytes)
-		if completedLength > length {
-			completedLength = length
-		}
-	} else if totalBytes == bytes {
-		completedLength = length
-	}
-
-	t.write("[")
-	t.write(times("=", completedLength))
-	t.write(string(arrowPositions[changeCount%len(arrowPositions)]))
-	t.write(times("-", length-completedLength))
-	t.write("]")
-}
-
-func (t *tui) printFilesystemStatus(rep *report.FilesystemReport, active bool, maxFS int) {
-
-	expected, replicated, containsInvalidSizeEstimates := rep.BytesSum()
-	sizeEstimationImpreciseNotice := ""
-	if containsInvalidSizeEstimates {
-		sizeEstimationImpreciseNotice = " (some steps lack size estimation)"
-	}
-	if rep.CurrentStep < len(rep.Steps) && rep.Steps[rep.CurrentStep].Info.BytesExpected == 0 {
-		sizeEstimationImpreciseNotice = " (step lacks size estimation)"
-	}
-
-	status := fmt.Sprintf("%s (step %d/%d, %s/%s)%s",
-		strings.ToUpper(string(rep.State)),
-		rep.CurrentStep, len(rep.Steps),
-		ByteCountBinary(replicated), ByteCountBinary(expected),
-		sizeEstimationImpreciseNotice,
-	)
-
-	activeIndicator := " "
-	if active {
-		activeIndicator = "*"
-	}
-	t.printf("%s %s %s ",
-		activeIndicator,
-		rightPad(rep.Info.Name, maxFS, " "),
-		status)
-
-	next := ""
-	if err := rep.Error(); err != nil {
-		next = err.Err
-	} else if rep.State != report.FilesystemDone {
-		if nextStep := rep.NextStep(); nextStep != nil {
-			if nextStep.IsIncremental() {
-				next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
-			} else {
-				next = fmt.Sprintf("next: full send %s", nextStep.Info.To)
-			}
-			attribs := []string{}
-
-			if nextStep.Info.Resumed {
-				attribs = append(attribs, "resumed")
-			}
-
-			attribs = append(attribs, fmt.Sprintf("encrypted=%s", nextStep.Info.Encrypted))
-
-			next += fmt.Sprintf(" (%s)", strings.Join(attribs, ", "))
-		} else {
-			next = "" // individual FSes may still be in planning state
-		}
-
-	}
-	t.printfDrawIndentedAndWrappedIfMultiline("%s", next)
-
-	t.newline()
-}
-
-func ByteCountBinary(b int64) string {
-	const unit = 1024
-	if b < unit {
-		return fmt.Sprintf("%d B", b)
-	}
-	div, exp := int64(unit), 0
-	for n := b / unit; n >= unit; n /= unit {
-		div *= unit
-		exp++
-	}
-	return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
-}
-
-func humanizeDuration(duration time.Duration) string {
-	days := int64(duration.Hours() / 24)
-	hours := int64(math.Mod(duration.Hours(), 24))
-	minutes := int64(math.Mod(duration.Minutes(), 60))
-	seconds := int64(math.Mod(duration.Seconds(), 60))
-
-	var parts []string
-
-	force := false
-	chunks := []int64{days, hours, minutes, seconds}
-	for i, chunk := range chunks {
-		if force || chunk > 0 {
-			padding := 0
-			if force {
-				padding = 2
-			}
-			parts = append(parts, fmt.Sprintf("%*d%c", padding, chunk, "dhms"[i]))
-			force = true
-		}
-	}
-
-	return strings.Join(parts, " ")
-}

client/stdinserver.go

@@ -1,51 +0,0 @@
-package client
-
-import (
-	"os"
-
-	"github.com/problame/go-netssh"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-
-	"context"
-	"errors"
-	"log"
-	"path"
-)
-
-var StdinserverCmd = &cli.Subcommand{
-	Use:   "stdinserver CLIENT_IDENTITY",
-	Short: "stdinserver transport mode (started from authorized_keys file as forced command)",
-	Run: func(subcommand *cli.Subcommand, args []string) error {
-		return runStdinserver(subcommand.Config(), args)
-	},
-}
-
-func runStdinserver(config *config.Config, args []string) error {
-
-	// NOTE: the netssh proxying protocol requires exiting with non-zero status if anything goes wrong
-	defer os.Exit(1)
-
-	log := log.New(os.Stderr, "", log.LUTC|log.Ldate|log.Ltime)
-
-	if len(args) != 1 || args[0] == "" {
-		err := errors.New("must specify client_identity as positional argument")
-		return err
-	}
-
-	identity := args[0]
-	unixaddr := path.Join(config.Global.Serve.StdinServer.SockDir, identity)
-
-	log.Printf("proxying client identity '%s' to zrepl daemon '%s'", identity, unixaddr)
-
-	ctx := netssh.ContextWithLog(context.TODO(), log)
-
-	err := netssh.Proxy(ctx, unixaddr)
-	if err == nil {
-		log.Print("proxying finished successfully, exiting with status 0")
-		os.Exit(0)
-	}
-	log.Printf("error proxying: %s", err)
-	return nil
-}

client/testcmd.go

@@ -1,210 +0,0 @@
-package client
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"os"
-
-	"github.com/pkg/errors"
-	"github.com/spf13/pflag"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-var TestCmd = &cli.Subcommand{
-	Use: "test",
-	SetupSubcommands: func() []*cli.Subcommand {
-		return []*cli.Subcommand{testFilter, testPlaceholder, testDecodeResumeToken}
-	},
-}
-
-var testFilterArgs struct {
-	job   string
-	all   bool
-	input string
-}
-
-var testFilter = &cli.Subcommand{
-	Use:   "filesystems --job JOB [--all | --input INPUT]",
-	Short: "test filesystems filter specified in push or source job",
-	SetupFlags: func(f *pflag.FlagSet) {
-		f.StringVar(&testFilterArgs.job, "job", "", "the name of the push or source job")
-		f.StringVar(&testFilterArgs.input, "input", "", "a filesystem name to test against the job's filters")
-		f.BoolVar(&testFilterArgs.all, "all", false, "test all local filesystems")
-	},
-	Run: runTestFilterCmd,
-}
-
-func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
-
-	if testFilterArgs.job == "" {
-		return fmt.Errorf("must specify --job flag")
-	}
-	if !(testFilterArgs.all != (testFilterArgs.input != "")) { // xor
-		return fmt.Errorf("must set one: --all or --input")
-	}
-
-	conf := subcommand.Config()
-
-	var confFilter config.FilesystemsFilter
-	job, err := conf.Job(testFilterArgs.job)
-	if err != nil {
-		return err
-	}
-	switch j := job.Ret.(type) {
-	case *config.SourceJob:
-		confFilter = j.Filesystems
-	case *config.PushJob:
-		confFilter = j.Filesystems
-	default:
-		return fmt.Errorf("job type %T does not have filesystems filter", j)
-	}
-
-	f, err := filters.DatasetMapFilterFromConfig(confFilter)
-	if err != nil {
-		return fmt.Errorf("filter invalid: %s", err)
-	}
-
-	var fsnames []string
-	if testFilterArgs.input != "" {
-		fsnames = []string{testFilterArgs.input}
-	} else {
-		out, err := zfs.ZFSList([]string{"name"})
-		if err != nil {
-			return fmt.Errorf("could not list ZFS filesystems: %s", err)
-		}
-		for _, row := range out {
-
-			fsnames = append(fsnames, row[0])
-		}
-	}
-
-	fspaths := make([]*zfs.DatasetPath, len(fsnames))
-	for i, fsname := range fsnames {
-		path, err := zfs.NewDatasetPath(fsname)
-		if err != nil {
-			return err
-		}
-		fspaths[i] = path
-	}
-
-	hadFilterErr := false
-	for _, in := range fspaths {
-		var res string
-		var errStr string
-		pass, err := f.Filter(in)
-		if err != nil {
-			res = "ERROR"
-			errStr = err.Error()
-			hadFilterErr = true
-		} else if pass {
-			res = "ACCEPT"
-		} else {
-			res = "REJECT"
-		}
-		fmt.Printf("%s\t%s\t%s\n", res, in.ToString(), errStr)
-	}
-
-	if hadFilterErr {
-		return fmt.Errorf("filter errors occurred")
-	}
-	return nil
-}
-
-var testPlaceholderArgs struct {
-	ds  string
-	all bool
-}
-
-var testPlaceholder = &cli.Subcommand{
-	Use:   "placeholder [--all | --dataset DATASET]",
-	Short: fmt.Sprintf("list received placeholder filesystems (zfs property %q)", zfs.PlaceholderPropertyName),
-	Example: `
-	placeholder --all
-	placeholder --dataset path/to/sink/clientident/fs`,
-	NoRequireConfig: true,
-	SetupFlags: func(f *pflag.FlagSet) {
-		f.StringVar(&testPlaceholderArgs.ds, "dataset", "", "dataset path (not required to exist)")
-		f.BoolVar(&testPlaceholderArgs.all, "all", false, "list tab-separated placeholder status of all filesystems")
-	},
-	Run: runTestPlaceholder,
-}
-
-func runTestPlaceholder(subcommand *cli.Subcommand, args []string) error {
-
-	var checkDPs []*zfs.DatasetPath
-
-	// all actions first
-	if testPlaceholderArgs.all {
-		out, err := zfs.ZFSList([]string{"name"})
-		if err != nil {
-			return errors.Wrap(err, "could not list ZFS filesystems")
-		}
-		for _, row := range out {
-			dp, err := zfs.NewDatasetPath(row[0])
-			if err != nil {
-				panic(err)
-			}
-			checkDPs = append(checkDPs, dp)
-		}
-	} else {
-		dp, err := zfs.NewDatasetPath(testPlaceholderArgs.ds)
-		if err != nil {
-			return err
-		}
-		if dp.Empty() {
-			return fmt.Errorf("must specify --dataset DATASET or --all")
-		}
-		checkDPs = append(checkDPs, dp)
-	}
-
-	fmt.Printf("IS_PLACEHOLDER\tDATASET\tzrepl:placeholder\n")
-	for _, dp := range checkDPs {
-		ph, err := zfs.ZFSGetFilesystemPlaceholderState(dp)
-		if err != nil {
-			return errors.Wrap(err, "cannot get placeholder state")
-		}
-		if !ph.FSExists {
-			panic("placeholder state inconsistent: filesystem " + ph.FS + " must exist in this context")
-		}
-		is := "yes"
-		if !ph.IsPlaceholder {
-			is = "no"
-		}
-		fmt.Printf("%s\t%s\t%s\n", is, dp.ToString(), ph.RawLocalPropertyValue)
-	}
-	return nil
-}
-
-var testDecodeResumeTokenArgs struct {
-	token string
-}
-
-var testDecodeResumeToken = &cli.Subcommand{
-	Use:   "decoderesumetoken --token TOKEN",
-	Short: "decode resume token",
-	SetupFlags: func(f *pflag.FlagSet) {
-		f.StringVar(&testDecodeResumeTokenArgs.token, "token", "", "the resume token obtained from the receive_resume_token property")
-	},
-	Run: runTestDecodeResumeTokenCmd,
-}
-
-func runTestDecodeResumeTokenCmd(subcommand *cli.Subcommand, args []string) error {
-	if testDecodeResumeTokenArgs.token == "" {
-		return fmt.Errorf("token argument must be specified")
-	}
-	token, err := zfs.ParseResumeToken(context.Background(), testDecodeResumeTokenArgs.token)
-	if err != nil {
-		return err
-	}
-	enc := json.NewEncoder(os.Stdout)
-	enc.SetIndent("", " ")
-	if err := enc.Encode(&token); err != nil {
-		panic(err)
-	}
-	return nil
-}

client/version.go

@@ -1,74 +0,0 @@
-package client
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/spf13/pflag"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon"
-	"github.com/zrepl/zrepl/version"
-)
-
-var versionArgs struct {
-	Show      string
-	Config    *config.Config
-	ConfigErr error
-}
-
-var VersionCmd = &cli.Subcommand{
-	Use:             "version",
-	Short:           "print version of zrepl binary and running daemon",
-	NoRequireConfig: true,
-	SetupFlags: func(f *pflag.FlagSet) {
-		f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)")
-	},
-	Run: func(subcommand *cli.Subcommand, args []string) error {
-		versionArgs.Config = subcommand.Config()
-		versionArgs.ConfigErr = subcommand.ConfigParsingError()
-		return runVersionCmd()
-	},
-}
-
-func runVersionCmd() error {
-	args := versionArgs
-
-	if args.Show != "daemon" && args.Show != "client" && args.Show != "" {
-		return fmt.Errorf("show flag must be 'client' or 'server' or be left empty")
-	}
-
-	var clientVersion, daemonVersion *version.ZreplVersionInformation
-	if args.Show == "client" || args.Show == "" {
-		clientVersion = version.NewZreplVersionInformation()
-		fmt.Printf("client: %s\n", clientVersion.String())
-	}
-	if args.Show == "daemon" || args.Show == "" {
-
-		if args.ConfigErr != nil {
-			return fmt.Errorf("config parsing error: %s", args.ConfigErr)
-		}
-
-		httpc, err := controlHttpClient(args.Config.Global.Control.SockPath)
-		if err != nil {
-			return fmt.Errorf("server: error: %s\n", err)
-		}
-
-		var info version.ZreplVersionInformation
-		err = jsonRequestResponse(httpc, daemon.ControlJobEndpointVersion, "", &info)
-		if err != nil {
-			return fmt.Errorf("server: error: %s\n", err)
-		}
-		daemonVersion = &info
-		fmt.Printf("server: %s\n", daemonVersion.String())
-	}
-
-	if args.Show == "" {
-		if clientVersion.Version != daemonVersion.Version {
-			fmt.Fprintf(os.Stderr, "WARNING: client version != daemon version, restart zrepl daemon\n")
-		}
-	}
-
-	return nil
-}

cmd/autosnap.go

@@ -0,0 +1,91 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/jobrun"
+	"github.com/zrepl/zrepl/zfs"
+	"os"
+	"sync"
+	"time"
+)
+
+var AutosnapCmd = &cobra.Command{
+	Use:   "autosnap",
+	Short: "perform automatic snapshotting",
+	Run:   cmdAutosnap,
+}
+
+func init() {
+	RootCmd.AddCommand(AutosnapCmd)
+}
+
+func cmdAutosnap(cmd *cobra.Command, args []string) {
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		runner.Start()
+	}()
+
+	if len(args) < 1 {
+		log.Printf("must specify exactly one job as positional argument")
+		os.Exit(1)
+	}
+
+	snap, ok := conf.Autosnaps[args[0]]
+	if !ok {
+		log.Printf("could not find autosnap job: %s", args[0])
+		os.Exit(1)
+	}
+
+	job := jobrun.Job{
+		Name:           snap.JobName,
+		RepeatStrategy: snap.Interval,
+		RunFunc: func(log jobrun.Logger) error {
+			log.Printf("doing autosnap: %v", snap)
+			ctx := AutosnapContext{snap}
+			return doAutosnap(ctx, log)
+		},
+	}
+	runner.AddJob(job)
+
+	wg.Wait()
+
+}
+
+type AutosnapContext struct {
+	Autosnap *Autosnap
+}
+
+func doAutosnap(ctx AutosnapContext, log Logger) (err error) {
+
+	snap := ctx.Autosnap
+
+	filesystems, err := zfs.ZFSListMapping(snap.DatasetFilter)
+	if err != nil {
+		return fmt.Errorf("cannot filter datasets: %s", err)
+	}
+
+	suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
+	snapname := fmt.Sprintf("%s%s", snap.Prefix, suffix)
+
+	hadError := false
+
+	for _, fs := range filesystems { // optimization: use recursive snapshots / channel programs here
+		log.Printf("snapshotting filesystem %s@%s", fs, snapname)
+		err := zfs.ZFSSnapshot(fs, snapname, false)
+		if err != nil {
+			log.Printf("error snapshotting %s: %s", fs, err)
+			hadError = true
+		}
+	}
+
+	if hadError {
+		err = fmt.Errorf("errors occurred during autosnap, check logs for details")
+	}
+
+	return
+
+}

cmd/config.go

@@ -0,0 +1,730 @@
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"github.com/jinzhu/copier"
+	"github.com/mitchellh/mapstructure"
+	"github.com/zrepl/zrepl/jobrun"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/sshbytestream"
+	. "github.com/zrepl/zrepl/util"
+	"github.com/zrepl/zrepl/zfs"
+	yaml "gopkg.in/yaml.v2"
+	"io"
+	"io/ioutil"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+var (
+	JobSectionPush     string = "push"
+	JobSectionPull     string = "pull"
+	JobSectionPrune    string = "prune"
+	JobSectionAutosnap string = "autosnap"
+)
+
+type Remote struct {
+	Name      string
+	Transport Transport
+}
+
+type Transport interface {
+	Connect(rpcLog Logger) (rpc.RPCRequester, error)
+}
+type LocalTransport struct {
+	Handler rpc.RPCHandler
+}
+type SSHTransport struct {
+	Host                 string
+	User                 string
+	Port                 uint16
+	IdentityFile         string   `mapstructure:"identity_file"`
+	TransportOpenCommand []string `mapstructure:"transport_open_command"`
+	SSHCommand           string   `mapstructure:"ssh_command"`
+	Options              []string
+	ConnLogReadFile      string `mapstructure:"connlog_read_file"`
+	ConnLogWriteFile     string `mapstructure:"connlog_write_file"`
+}
+
+type Push struct {
+	JobName           string // for use with jobrun package
+	To                *Remote
+	Filter            zfs.DatasetFilter
+	InitialReplPolicy rpc.InitialReplPolicy
+	RepeatStrategy    jobrun.RepeatStrategy
+}
+type Pull struct {
+	JobName           string // for use with jobrun package
+	From              *Remote
+	Mapping           DatasetMapFilter
+	InitialReplPolicy rpc.InitialReplPolicy
+	RepeatStrategy    jobrun.RepeatStrategy
+}
+
+type Prune struct {
+	JobName         string // for use with jobrun package
+	DatasetFilter   zfs.DatasetFilter
+	SnapshotFilter  zfs.FilesystemVersionFilter
+	RetentionPolicy *RetentionGrid // TODO abstract interface to support future policies?
+}
+
+type Autosnap struct {
+	JobName       string // for use with jobrun package
+	Prefix        string
+	Interval      jobrun.RepeatStrategy
+	DatasetFilter zfs.DatasetFilter
+}
+
+type Config struct {
+	Remotes   map[string]*Remote
+	Pushs     map[string]*Push            // job name -> job
+	Pulls     map[string]*Pull            // job name -> job
+	Sinks     map[string]DatasetMapFilter // client identity -> mapping
+	PullACLs  map[string]DatasetMapFilter // client identity -> filter
+	Prunes    map[string]*Prune           // job name -> job
+	Autosnaps map[string]*Autosnap        // job name -> job
+}
+
+func ParseConfig(path string) (config Config, err error) {
+
+	c := make(map[string]interface{}, 0)
+
+	var bytes []byte
+
+	if bytes, err = ioutil.ReadFile(path); err != nil {
+		return
+	}
+
+	if err = yaml.Unmarshal(bytes, &c); err != nil {
+		return
+	}
+
+	return parseMain(c)
+}
+
+func parseMain(root map[string]interface{}) (c Config, err error) {
+	if c.Remotes, err = parseRemotes(root["remotes"]); err != nil {
+		return
+	}
+
+	remoteLookup := func(name string) (remote *Remote, err error) {
+		remote = c.Remotes[name]
+		if remote == nil {
+			err = fmt.Errorf("remote '%s' not defined", name)
+		}
+		return
+	}
+
+	if c.Pushs, err = parsePushs(root["pushs"], remoteLookup); err != nil {
+		return
+	}
+	if c.Pulls, err = parsePulls(root["pulls"], remoteLookup); err != nil {
+		return
+	}
+	if c.Sinks, err = parseSinks(root["sinks"]); err != nil {
+		return
+	}
+	if c.PullACLs, err = parsePullACLs(root["pull_acls"]); err != nil {
+		return
+	}
+	if c.Prunes, err = parsePrunes(root["prune"]); err != nil {
+		return
+	}
+	if c.Autosnaps, err = parseAutosnaps(root["autosnap"]); err != nil {
+		return
+	}
+
+	return
+}
+
+func fullJobName(section, name string) (full string, err error) {
+	if len(name) < 1 {
+		err = fmt.Errorf("job name not set")
+		return
+	}
+	full = fmt.Sprintf("%s.%s", section, name)
+	return
+}
+
+func parseRemotes(v interface{}) (remotes map[string]*Remote, err error) {
+
+	asMap := make(map[string]struct {
+		Transport map[string]interface{}
+	}, 0)
+	if err = mapstructure.Decode(v, &asMap); err != nil {
+		return
+	}
+
+	remotes = make(map[string]*Remote, len(asMap))
+	for name, p := range asMap {
+
+		if name == rpc.LOCAL_TRANSPORT_IDENTITY {
+			err = errors.New(fmt.Sprintf("remote name '%s' reserved for local pulls", rpc.LOCAL_TRANSPORT_IDENTITY))
+			return
+		}
+
+		var transport Transport
+		if transport, err = parseTransport(p.Transport); err != nil {
+			return
+		}
+		remotes[name] = &Remote{
+			Name:      name,
+			Transport: transport,
+		}
+	}
+
+	return
+}
+
+func parseTransport(it map[string]interface{}) (t Transport, err error) {
+
+	if len(it) != 1 {
+		err = errors.New("ambiguous transport type")
+		return
+	}
+
+	for key, val := range it {
+		switch key {
+		case "ssh":
+			t := SSHTransport{}
+			if err = mapstructure.Decode(val, &t); err != nil {
+				err = errors.New(fmt.Sprintf("could not parse ssh transport: %s", err))
+				return nil, err
+			}
+			return t, nil
+		default:
+			return nil, errors.New(fmt.Sprintf("unknown transport type '%s'\n", key))
+		}
+	}
+
+	return // unreachable
+
+}
+
+type remoteLookup func(name string) (*Remote, error)
+
+func parsePushs(v interface{}, rl remoteLookup) (p map[string]*Push, err error) {
+
+	asMap := make(map[string]struct {
+		To                string
+		Filter            map[string]string
+		InitialReplPolicy string
+		Repeat            map[string]string
+	}, 0)
+
+	if err = mapstructure.Decode(v, &asMap); err != nil {
+		return
+	}
+
+	p = make(map[string]*Push, len(asMap))
+
+	for name, e := range asMap {
+
+		var toRemote *Remote
+		if toRemote, err = rl(e.To); err != nil {
+			return
+		}
+		push := &Push{
+			To: toRemote,
+		}
+
+		if push.JobName, err = fullJobName(JobSectionPush, name); err != nil {
+			return
+		}
+		if push.Filter, err = parseDatasetMapFilter(e.Filter, true); err != nil {
+			return
+		}
+
+		if push.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, rpc.DEFAULT_INITIAL_REPL_POLICY); err != nil {
+			return
+		}
+
+		if push.RepeatStrategy, err = parseRepeatStrategy(e.Repeat); err != nil {
+			return
+		}
+
+		p[name] = push
+	}
+
+	return
+}
+
+func parsePulls(v interface{}, rl remoteLookup) (p map[string]*Pull, err error) {
+
+	asMap := make(map[string]struct {
+		From              string
+		Mapping           map[string]string
+		InitialReplPolicy string
+		Repeat            map[string]string
+	}, 0)
+
+	if err = mapstructure.Decode(v, &asMap); err != nil {
+		return
+	}
+
+	p = make(map[string]*Pull, len(asMap))
+
+	for name, e := range asMap {
+
+		if len(e.From) < 1 {
+			err = fmt.Errorf("source not set ('from' attribute is empty)")
+			return
+		}
+
+		var fromRemote *Remote
+
+		if e.From == rpc.LOCAL_TRANSPORT_IDENTITY {
+			fromRemote = &Remote{
+				Name:      rpc.LOCAL_TRANSPORT_IDENTITY,
+				Transport: LocalTransport{},
+			}
+		} else {
+			if fromRemote, err = rl(e.From); err != nil {
+				return
+			}
+		}
+
+		pull := &Pull{
+			From: fromRemote,
+		}
+		if pull.JobName, err = fullJobName(JobSectionPull, name); err != nil {
+			return
+		}
+		if pull.Mapping, err = parseDatasetMapFilter(e.Mapping, false); err != nil {
+			return
+		}
+		if pull.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, rpc.DEFAULT_INITIAL_REPL_POLICY); err != nil {
+			return
+		}
+		if pull.RepeatStrategy, err = parseRepeatStrategy(e.Repeat); err != nil {
+			return
+		}
+
+		p[name] = pull
+	}
+
+	return
+}
+
+func parseInitialReplPolicy(v interface{}, defaultPolicy rpc.InitialReplPolicy) (p rpc.InitialReplPolicy, err error) {
+	s, ok := v.(string)
+	if !ok {
+		goto err
+	}
+
+	switch {
+	case s == "":
+		p = defaultPolicy
+	case s == "most_recent":
+		p = rpc.InitialReplPolicyMostRecent
+	case s == "all":
+		p = rpc.InitialReplPolicyAll
+	default:
+		goto err
+	}
+
+	return
+
+err:
+	err = errors.New(fmt.Sprintf("expected InitialReplPolicy, got %#v", v))
+	return
+}
+
+func parseRepeatStrategy(r map[string]string) (s jobrun.RepeatStrategy, err error) {
+
+	if r == nil {
+		return jobrun.NoRepeatStrategy{}, nil
+	}
+
+	if repeatStr, ok := r["interval"]; ok {
+		d, err := parseDuration(repeatStr)
+		if err != nil {
+			return nil, err
+		}
+		s = &jobrun.PeriodicRepeatStrategy{d}
+		return s, err
+	} else {
+		return nil, fmt.Errorf("attribute 'interval' not found but required in repeat specification")
+	}
+
+}
+
+func expectList(v interface{}) (asList []interface{}, err error) {
+	var ok bool
+	if asList, ok = v.([]interface{}); !ok {
+		err = errors.New("expected list")
+	}
+	return
+}
+
+func parseSinks(v interface{}) (m map[string]DatasetMapFilter, err error) {
+	var asMap map[string]map[string]interface{}
+	if err = mapstructure.Decode(v, &asMap); err != nil {
+		return
+	}
+
+	m = make(map[string]DatasetMapFilter, len(asMap))
+
+	for identity, entry := range asMap {
+		parseSink := func() (mapping DatasetMapFilter, err error) {
+			mappingMap, ok := entry["mapping"]
+			if !ok {
+				err = fmt.Errorf("no mapping specified")
+				return
+			}
+			mapping, err = parseDatasetMapFilter(mappingMap, false)
+			return
+		}
+		mapping, sinkErr := parseSink()
+		if sinkErr != nil {
+			err = fmt.Errorf("cannot parse sink for identity '%s': %s", identity, sinkErr)
+			return
+		}
+		m[identity] = mapping
+	}
+	return
+}
+
+func parsePullACLs(v interface{}) (m map[string]DatasetMapFilter, err error) {
+	var asMap map[string]map[string]interface{}
+	if err = mapstructure.Decode(v, &asMap); err != nil {
+		return
+	}
+
+	m = make(map[string]DatasetMapFilter, len(asMap))
+
+	for identity, entry := range asMap {
+		parsePullACL := func() (filter DatasetMapFilter, err error) {
+			filterMap, ok := entry["filter"]
+			if !ok {
+				err = fmt.Errorf("no filter specified")
+				return
+			}
+			filter, err = parseDatasetMapFilter(filterMap, true)
+			return
+		}
+		filter, filterErr := parsePullACL()
+		if filterErr != nil {
+			err = fmt.Errorf("cannot parse pull-ACL for identity '%s': %s", identity, filterErr)
+			return
+		}
+		m[identity] = filter
+	}
+	return
+}
+
+func parseDatasetMapFilter(mi interface{}, filterOnly bool) (f DatasetMapFilter, err error) {
+
+	var m map[string]string
+	if err = mapstructure.Decode(mi, &m); err != nil {
+		err = fmt.Errorf("maps / filters must be specified as map[string]string: %s", err)
+		return
+	}
+
+	f = NewDatasetMapFilter(len(m), filterOnly)
+	for pathPattern, mapping := range m {
+		if err = f.Add(pathPattern, mapping); err != nil {
+			err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err)
+			return
+		}
+	}
+	return
+}
+
+func (t SSHTransport) Connect(rpcLog Logger) (r rpc.RPCRequester, err error) {
+	var stream io.ReadWriteCloser
+	var rpcTransport sshbytestream.SSHTransport
+	if err = copier.Copy(&rpcTransport, t); err != nil {
+		return
+	}
+	if stream, err = sshbytestream.Outgoing(rpcTransport); err != nil {
+		return
+	}
+	stream, err = NewReadWriteCloserLogger(stream, t.ConnLogReadFile, t.ConnLogWriteFile)
+	if err != nil {
+		return
+	}
+	return rpc.ConnectByteStreamRPC(stream, rpcLog)
+}
+
+func (t LocalTransport) Connect(rpcLog Logger) (r rpc.RPCRequester, err error) {
+	if t.Handler == nil {
+		panic("local transport with uninitialized handler")
+	}
+	return rpc.ConnectLocalRPC(t.Handler), nil
+}
+
+func (t *LocalTransport) SetHandler(handler rpc.RPCHandler) {
+	t.Handler = handler
+}
+
+func parsePrunes(m interface{}) (rets map[string]*Prune, err error) {
+
+	asList := make(map[string]map[string]interface{}, 0)
+	if err = mapstructure.Decode(m, &asList); err != nil {
+		return
+	}
+
+	rets = make(map[string]*Prune, len(asList))
+
+	for name, e := range asList {
+		var prune *Prune
+		if prune, err = parsePrune(e, name); err != nil {
+			err = fmt.Errorf("cannot parse prune job %s: %s", name, err)
+			return
+		}
+		rets[name] = prune
+	}
+
+	return
+}
+
+func parsePrune(e map[string]interface{}, name string) (prune *Prune, err error) {
+
+	// Only support grid policy for now
+	policyName, ok := e["policy"]
+	if !ok || policyName != "grid" {
+		err = fmt.Errorf("prune job with unimplemented policy '%s'", policyName)
+		return
+	}
+
+	var i struct {
+		Grid           string
+		DatasetFilter  map[string]string `mapstructure:"dataset_filter"`
+		SnapshotFilter map[string]string `mapstructure:"snapshot_filter"`
+	}
+
+	if err = mapstructure.Decode(e, &i); err != nil {
+		return
+	}
+
+	prune = &Prune{}
+
+	if prune.JobName, err = fullJobName(JobSectionPrune, name); err != nil {
+		return
+	}
+
+	// Parse grid policy
+	intervals, err := parseRetentionGridIntervalsString(i.Grid)
+	if err != nil {
+		err = fmt.Errorf("cannot parse retention grid: %s", err)
+		return
+	}
+	// Assert intervals are of increasing length (not necessarily required, but indicates config mistake)
+	lastDuration := time.Duration(0)
+	for i := range intervals {
+		if intervals[i].Length < lastDuration {
+			err = fmt.Errorf("retention grid interval length must be monotonically increasing:"+
+				"interval %d is shorter than %d", i+1, i)
+			return
+		} else {
+			lastDuration = intervals[i].Length
+		}
+	}
+	prune.RetentionPolicy = NewRetentionGrid(intervals)
+
+	// Parse filters
+	if prune.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil {
+		err = fmt.Errorf("cannot parse dataset filter: %s", err)
+		return
+	}
+	if prune.SnapshotFilter, err = parseSnapshotFilter(i.SnapshotFilter); err != nil {
+		err = fmt.Errorf("cannot parse snapshot filter: %s", err)
+		return
+	}
+
+	return
+}
+
+var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
+
+func parseDuration(e string) (d time.Duration, err error) {
+	comps := durationStringRegex.FindStringSubmatch(e)
+	if len(comps) != 3 {
+		err = fmt.Errorf("does not match regex: %s %#v", e, comps)
+		return
+	}
+
+	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
+	if err != nil {
+		return
+	}
+
+	var durationUnit time.Duration
+	switch comps[2] {
+	case "s":
+		durationUnit = time.Second
+	case "m":
+		durationUnit = time.Minute
+	case "h":
+		durationUnit = time.Hour
+	case "d":
+		durationUnit = 24 * time.Hour
+	case "w":
+		durationUnit = 24 * 7 * time.Hour
+	default:
+		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
+		return
+	}
+
+	d = time.Duration(durationFactor) * durationUnit
+	return
+
+}
+
+var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
+
+func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {
+
+	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
+	if comps == nil {
+		err = fmt.Errorf("retention string does not match expected format")
+		return
+	}
+
+	times, err := strconv.Atoi(comps[1])
+	if err != nil {
+		return nil, err
+	} else if times <= 0 {
+		return nil, fmt.Errorf("contains factor <= 0")
+	}
+
+	duration, err := parseDuration(comps[2])
+	if err != nil {
+		return nil, err
+	}
+
+	keepCount := 1
+	if comps[3] != "" {
+		// Decompose key=value, comma separated
+		// For now, only keep_count is supported
+		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
+		res := re.FindStringSubmatch(comps[4])
+		if res == nil || len(res) != 2 {
+			err = fmt.Errorf("interval parameter contains unknown parameters")
+			return
+		}
+		if res[1] == "all" {
+			keepCount = RetentionGridKeepCountAll
+		} else {
+			keepCount, err = strconv.Atoi(res[1])
+			if err != nil {
+				err = fmt.Errorf("cannot parse keep_count value")
+				return
+			}
+		}
+	}
+
+	intervals = make([]RetentionInterval, times)
+	for i := range intervals {
+		intervals[i] = RetentionInterval{
+			Length:    duration,
+			KeepCount: keepCount,
+		}
+	}
+
+	return
+
+}
+
+func parseRetentionGridIntervalsString(s string) (intervals []RetentionInterval, err error) {
+
+	ges := strings.Split(s, "|")
+	intervals = make([]RetentionInterval, 0, 7*len(ges))
+
+	for intervalIdx, e := range ges {
+		parsed, err := parseRetentionGridIntervalString(e)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
+		}
+		intervals = append(intervals, parsed...)
+	}
+
+	return
+}
+
+type prefixSnapshotFilter struct {
+	prefix string
+}
+
+func (f prefixSnapshotFilter) Filter(fsv zfs.FilesystemVersion) (accept bool, err error) {
+	return fsv.Type == zfs.Snapshot && strings.HasPrefix(fsv.Name, f.prefix), nil
+}
+
+func parseSnapshotFilter(fm map[string]string) (snapFilter zfs.FilesystemVersionFilter, err error) {
+	prefix, ok := fm["prefix"]
+	if !ok {
+		err = fmt.Errorf("unsupported snapshot filter")
+		return
+	}
+	snapFilter = prefixSnapshotFilter{prefix}
+	return
+}
+
+func parseAutosnaps(m interface{}) (snaps map[string]*Autosnap, err error) {
+
+	asMap := make(map[string]interface{}, 0)
+	if err = mapstructure.Decode(m, &asMap); err != nil {
+		return
+	}
+
+	snaps = make(map[string]*Autosnap, len(asMap))
+
+	for name, e := range asMap {
+		var snap *Autosnap
+		if snap, err = parseAutosnap(e, name); err != nil {
+			err = fmt.Errorf("cannot parse autonsap job %s: %s", name, err)
+			return
+		}
+		snaps[name] = snap
+	}
+
+	return
+
+}
+
+func parseAutosnap(m interface{}, name string) (a *Autosnap, err error) {
+
+	var i struct {
+		Prefix        string
+		Interval      string
+		DatasetFilter map[string]string `mapstructure:"dataset_filter"`
+	}
+
+	if err = mapstructure.Decode(m, &i); err != nil {
+		err = fmt.Errorf("structure unfit: %s", err)
+		return
+	}
+
+	a = &Autosnap{}
+
+	if a.JobName, err = fullJobName(JobSectionAutosnap, name); err != nil {
+		return
+	}
+
+	if len(i.Prefix) < 1 {
+		err = fmt.Errorf("prefix must not be empty")
+		return
+	}
+	a.Prefix = i.Prefix
+
+	var interval time.Duration
+	if interval, err = parseDuration(i.Interval); err != nil {
+		err = fmt.Errorf("cannot parse interval: %s", err)
+		return
+	}
+	a.Interval = &jobrun.PeriodicRepeatStrategy{interval}
+
+	if len(i.DatasetFilter) == 0 {
+		err = fmt.Errorf("dataset_filter not specified")
+		return
+	}
+	if a.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil {
+		err = fmt.Errorf("cannot parse dataset filter: %s", err)
+	}
+
+	return
+}

cmd/config_mapfilter.go

@@ -0,0 +1,162 @@
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type DatasetMapFilter struct {
+	entries []datasetMapFilterEntry
+
+	// if set, only valid filter entries can be added using Add()
+	// and Map() will always return an error
+	filterOnly bool
+}
+
+type datasetMapFilterEntry struct {
+	path *zfs.DatasetPath
+	// the mapping. since this datastructure acts as both mapping and filter
+	// we have to convert it to the desired rep dynamically
+	mapping      string
+	subtreeMatch bool
+}
+
+var NoMatchError error = errors.New("no match found in mapping")
+
+func NewDatasetMapFilter(capacity int, filterOnly bool) DatasetMapFilter {
+	return DatasetMapFilter{
+		entries: make([]datasetMapFilterEntry, 0, capacity),
+	}
+}
+
+func (m *DatasetMapFilter) Add(pathPattern, mapping string) (err error) {
+
+	if m.filterOnly {
+		if _, err = parseDatasetFilterResult(mapping); err != nil {
+			return
+		}
+	}
+
+	// assert path glob adheres to spec
+	const SUBTREE_PATTERN string = "<"
+	patternCount := strings.Count(pathPattern, SUBTREE_PATTERN)
+	switch {
+	case patternCount > 1:
+	case patternCount == 1 && !strings.HasSuffix(pathPattern, SUBTREE_PATTERN):
+		err = fmt.Errorf("pattern invalid: only one '<' at end of string allowed")
+		return
+	}
+
+	pathStr := strings.TrimSuffix(pathPattern, SUBTREE_PATTERN)
+	path, err := zfs.NewDatasetPath(pathStr)
+	if err != nil {
+		return fmt.Errorf("pattern is not a dataset path: %s", err)
+	}
+
+	entry := datasetMapFilterEntry{
+		path:         path,
+		mapping:      mapping,
+		subtreeMatch: patternCount > 0,
+	}
+	m.entries = append(m.entries, entry)
+	return
+
+}
+
+// find the most specific prefix mapping we have
+//
+// longer prefix wins over shorter prefix, direct wins over glob
+func (m DatasetMapFilter) mostSpecificPrefixMapping(path *zfs.DatasetPath) (idx int, found bool) {
+	lcp, lcp_entry_idx := -1, -1
+	direct_idx := -1
+	for e := range m.entries {
+		entry := m.entries[e]
+		ep := m.entries[e].path
+		lep := ep.Length()
+
+		switch {
+		case !entry.subtreeMatch && ep.Equal(path):
+			direct_idx = e
+			continue
+		case entry.subtreeMatch && path.HasPrefix(ep) && lep > lcp:
+			lcp = lep
+			lcp_entry_idx = e
+		default:
+			continue
+		}
+	}
+
+	if lcp_entry_idx >= 0 || direct_idx >= 0 {
+		found = true
+		switch {
+		case direct_idx >= 0:
+			idx = direct_idx
+		case lcp_entry_idx >= 0:
+			idx = lcp_entry_idx
+		}
+	}
+	return
+}
+
+func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error) {
+
+	if m.filterOnly {
+		err = fmt.Errorf("using a filter for mapping simply does not work")
+		return
+	}
+
+	mi, hasMapping := m.mostSpecificPrefixMapping(source)
+	if !hasMapping {
+		err = NoMatchError
+		return
+	}
+	me := m.entries[mi]
+
+	target, err = zfs.NewDatasetPath(me.mapping)
+	if err != nil {
+		err = fmt.Errorf("mapping target is not a dataset path: %s", err)
+		return
+	}
+	if m.entries[mi].subtreeMatch {
+		// strip common prefix
+		extendComps := source.Copy()
+		if me.path.Empty() {
+			// special case: trying to map the root => strip first component
+			extendComps.TrimNPrefixComps(1)
+		} else {
+			extendComps.TrimPrefix(me.path)
+		}
+		target.Extend(extendComps)
+	}
+	return
+}
+
+func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
+	mi, hasMapping := m.mostSpecificPrefixMapping(p)
+	if !hasMapping {
+		pass = false
+		return
+	}
+	me := m.entries[mi]
+	pass, err = parseDatasetFilterResult(me.mapping)
+	return
+}
+
+// Parse a dataset filter result
+func parseDatasetFilterResult(result string) (pass bool, err error) {
+	l := strings.ToLower(result)
+	switch strings.ToLower(l) {
+	case "ok":
+		pass = true
+		return
+	case "omit":
+		return
+	default:
+		err = fmt.Errorf("'%s' is not a valid filter result", result)
+		return
+	}
+	return
+}

cmd/config_test.go

@@ -0,0 +1,123 @@
+package cmd
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/zrepl/zrepl/util"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func TestSampleConfigFileIsParsedWithoutErrors(t *testing.T) {
+	_, err := ParseConfig("./sampleconf/zrepl.yml")
+	assert.Nil(t, err)
+}
+
+func TestParseRetentionGridStringParsing(t *testing.T) {
+
+	intervals, err := parseRetentionGridIntervalsString("2x10m(keep=2) | 1x1h | 3x1w")
+
+	assert.Nil(t, err)
+	assert.Len(t, intervals, 6)
+	proto := util.RetentionInterval{
+		KeepCount: 2,
+		Length:    10 * time.Minute,
+	}
+	assert.EqualValues(t, proto, intervals[0])
+	assert.EqualValues(t, proto, intervals[1])
+
+	proto.KeepCount = 1
+	proto.Length = 1 * time.Hour
+	assert.EqualValues(t, proto, intervals[2])
+
+	proto.Length = 7 * 24 * time.Hour
+	assert.EqualValues(t, proto, intervals[3])
+	assert.EqualValues(t, proto, intervals[4])
+	assert.EqualValues(t, proto, intervals[5])
+
+	intervals, err = parseRetentionGridIntervalsString("|")
+	assert.Error(t, err)
+	intervals, err = parseRetentionGridIntervalsString("2x10m")
+	assert.NoError(t, err)
+
+	intervals, err = parseRetentionGridIntervalsString("1x10m(keep=all)")
+	assert.NoError(t, err)
+	assert.Len(t, intervals, 1)
+	assert.EqualValues(t, util.RetentionGridKeepCountAll, intervals[0].KeepCount)
+
+}
+
+func TestDatasetMapFilter(t *testing.T) {
+
+	expectMapping := func(m map[string]string, from, to string) {
+		dmf, err := parseDatasetMapFilter(m, false)
+		if err != nil {
+			t.Logf("expect test map to be valid: %s", err)
+			t.FailNow()
+		}
+		fromPath, err := zfs.NewDatasetPath(from)
+		if err != nil {
+			t.Logf("expect test from path to be valid: %s", err)
+			t.FailNow()
+		}
+		toPath, err := zfs.NewDatasetPath(to)
+		if err != nil {
+			t.Logf("expect test to path to be valid: %s", err)
+			t.FailNow()
+		}
+
+		res, err := dmf.Map(fromPath)
+		t.Logf("%s => %s", fromPath.ToString(), res.ToString())
+		assert.Nil(t, err)
+		assert.True(t, res.Equal(toPath))
+	}
+
+	expectFilter := func(m map[string]string, path string, pass bool) {
+		dmf, err := parseDatasetMapFilter(m, true)
+		if err != nil {
+			t.Logf("expect test filter to be valid: %s", err)
+			t.FailNow()
+		}
+		p, err := zfs.NewDatasetPath(path)
+		if err != nil {
+			t.Logf("expect test path to be valid: %s", err)
+			t.FailNow()
+		}
+		res, err := dmf.Filter(p)
+		assert.Nil(t, err)
+		assert.Equal(t, pass, res)
+	}
+
+	map1 := map[string]string{
+		"a/b/c<": "root1",
+		"a/b<":   "root2",
+		"<":      "root3/b/c",
+		"q<":     "root4/1/2",
+	}
+
+	expectMapping(map1, "a/b/c", "root1")
+	expectMapping(map1, "a/b/c/d", "root1/d")
+	expectMapping(map1, "a/b/e", "root2/e")
+	expectMapping(map1, "a/b", "root2")
+	expectMapping(map1, "x", "root3/b/c")
+	expectMapping(map1, "x/y", "root3/b/c/y")
+	expectMapping(map1, "q", "root4/1/2")
+	expectMapping(map1, "q/r", "root4/1/2/r")
+
+	filter1 := map[string]string{
+		"<":    "omit",
+		"a<":   "ok",
+		"a/b<": "omit",
+	}
+
+	expectFilter(filter1, "b", false)
+	expectFilter(filter1, "a", true)
+	expectFilter(filter1, "a/d", true)
+	expectFilter(filter1, "a/b", false)
+	expectFilter(filter1, "a/b/c", false)
+
+	filter2 := map[string]string{}
+	expectFilter(filter2, "foo", false) // default to omit
+
+}

cmd/handler.go

@@ -0,0 +1,177 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/zfs"
+	"io"
+)
+
+type DatasetMapping interface {
+	Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error)
+}
+
+type Handler struct {
+	Logger          Logger
+	PullACL         zfs.DatasetFilter
+	SinkMappingFunc func(clientIdentity string) (mapping DatasetMapping, err error)
+}
+
+func (h Handler) HandleFilesystemRequest(r rpc.FilesystemRequest) (roots []*zfs.DatasetPath, err error) {
+
+	h.Logger.Printf("handling fsr: %#v", r)
+
+	h.Logger.Printf("using PullACL: %#v", h.PullACL)
+
+	if roots, err = zfs.ZFSListMapping(h.PullACL); err != nil {
+		h.Logger.Printf("handle fsr err: %v\n", err)
+		return
+	}
+
+	h.Logger.Printf("returning: %#v", roots)
+
+	return
+}
+
+func (h Handler) HandleFilesystemVersionsRequest(r rpc.FilesystemVersionsRequest) (versions []zfs.FilesystemVersion, err error) {
+
+	h.Logger.Printf("handling filesystem versions request: %#v", r)
+
+	// allowed to request that?
+	if h.pullACLCheck(r.Filesystem); err != nil {
+		return
+	}
+
+	// find our versions
+	if versions, err = zfs.ZFSListFilesystemVersions(r.Filesystem, nil); err != nil {
+		h.Logger.Printf("our versions error: %#v\n", err)
+		return
+	}
+
+	h.Logger.Printf("our versions: %#v\n", versions)
+	return
+
+}
+
+func (h Handler) HandleInitialTransferRequest(r rpc.InitialTransferRequest) (stream io.Reader, err error) {
+
+	h.Logger.Printf("handling initial transfer request: %#v", r)
+	if err = h.pullACLCheck(r.Filesystem); err != nil {
+		return
+	}
+
+	h.Logger.Printf("invoking zfs send")
+
+	if stream, err = zfs.ZFSSend(r.Filesystem, &r.FilesystemVersion, nil); err != nil {
+		h.Logger.Printf("error sending filesystem: %#v", err)
+	}
+
+	return
+
+}
+
+func (h Handler) HandleIncrementalTransferRequest(r rpc.IncrementalTransferRequest) (stream io.Reader, err error) {
+
+	h.Logger.Printf("handling incremental transfer request: %#v", r)
+	if err = h.pullACLCheck(r.Filesystem); err != nil {
+		return
+	}
+
+	h.Logger.Printf("invoking zfs send")
+
+	if stream, err = zfs.ZFSSend(r.Filesystem, &r.From, &r.To); err != nil {
+		h.Logger.Printf("error sending filesystem: %#v", err)
+	}
+
+	return
+
+}
+
+func (h Handler) HandleResumeTransferRequest(r rpc.ResumeTransferRequest) (stream io.Reader, err error) {
+	// decode receive_resume_token: zfs send -nvP <token>
+	// A) use exit code to determine if could send (exit != 0 means could not send)
+	// B) check ACL if the filesystem in toname field (it is a snapshot so strip its snapshot name first)
+	//    is allowed for the given client
+	/*
+
+			# zfs send -nvt will print nvlist contents on ZoL and FreeBSD, should be good enough
+			# will print regardless of whether it can send or not -> good for us
+			# need clean way to extract that...
+				# expect 'resume token contents:\nnvlist version: 0\n'
+				# parse everything after that is tab-indented as key value pairs separated by = sign
+				# => return dict
+
+			zfs send -nv -t 1-c6491acbe-c8-789c636064000310a500c4ec50360710e72765a526973030b0419460caa7a515a79630c001489e0d493ea9b224b5182451885d7f497e7a69660a0343f79b1b9a8a2b3db65b20c97382e5f312735319188a4bf28b12d353f5931293b34b0b8af5ab8a520b72f4d3f2f31d8a53f35220660300c1091dbe
+		resume token contents:
+		nvlist version: 0
+		        object = 0x6
+		        offset = 0x0
+		        bytes = 0x7100
+		        toguid = 0xb748a92129d8ec8b
+		        toname = storage/backups/zrepl/foo@send
+		cannot resume send: 'storage/backups/zrepl/foo@send' used in the initial send no longer exists
+
+		zfs send -nvt 1-ebbbbea7e-f0-789c636064000310a501c49c50360710a715e5e7a69766a6304041f79b1b9a8a2b3db62b00d9ec48eaf293b252934b181858a0ea30e4d3d28a534b18e00024cf86249f5459925a0ca44fc861d75f920f71c59c5fdf6f7b3eea32b24092e704cbe725e6a632301497e41725a6a7ea27252667971614eb5715a516e4e8a7e5e73b14a7e6a51881cd06005a222749
+		resume token contents:
+		nvlist version: 0
+		        fromguid = 0xb748a92129d8ec8b #NOTE the fromguid field which is only in this one, so don't hardcode
+		        object = 0x4
+		        offset = 0x0
+		        bytes = 0x1ec8
+		        toguid = 0x328ae249dbf7fa9c
+		        toname = storage/backups/zrepl/foo@send2
+		send from storage/backups/zrepl/foo@send to storage/backups/zrepl/foo@send2 estimated size is 1.02M
+
+	*/
+	panic("not implemented")
+	return nil, nil
+}
+
+func (h Handler) HandlePullMeRequest(r rpc.PullMeRequest, clientIdentity string, client rpc.RPCRequester) (err error) {
+
+	// Check if we have a sink for this request
+	// Use that mapping to do what happens in doPull
+
+	h.Logger.Printf("handling PullMeRequest: %#v", r)
+
+	var sinkMapping DatasetMapping
+	sinkMapping, err = h.SinkMappingFunc(clientIdentity)
+	if err != nil {
+		h.Logger.Printf("no sink mapping for client identity '%s', denying PullMeRequest", clientIdentity)
+		err = fmt.Errorf("no sink for client identity '%s'", clientIdentity)
+		return
+	}
+
+	h.Logger.Printf("doing pull...")
+
+	err = doPull(PullContext{
+		Remote:            client,
+		Log:               h.Logger,
+		Mapping:           sinkMapping,
+		InitialReplPolicy: r.InitialReplPolicy,
+	})
+	if err != nil {
+		h.Logger.Printf("PullMeRequest failed with error: %s", err)
+		return
+	}
+
+	h.Logger.Printf("finished handling PullMeRequest: %#v", r)
+
+	return
+}
+
+func (h Handler) pullACLCheck(p *zfs.DatasetPath) (err error) {
+	var allowed bool
+	allowed, err = h.PullACL.Filter(p)
+	if err != nil {
+		err = fmt.Errorf("error evaluating ACL: %s", err)
+		h.Logger.Printf(err.Error())
+		return
+	}
+	if !allowed {
+		err = fmt.Errorf("ACL prohibits access to %s", p.ToString())
+		h.Logger.Printf(err.Error())
+		return
+	}
+	return
+}

cmd/main.go

@@ -0,0 +1,104 @@
+// zrepl replicates ZFS filesystems & volumes between pools
+//
+// Code Organization
+//
+// The cmd package uses github.com/spf13/cobra for its CLI.
+//
+// It combines the other packages in the zrepl project to implement zrepl functionality.
+//
+// Each subcommand's code is in the corresponding *.go file.
+// All other *.go files contain code shared by the subcommands.
+package cmd
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/jobrun"
+	"golang.org/x/sys/unix"
+	"io"
+	golog "log"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+)
+
+type Logger interface {
+	Printf(format string, v ...interface{})
+}
+
+// global state / facilities
+var (
+	conf     Config
+	runner   *jobrun.JobRunner
+	logFlags int = golog.LUTC | golog.Ldate | golog.Ltime
+	logOut   io.Writer
+	log      Logger
+)
+
+var RootCmd = &cobra.Command{
+	Use:   "zrepl",
+	Short: "ZFS dataset replication",
+	Long: `Replicate ZFS filesystems & volumes between pools:
+
+  - push & pull mode
+  - automatic snapshot creation & pruning
+  - local / over the network
+  - ACLs instead of blank SSH access`,
+}
+
+var rootArgs struct {
+	configFile string
+	stderrFile string
+	httpPprof  string
+}
+
+func init() {
+	cobra.OnInitialize(initConfig)
+	RootCmd.PersistentFlags().StringVar(&rootArgs.configFile, "config", "", "config file path")
+	RootCmd.PersistentFlags().StringVar(&rootArgs.stderrFile, "stderrFile", "-", "redirect stderr to given path")
+	RootCmd.PersistentFlags().StringVar(&rootArgs.httpPprof, "debug.pprof.http", "", "run pprof http server on given port")
+}
+
+func initConfig() {
+
+	// Logging & stderr redirection
+	if rootArgs.stderrFile != "-" {
+		file, err := os.OpenFile(rootArgs.stderrFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
+		if err != nil {
+			return
+		}
+
+		if err = unix.Dup2(int(file.Fd()), int(os.Stderr.Fd())); err != nil {
+			file.WriteString(fmt.Sprintf("error redirecting stderr file %s: %s\n", rootArgs.stderrFile, err))
+			return
+		}
+		logOut = file
+	} else {
+		logOut = os.Stderr
+	}
+
+	log = golog.New(logOut, "", logFlags)
+
+	// CPU profiling
+	if rootArgs.httpPprof != "" {
+		go func() {
+			http.ListenAndServe(rootArgs.httpPprof, nil)
+		}()
+	}
+
+	// Config
+	if rootArgs.configFile == "" {
+		log.Printf("config file not set")
+		os.Exit(1)
+	}
+	var err error
+	if conf, err = ParseConfig(rootArgs.configFile); err != nil {
+		log.Printf("error parsing config: %s", err)
+		os.Exit(1)
+	}
+
+	jobrunLogger := golog.New(os.Stderr, "jobrun ", logFlags)
+	runner = jobrun.NewJobRunner(jobrunLogger)
+	return
+
+}

cmd/prune.go

@@ -0,0 +1,139 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/util"
+	"github.com/zrepl/zrepl/zfs"
+	"os"
+	"sort"
+	"time"
+)
+
+var pruneArgs struct {
+	job    string
+	dryRun bool
+}
+
+var PruneCmd = &cobra.Command{
+	Use:   "prune",
+	Short: "perform pruning",
+	Run:   cmdPrune,
+}
+
+func init() {
+	PruneCmd.Flags().StringVar(&pruneArgs.job, "job", "", "job to run")
+	PruneCmd.Flags().BoolVarP(&pruneArgs.dryRun, "dryrun", "n", false, "dry run")
+	RootCmd.AddCommand(PruneCmd)
+}
+
+func cmdPrune(cmd *cobra.Command, args []string) {
+
+	if len(args) < 1 {
+		log.Printf("must specify exactly one job as positional argument")
+		os.Exit(1)
+	}
+
+	job, ok := conf.Prunes[args[0]]
+	if !ok {
+		log.Printf("could not find prune job: %s", args[0])
+		os.Exit(1)
+	}
+
+	log.Printf("Beginning prune job:\n%s", job)
+	ctx := PruneContext{job, time.Now(), pruneArgs.dryRun}
+	err := doPrune(ctx, log)
+	if err != nil {
+		log.Printf("Prune job failed with error: %s", err)
+		os.Exit(1)
+	}
+
+}
+
+type PruneContext struct {
+	Prune  *Prune
+	Now    time.Time
+	DryRun bool
+}
+
+type retentionGridAdaptor struct {
+	zfs.FilesystemVersion
+}
+
+func (a retentionGridAdaptor) Date() time.Time {
+	return a.Creation
+}
+
+func (a retentionGridAdaptor) LessThan(b util.RetentionGridEntry) bool {
+	return a.CreateTXG < b.(retentionGridAdaptor).CreateTXG
+}
+
+func doPrune(ctx PruneContext, log Logger) error {
+
+	if ctx.DryRun {
+		log.Printf("doing dry run")
+	}
+
+	prune := ctx.Prune
+
+	// ZFSListSnapsFiltered --> todo can extend fsfilter or need new? Have already something per fs
+	// Dedicated snapshot object? Adaptor object to FilesystemVersion?
+
+	filesystems, err := zfs.ZFSListMapping(prune.DatasetFilter)
+	if err != nil {
+		return fmt.Errorf("error applying filesystem filter: %s", err)
+	}
+
+	for _, fs := range filesystems {
+
+		fsversions, err := zfs.ZFSListFilesystemVersions(fs, prune.SnapshotFilter)
+		if err != nil {
+			return fmt.Errorf("error listing filesytem versions of %s: %s", fs, err)
+		}
+
+		if len(fsversions) == 0 {
+			continue
+		}
+
+		adaptors := make([]util.RetentionGridEntry, len(fsversions))
+		for fsv := range fsversions {
+			adaptors[fsv] = retentionGridAdaptor{fsversions[fsv]}
+		}
+
+		sort.SliceStable(adaptors, func(i, j int) bool {
+			return adaptors[i].LessThan(adaptors[j])
+		})
+		ctx.Now = adaptors[len(adaptors)-1].Date()
+
+		describe := func(a retentionGridAdaptor) string {
+			timeSince := a.Creation.Sub(ctx.Now)
+			const day time.Duration = 24 * time.Hour
+			days := timeSince / day
+			remainder := timeSince % day
+			return fmt.Sprintf("%s@%dd%s from now", a.ToAbsPath(fs), days, remainder)
+		}
+
+		keep, remove := prune.RetentionPolicy.FitEntries(ctx.Now, adaptors)
+		for _, a := range remove {
+			r := a.(retentionGridAdaptor)
+			log.Printf("remove %s", describe(r))
+			// do echo what we'll do and exec zfs destroy if not dry run
+			// special handling for EBUSY (zfs hold)
+			// error handling for clones? just echo to cli, skip over, and exit with non-zero status code (we're idempotent)
+			if !ctx.DryRun {
+				err := zfs.ZFSDestroy(r.ToAbsPath(fs))
+				if err != nil {
+					// handle
+					log.Printf("error: %s", err)
+				}
+			}
+		}
+		for _, a := range keep {
+			r := a.(retentionGridAdaptor)
+			log.Printf("would keep %s", describe(r))
+		}
+	}
+
+	return nil
+
+}

cmd/replication.go

@@ -0,0 +1,587 @@
+package cmd
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/jobrun"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/util"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var runArgs struct {
+	job  string
+	once bool
+}
+
+var RunCmd = &cobra.Command{
+	Use:   "run",
+	Short: "run push & pull replication",
+	Run:   cmdRun,
+}
+
+var PushCmd = &cobra.Command{
+	Use:   "push",
+	Short: "run push job (first positional argument)",
+	Run:   cmdPush,
+}
+
+var PullCmd = &cobra.Command{
+	Use:   "pull",
+	Short: "run pull job (first positional argument)",
+	Run:   cmdPull,
+}
+
+func init() {
+	RootCmd.AddCommand(RunCmd)
+	RunCmd.Flags().BoolVar(&runArgs.once, "once", false, "run jobs only once, regardless of configured repeat behavior")
+	RunCmd.Flags().StringVar(&runArgs.job, "job", "", "run only the given job")
+
+	RootCmd.AddCommand(PushCmd)
+	RootCmd.AddCommand(PullCmd)
+}
+
+func cmdPush(cmd *cobra.Command, args []string) {
+
+	if len(args) != 1 {
+		log.Printf("must specify exactly one job as positional argument")
+		os.Exit(1)
+	}
+	job, ok := conf.Pushs[args[0]]
+	if !ok {
+		log.Printf("could not find push job %s", args[0])
+		os.Exit(1)
+	}
+	if err := jobPush(job, log); err != nil {
+		log.Printf("error doing push: %s", err)
+		os.Exit(1)
+	}
+
+}
+
+func cmdPull(cmd *cobra.Command, args []string) {
+
+	if len(args) != 1 {
+		log.Printf("must specify exactly one job as positional argument")
+		os.Exit(1)
+	}
+	job, ok := conf.Pulls[args[0]]
+	if !ok {
+		log.Printf("could not find pull job %s", args[0])
+		os.Exit(1)
+	}
+
+	if err := jobPull(job, log); err != nil {
+		log.Printf("error doing pull: %s", err)
+		os.Exit(1)
+	}
+
+}
+
+func cmdRun(cmd *cobra.Command, args []string) {
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		runner.Start()
+	}()
+
+	jobs := make([]jobrun.Job, len(conf.Pulls)+len(conf.Pushs))
+	i := 0
+	for _, pull := range conf.Pulls {
+		jobs[i] = jobrun.Job{
+			Name:           fmt.Sprintf("pull.%d", i),
+			RepeatStrategy: pull.RepeatStrategy,
+			RunFunc: func(log jobrun.Logger) error {
+				log.Printf("doing pull: %v", pull)
+				return jobPull(pull, log)
+			},
+		}
+		i++
+	}
+	for _, push := range conf.Pushs {
+		jobs[i] = jobrun.Job{
+			Name:           fmt.Sprintf("push.%d", i),
+			RepeatStrategy: push.RepeatStrategy,
+			RunFunc: func(log jobrun.Logger) error {
+				log.Printf("doing push: %v", push)
+				return jobPush(push, log)
+			},
+		}
+		i++
+	}
+
+	for _, j := range jobs {
+		if runArgs.once {
+			j.RepeatStrategy = jobrun.NoRepeatStrategy{}
+		}
+		if runArgs.job != "" {
+			if runArgs.job == j.Name {
+				runner.AddJob(j)
+				break
+			}
+			continue
+		}
+		runner.AddJob(j)
+	}
+
+	for {
+		select {
+		case job := <-runner.NotificationChan():
+			log.Printf("job %s reported error: %v\n", job.Name, job.LastError)
+		}
+	}
+
+	wg.Wait()
+
+}
+
+type localPullACL struct{}
+
+func (a localPullACL) Filter(p *zfs.DatasetPath) (pass bool, err error) {
+	return true, nil
+}
+
+func jobPull(pull *Pull, log jobrun.Logger) (err error) {
+
+	if lt, ok := pull.From.Transport.(LocalTransport); ok {
+
+		lt.SetHandler(Handler{
+			Logger: log,
+			// Allow access to any dataset since we control what mapping
+			// is passed to the pull routine.
+			// All local datasets will be passed to its Map() function,
+			// but only those for which a mapping exists will actually be pulled.
+			// We can pay this small performance penalty for now.
+			PullACL: localPullACL{},
+		})
+		pull.From.Transport = lt
+		log.Printf("fixing up local transport: %#v", pull.From.Transport)
+	}
+
+	var remote rpc.RPCRequester
+
+	if remote, err = pull.From.Transport.Connect(log); err != nil {
+		return
+	}
+
+	defer closeRPCWithTimeout(log, remote, time.Second*10, "")
+
+	return doPull(PullContext{remote, log, pull.Mapping, pull.InitialReplPolicy})
+}
+
+func jobPush(push *Push, log jobrun.Logger) (err error) {
+
+	if _, ok := push.To.Transport.(LocalTransport); ok {
+		panic("no support for local pushs")
+	}
+
+	var remote rpc.RPCRequester
+	if remote, err = push.To.Transport.Connect(log); err != nil {
+		return err
+	}
+
+	defer closeRPCWithTimeout(log, remote, time.Second*10, "")
+
+	log.Printf("building handler for PullMeRequest")
+	handler := Handler{
+		Logger:          log,
+		PullACL:         push.Filter,
+		SinkMappingFunc: nil, // no need for that in the handler for PullMe
+	}
+	log.Printf("handler: %#v", handler)
+
+	r := rpc.PullMeRequest{
+		InitialReplPolicy: push.InitialReplPolicy,
+	}
+	log.Printf("doing PullMeRequest: %#v", r)
+
+	if err = remote.PullMeRequest(r, handler); err != nil {
+		log.Printf("PullMeRequest failed: %s", err)
+		return
+	}
+
+	log.Printf("push job finished")
+	return
+
+}
+
+func closeRPCWithTimeout(log Logger, remote rpc.RPCRequester, timeout time.Duration, goodbye string) {
+	log.Printf("closing rpc connection")
+
+	ch := make(chan error)
+	go func() {
+		ch <- remote.CloseRequest(rpc.CloseRequest{goodbye})
+		close(ch)
+	}()
+
+	var err error
+	select {
+	case <-time.After(timeout):
+		err = fmt.Errorf("timeout exceeded (%s)", timeout)
+	case closeRequestErr := <-ch:
+		err = closeRequestErr
+	}
+
+	if err != nil {
+		log.Printf("error closing connection: %s", err)
+		err = remote.ForceClose()
+		if err != nil {
+			log.Printf("error force-closing connection: %s", err)
+		}
+	}
+	return
+}
+
+type PullContext struct {
+	Remote            rpc.RPCRequester
+	Log               Logger
+	Mapping           DatasetMapping
+	InitialReplPolicy rpc.InitialReplPolicy
+}
+
+func doPull(pull PullContext) (err error) {
+
+	remote := pull.Remote
+	log := pull.Log
+
+	log.Printf("requesting remote filesystem list")
+	fsr := rpc.FilesystemRequest{}
+	var remoteFilesystems []*zfs.DatasetPath
+	if remoteFilesystems, err = remote.FilesystemRequest(fsr); err != nil {
+		return
+	}
+
+	log.Printf("map remote filesystems to local paths and determine order for per-filesystem sync")
+	type RemoteLocalMapping struct {
+		Remote *zfs.DatasetPath
+		Local  *zfs.DatasetPath
+	}
+	replMapping := make(map[string]RemoteLocalMapping, len(remoteFilesystems))
+	localTraversal := zfs.NewDatasetPathForest()
+	for fs := range remoteFilesystems {
+		var err error
+		var localFs *zfs.DatasetPath
+		localFs, err = pull.Mapping.Map(remoteFilesystems[fs])
+		if err != nil {
+			if err != NoMatchError {
+				err := fmt.Errorf("error mapping %s: %s", remoteFilesystems[fs], err)
+				log.Printf("%s", err)
+				return err
+			}
+			continue
+		}
+		log.Printf("%s => %s", remoteFilesystems[fs].ToString(), localFs.ToString())
+		m := RemoteLocalMapping{remoteFilesystems[fs], localFs}
+		replMapping[m.Local.ToString()] = m
+		localTraversal.Add(m.Local)
+	}
+
+	log.Printf("build cache for already present local filesystem state")
+	localFilesystemState, err := zfs.ZFSListFilesystemState()
+	if err != nil {
+		log.Printf("error requesting local filesystem state: %s", err)
+		return err
+	}
+
+	log.Printf("start per-filesystem sync")
+	localTraversal.WalkTopDown(func(v zfs.DatasetPathVisit) bool {
+
+		if v.FilledIn {
+			if _, exists := localFilesystemState[v.Path.ToString()]; exists {
+				// No need to verify if this is a placeholder or not. It is sufficient
+				// to know we can add child filesystems to it
+				return true
+			}
+			log.Printf("creating placeholder filesystem %s", v.Path.ToString())
+			err = zfs.ZFSCreatePlaceholderFilesystem(v.Path)
+			if err != nil {
+				err = fmt.Errorf("aborting, cannot create placeholder filesystem %s: %s", v.Path, err)
+				return false
+			}
+			return true
+		}
+
+		m, ok := replMapping[v.Path.ToString()]
+		if !ok {
+			panic("internal inconsistency: replMapping should contain mapping for any path that was not filled in by WalkTopDown()")
+		}
+
+		log := func(format string, args ...interface{}) {
+			log.Printf("[%s => %s]: %s", m.Remote.ToString(), m.Local.ToString(), fmt.Sprintf(format, args...))
+		}
+
+		log("examing local filesystem state")
+		localState, localExists := localFilesystemState[m.Local.ToString()]
+		var versions []zfs.FilesystemVersion
+		switch {
+		case !localExists:
+			log("local filesystem does not exist")
+		case localState.Placeholder:
+			log("local filesystem is marked as placeholder")
+		case localState.ResumeToken != "":
+
+			log("local filesystem has receive_resume_token")
+			if false { // TODO Check if managed resume is disabled (explain information leakage in docs!)
+				log("managed resume tokens are disabled via zrepl config, assuming config change or external administrative action")
+				log("policy forbids aborting the partial recv automatically, skipping this filesystem")
+				log("for zrepl to resume replication for this dataset, administrators should finishing the recv manually or abort the recv via `zfs recv -A %s`", m.Local.ToString())
+				return true // TODO right choice to allow children? ... should be, since the fs exists...
+			}
+
+			log("decoding receive_resume_token")
+			// TODO, see handler comments
+			// TODO override logger with fromguid toguid
+
+			log("requesting resume of transfer")
+			r := rpc.ResumeTransferRequest{m.Remote, localState.ResumeToken}
+			stream, err := remote.ResumeTransferRequest(r)
+			if err != nil {
+
+				log("resume transfer request failed: %s", err)
+				rre, ok := err.(*rpc.ResumeTransferError)
+				if !ok {
+					log("skipping this filesystem, could be temporary issue")
+					return true // TODO right choice to allow children
+				}
+
+				// Determine if we should clear the resume token
+				clearAndUseSnaps := false
+				switch rre.Reason {
+				case rpc.ResumeTransferErrorReasonNotImplemented:
+					fallthrough
+				case rpc.ResumeTransferErrorReasonDisabled:
+					fallthrough
+				case rpc.ResumeTransferErrorReasonZFSErrorPermanent:
+					clearAndUseSnaps = true
+
+				case rpc.ResumeTransferErrorReasonZFSErrorMaybeTemporary:
+					fallthrough
+				default:
+					clearAndUseSnaps = false
+				}
+
+				if !clearAndUseSnaps {
+					log("skipping this filesystem, error identified as temporary")
+					return true // TODO right choice to allow children
+				}
+
+				log("clearing local receive_resume_token")
+				if err := zfs.ZFSRecvAbort(m.Local); err != nil {
+					log("error clearing receive_resume_token: %s", err)
+					return true // TODO right choice to allow children? the filesystem exists, so it should be ok
+				}
+				// TODO go back to top of function (put all this into separate function, then tail recursive call)
+				// TODO return this_function()
+			}
+
+			// TODO warning code duplication, see below. need to unify this
+			log("invoking zfs receive")
+			watcher := util.IOProgressWatcher{Reader: stream}
+			watcher.KickOff(1*time.Second, func(p util.IOProgress) {
+				log("progress on receive operation: %v bytes received", p.TotalRX)
+			})
+
+			recvArgs := []string{"-u"}
+			if localState.Placeholder {
+				log("receive with forced rollback to replace placeholder filesystem")
+				recvArgs = append(recvArgs, "-F")
+			}
+
+			if err = zfs.ZFSRecv(m.Local, &watcher, recvArgs...); err != nil {
+				log("error receiving stream: %s", err)
+				return false
+			}
+			log("finished receiving stream, %v bytes total", watcher.Progress().TotalRX)
+
+			// TODO further problem property handling code must be duplicated here...
+
+			// TODO tail recursive call to this function, we might not be dony syncing yet
+			// TODO return this_function()
+			return true
+
+		default:
+			log("local filesystem exists")
+			log("requesting local filesystem versions")
+			if versions, err = zfs.ZFSListFilesystemVersions(m.Local, nil); err != nil {
+				log("cannot get local filesystem versions: %s", err)
+				return false
+			}
+		}
+
+		log("requesting remote filesystem versions")
+		var theirVersions []zfs.FilesystemVersion
+		theirVersions, err = remote.FilesystemVersionsRequest(rpc.FilesystemVersionsRequest{
+			Filesystem: m.Remote,
+		})
+		if err != nil {
+			log("error requesting remote filesystem versions: %s", err)
+			log("stopping replication for all filesystems mapped as children of %s", m.Local.ToString())
+			return false
+		}
+
+		log("computing diff between remote and local filesystem versions")
+		diff := zfs.MakeFilesystemDiff(versions, theirVersions)
+		log("%s", diff)
+
+		if localState.Placeholder && diff.Conflict != zfs.ConflictAllRight {
+			panic("internal inconsistency: local placeholder implies ConflictAllRight")
+		}
+
+		switch diff.Conflict {
+		case zfs.ConflictAllRight:
+
+			log("performing initial sync, following policy: '%s'", pull.InitialReplPolicy)
+
+			if pull.InitialReplPolicy != rpc.InitialReplPolicyMostRecent {
+				panic(fmt.Sprintf("policy '%s' not implemented", pull.InitialReplPolicy))
+			}
+
+			snapsOnly := make([]zfs.FilesystemVersion, 0, len(diff.MRCAPathRight))
+			for s := range diff.MRCAPathRight {
+				if diff.MRCAPathRight[s].Type == zfs.Snapshot {
+					snapsOnly = append(snapsOnly, diff.MRCAPathRight[s])
+				}
+			}
+
+			if len(snapsOnly) < 1 {
+				log("cannot perform initial sync: no remote snapshots. stopping...")
+				return false
+			}
+
+			r := rpc.InitialTransferRequest{
+				Filesystem:        m.Remote,
+				FilesystemVersion: snapsOnly[len(snapsOnly)-1],
+			}
+
+			log("requesting snapshot stream for %s", r.FilesystemVersion)
+
+			var stream io.Reader
+			if stream, err = remote.InitialTransferRequest(r); err != nil {
+				log("error requesting initial transfer: %s", err)
+				return false
+			}
+			log("received initial transfer request response")
+
+			log("invoking zfs receive")
+			watcher := util.IOProgressWatcher{Reader: stream}
+			watcher.KickOff(1*time.Second, func(p util.IOProgress) {
+				log("progress on receive operation: %v bytes received", p.TotalRX)
+			})
+
+			recvArgs := []string{"-u"}
+			if localState.Placeholder {
+				log("receive with forced rollback to replace placeholder filesystem")
+				recvArgs = append(recvArgs, "-F")
+			}
+
+			if err = zfs.ZFSRecv(m.Local, &watcher, recvArgs...); err != nil {
+				log("error receiving stream: %s", err)
+				return false
+			}
+			log("finished receiving stream, %v bytes total", watcher.Progress().TotalRX)
+
+			log("configuring properties of received filesystem")
+			if err = zfs.ZFSSet(m.Local, "readonly", "on"); err != nil {
+
+			}
+
+			log("finished initial transfer")
+			return true
+
+		case zfs.ConflictIncremental:
+
+			if len(diff.IncrementalPath) < 2 {
+				log("remote and local are in sync")
+				return true
+			}
+
+			log("following incremental path from diff")
+			var pathRx uint64
+
+			for i := 0; i < len(diff.IncrementalPath)-1; i++ {
+
+				from, to := diff.IncrementalPath[i], diff.IncrementalPath[i+1]
+
+				log := func(format string, args ...interface{}) {
+					log("[%v/%v][%s => %s]: %s", i+1, len(diff.IncrementalPath)-1,
+						from.Name, to.Name, fmt.Sprintf(format, args...))
+				}
+
+				log("requesting incremental snapshot stream")
+				r := rpc.IncrementalTransferRequest{
+					Filesystem: m.Remote,
+					From:       from,
+					To:         to,
+				}
+				var stream io.Reader
+				if stream, err = remote.IncrementalTransferRequest(r); err != nil {
+					log("error requesting incremental snapshot stream: %s", err)
+					return false
+				}
+
+				log("invoking zfs receive")
+				watcher := util.IOProgressWatcher{Reader: stream}
+				watcher.KickOff(1*time.Second, func(p util.IOProgress) {
+					log("progress on receive operation: %v bytes received", p.TotalRX)
+				})
+
+				if err = zfs.ZFSRecv(m.Local, &watcher); err != nil {
+					log("error receiving stream: %s", err)
+					return false
+				}
+
+				totalRx := watcher.Progress().TotalRX
+				pathRx += totalRx
+				log("finished incremental transfer, %v bytes total", totalRx)
+
+			}
+
+			log("finished following incremental path, %v bytes total", pathRx)
+			return true
+
+		case zfs.ConflictNoCommonAncestor:
+
+			log("remote and local filesystem have snapshots, but no common one")
+			log("perform manual replication to establish a common snapshot history")
+			log("remote versions:")
+			for _, v := range diff.MRCAPathRight {
+				log(" %s (GUID %v)", v, v.Guid)
+			}
+			log("local versions:")
+			for _, v := range diff.MRCAPathLeft {
+				log(" %s (GUID %v)", v, v.Guid)
+			}
+			return false
+
+		case zfs.ConflictDiverged:
+
+			log("remote and local filesystem share a history but have diverged")
+			log("perform manual replication or delete snapshots on the receiving" +
+				"side  to establish an incremental replication parse")
+			log("remote-only versions:")
+			for _, v := range diff.MRCAPathRight {
+				log(" %s (GUID %v)", v, v.Guid)
+			}
+			log("local-only versions:")
+			for _, v := range diff.MRCAPathLeft {
+				log(" %s (GUID %v)", v, v.Guid)
+			}
+			return false
+
+		}
+
+		panic("implementation error: this should not be reached")
+		return false
+
+	})
+
+	return
+
+}

cmd/sampleconf/zrepl.yml

@@ -0,0 +1,109 @@
+remotes:
+  offsite_backups:
+    transport:
+      ssh:
+        host: 192.168.122.6
+        user: root
+        port: 22
+        identity_file: /etc/zrepl/identities/offsite_backups
+
+pushs:
+
+  offsite:
+    to: offsite_backups
+    filter: {
+      # like in pull_acls
+      "tank/var/db<":   ok,
+      "tank/usr/home<": ok,
+    }
+
+pulls:
+
+  offsite:
+    from: offsite_backups
+    mapping: {
+      # like in sinks
+    }
+
+  # local replication, only allowed in pull mode
+  # the from name 'local' is reserved for this purpose
+  homemirror:
+    from: local
+    repeat:
+      interval: 15m
+    mapping: {
+      "tank/usr/home":"mirrorpool/foo/bar"
+    }
+
+sinks:
+
+  db1:
+    mapping: {
+      # direct mapping
+      "ssdpool/var/db/postgresql9.6":"zroot/backups/db1/pg_data"
+    }
+
+  mirror1:
+    mapping: {
+      # "<" subtree wildcard matches the dataset left of < and all its children
+      "tank/foo/bar<":"zroot/backups/mirror1" 
+    }
+
+  mirror2:
+    # more specific path patterns win over less specific ones
+    # direct mappings win over subtree wildcards
+    # detailed rule precedence: check unit tests & docs for exact behavior
+    # TODO subcommand to test a mapping & filter
+    mapping: {
+      "tank<":                       "zroot/backups/mirror1/tank1",
+      "tank/cdn/root<":              "storage/cdn/root",
+      "tank/legacydb":               "legacypool/backups/legacydb",
+    }
+
+
+pull_acls:
+
+  office_backup:
+    filter: {
+      # valid filter results (right hand side): ok, omit
+      # default is to omit
+      # rule precedence is same as for mappings
+      "tank<":         omit,
+      "tank/usr/home": ok,
+    }
+
+
+prune:
+
+  clean_backups:
+    policy: grid
+    grid: 6x10m | 24x1h | 7x1d | 5 x 1w | 4 x 5w
+    dataset_filter: {
+      "tank/backups/legacyscript<": omit,
+      "tank/backups<":              ok,
+    }
+    snapshot_filter: {
+      prefix: zrepl_
+    }
+
+  hfbak_prune: # cleans up after hfbak autosnap job
+    policy: grid
+    grid: 1x1m(keep=all)
+    dataset_filter: {
+      "pool1<": ok
+    }
+    snapshot_filter: {
+      prefix: zrepl_hfbak_
+    }
+
+autosnap:
+
+  hfbak:
+    prefix: zrepl_hfbak_
+    interval: 1s
+    dataset_filter: {
+      "pool1<": ok
+    }
+    # prune: hfbak_prune
+    # future versions may inline the retention policy here, but for now,
+    # pruning has to be triggered manually (it's safe to run autosnap + prune in parallel)

cmd/stdinserver.go

@@ -0,0 +1,72 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/sshbytestream"
+	"io"
+	golog "log"
+	"os"
+)
+
+var StdinserverCmd = &cobra.Command{
+	Use:   "stdinserver CLIENT_IDENTITY",
+	Short: "start in stdin server mode (from authorized_keys file)",
+	Run:   cmdStdinServer,
+}
+
+func init() {
+	RootCmd.AddCommand(StdinserverCmd)
+}
+
+func cmdStdinServer(cmd *cobra.Command, args []string) {
+
+	var err error
+	defer func() {
+		if err != nil {
+			log.Printf("stdinserver exiting with error: %s", err)
+			os.Exit(1)
+		}
+	}()
+
+	if len(args) != 1 || args[0] == "" {
+		err = fmt.Errorf("must specify client identity as positional argument")
+		return
+	}
+	identity := args[0]
+
+	pullACL, ok := conf.PullACLs[identity]
+	if !ok {
+		err = fmt.Errorf("could not find PullACL for identity '%s'", identity)
+		return
+	}
+
+	var sshByteStream io.ReadWriteCloser
+	if sshByteStream, err = sshbytestream.Incoming(); err != nil {
+		return
+	}
+
+	sinkMapping := func(identity string) (m DatasetMapping, err error) {
+		sink, ok := conf.Sinks[identity]
+		if !ok {
+			return nil, fmt.Errorf("could not find sink for identity '%s'", identity)
+		}
+		return sink, nil
+	}
+
+	sinkLogger := golog.New(logOut, fmt.Sprintf("sink[%s] ", identity), logFlags)
+	handler := Handler{
+		Logger:          sinkLogger,
+		SinkMappingFunc: sinkMapping,
+		PullACL:         pullACL,
+	}
+
+	if err = rpc.ListenByteStreamRPC(sshByteStream, identity, handler, sinkLogger); err != nil {
+		log.Printf("listenbytestreamerror: %#v\n", err)
+		os.Exit(1)
+	}
+
+	return
+
+}

config/config.go

@@ -1,657 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"io/ioutil"
-	"log/syslog"
-	"os"
-	"reflect"
-	"regexp"
-	"strconv"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/zrepl/yaml-config"
-)
-
-type Config struct {
-	Jobs   []JobEnum `yaml:"jobs"`
-	Global *Global   `yaml:"global,optional,fromdefaults"`
-}
-
-func (c *Config) Job(name string) (*JobEnum, error) {
-	for _, j := range c.Jobs {
-		if j.Name() == name {
-			return &j, nil
-		}
-	}
-	return nil, fmt.Errorf("job %q not defined in config", name)
-}
-
-type JobEnum struct {
-	Ret interface{}
-}
-
-func (j JobEnum) Name() string {
-	var name string
-	switch v := j.Ret.(type) {
-	case *SnapJob:
-		name = v.Name
-	case *PushJob:
-		name = v.Name
-	case *SinkJob:
-		name = v.Name
-	case *PullJob:
-		name = v.Name
-	case *SourceJob:
-		name = v.Name
-	default:
-		panic(fmt.Sprintf("unknown job type %T", v))
-	}
-	return name
-}
-
-type ActiveJob struct {
-	Type    string                `yaml:"type"`
-	Name    string                `yaml:"name"`
-	Connect ConnectEnum           `yaml:"connect"`
-	Pruning PruningSenderReceiver `yaml:"pruning"`
-	Debug   JobDebugSettings      `yaml:"debug,optional"`
-}
-
-type PassiveJob struct {
-	Type  string           `yaml:"type"`
-	Name  string           `yaml:"name"`
-	Serve ServeEnum        `yaml:"serve"`
-	Debug JobDebugSettings `yaml:"debug,optional"`
-}
-
-type SnapJob struct {
-	Type         string            `yaml:"type"`
-	Name         string            `yaml:"name"`
-	Pruning      PruningLocal      `yaml:"pruning"`
-	Debug        JobDebugSettings  `yaml:"debug,optional"`
-	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
-	Filesystems  FilesystemsFilter `yaml:"filesystems"`
-}
-
-type SendOptions struct {
-	Encrypted bool `yaml:"encrypted"`
-}
-
-var _ yaml.Defaulter = (*SendOptions)(nil)
-
-func (l *SendOptions) SetDefault() {
-	*l = SendOptions{Encrypted: false}
-}
-
-type RecvOptions struct {
-	// Note: we cannot enforce encrypted recv as the ZFS cli doesn't provide a mechanism for it
-	// Encrypted bool `yaml:"may_encrypted"`
-
-	// Future:
-	// Reencrypt bool `yaml:"reencrypt"`
-}
-
-var _ yaml.Defaulter = (*RecvOptions)(nil)
-
-func (l *RecvOptions) SetDefault() {
-	*l = RecvOptions{}
-}
-
-type PushJob struct {
-	ActiveJob    `yaml:",inline"`
-	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
-	Filesystems  FilesystemsFilter `yaml:"filesystems"`
-	Send         *SendOptions      `yaml:"send,fromdefaults,optional"`
-}
-
-type PullJob struct {
-	ActiveJob `yaml:",inline"`
-	RootFS    string                   `yaml:"root_fs"`
-	Interval  PositiveDurationOrManual `yaml:"interval"`
-	Recv      *RecvOptions             `yaml:"recv,fromdefaults,optional"`
-}
-
-type PositiveDurationOrManual struct {
-	Interval time.Duration
-	Manual   bool
-}
-
-var _ yaml.Unmarshaler = (*PositiveDurationOrManual)(nil)
-
-func (i *PositiveDurationOrManual) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	var s string
-	if err := u(&s, true); err != nil {
-		return err
-	}
-	switch s {
-	case "manual":
-		i.Manual = true
-		i.Interval = 0
-	case "":
-		return fmt.Errorf("value must not be empty")
-	default:
-		i.Manual = false
-		i.Interval, err = time.ParseDuration(s)
-		if err != nil {
-			return err
-		}
-		if i.Interval <= 0 {
-			return fmt.Errorf("value must be a positive duration, got %q", s)
-		}
-	}
-	return nil
-}
-
-type SinkJob struct {
-	PassiveJob `yaml:",inline"`
-	RootFS     string       `yaml:"root_fs"`
-	Recv       *RecvOptions `yaml:"recv,optional,fromdefaults"`
-}
-
-type SourceJob struct {
-	PassiveJob   `yaml:",inline"`
-	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
-	Filesystems  FilesystemsFilter `yaml:"filesystems"`
-	Send         *SendOptions      `yaml:"send,optional,fromdefaults"`
-}
-
-type FilesystemsFilter map[string]bool
-
-type SnapshottingEnum struct {
-	Ret interface{}
-}
-
-type SnapshottingPeriodic struct {
-	Type     string        `yaml:"type"`
-	Prefix   string        `yaml:"prefix"`
-	Interval time.Duration `yaml:"interval,positive"`
-	Hooks    HookList      `yaml:"hooks,optional"`
-}
-
-type SnapshottingManual struct {
-	Type string `yaml:"type"`
-}
-
-type PruningSenderReceiver struct {
-	KeepSender   []PruningEnum `yaml:"keep_sender"`
-	KeepReceiver []PruningEnum `yaml:"keep_receiver"`
-}
-
-type PruningLocal struct {
-	Keep []PruningEnum `yaml:"keep"`
-}
-
-type LoggingOutletEnumList []LoggingOutletEnum
-
-func (l *LoggingOutletEnumList) SetDefault() {
-	def := `
-type: "stdout"
-time: true
-level: "warn"
-format: "human"
-`
-	s := &StdoutLoggingOutlet{}
-	err := yaml.UnmarshalStrict([]byte(def), &s)
-	if err != nil {
-		panic(err)
-	}
-	*l = []LoggingOutletEnum{LoggingOutletEnum{Ret: s}}
-}
-
-var _ yaml.Defaulter = &LoggingOutletEnumList{}
-
-type Global struct {
-	Logging    *LoggingOutletEnumList `yaml:"logging,optional,fromdefaults"`
-	Monitoring []MonitoringEnum       `yaml:"monitoring,optional"`
-	Control    *GlobalControl         `yaml:"control,optional,fromdefaults"`
-	Serve      *GlobalServe           `yaml:"serve,optional,fromdefaults"`
-}
-
-func Default(i interface{}) {
-	v := reflect.ValueOf(i)
-	if v.Kind() != reflect.Ptr {
-		panic(v)
-	}
-	y := `{}`
-	err := yaml.Unmarshal([]byte(y), v.Interface())
-	if err != nil {
-		panic(err)
-	}
-}
-
-type ConnectEnum struct {
-	Ret interface{}
-}
-
-type ConnectCommon struct {
-	Type string `yaml:"type"`
-}
-
-type TCPConnect struct {
-	ConnectCommon `yaml:",inline"`
-	Address       string        `yaml:"address,hostport"`
-	DialTimeout   time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
-}
-
-type TLSConnect struct {
-	ConnectCommon `yaml:",inline"`
-	Address       string        `yaml:"address,hostport"`
-	Ca            string        `yaml:"ca"`
-	Cert          string        `yaml:"cert"`
-	Key           string        `yaml:"key"`
-	ServerCN      string        `yaml:"server_cn"`
-	DialTimeout   time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
-}
-
-type SSHStdinserverConnect struct {
-	ConnectCommon        `yaml:",inline"`
-	Host                 string        `yaml:"host"`
-	User                 string        `yaml:"user"`
-	Port                 uint16        `yaml:"port"`
-	IdentityFile         string        `yaml:"identity_file"`
-	TransportOpenCommand []string      `yaml:"transport_open_command,optional"` //TODO unused
-	SSHCommand           string        `yaml:"ssh_command,optional"`            //TODO unused
-	Options              []string      `yaml:"options,optional"`
-	DialTimeout          time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
-}
-
-type LocalConnect struct {
-	ConnectCommon  `yaml:",inline"`
-	ListenerName   string        `yaml:"listener_name"`
-	ClientIdentity string        `yaml:"client_identity"`
-	DialTimeout    time.Duration `yaml:"dial_timeout,zeropositive,default=2s"`
-}
-
-type ServeEnum struct {
-	Ret interface{}
-}
-
-type ServeCommon struct {
-	Type string `yaml:"type"`
-}
-
-type TCPServe struct {
-	ServeCommon    `yaml:",inline"`
-	Listen         string            `yaml:"listen,hostport"`
-	ListenFreeBind bool              `yaml:"listen_freebind,default=false"`
-	Clients        map[string]string `yaml:"clients"`
-}
-
-type TLSServe struct {
-	ServeCommon      `yaml:",inline"`
-	Listen           string        `yaml:"listen,hostport"`
-	ListenFreeBind   bool          `yaml:"listen_freebind,default=false"`
-	Ca               string        `yaml:"ca"`
-	Cert             string        `yaml:"cert"`
-	Key              string        `yaml:"key"`
-	ClientCNs        []string      `yaml:"client_cns"`
-	HandshakeTimeout time.Duration `yaml:"handshake_timeout,zeropositive,default=10s"`
-}
-
-type StdinserverServer struct {
-	ServeCommon      `yaml:",inline"`
-	ClientIdentities []string `yaml:"client_identities"`
-}
-
-type LocalServe struct {
-	ServeCommon  `yaml:",inline"`
-	ListenerName string `yaml:"listener_name"`
-}
-
-type PruningEnum struct {
-	Ret interface{}
-}
-
-type PruneKeepNotReplicated struct {
-	Type                 string `yaml:"type"`
-	KeepSnapshotAtCursor bool   `yaml:"keep_snapshot_at_cursor,optional,default=true"`
-}
-
-type PruneKeepLastN struct {
-	Type  string `yaml:"type"`
-	Count int    `yaml:"count"`
-}
-
-type PruneKeepRegex struct { // FIXME rename to KeepRegex
-	Type   string `yaml:"type"`
-	Regex  string `yaml:"regex"`
-	Negate bool   `yaml:"negate,optional,default=false"`
-}
-
-type LoggingOutletEnum struct {
-	Ret interface{}
-}
-
-type LoggingOutletCommon struct {
-	Type   string `yaml:"type"`
-	Level  string `yaml:"level"`
-	Format string `yaml:"format"`
-}
-
-type StdoutLoggingOutlet struct {
-	LoggingOutletCommon `yaml:",inline"`
-	Time                bool `yaml:"time,default=true"`
-	Color               bool `yaml:"color,default=true"`
-}
-
-type SyslogLoggingOutlet struct {
-	LoggingOutletCommon `yaml:",inline"`
-	Facility            *SyslogFacility `yaml:"facility,optional,fromdefaults"`
-	RetryInterval       time.Duration   `yaml:"retry_interval,positive,default=10s"`
-}
-
-type TCPLoggingOutlet struct {
-	LoggingOutletCommon `yaml:",inline"`
-	Address             string               `yaml:"address,hostport"`
-	Net                 string               `yaml:"net,default=tcp"`
-	RetryInterval       time.Duration        `yaml:"retry_interval,positive,default=10s"`
-	TLS                 *TCPLoggingOutletTLS `yaml:"tls,optional"`
-}
-
-type TCPLoggingOutletTLS struct {
-	CA   string `yaml:"ca"`
-	Cert string `yaml:"cert"`
-	Key  string `yaml:"key"`
-}
-
-type MonitoringEnum struct {
-	Ret interface{}
-}
-
-type PrometheusMonitoring struct {
-	Type           string `yaml:"type"`
-	Listen         string `yaml:"listen,hostport"`
-	ListenFreeBind bool   `yaml:"listen_freebind,default=false"`
-}
-
-type SyslogFacility syslog.Priority
-
-func (f *SyslogFacility) SetDefault() {
-	*f = SyslogFacility(syslog.LOG_LOCAL0)
-}
-
-var _ yaml.Defaulter = (*SyslogFacility)(nil)
-
-type GlobalControl struct {
-	SockPath string `yaml:"sockpath,default=/var/run/zrepl/control"`
-}
-
-type GlobalServe struct {
-	StdinServer *GlobalStdinServer `yaml:"stdinserver,optional,fromdefaults"`
-}
-
-type GlobalStdinServer struct {
-	SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
-}
-
-type JobDebugSettings struct {
-	Conn *struct {
-		ReadDump  string `yaml:"read_dump"`
-		WriteDump string `yaml:"write_dump"`
-	} `yaml:"conn,optional"`
-	RPCLog bool `yaml:"rpc_log,optional,default=false"`
-}
-
-type HookList []HookEnum
-
-type HookEnum struct {
-	Ret interface{}
-}
-
-type HookCommand struct {
-	Path               string            `yaml:"path"`
-	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
-	Filesystems        FilesystemsFilter `yaml:"filesystems,optional,default={'<': true}"`
-	HookSettingsCommon `yaml:",inline"`
-}
-
-type HookPostgresCheckpoint struct {
-	HookSettingsCommon `yaml:",inline"`
-	DSN                string            `yaml:"dsn"`
-	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
-	Filesystems        FilesystemsFilter `yaml:"filesystems"` // required, user should not CHECKPOINT for every FS
-}
-
-type HookMySQLLockTables struct {
-	HookSettingsCommon `yaml:",inline"`
-	DSN                string            `yaml:"dsn"`
-	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
-	Filesystems        FilesystemsFilter `yaml:"filesystems"`
-}
-
-type HookSettingsCommon struct {
-	Type       string `yaml:"type"`
-	ErrIsFatal bool   `yaml:"err_is_fatal,optional,default=false"`
-}
-
-func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) {
-	var in struct {
-		Type string
-	}
-	if err := u(&in, true); err != nil {
-		return nil, err
-	}
-	if in.Type == "" {
-		return nil, &yaml.TypeError{Errors: []string{"must specify type"}}
-	}
-
-	v, ok := types[in.Type]
-	if !ok {
-		return nil, &yaml.TypeError{Errors: []string{fmt.Sprintf("invalid type name %q", in.Type)}}
-	}
-	if err := u(v, false); err != nil {
-		return nil, err
-	}
-	return v, nil
-}
-
-func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"snap":   &SnapJob{},
-		"push":   &PushJob{},
-		"sink":   &SinkJob{},
-		"pull":   &PullJob{},
-		"source": &SourceJob{},
-	})
-	return
-}
-
-func (t *ConnectEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"tcp":             &TCPConnect{},
-		"tls":             &TLSConnect{},
-		"ssh+stdinserver": &SSHStdinserverConnect{},
-		"local":           &LocalConnect{},
-	})
-	return
-}
-
-func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"tcp":         &TCPServe{},
-		"tls":         &TLSServe{},
-		"stdinserver": &StdinserverServer{},
-		"local":       &LocalServe{},
-	})
-	return
-}
-
-func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"not_replicated": &PruneKeepNotReplicated{},
-		"last_n":         &PruneKeepLastN{},
-		"grid":           &PruneGrid{},
-		"regex":          &PruneKeepRegex{},
-	})
-	return
-}
-
-func (t *SnapshottingEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"periodic": &SnapshottingPeriodic{},
-		"manual":   &SnapshottingManual{},
-	})
-	return
-}
-
-func (t *LoggingOutletEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"stdout": &StdoutLoggingOutlet{},
-		"syslog": &SyslogLoggingOutlet{},
-		"tcp":    &TCPLoggingOutlet{},
-	})
-	return
-}
-
-func (t *MonitoringEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"prometheus": &PrometheusMonitoring{},
-	})
-	return
-}
-
-func (t *SyslogFacility) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	var s string
-	if err := u(&s, true); err != nil {
-		return err
-	}
-	var level syslog.Priority
-	switch s {
-	case "kern":
-		level = syslog.LOG_KERN
-	case "user":
-		level = syslog.LOG_USER
-	case "mail":
-		level = syslog.LOG_MAIL
-	case "daemon":
-		level = syslog.LOG_DAEMON
-	case "auth":
-		level = syslog.LOG_AUTH
-	case "syslog":
-		level = syslog.LOG_SYSLOG
-	case "lpr":
-		level = syslog.LOG_LPR
-	case "news":
-		level = syslog.LOG_NEWS
-	case "uucp":
-		level = syslog.LOG_UUCP
-	case "cron":
-		level = syslog.LOG_CRON
-	case "authpriv":
-		level = syslog.LOG_AUTHPRIV
-	case "ftp":
-		level = syslog.LOG_FTP
-	case "local0":
-		level = syslog.LOG_LOCAL0
-	case "local1":
-		level = syslog.LOG_LOCAL1
-	case "local2":
-		level = syslog.LOG_LOCAL2
-	case "local3":
-		level = syslog.LOG_LOCAL3
-	case "local4":
-		level = syslog.LOG_LOCAL4
-	case "local5":
-		level = syslog.LOG_LOCAL5
-	case "local6":
-		level = syslog.LOG_LOCAL6
-	case "local7":
-		level = syslog.LOG_LOCAL7
-	default:
-		return fmt.Errorf("invalid syslog level: %q", s)
-	}
-	*t = SyslogFacility(level)
-	return nil
-}
-
-func (t *HookEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"command":             &HookCommand{},
-		"postgres-checkpoint": &HookPostgresCheckpoint{},
-		"mysql-lock-tables":   &HookMySQLLockTables{},
-	})
-	return
-}
-
-var ConfigFileDefaultLocations = []string{
-	"/etc/zrepl/zrepl.yml",
-	"/usr/local/etc/zrepl/zrepl.yml",
-}
-
-func ParseConfig(path string) (i *Config, err error) {
-
-	if path == "" {
-		// Try default locations
-		for _, l := range ConfigFileDefaultLocations {
-			stat, statErr := os.Stat(l)
-			if statErr != nil {
-				continue
-			}
-			if !stat.Mode().IsRegular() {
-				err = errors.Errorf("file at default location is not a regular file: %s", l)
-				return
-			}
-			path = l
-			break
-		}
-	}
-
-	var bytes []byte
-
-	if bytes, err = ioutil.ReadFile(path); err != nil {
-		return
-	}
-
-	return ParseConfigBytes(bytes)
-}
-
-func ParseConfigBytes(bytes []byte) (*Config, error) {
-	var c *Config
-	if err := yaml.UnmarshalStrict(bytes, &c); err != nil {
-		return nil, err
-	}
-	if c == nil {
-		return nil, fmt.Errorf("config is empty or only consists of comments")
-	}
-	return c, nil
-}
-
-var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
-
-func parsePostitiveDuration(e string) (d time.Duration, err error) {
-	comps := durationStringRegex.FindStringSubmatch(e)
-	if len(comps) != 3 {
-		err = fmt.Errorf("does not match regex: %s %#v", e, comps)
-		return
-	}
-
-	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
-	if err != nil {
-		return 0, err
-	}
-	if durationFactor <= 0 {
-		return 0, errors.New("duration must be positive integer")
-	}
-
-	var durationUnit time.Duration
-	switch comps[2] {
-	case "s":
-		durationUnit = time.Second
-	case "m":
-		durationUnit = time.Minute
-	case "h":
-		durationUnit = time.Hour
-	case "d":
-		durationUnit = 24 * time.Hour
-	case "w":
-		durationUnit = 24 * 7 * time.Hour
-	default:
-		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
-		return
-	}
-
-	d = time.Duration(durationFactor) * durationUnit
-	return
-}

config/config_global_test.go

@@ -1,115 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"log/syslog"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"github.com/zrepl/yaml-config"
-)
-
-func testValidGlobalSection(t *testing.T, s string) *Config {
-	jobdef := `
-jobs:
-- name: dummyjob
-  type: sink
-  serve:
-    type: tcp
-    listen: ":2342"
-    clients: {
-      "10.0.0.1":"foo"
-    }
-  root_fs: zoot/foo
-`
-	_, err := ParseConfigBytes([]byte(jobdef))
-	require.NoError(t, err)
-	return testValidConfig(t, s+jobdef)
-}
-
-func TestOutletTypes(t *testing.T) {
-	conf := testValidGlobalSection(t, `
-global:
-  logging:
-  - type: stdout
-    level: debug
-    format: human
-  - type: syslog
-    level: info
-    retry_interval: 20s
-    format: human
-  - type: tcp
-    level: debug
-    format: json
-    address: logserver.example.com:1234
-  - type: tcp
-    level: debug
-    format: json
-    address: encryptedlogserver.example.com:1234
-    retry_interval: 20s 
-    tls:
-      ca: /etc/zrepl/log/ca.crt
-      cert: /etc/zrepl/log/key.pem
-      key: /etc/zrepl/log/cert.pem
-`)
-	assert.Equal(t, 4, len(*conf.Global.Logging))
-	assert.NotNil(t, (*conf.Global.Logging)[3].Ret.(*TCPLoggingOutlet).TLS)
-}
-
-func TestDefaultLoggingOutlet(t *testing.T) {
-	conf := testValidGlobalSection(t, "")
-	assert.Equal(t, 1, len(*conf.Global.Logging))
-	o := (*conf.Global.Logging)[0].Ret.(*StdoutLoggingOutlet)
-	assert.Equal(t, "warn", o.Level)
-	assert.Equal(t, "human", o.Format)
-}
-
-func TestPrometheusMonitoring(t *testing.T) {
-	conf := testValidGlobalSection(t, `
-global:
-  monitoring:
-    - type: prometheus
-      listen: ':9091'
-`)
-	assert.Equal(t, ":9091", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
-}
-
-func TestSyslogLoggingOutletFacility(t *testing.T) {
-	type SyslogFacilityPriority struct {
-		Facility string
-		Priority syslog.Priority
-	}
-	syslogFacilitiesPriorities := []SyslogFacilityPriority{
-		{"", syslog.LOG_LOCAL0}, // default
-		{"kern", syslog.LOG_KERN}, {"daemon", syslog.LOG_DAEMON}, {"auth", syslog.LOG_AUTH},
-		{"syslog", syslog.LOG_SYSLOG}, {"lpr", syslog.LOG_LPR}, {"news", syslog.LOG_NEWS},
-		{"uucp", syslog.LOG_UUCP}, {"cron", syslog.LOG_CRON}, {"authpriv", syslog.LOG_AUTHPRIV},
-		{"ftp", syslog.LOG_FTP}, {"local0", syslog.LOG_LOCAL0}, {"local1", syslog.LOG_LOCAL1},
-		{"local2", syslog.LOG_LOCAL2}, {"local3", syslog.LOG_LOCAL3}, {"local4", syslog.LOG_LOCAL4},
-		{"local5", syslog.LOG_LOCAL5}, {"local6", syslog.LOG_LOCAL6}, {"local7", syslog.LOG_LOCAL7},
-	}
-
-	for _, sFP := range syslogFacilitiesPriorities {
-		logcfg := fmt.Sprintf(`
-global:
-  logging:
-  - type: syslog
-    level: info
-    format: human
-    facility: %s
-`, sFP.Facility)
-		conf := testValidGlobalSection(t, logcfg)
-		assert.Equal(t, 1, len(*conf.Global.Logging))
-		assert.True(t, SyslogFacility(sFP.Priority) == *(*conf.Global.Logging)[0].Ret.(*SyslogLoggingOutlet).Facility)
-	}
-}
-
-func TestLoggingOutletEnumList_SetDefaults(t *testing.T) {
-	e := &LoggingOutletEnumList{}
-	var i yaml.Defaulter = e
-	require.NotPanics(t, func() {
-		i.SetDefault()
-		assert.Equal(t, "warn", (*e)[0].Ret.(*StdoutLoggingOutlet).Level)
-	})
-}

config/config_jobid_test.go

@@ -1 +0,0 @@
-package config

config/config_minimal_test.go

@@ -1,40 +0,0 @@
-package config
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestConfigEmptyFails(t *testing.T) {
-	conf, err := testConfig(t, "\n")
-	assert.Nil(t, conf)
-	assert.Error(t, err)
-}
-
-func TestJobsOnlyWorks(t *testing.T) {
-	testValidConfig(t, `
-jobs:
-- name: push
-  type: push
-  # snapshot the filesystems matched by the left-hand-side of the mapping
-  # every 10m with zrepl_ as prefix
-  connect:
-    type: tcp
-    address: localhost:2342
-  filesystems: {
-    "pool1/var/db<": true,
-    "pool1/usr/home<": true,
-    "pool1/usr/home/paranoid": false, #don't backup paranoid user
-    "pool1/poudriere/ports<": false #don't backup the ports trees
-  }
-  snapshotting:
-    type: manual
-  pruning:
-    keep_sender:
-    - type: not_replicated
-    keep_receiver:
-    - type: last_n 
-      count: 1
-`)
-}

config/config_positiveintervalormanual_test.go

@@ -1,41 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/zrepl/yaml-config"
-)
-
-func TestPositiveDurationOrManual(t *testing.T) {
-	cases := []struct {
-		Comment, Input string
-		Result         *PositiveDurationOrManual
-	}{
-		{"empty is error", "", nil},
-		{"negative is error", "-1s", nil},
-		{"zero seconds is error", "0s", nil},
-		{"zero is error", "0", nil},
-		{"non-manual is error", "something", nil},
-		{"positive seconds works", "1s", &PositiveDurationOrManual{Manual: false, Interval: 1 * time.Second}},
-		{"manual works", "manual", &PositiveDurationOrManual{Manual: true, Interval: 0}},
-	}
-	for _, tc := range cases {
-		t.Run(tc.Comment, func(t *testing.T) {
-			var out struct {
-				FieldName PositiveDurationOrManual `yaml:"fieldname"`
-			}
-			input := fmt.Sprintf("\nfieldname: %s\n", tc.Input)
-			err := yaml.UnmarshalStrict([]byte(input), &out)
-			if tc.Result == nil {
-				assert.Error(t, err)
-				t.Logf("%#v", out)
-			} else {
-				assert.Equal(t, *tc.Result, out.FieldName)
-			}
-		})
-	}
-
-}

config/config_send_test.go

@@ -1,74 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestSendOptions(t *testing.T) {
-	tmpl := `
-jobs:
-- name: foo
-  type: push
-  connect:
-    type: local
-    listener_name: foo
-    client_identity: bar
-  filesystems: {"<": true}
-  %s
-  snapshotting:
-    type: manual
-  pruning:
-    keep_sender:
-    - type: last_n
-      count: 10
-    keep_receiver:
-    - type: last_n
-      count: 10
-`
-	encrypted_false := `
-  send:
-    encrypted: false
-`
-
-	encrypted_true := `
-  send:
-    encrypted: true
-`
-
-	encrypted_unspecified := `
-  send: {}
-`
-
-	send_not_specified := `
-`
-
-	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
-	var c *Config
-
-	t.Run("encrypted_false", func(t *testing.T) {
-		c = testValidConfig(t, fill(encrypted_false))
-		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
-		assert.Equal(t, false, encrypted)
-	})
-	t.Run("encrypted_true", func(t *testing.T) {
-		c = testValidConfig(t, fill(encrypted_true))
-		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
-		assert.Equal(t, true, encrypted)
-	})
-
-	t.Run("encrypted_unspecified", func(t *testing.T) {
-		c, err := testConfig(t, fill(encrypted_unspecified))
-		assert.Error(t, err)
-		assert.Nil(t, c)
-	})
-
-	t.Run("send_not_specified", func(t *testing.T) {
-		c, err := testConfig(t, fill(send_not_specified))
-		assert.NoError(t, err)
-		assert.NotNil(t, c)
-	})
-
-}

config/config_snapshotting_test.go

@@ -1,90 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestSnapshotting(t *testing.T) {
-	tmpl := `
-jobs:
-- name: foo
-  type: push
-  connect:
-    type: local
-    listener_name: foo
-    client_identity: bar
-  filesystems: {"<": true}
-  %s
-  pruning:
-    keep_sender:
-    - type: last_n
-      count: 10
-    keep_receiver:
-    - type: last_n
-      count: 10
-`
-	manual := `
-  snapshotting:
-    type: manual
-`
-	periodic := `
-  snapshotting:
-    type: periodic
-    prefix: zrepl_
-    interval: 10m
-`
-
-	hooks := `
-  snapshotting:
-    type: periodic
-    prefix: zrepl_
-    interval: 10m
-    hooks:
-    - type: command
-      path: /tmp/path/to/command
-    - type: command
-      path: /tmp/path/to/command
-      filesystems: { "zroot<": true, "<": false }
-    - type: postgres-checkpoint
-      dsn: "host=localhost port=5432 user=postgres sslmode=disable"
-      filesystems: {
-          "tank/postgres/data11": true
-      }
-    - type: mysql-lock-tables
-      dsn: "root@tcp(localhost)/"
-      filesystems: {
-        "tank/mysql": true
-      }
-`
-
-	fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
-	var c *Config
-
-	t.Run("manual", func(t *testing.T) {
-		c = testValidConfig(t, fillSnapshotting(manual))
-		snm := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingManual)
-		assert.Equal(t, "manual", snm.Type)
-	})
-
-	t.Run("periodic", func(t *testing.T) {
-		c = testValidConfig(t, fillSnapshotting(periodic))
-		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
-		assert.Equal(t, "periodic", snp.Type)
-		assert.Equal(t, 10*time.Minute, snp.Interval)
-		assert.Equal(t, "zrepl_", snp.Prefix)
-	})
-
-	t.Run("hooks", func(t *testing.T) {
-		c = testValidConfig(t, fillSnapshotting(hooks))
-		hs := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic).Hooks
-		assert.Equal(t, hs[0].Ret.(*HookCommand).Filesystems["<"], true)
-		assert.Equal(t, hs[1].Ret.(*HookCommand).Filesystems["zroot<"], true)
-		assert.Equal(t, hs[2].Ret.(*HookPostgresCheckpoint).Filesystems["tank/postgres/data11"], true)
-		assert.Equal(t, hs[3].Ret.(*HookMySQLLockTables).Filesystems["tank/mysql"], true)
-	})
-
-}

config/config_test.go

@@ -1,88 +0,0 @@
-package config
-
-import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"path"
-	"path/filepath"
-	"strings"
-	"testing"
-	"text/template"
-
-	"github.com/kr/pretty"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
-	paths, err := filepath.Glob("./samples/*")
-	if err != nil {
-		t.Errorf("glob failed: %+v", err)
-	}
-
-	for _, p := range paths {
-
-		if path.Ext(p) != ".yml" {
-			t.Logf("skipping file %s", p)
-			continue
-		}
-
-		t.Run(p, func(t *testing.T) {
-			c, err := ParseConfig(p)
-			if err != nil {
-				t.Errorf("error parsing %s:\n%+v", p, err)
-			}
-
-			t.Logf("file: %s", p)
-			t.Log(pretty.Sprint(c))
-		})
-
-	}
-
-}
-
-// template must be a template/text template with a single '{{ . }}' as placehodler for val
-//nolint[:deadcode,unused]
-func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
-	tmp, err := template.New("master").Parse(tmpl)
-	if err != nil {
-		panic(err)
-	}
-	var buf bytes.Buffer
-	err = tmp.Execute(&buf, val)
-	if err != nil {
-		panic(err)
-	}
-	return testValidConfig(t, buf.String())
-}
-
-func testValidConfig(t *testing.T, input string) *Config {
-	t.Helper()
-	conf, err := testConfig(t, input)
-	require.NoError(t, err)
-	require.NotNil(t, conf)
-	return conf
-}
-
-func testConfig(t *testing.T, input string) (*Config, error) {
-	t.Helper()
-	return ParseConfigBytes([]byte(input))
-}
-
-func trimSpaceEachLineAndPad(s, pad string) string {
-	var out strings.Builder
-	scan := bufio.NewScanner(strings.NewReader(s))
-	for scan.Scan() {
-		fmt.Fprintf(&out, "%s%s\n", pad, bytes.TrimSpace(scan.Bytes()))
-	}
-	return out.String()
-}
-
-func TestTrimSpaceEachLineAndPad(t *testing.T) {
-	foo := `
-	foo
-	bar baz 
-	`
-	assert.Equal(t, "  \n  foo\n  bar baz\n  \n", trimSpaceEachLineAndPad(foo, "  "))
-}

config/config_transport_test.go

@@ -1,99 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestTransportConnect(t *testing.T) {
-	tmpl := `
-jobs:
-- name: foo
-  type: push
-  connect:
-%s
-  filesystems: {"<": true}
-  snapshotting:
-    type: manual
-  pruning:
-    keep_sender:
-    - type: last_n
-      count: 10
-    keep_receiver:
-    - type: last_n
-      count: 10
-`
-
-	mconf := func(s string) string { return fmt.Sprintf(tmpl, s) }
-
-	type test struct {
-		Name        string
-		ExpectError bool
-		Connect     string
-	}
-
-	testTable := []test{
-		{
-			Name:        "tcp_with_address_and_port",
-			ExpectError: false,
-			Connect: `
-			type: tcp
-			address: 10.0.0.23:42
-			`,
-		},
-		{
-			Name:        "tls_with_host_and_port",
-			ExpectError: false,
-			Connect: `
-			type: tls
-			address: "server1.foo.bar:8888"
-			ca:   /etc/zrepl/ca.crt
-			cert: /etc/zrepl/backupserver.fullchain
-			key:  /etc/zrepl/backupserver.key
-			server_cn: "server1"
-			`,
-		},
-		{
-			Name:        "tcp_without_port",
-			ExpectError: true,
-			Connect: `
-			type: tcp
-			address: 10.0.0.23
-			`,
-		},
-		{
-			Name:        "tls_without_port",
-			ExpectError: true,
-			Connect: `
-			type: tls
-			address: 10.0.0.23
-			ca:   /etc/zrepl/ca.crt
-			cert: /etc/zrepl/backupserver.fullchain
-			key:  /etc/zrepl/backupserver.key
-			server_cn: "server1"
-			`,
-		},
-	}
-
-	for _, tc := range testTable {
-		t.Run(tc.Name, func(t *testing.T) {
-			require.NotEmpty(t, tc.Connect)
-			connect := trimSpaceEachLineAndPad(tc.Connect, "    ")
-			conf := mconf(connect)
-			config, err := testConfig(t, conf)
-			if tc.ExpectError && err == nil {
-				t.Errorf("expected test failure, but got valid config %v", config)
-				return
-			}
-			if !tc.ExpectError && err != nil {
-				t.Errorf("not expecint test failure but got error: %s", err)
-				return
-			}
-			t.Logf("error=%v", err)
-			t.Logf("config=%v", config)
-		})
-	}
-
-}

config/retentiongrid.go

@@ -1,119 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"regexp"
-	"strconv"
-	"strings"
-	"time"
-)
-
-type RetentionIntervalList []RetentionInterval
-
-type PruneGrid struct {
-	Type  string                `yaml:"type"`
-	Grid  RetentionIntervalList `yaml:"grid"`
-	Regex string                `yaml:"regex"`
-}
-
-type RetentionInterval struct {
-	length    time.Duration
-	keepCount int
-}
-
-func (i *RetentionInterval) Length() time.Duration {
-	return i.length
-}
-
-func (i *RetentionInterval) KeepCount() int {
-	return i.keepCount
-}
-
-const RetentionGridKeepCountAll int = -1
-
-func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	var in string
-	if err := u(&in, true); err != nil {
-		return err
-	}
-
-	intervals, err := parseRetentionGridIntervalsString(in)
-	if err != nil {
-		return err
-	}
-
-	*t = intervals
-
-	return nil
-}
-
-var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
-
-func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {
-
-	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
-	if comps == nil {
-		err = fmt.Errorf("retention string does not match expected format")
-		return
-	}
-
-	times, err := strconv.Atoi(comps[1])
-	if err != nil {
-		return nil, err
-	} else if times <= 0 {
-		return nil, fmt.Errorf("contains factor <= 0")
-	}
-
-	duration, err := parsePostitiveDuration(comps[2])
-	if err != nil {
-		return nil, err
-	}
-
-	keepCount := 1
-	if comps[3] != "" {
-		// Decompose key=value, comma separated
-		// For now, only keep_count is supported
-		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
-		res := re.FindStringSubmatch(comps[4])
-		if res == nil || len(res) != 2 {
-			err = fmt.Errorf("interval parameter contains unknown parameters")
-			return
-		}
-		if res[1] == "all" {
-			keepCount = RetentionGridKeepCountAll
-		} else {
-			keepCount, err = strconv.Atoi(res[1])
-			if err != nil {
-				err = fmt.Errorf("cannot parse keep_count value")
-				return
-			}
-		}
-	}
-
-	intervals = make([]RetentionInterval, times)
-	for i := range intervals {
-		intervals[i] = RetentionInterval{
-			length:    duration,
-			keepCount: keepCount,
-		}
-	}
-
-	return
-
-}
-
-func parseRetentionGridIntervalsString(s string) (intervals []RetentionInterval, err error) {
-
-	ges := strings.Split(s, "|")
-	intervals = make([]RetentionInterval, 0, 7*len(ges))
-
-	for intervalIdx, e := range ges {
-		parsed, err := parseRetentionGridIntervalString(e)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
-		}
-		intervals = append(intervals, parsed...)
-	}
-
-	return
-}

config/samples/hooks/template.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-[ "$ZREPL_DRYRUN" = "true" ] && DRYRUN="echo DRYRUN: "
-
-pre_snapshot() {
-    $DRYRUN date
-}
-
-post_snapshot() {
-    $DRYRUN date
-}
-
-case "$ZREPL_HOOKTYPE" in
-    pre_snapshot|post_snapshot)
-        "$ZREPL_HOOKTYPE"
-        ;;
-    *)
-        printf 'Unrecognized hook type: %s\n' "$ZREPL_HOOKTYPE"
-        exit 255
-        ;;
-esac

config/samples/local.yml

@@ -1,31 +0,0 @@
-
-jobs:
-  - type: sink
-    name: "local_sink"
-    root_fs: "storage/zrepl/sink"
-    serve:
-      type: local
-      listener_name: localsink
-
-  - type: push
-    name: "backup_system"
-    connect:
-      type: local
-      listener_name: localsink
-      client_identity: local_backup
-    filesystems: {
-      "system<": true,
-    }
-    snapshotting:
-      type: periodic
-      interval: 10m
-      prefix: zrepl_
-    pruning:
-      keep_sender:
-      - type: not_replicated
-      - type: last_n
-        count: 10
-      keep_receiver:
-      - type: grid
-        grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-        regex: "zrepl_.*"

config/samples/pull.yml

@@ -1,24 +0,0 @@
-jobs:
-- name: pull_servers
-  type: pull
-  connect:
-    type: tls
-    address: "server1.foo.bar:8888"
-    ca: "/certs/ca.crt"
-    cert: "/certs/cert.crt"
-    key: "/certs/key.pem"
-    server_cn: "server1"
-  root_fs: "pool2/backup_servers"
-  interval: 10m
-  pruning:
-    keep_sender:
-    - type: not_replicated
-    - type: last_n
-      count: 10
-    - type: grid
-      grid: 1x1h(keep=all) | 24x1h | 14x1d
-      regex: "zrepl_.*"
-    keep_receiver:
-    - type: grid
-      grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-      regex: "zrepl_.*"

config/samples/pull_ssh.yml

@@ -1,28 +0,0 @@
-jobs:
-
-- name: pull_servers
-  type: pull
-  connect:
-    type: ssh+stdinserver
-    host: app-srv.example.com
-    user: root
-    port: 22
-    identity_file: /etc/zrepl/ssh/identity
-    options: # optional, default [], `-o` arguments passed to ssh
-    - "Compression=yes"
-  root_fs: "pool2/backup_servers"
-  interval: 10m
-  pruning:
-    keep_sender:
-    - type: not_replicated
-    - type: last_n
-      count: 10
-    - type: grid
-      grid: 1x1h(keep=all) | 24x1h | 14x1d
-      regex: "^zrepl_.*"
-    keep_receiver:
-    - type: regex
-      regex: keep_
-    - type: grid
-      grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-      regex: "^zrepl_.*"

config/samples/push.yml

@@ -1,26 +0,0 @@
-jobs:
-  - type: push
-    name: "push"
-    filesystems: {
-      "<": true,
-      "tmp": false
-    }
-    connect:
-      type: tcp
-      address: "backup-server.foo.bar:8888"
-    snapshotting:
-      type: manual
-    send:
-      encrypted: false
-    pruning:
-      keep_sender:
-        - type: not_replicated
-        - type: last_n
-          count: 10
-        - type: grid
-          grid: 1x1h(keep=all) | 24x1h | 14x1d
-          regex: "^zrepl_.*"
-      keep_receiver:
-        - type: grid
-          grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-          regex: "^zrepl_.*"

config/samples/sink.yml

@@ -1,13 +0,0 @@
-jobs:
-  - type: sink
-    name: "laptop_sink"
-    root_fs: "pool2/backup_laptops"
-    serve:
-      type: tls
-      listen: "192.168.122.189:8888"
-      ca: "ca.pem"
-      cert: "cert.pem"
-      key: "key.pem"
-      client_cns:
-        - "laptop1"
-        - "homeserver"

config/samples/snap.yml

@@ -1,14 +0,0 @@
-jobs:
-- name: snapjob
-  type: snap
-  filesystems: {
-    "tank/frequently_changed<": true,
-  }
-  snapshotting:
-    type: periodic
-    interval: 2m
-    prefix: zrepl_snapjob_
-  pruning:
-    keep:
-      - type: last_n
-        count: 60

config/samples/source.yml

@@ -1,17 +0,0 @@
-jobs:
-- name: pull_source
-  type: source
-  serve:
-    type: tcp
-    listen: "0.0.0.0:8888"
-    clients: {
-      "192.168.122.123" : "client1"
-    }
-  filesystems: {
-    "<": true,
-    "secret": false
-  }
-  snapshotting:
-    type: periodic
-    interval: 10m
-    prefix: zrepl_

config/samples/source_ssh.yml

@@ -1,17 +0,0 @@
-jobs:
-- name: pull_source
-  type: source
-  serve:
-    type: stdinserver
-    client_identities:
-      - "client1"
-      - "client2"
-  filesystems: {
-    "<": true,
-    "secret": false
-  }
-  snapshotting:
-    type: periodic
-    interval: 10m
-    prefix: zrepl_
-

daemon/control.go

@@ -1,282 +0,0 @@
-package daemon
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net"
-	"net/http"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/daemon/nethelpers"
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/util/envconst"
-	"github.com/zrepl/zrepl/version"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type controlJob struct {
-	sockaddr *net.UnixAddr
-	jobs     *jobs
-}
-
-func newControlJob(sockpath string, jobs *jobs) (j *controlJob, err error) {
-	j = &controlJob{jobs: jobs}
-
-	j.sockaddr, err = net.ResolveUnixAddr("unix", sockpath)
-	if err != nil {
-		err = errors.Wrap(err, "cannot resolve unix address")
-		return
-	}
-
-	return
-}
-
-func (j *controlJob) Name() string { return jobNameControl }
-
-func (j *controlJob) Status() *job.Status { return &job.Status{Type: job.TypeInternal} }
-
-func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
-
-func (j *controlJob) SenderConfig() *endpoint.SenderConfig { return nil }
-
-var promControl struct {
-	requestBegin    *prometheus.CounterVec
-	requestFinished *prometheus.HistogramVec
-}
-
-func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) {
-	promControl.requestBegin = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "zrepl",
-		Subsystem: "control",
-		Name:      "request_begin",
-		Help:      "number of request we started to handle",
-	}, []string{"endpoint"})
-
-	promControl.requestFinished = prometheus.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace: "zrepl",
-		Subsystem: "control",
-		Name:      "request_finished",
-		Help:      "time it took a request to finih",
-		Buckets:   []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3, 10e-3, 100e-3, 200e-3, 400e-3, 800e-3, 1, 10, 20},
-	}, []string{"endpoint"})
-	registerer.MustRegister(promControl.requestBegin)
-	registerer.MustRegister(promControl.requestFinished)
-}
-
-const (
-	ControlJobEndpointPProf   string = "/debug/pprof"
-	ControlJobEndpointVersion string = "/version"
-	ControlJobEndpointStatus  string = "/status"
-	ControlJobEndpointSignal  string = "/signal"
-)
-
-func (j *controlJob) Run(ctx context.Context) {
-
-	log := job.GetLogger(ctx)
-	defer log.Info("control job finished")
-
-	l, err := nethelpers.ListenUnixPrivate(j.sockaddr)
-	if err != nil {
-		log.WithError(err).Error("error listening")
-		return
-	}
-
-	pprofServer := NewPProfServer(ctx)
-	if listen := envconst.String("ZREPL_DAEMON_AUTOSTART_PPROF_SERVER", ""); listen != "" {
-		pprofServer.Control(PprofServerControlMsg{
-			Run:               true,
-			HttpListenAddress: listen,
-		})
-	}
-
-	mux := http.NewServeMux()
-	mux.Handle(ControlJobEndpointPProf,
-		requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) {
-			var msg PprofServerControlMsg
-			err := decoder(&msg)
-			if err != nil {
-				return nil, errors.Errorf("decode failed")
-			}
-			pprofServer.Control(msg)
-			return struct{}{}, nil
-		}}})
-
-	mux.Handle(ControlJobEndpointVersion,
-		requestLogger{log: log, handler: jsonResponder{log, func() (interface{}, error) {
-			return version.NewZreplVersionInformation(), nil
-		}}})
-
-	mux.Handle(ControlJobEndpointStatus,
-		// don't log requests to status endpoint, too spammy
-		jsonResponder{log, func() (interface{}, error) {
-			s := j.jobs.status()
-			return s, nil
-		}})
-
-	mux.Handle(ControlJobEndpointSignal,
-		requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) {
-			type reqT struct {
-				Name string
-				Op   string
-			}
-			var req reqT
-			if decoder(&req) != nil {
-				return nil, errors.Errorf("decode failed")
-			}
-
-			var err error
-			switch req.Op {
-			case "wakeup":
-				err = j.jobs.wakeup(req.Name)
-			case "reset":
-				err = j.jobs.reset(req.Name)
-			default:
-				err = fmt.Errorf("operation %q is invalid", req.Op)
-			}
-
-			return struct{}{}, err
-		}}})
-	server := http.Server{
-		Handler: mux,
-		// control socket is local, 1s timeout should be more than sufficient, even on a loaded system
-		WriteTimeout: 1 * time.Second,
-		ReadTimeout:  1 * time.Second,
-	}
-
-outer:
-	for {
-
-		served := make(chan error)
-		go func() {
-			served <- server.Serve(l)
-			close(served)
-		}()
-
-		select {
-		case <-ctx.Done():
-			log.WithError(ctx.Err()).Info("context done")
-			err := server.Shutdown(context.Background())
-			if err != nil {
-				log.WithError(err).Error("cannot shutdown server")
-			}
-			break outer
-		case err = <-served:
-			if err != nil {
-				log.WithError(err).Error("error serving")
-				break outer
-			}
-		}
-
-	}
-
-}
-
-type jsonResponder struct {
-	log      Logger
-	producer func() (interface{}, error)
-}
-
-func (j jsonResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	logIoErr := func(err error) {
-		if err != nil {
-			j.log.WithError(err).Error("control handler io error")
-		}
-	}
-	res, err := j.producer()
-	if err != nil {
-		j.log.WithError(err).Error("control handler error")
-		w.WriteHeader(http.StatusInternalServerError)
-		_, err = io.WriteString(w, err.Error())
-		logIoErr(err)
-		return
-	}
-	var buf bytes.Buffer
-	err = json.NewEncoder(&buf).Encode(res)
-	if err != nil {
-		j.log.WithError(err).Error("control handler json marshal error")
-		w.WriteHeader(http.StatusInternalServerError)
-		_, err = io.WriteString(w, err.Error())
-	} else {
-		_, err = io.Copy(w, &buf)
-	}
-	logIoErr(err)
-}
-
-type jsonDecoder = func(interface{}) error
-
-type jsonRequestResponder struct {
-	log      Logger
-	producer func(decoder jsonDecoder) (interface{}, error)
-}
-
-func (j jsonRequestResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	logIoErr := func(err error) {
-		if err != nil {
-			j.log.WithError(err).Error("control handler io error")
-		}
-	}
-
-	var decodeError error
-	decoder := func(i interface{}) error {
-		err := json.NewDecoder(r.Body).Decode(&i)
-		decodeError = err
-		return err
-	}
-	res, producerErr := j.producer(decoder)
-
-	//If we had a decode error ignore output of producer and return error
-	if decodeError != nil {
-		w.WriteHeader(http.StatusBadRequest)
-		_, err := io.WriteString(w, decodeError.Error())
-		logIoErr(err)
-		return
-	}
-	if producerErr != nil {
-		j.log.WithError(producerErr).Error("control handler error")
-		w.WriteHeader(http.StatusInternalServerError)
-		_, err := io.WriteString(w, producerErr.Error())
-		logIoErr(err)
-		return
-	}
-
-	var buf bytes.Buffer
-	encodeErr := json.NewEncoder(&buf).Encode(res)
-	if encodeErr != nil {
-		j.log.WithError(producerErr).Error("control handler json marhsal error")
-		w.WriteHeader(http.StatusInternalServerError)
-		_, err := io.WriteString(w, encodeErr.Error())
-		logIoErr(err)
-	} else {
-		_, err := io.Copy(w, &buf)
-		logIoErr(err)
-	}
-}
-
-type requestLogger struct {
-	log         logger.Logger
-	handler     http.Handler
-	handlerFunc http.HandlerFunc
-}
-
-func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	log := l.log.WithField("method", r.Method).WithField("url", r.URL)
-	log.Debug("start")
-	promControl.requestBegin.WithLabelValues(r.URL.Path).Inc()
-	defer prometheus.NewTimer(promControl.requestFinished.WithLabelValues(r.URL.Path)).ObserveDuration()
-	if l.handlerFunc != nil {
-		l.handlerFunc(w, r)
-	} else if l.handler != nil {
-		l.handler.ServeHTTP(w, r)
-	} else {
-		log.Error("no handler or handlerFunc configured")
-	}
-	log.Debug("finish")
-}

daemon/daemon.go

@@ -1,222 +0,0 @@
-package daemon
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"os/signal"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/daemon/job/reset"
-	"github.com/zrepl/zrepl/daemon/job/wakeup"
-	"github.com/zrepl/zrepl/daemon/logging"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/version"
-)
-
-func Run(conf *config.Config) error {
-
-	ctx, cancel := context.WithCancel(context.Background())
-
-	defer cancel()
-	sigChan := make(chan os.Signal, 1)
-	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
-	go func() {
-		<-sigChan
-		cancel()
-	}()
-
-	outlets, err := logging.OutletsFromConfig(*conf.Global.Logging)
-	if err != nil {
-		return errors.Wrap(err, "cannot build logging from config")
-	}
-
-	confJobs, err := job.JobsFromConfig(conf)
-	if err != nil {
-		return errors.Wrap(err, "cannot build jobs from config")
-	}
-
-	log := logger.NewLogger(outlets, 1*time.Second)
-	log.Info(version.NewZreplVersionInformation().String())
-
-	for _, job := range confJobs {
-		if IsInternalJobName(job.Name()) {
-			panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME
-		}
-	}
-
-	ctx = job.WithLogger(ctx, log)
-
-	jobs := newJobs()
-
-	// start control socket
-	controlJob, err := newControlJob(conf.Global.Control.SockPath, jobs)
-	if err != nil {
-		panic(err) // FIXME
-	}
-	jobs.start(ctx, controlJob, true)
-
-	for i, jc := range conf.Global.Monitoring {
-		var (
-			job job.Job
-			err error
-		)
-		switch v := jc.Ret.(type) {
-		case *config.PrometheusMonitoring:
-			job, err = newPrometheusJobFromConfig(v)
-		default:
-			return errors.Errorf("unknown monitoring job #%d (type %T)", i, v)
-		}
-		if err != nil {
-			return errors.Wrapf(err, "cannot build monitoring job #%d", i)
-		}
-		jobs.start(ctx, job, true)
-	}
-
-	log.Info("starting daemon")
-
-	// start regular jobs
-	for _, j := range confJobs {
-		jobs.start(ctx, j, false)
-	}
-
-	select {
-	case <-jobs.wait():
-		log.Info("all jobs finished")
-	case <-ctx.Done():
-		log.WithError(ctx.Err()).Info("context finished")
-	}
-	log.Info("daemon exiting")
-	return nil
-}
-
-type jobs struct {
-	wg sync.WaitGroup
-
-	// m protects all fields below it
-	m       sync.RWMutex
-	wakeups map[string]wakeup.Func // by Job.Name
-	resets  map[string]reset.Func  // by Job.Name
-	jobs    map[string]job.Job
-}
-
-func newJobs() *jobs {
-	return &jobs{
-		wakeups: make(map[string]wakeup.Func),
-		resets:  make(map[string]reset.Func),
-		jobs:    make(map[string]job.Job),
-	}
-}
-
-const (
-	logJobField string = "job"
-)
-
-func (s *jobs) wait() <-chan struct{} {
-	ch := make(chan struct{})
-	go func() {
-		s.wg.Wait()
-	}()
-	return ch
-}
-
-func (s *jobs) status() map[string]*job.Status {
-	s.m.RLock()
-	defer s.m.RUnlock()
-
-	type res struct {
-		name   string
-		status *job.Status
-	}
-	var wg sync.WaitGroup
-	c := make(chan res, len(s.jobs))
-	for name, j := range s.jobs {
-		wg.Add(1)
-		go func(name string, j job.Job) {
-			defer wg.Done()
-			c <- res{name: name, status: j.Status()}
-		}(name, j)
-	}
-	wg.Wait()
-	close(c)
-	ret := make(map[string]*job.Status, len(s.jobs))
-	for res := range c {
-		ret[res.name] = res.status
-	}
-	return ret
-}
-
-func (s *jobs) wakeup(job string) error {
-	s.m.RLock()
-	defer s.m.RUnlock()
-
-	wu, ok := s.wakeups[job]
-	if !ok {
-		return errors.Errorf("Job %s does not exist", job)
-	}
-	return wu()
-}
-
-func (s *jobs) reset(job string) error {
-	s.m.RLock()
-	defer s.m.RUnlock()
-
-	wu, ok := s.resets[job]
-	if !ok {
-		return errors.Errorf("Job %s does not exist", job)
-	}
-	return wu()
-}
-
-const (
-	jobNamePrometheus = "_prometheus"
-	jobNameControl    = "_control"
-)
-
-func IsInternalJobName(s string) bool {
-	return strings.HasPrefix(s, "_")
-}
-
-func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
-	s.m.Lock()
-	defer s.m.Unlock()
-
-	jobLog := job.GetLogger(ctx).
-		WithField(logJobField, j.Name()).
-		WithOutlet(newPrometheusLogOutlet(j.Name()), logger.Debug)
-	jobName := j.Name()
-	if !internal && IsInternalJobName(jobName) {
-		panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName))
-	}
-	if internal && !IsInternalJobName(jobName) {
-		panic(fmt.Sprintf("internal job does not use internal job name %s", jobName))
-	}
-	if _, ok := s.jobs[jobName]; ok {
-		panic(fmt.Sprintf("duplicate job name %s", jobName))
-	}
-
-	j.RegisterMetrics(prometheus.DefaultRegisterer)
-
-	s.jobs[jobName] = j
-	ctx = job.WithLogger(ctx, jobLog)
-	ctx, wakeup := wakeup.Context(ctx)
-	ctx, resetFunc := reset.Context(ctx)
-	s.wakeups[jobName] = wakeup
-	s.resets[jobName] = resetFunc
-
-	s.wg.Add(1)
-	go func() {
-		defer s.wg.Done()
-		jobLog.Info("starting job")
-		defer jobLog.Info("job exited")
-		j.Run(ctx)
-	}()
-}

daemon/filters/fsmapfilter.go

@@ -1,275 +0,0 @@
-package filters
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type DatasetMapFilter struct {
-	entries []datasetMapFilterEntry
-
-	// if set, only valid filter entries can be added using Add()
-	// and Map() will always return an error
-	filterMode bool
-}
-
-type datasetMapFilterEntry struct {
-	path *zfs.DatasetPath
-	// the mapping. since this datastructure acts as both mapping and filter
-	// we have to convert it to the desired rep dynamically
-	mapping      string
-	subtreeMatch bool
-}
-
-func NewDatasetMapFilter(capacity int, filterMode bool) *DatasetMapFilter {
-	return &DatasetMapFilter{
-		entries:    make([]datasetMapFilterEntry, 0, capacity),
-		filterMode: filterMode,
-	}
-}
-
-func (m *DatasetMapFilter) Add(pathPattern, mapping string) (err error) {
-
-	if m.filterMode {
-		if _, err = m.parseDatasetFilterResult(mapping); err != nil {
-			return
-		}
-	}
-
-	// assert path glob adheres to spec
-	const SUBTREE_PATTERN string = "<"
-	patternCount := strings.Count(pathPattern, SUBTREE_PATTERN)
-	switch {
-	case patternCount > 1:
-	case patternCount == 1 && !strings.HasSuffix(pathPattern, SUBTREE_PATTERN):
-		err = fmt.Errorf("pattern invalid: only one '<' at end of string allowed")
-		return
-	}
-
-	pathStr := strings.TrimSuffix(pathPattern, SUBTREE_PATTERN)
-	path, err := zfs.NewDatasetPath(pathStr)
-	if err != nil {
-		return fmt.Errorf("pattern is not a dataset path: %s", err)
-	}
-
-	entry := datasetMapFilterEntry{
-		path:         path,
-		mapping:      mapping,
-		subtreeMatch: patternCount > 0,
-	}
-	m.entries = append(m.entries, entry)
-	return
-
-}
-
-// find the most specific prefix mapping we have
-//
-// longer prefix wins over shorter prefix, direct wins over glob
-func (m DatasetMapFilter) mostSpecificPrefixMapping(path *zfs.DatasetPath) (idx int, found bool) {
-	lcp, lcp_entry_idx := -1, -1
-	direct_idx := -1
-	for e := range m.entries {
-		entry := m.entries[e]
-		ep := m.entries[e].path
-		lep := ep.Length()
-
-		switch {
-		case !entry.subtreeMatch && ep.Equal(path):
-			direct_idx = e
-			continue
-		case entry.subtreeMatch && path.HasPrefix(ep) && lep > lcp:
-			lcp = lep
-			lcp_entry_idx = e
-		default:
-			continue
-		}
-	}
-
-	if lcp_entry_idx >= 0 || direct_idx >= 0 {
-		found = true
-		switch {
-		case direct_idx >= 0:
-			idx = direct_idx
-		case lcp_entry_idx >= 0:
-			idx = lcp_entry_idx
-		}
-	}
-	return
-}
-
-// Returns target == nil if there is no mapping
-func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error) {
-
-	if m.filterMode {
-		err = fmt.Errorf("using a filter for mapping simply does not work")
-		return
-	}
-
-	mi, hasMapping := m.mostSpecificPrefixMapping(source)
-	if !hasMapping {
-		return nil, nil
-	}
-	me := m.entries[mi]
-
-	if me.mapping == "" {
-		// Special case treatment: 'foo/bar<' => ''
-		if !me.subtreeMatch {
-			return nil, fmt.Errorf("mapping to '' must be a subtree match")
-		}
-		// ok...
-	} else {
-		if strings.HasPrefix("!", me.mapping) {
-			// reject mapping
-			return nil, nil
-		}
-	}
-
-	target, err = zfs.NewDatasetPath(me.mapping)
-	if err != nil {
-		err = fmt.Errorf("mapping target is not a dataset path: %s", err)
-		return
-	}
-	if me.subtreeMatch {
-		// strip common prefix ('<' wildcards are no special case here)
-		extendComps := source.Copy()
-		extendComps.TrimPrefix(me.path)
-		target.Extend(extendComps)
-	}
-	return
-}
-
-func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
-
-	if !m.filterMode {
-		err = fmt.Errorf("using a mapping as a filter does not work")
-		return
-	}
-
-	mi, hasMapping := m.mostSpecificPrefixMapping(p)
-	if !hasMapping {
-		pass = false
-		return
-	}
-	me := m.entries[mi]
-	pass, err = m.parseDatasetFilterResult(me.mapping)
-	return
-}
-
-// Construct a new filter-only DatasetMapFilter from a mapping
-// The new filter allows excactly those paths that were not forbidden by the mapping.
-func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
-
-	if m.filterMode {
-		err = errors.Errorf("can only invert mappings")
-		return
-	}
-
-	inv = &DatasetMapFilter{
-		make([]datasetMapFilterEntry, len(m.entries)),
-		true,
-	}
-
-	for i, e := range m.entries {
-		inv.entries[i].path, err = zfs.NewDatasetPath(e.mapping)
-		if err != nil {
-			err = errors.Wrapf(err, "mapping cannot be inverted: '%s' is not a dataset path", e.mapping)
-			return
-		}
-		inv.entries[i].mapping = MapFilterResultOk
-		inv.entries[i].subtreeMatch = e.subtreeMatch
-	}
-
-	return inv, nil
-}
-
-// FIXME investigate whether we can support more...
-func (m DatasetMapFilter) Invert() (endpoint.FSMap, error) {
-
-	if m.filterMode {
-		return nil, errors.Errorf("can only invert mappings")
-	}
-
-	if len(m.entries) != 1 {
-		return nil, errors.Errorf("inversion of complicated mappings is not implemented") // FIXME
-	}
-
-	e := m.entries[0]
-
-	inv := &DatasetMapFilter{
-		make([]datasetMapFilterEntry, len(m.entries)),
-		false,
-	}
-	mp, err := zfs.NewDatasetPath(e.mapping)
-	if err != nil {
-		return nil, err
-	}
-
-	inv.entries[0] = datasetMapFilterEntry{
-		path:         mp,
-		mapping:      e.path.ToString(),
-		subtreeMatch: e.subtreeMatch,
-	}
-
-	return inv, nil
-}
-
-// Creates a new DatasetMapFilter in filter mode from a mapping
-// All accepting mapping results are mapped to accepting filter results
-// All rejecting mapping results are mapped to rejecting filter results
-func (m DatasetMapFilter) AsFilter() endpoint.FSFilter {
-
-	f := &DatasetMapFilter{
-		make([]datasetMapFilterEntry, len(m.entries)),
-		true,
-	}
-
-	for i, e := range m.entries {
-		var newe datasetMapFilterEntry = e
-		if strings.HasPrefix(newe.mapping, "!") {
-			newe.mapping = MapFilterResultOmit
-		} else {
-			newe.mapping = MapFilterResultOk
-		}
-		f.entries[i] = newe
-	}
-
-	return f
-}
-
-const (
-	MapFilterResultOk   string = "ok"
-	MapFilterResultOmit string = "!"
-)
-
-// Parse a dataset filter result
-func (m DatasetMapFilter) parseDatasetFilterResult(result string) (pass bool, err error) {
-	l := strings.ToLower(result)
-	if l == MapFilterResultOk {
-		return true, nil
-	}
-	if l == MapFilterResultOmit {
-		return false, nil
-	}
-	return false, fmt.Errorf("'%s' is not a valid filter result", result)
-}
-
-func DatasetMapFilterFromConfig(in map[string]bool) (f *DatasetMapFilter, err error) {
-
-	f = NewDatasetMapFilter(len(in), true)
-	for pathPattern, accept := range in {
-		mapping := MapFilterResultOmit
-		if accept {
-			mapping = MapFilterResultOk
-		}
-		if err = f.Add(pathPattern, mapping); err != nil {
-			err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err)
-			return
-		}
-	}
-	return
-}

daemon/filters/fsmapfilter_test.go

@@ -1,89 +0,0 @@
-package filters
-
-import (
-	"testing"
-
-	"github.com/zrepl/zrepl/zfs"
-)
-
-func TestDatasetMapFilter(t *testing.T) {
-
-	type testCase struct {
-		name      string
-		filter    map[string]string
-		checkPass map[string]bool // each entry is checked to match the filter's `pass` return value
-	}
-
-	tcs := []testCase{
-		{
-			"default_no_match",
-			map[string]string{},
-			map[string]bool{
-				"":      false,
-				"foo":   false,
-				"zroot": false,
-			},
-		},
-		{
-			"more_specific_path_has_precedence",
-			map[string]string{
-				"tank<":         "ok",
-				"tank/tmp<":     "!",
-				"tank/home/x<":  "!",
-				"tank/home/x/1": "ok",
-			},
-			map[string]bool{
-				"zroot":         false,
-				"tank":          true,
-				"tank/tmp":      false,
-				"tank/tmp/foo":  false,
-				"tank/home/x":   false,
-				"tank/home/y":   true,
-				"tank/home/x/1": true,
-				"tank/home/x/2": false,
-			},
-		},
-		{
-			"precedence_of_specific_over_subtree_wildcard_on_same_path",
-			map[string]string{
-				"tank/home/bob":  "ok",
-				"tank/home/bob<": "!",
-			},
-			map[string]bool{
-				"tank/home/bob":           true,
-				"tank/home/bob/downloads": false,
-			},
-		},
-	}
-
-	for tc := range tcs {
-		t.Run(tcs[tc].name, func(t *testing.T) {
-			c := tcs[tc]
-			f := NewDatasetMapFilter(len(c.filter), true)
-			for p, a := range c.filter {
-				err := f.Add(p, a)
-				if err != nil {
-					t.Fatalf("incorrect filter spec: %s", err)
-				}
-			}
-			for p, checkPass := range c.checkPass {
-				zp, err := zfs.NewDatasetPath(p)
-				if err != nil {
-					t.Fatalf("incorrect path spec: %s", err)
-				}
-				pass, err := f.Filter(zp)
-				if err != nil {
-					t.Fatalf("unexpected filter error: %s", err)
-				}
-				ok := pass == checkPass
-				failstr := "OK"
-				if !ok {
-					failstr = "FAIL"
-					t.Fail()
-				}
-				t.Logf("%-40q  %5v  (exp=%v act=%v)", p, failstr, checkPass, pass)
-			}
-		})
-	}
-
-}

daemon/filters/fsvfilter.go

@@ -1,41 +0,0 @@
-package filters
-
-import (
-	"strings"
-
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type AnyFSVFilter struct{}
-
-func NewAnyFSVFilter() AnyFSVFilter {
-	return AnyFSVFilter{}
-}
-
-var _ zfs.FilesystemVersionFilter = AnyFSVFilter{}
-
-func (AnyFSVFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) {
-	return true, nil
-}
-
-type PrefixFilter struct {
-	prefix    string
-	fstype    zfs.VersionType
-	fstypeSet bool // optionals anyone?
-}
-
-var _ zfs.FilesystemVersionFilter = &PrefixFilter{}
-
-func NewPrefixFilter(prefix string) *PrefixFilter {
-	return &PrefixFilter{prefix: prefix}
-}
-
-func NewTypedPrefixFilter(prefix string, versionType zfs.VersionType) *PrefixFilter {
-	return &PrefixFilter{prefix, versionType, true}
-}
-
-func (f *PrefixFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) {
-	fstypeMatches := (!f.fstypeSet || t == f.fstype)
-	prefixMatches := strings.HasPrefix(name, f.prefix)
-	return fstypeMatches && prefixMatches, nil
-}

daemon/hooks/edge_string.go

@@ -1,35 +0,0 @@
-// Code generated by "stringer -type=Edge"; DO NOT EDIT.
-
-package hooks
-
-import "strconv"
-
-func _() {
-	// An "invalid array index" compiler error signifies that the constant values have changed.
-	// Re-run the stringer command to generate them again.
-	var x [1]struct{}
-	_ = x[Pre-1]
-	_ = x[Callback-2]
-	_ = x[Post-4]
-}
-
-const (
-	_Edge_name_0 = "PreCallback"
-	_Edge_name_1 = "Post"
-)
-
-var (
-	_Edge_index_0 = [...]uint8{0, 3, 11}
-)
-
-func (i Edge) String() string {
-	switch {
-	case 1 <= i && i <= 2:
-		i -= 1
-		return _Edge_name_0[_Edge_index_0[i]:_Edge_index_0[i+1]]
-	case i == 4:
-		return _Edge_name_1
-	default:
-		return "Edge(" + strconv.FormatInt(int64(i), 10) + ")"
-	}
-}

daemon/hooks/hook_config.go

@@ -1,52 +0,0 @@
-package hooks
-
-import (
-	"fmt"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type List []Hook
-
-func HookFromConfig(in config.HookEnum) (Hook, error) {
-	switch v := in.Ret.(type) {
-	case *config.HookCommand:
-		return NewCommandHook(v)
-	case *config.HookPostgresCheckpoint:
-		return PgChkptHookFromConfig(v)
-	case *config.HookMySQLLockTables:
-		return MyLockTablesFromConfig(v)
-	default:
-		return nil, fmt.Errorf("unknown hook type %T", v)
-	}
-}
-
-func ListFromConfig(in *config.HookList) (r *List, err error) {
-	hl := make(List, len(*in))
-
-	for i, h := range *in {
-		hl[i], err = HookFromConfig(h)
-		if err != nil {
-			return nil, fmt.Errorf("create hook #%d: %s", i+1, err)
-		}
-	}
-
-	return &hl, nil
-}
-
-func (l List) CopyFilteredForFilesystem(fs *zfs.DatasetPath) (ret List, err error) {
-	ret = make(List, 0, len(l))
-
-	for _, h := range l {
-		var passFilesystem bool
-		if passFilesystem, err = h.Filesystems().Filter(fs); err != nil {
-			return nil, err
-		}
-		if passFilesystem {
-			ret = append(ret, h)
-		}
-	}
-
-	return ret, nil
-}

daemon/hooks/hook_docs.go

@@ -1,34 +0,0 @@
-// Package hooks implements pre- and post snapshot hooks.
-//
-// Plan is a generic executor for ExpectStepReports before and after an activity specified in a callback.
-// It provides a reporting facility that can be polled while the plan is executing to gather progress information.
-//
-// This package also provides all supported hook type implementations and abstractions around them.
-//
-// Use For Other Kinds Of ExpectStepReports
-//
-// This package REQUIRES REFACTORING before it can be used for other activities than snapshots, e.g. pre- and post-replication:
-//
-// The Hook interface requires a hook to provide a Filesystems() filter, which doesn't make sense for
-// all kinds of activities.
-//
-// The hook implementations should move out of this package.
-// However, there is a lot of tight coupling which to untangle isn't worth it ATM.
-//
-// How This Package Is Used By Package Snapper
-//
-// Deserialize a config.List using ListFromConfig().
-// Then it MUST filter the list to only contain hooks for a particular filesystem using
-// hooksList.CopyFilteredForFilesystem(fs).
-//
-// Then create a CallbackHook using NewCallbackHookForFilesystem().
-//
-// Pass all of the above to NewPlan() which provides a Report() and Run() method:
-//
-// Plan.Run(ctx context.Context,dryRun bool) executes the plan and take a context as argument that should contain a logger added using hooks.WithLogger()).
-// The value of dryRun is passed through to the hooks' Run() method.
-// Command hooks make it available in the environment variable ZREPL_DRYRUN.
-//
-// Plan.Report() can be called while Plan.Run() is executing to give an overview of plan execution progress (future use in "zrepl status").
-//
-package hooks

daemon/hooks/hook_exec.go

@@ -1,292 +0,0 @@
-package hooks
-
-import (
-	"context"
-	"fmt"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/zrepl/zrepl/zfs"
-)
-
-// Re-export type here so that
-// every file in package hooks doesn't
-// have to import github.com/zrepl/zrepl/zfs
-type Filter zfs.DatasetFilter
-
-type Hook interface {
-	Filesystems() Filter
-
-	// If true and the Pre edge invocation of Run fails, Post edge will not run and other Pre edges will not run.
-	ErrIsFatal() bool
-
-	// Run is invoked by HookPlan for a Pre edge.
-	// If HookReport.HadError() == false, the Post edge will be invoked, too.
-	Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport
-
-	String() string
-}
-
-type Phase string
-
-const (
-	PhaseSnapshot = Phase("snapshot")
-	PhaseTesting  = Phase("testing")
-)
-
-func (p Phase) String() string {
-	return string(p)
-}
-
-//go:generate stringer -type=Edge
-type Edge uint
-
-const (
-	Pre = Edge(1 << iota)
-	Callback
-	Post
-)
-
-func (e Edge) StringForPhase(phase Phase) string {
-	return fmt.Sprintf("%s_%s", e.String(), phase.String())
-}
-
-//go:generate enumer -type=StepStatus -trimprefix=Step
-type StepStatus int
-
-const (
-	StepPending StepStatus = 1 << iota
-	StepExec
-	StepOk
-	StepErr
-	StepSkippedDueToFatalErr
-	StepSkippedDueToPreErr
-)
-
-type HookReport interface {
-	String() string
-	HadError() bool
-	Error() string
-}
-
-type Step struct {
-	Hook       Hook
-	Edge       Edge
-	Status     StepStatus
-	Begin, End time.Time
-	// Report may be nil
-	// FIXME cannot serialize this for client status, but contains interesting info (like what error happened)
-	Report HookReport
-	state  map[interface{}]interface{}
-}
-
-func (s Step) String() (out string) {
-	fatal := "~"
-	if s.Hook.ErrIsFatal() && s.Edge == Pre {
-		fatal = "!"
-	}
-	runTime := "..."
-	if s.Status != StepPending {
-		t := s.End.Sub(s.Begin)
-		runTime = t.Round(time.Millisecond).String()
-	}
-	return fmt.Sprintf("[%s] [%5s] %s [%s]  %s", s.Status, runTime, fatal, s.Edge, s.Hook)
-}
-
-type Plan struct {
-	mtx sync.RWMutex
-
-	steps []*Step
-	pre   []*Step // protected by mtx
-	cb    *Step
-	post  []*Step // not reversed, i.e. entry at index i corresponds to pre-edge in pre[i]
-
-	phase Phase
-	env   Env
-}
-
-func NewPlan(hooks *List, phase Phase, cb *CallbackHook, extra Env) (*Plan, error) {
-
-	var pre, post []*Step
-	// TODO sanity check unique name of hook?
-	for _, hook := range *hooks {
-		state := make(map[interface{}]interface{})
-		preE := &Step{
-			Hook:   hook,
-			Edge:   Pre,
-			Status: StepPending,
-			state:  state,
-		}
-		pre = append(pre, preE)
-		postE := &Step{
-			Hook:   hook,
-			Edge:   Post,
-			Status: StepPending,
-			state:  state,
-		}
-		post = append(post, postE)
-	}
-
-	cbE := &Step{
-		Hook:   cb,
-		Edge:   Callback,
-		Status: StepPending,
-	}
-
-	steps := make([]*Step, 0, len(pre)+len(post)+1)
-	steps = append(steps, pre...)
-	steps = append(steps, cbE)
-	for i := len(post) - 1; i >= 0; i-- {
-		steps = append(steps, post[i])
-	}
-
-	plan := &Plan{
-		phase: phase,
-		env:   extra,
-		steps: steps,
-		pre:   pre,
-		post:  post,
-		cb:    cbE,
-	}
-
-	return plan, nil
-}
-
-type PlanReport []Step
-
-func (p *Plan) Report() PlanReport {
-	p.mtx.RLock()
-	defer p.mtx.RUnlock()
-	rep := make([]Step, len(p.steps))
-	for i := range rep {
-		rep[i] = *p.steps[i]
-	}
-	return rep
-}
-
-func (r PlanReport) HadError() bool {
-	for _, e := range r {
-		if e.Status == StepErr {
-			return true
-		}
-	}
-	return false
-}
-
-func (r PlanReport) HadFatalError() bool {
-	for _, e := range r {
-		if e.Status == StepSkippedDueToFatalErr {
-			return true
-		}
-	}
-	return false
-}
-
-func (r PlanReport) String() string {
-	stepStrings := make([]string, len(r))
-	for i, e := range r {
-		stepStrings[i] = fmt.Sprintf("%02d %s", i+1, e)
-	}
-	return strings.Join(stepStrings, "\n")
-}
-
-func (p *Plan) Run(ctx context.Context, dryRun bool) {
-	p.mtx.RLock()
-	defer p.mtx.RUnlock()
-	w := func(f func()) {
-		p.mtx.RUnlock()
-		defer p.mtx.RLock()
-		p.mtx.Lock()
-		defer p.mtx.Unlock()
-		f()
-	}
-	runHook := func(s *Step, ctx context.Context, edge Edge) HookReport {
-		w(func() { s.Status = StepExec })
-		begin := time.Now()
-		r := s.Hook.Run(ctx, edge, p.phase, dryRun, p.env, s.state)
-		end := time.Now()
-		w(func() {
-			s.Report = r
-			s.Status = StepOk
-			if r.HadError() {
-				s.Status = StepErr
-			}
-			s.Begin, s.End = begin, end
-		})
-		return r
-	}
-
-	l := getLogger(ctx)
-
-	// it's a stack, execute until we reach the end of the list (last item in)
-	// or fail inbetween
-	l.Info("run pre-edges in configuration order")
-	next := 0
-	for ; next < len(p.pre); next++ {
-		e := p.pre[next]
-		l := l.WithField("hook", e.Hook)
-		r := runHook(e, ctx, Pre)
-		if r.HadError() {
-			l.WithError(r).Error("hook invocation failed for pre-edge")
-			if e.Hook.ErrIsFatal() {
-				l.Error("the hook run was aborted due to a fatal error in this hook")
-				break
-			}
-		}
-	}
-
-	hadFatalErr := next != len(p.pre)
-	if hadFatalErr {
-		l.Error("fatal error in a pre-snapshot hook invocation")
-		l.Error("no snapshot will be taken")
-		l.Error("only running post-edges for successful pre-edges")
-		w(func() {
-			p.post[next].Status = StepSkippedDueToFatalErr
-			for i := next + 1; i < len(p.pre); i++ {
-				p.pre[i].Status = StepSkippedDueToFatalErr
-				p.post[i].Status = StepSkippedDueToFatalErr
-			}
-			p.cb.Status = StepSkippedDueToFatalErr
-		})
-		return
-	}
-
-	l.Info("running callback")
-	cbR := runHook(p.cb, ctx, Callback)
-	if cbR.HadError() {
-		l.WithError(cbR).Error("callback failed")
-	}
-
-	l.Info("run post-edges for successful pre-edges in reverse configuration order")
-
-	// the constructor produces pre and post entries
-	// post is NOT reversed
-	next-- // now at index of last executed pre-edge
-	for ; next >= 0; next-- {
-		e := p.post[next]
-		l := l.WithField("hook", e.Hook)
-
-		if p.pre[next].Status != StepOk {
-			if p.pre[next].Status != StepErr {
-				panic(fmt.Sprintf("expecting a pre-edge hook report to be either Ok or Err, got %s", p.pre[next].Status))
-			}
-			l.Info("skip post-edge because pre-edge failed")
-			w(func() {
-				e.Status = StepSkippedDueToPreErr
-			})
-			continue
-		}
-
-		report := runHook(e, ctx, Post)
-
-		if report.HadError() {
-			l.WithError(report).Error("hook invocation failed for post-edge")
-			l.Error("subsequent post-edges run regardless of this post-edge failure")
-		}
-
-		// ErrIsFatal is only relevant for Pre
-
-	}
-
-}

daemon/hooks/hook_logging.go

@@ -1,103 +0,0 @@
-package hooks
-
-import (
-	"bufio"
-	"bytes"
-	"context"
-	"sync"
-
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/util/envconst"
-)
-
-type contextKey int
-
-const (
-	contextKeyLog contextKey = 0
-)
-
-type Logger = logger.Logger
-
-func WithLogger(ctx context.Context, log Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLog, log)
-}
-
-func GetLogger(ctx context.Context) Logger { return getLogger(ctx) }
-
-func getLogger(ctx context.Context) Logger {
-	if log, ok := ctx.Value(contextKeyLog).(Logger); ok {
-		return log
-	}
-	return logger.NewNullLogger()
-}
-
-const MAX_HOOK_LOG_SIZE_DEFAULT int = 1 << 20
-
-type logWriter struct {
-	/*
-		Mutex prevents:
-			concurrent writes to buf, scanner in Write([]byte)
-			data race on scanner vs Write([]byte)
-				and concurrent write to buf (call to buf.Reset())
-				in Close()
-
-		(Also, Close() should generally block until any Write() call completes.)
-	*/
-	mtx     *sync.Mutex
-	buf     bytes.Buffer
-	scanner *bufio.Scanner
-	logger  Logger
-	level   logger.Level
-	field   string
-}
-
-func NewLogWriter(mtx *sync.Mutex, logger Logger, level logger.Level, field string) *logWriter {
-	w := new(logWriter)
-	w.mtx = mtx
-	w.scanner = bufio.NewScanner(&w.buf)
-	w.logger = logger
-	w.level = level
-	w.field = field
-	return w
-}
-
-func (w *logWriter) log(line string) {
-	w.logger.WithField(w.field, line).Log(w.level, "hook output")
-}
-
-func (w *logWriter) logUnreadBytes() error {
-	for w.scanner.Scan() {
-		w.log(w.scanner.Text())
-	}
-	if w.buf.Cap() > envconst.Int("ZREPL_MAX_HOOK_LOG_SIZE", MAX_HOOK_LOG_SIZE_DEFAULT) {
-		w.buf.Reset()
-	}
-
-	return nil
-}
-
-func (w *logWriter) Write(in []byte) (int, error) {
-	w.mtx.Lock()
-	defer w.mtx.Unlock()
-
-	n, bufErr := w.buf.Write(in)
-	if bufErr != nil {
-		return n, bufErr
-	}
-
-	err := w.logUnreadBytes()
-	if err != nil {
-		return n, err
-	}
-	// Always reset the scanner for the next Write
-	w.scanner = bufio.NewScanner(&w.buf)
-
-	return n, nil
-}
-
-func (w *logWriter) Close() (err error) {
-	w.mtx.Lock()
-	defer w.mtx.Unlock()
-
-	return w.logUnreadBytes()
-}

daemon/hooks/hook_type_callback.go

@@ -1,65 +0,0 @@
-package hooks
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type HookJobCallback func(ctx context.Context) error
-
-type CallbackHook struct {
-	cb            HookJobCallback
-	filter        Filter
-	displayString string
-}
-
-func NewCallbackHookForFilesystem(displayString string, fs *zfs.DatasetPath, cb HookJobCallback) *CallbackHook {
-	filter, _ := filters.DatasetMapFilterFromConfig(map[string]bool{fs.ToString(): true})
-	return NewCallbackHook(displayString, cb, filter)
-}
-
-func NewCallbackHook(displayString string, cb HookJobCallback, filter Filter) *CallbackHook {
-	return &CallbackHook{
-		cb:            cb,
-		filter:        filter,
-		displayString: displayString,
-	}
-}
-
-func (h *CallbackHook) Filesystems() Filter {
-	return h.filter
-}
-
-func (h *CallbackHook) ErrIsFatal() bool {
-	return false // callback is by definition
-}
-
-func (h *CallbackHook) String() string {
-	return h.displayString
-}
-
-type CallbackHookReport struct {
-	Name string
-	Err  error
-}
-
-func (r *CallbackHookReport) String() string {
-	if r.HadError() {
-		return r.Error()
-	}
-	return r.Name
-}
-
-func (r *CallbackHookReport) HadError() bool { return r.Err != nil }
-
-func (r *CallbackHookReport) Error() string {
-	return fmt.Sprintf("%s error: %s", r.Name, r.Err)
-}
-
-func (h *CallbackHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
-	err := h.cb(ctx)
-	return &CallbackHookReport{h.displayString, err}
-}

daemon/hooks/hook_type_command.go

@@ -1,192 +0,0 @@
-package hooks
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"math"
-	"os"
-	"os/exec"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/util/circlog"
-	"github.com/zrepl/zrepl/util/envconst"
-)
-
-type HookEnvVar string
-
-const (
-	EnvType     HookEnvVar = "ZREPL_HOOKTYPE"
-	EnvDryRun   HookEnvVar = "ZREPL_DRYRUN"
-	EnvFS       HookEnvVar = "ZREPL_FS"
-	EnvSnapshot HookEnvVar = "ZREPL_SNAPNAME"
-	EnvTimeout  HookEnvVar = "ZREPL_TIMEOUT"
-)
-
-type Env map[HookEnvVar]string
-
-func NewHookEnv(edge Edge, phase Phase, dryRun bool, timeout time.Duration, extra Env) Env {
-	r := Env{
-		EnvTimeout: fmt.Sprintf("%.f", math.Floor(timeout.Seconds())),
-	}
-
-	edgeString := edge.StringForPhase(phase)
-	r[EnvType] = strings.ToLower(edgeString)
-
-	var dryRunString string
-	if dryRun {
-		dryRunString = "true"
-	} else {
-		dryRunString = ""
-	}
-	r[EnvDryRun] = dryRunString
-
-	for k, v := range extra {
-		r[k] = v
-	}
-
-	return r
-}
-
-type CommandHook struct {
-	edge       Edge
-	filter     Filter
-	errIsFatal bool
-	command    string
-	timeout    time.Duration
-}
-
-type CommandHookReport struct {
-	Command                      string
-	Args                         []string // currently always empty
-	Env                          Env
-	Err                          error
-	CapturedStdoutStderrCombined []byte
-}
-
-func (r *CommandHookReport) String() string {
-	// Reproduces a POSIX shell-compatible command line
-	var cmdLine strings.Builder
-	sep := ""
-
-	// Make sure environment variables are always
-	// printed in the same order
-	var hookEnvKeys []HookEnvVar
-	for k := range r.Env {
-		hookEnvKeys = append(hookEnvKeys, k)
-	}
-	sort.Slice(hookEnvKeys, func(i, j int) bool { return string(hookEnvKeys[i]) < string(hookEnvKeys[j]) })
-
-	for _, k := range hookEnvKeys {
-		cmdLine.WriteString(fmt.Sprintf("%s%s='%s'", sep, k, r.Env[k]))
-		sep = " "
-	}
-
-	cmdLine.WriteString(fmt.Sprintf("%s%s", sep, r.Command))
-	for _, a := range r.Args {
-		cmdLine.WriteString(fmt.Sprintf("%s'%s'", sep, a))
-	}
-
-	return fmt.Sprintf("command hook invocation: \"%s\"", cmdLine.String()) // no %q to make copy-pastable
-}
-func (r *CommandHookReport) Error() string {
-	if r.Err == nil {
-		return ""
-	}
-	return fmt.Sprintf("%s FAILED with error: %s", r.String(), r.Err)
-}
-
-func (r *CommandHookReport) HadError() bool {
-	return r.Err != nil
-}
-
-func NewCommandHook(in *config.HookCommand) (r *CommandHook, err error) {
-	r = &CommandHook{
-		errIsFatal: in.ErrIsFatal,
-		command:    in.Path,
-		timeout:    in.Timeout,
-	}
-
-	r.filter, err = filters.DatasetMapFilterFromConfig(in.Filesystems)
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse filesystem filter: %s", err)
-	}
-
-	r.edge = Pre | Post
-
-	return r, nil
-}
-
-func (h *CommandHook) Filesystems() Filter {
-	return h.filter
-}
-
-func (h *CommandHook) ErrIsFatal() bool {
-	return h.errIsFatal
-}
-
-func (h *CommandHook) String() string {
-	return h.command
-}
-
-func (h *CommandHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
-	l := getLogger(ctx).WithField("command", h.command)
-
-	cmdCtx, cancel := context.WithTimeout(ctx, h.timeout)
-	defer cancel()
-
-	cmdExec := exec.CommandContext(cmdCtx, h.command)
-
-	hookEnv := NewHookEnv(edge, phase, dryRun, h.timeout, extra)
-	cmdEnv := os.Environ()
-	for k, v := range hookEnv {
-		cmdEnv = append(cmdEnv, fmt.Sprintf("%s=%s", k, v))
-	}
-	cmdExec.Env = cmdEnv
-
-	var scanMutex sync.Mutex
-	combinedOutput, err := circlog.NewCircularLog(envconst.Int("ZREPL_MAX_HOOK_LOG_SIZE", MAX_HOOK_LOG_SIZE_DEFAULT))
-	if err != nil {
-		return &CommandHookReport{Err: err}
-	}
-	logErrWriter := NewLogWriter(&scanMutex, l, logger.Warn, "stderr")
-	logOutWriter := NewLogWriter(&scanMutex, l, logger.Info, "stdout")
-	defer logErrWriter.Close()
-	defer logOutWriter.Close()
-
-	cmdExec.Stderr = io.MultiWriter(logErrWriter, combinedOutput)
-	cmdExec.Stdout = io.MultiWriter(logOutWriter, combinedOutput)
-
-	report := &CommandHookReport{
-		Command: h.command,
-		Env:     hookEnv,
-		// no report.Args
-	}
-
-	err = cmdExec.Start()
-	if err != nil {
-		report.Err = err
-		return report
-	}
-
-	err = cmdExec.Wait()
-	combinedOutputBytes := combinedOutput.Bytes()
-	report.CapturedStdoutStderrCombined = make([]byte, len(combinedOutputBytes))
-	copy(report.CapturedStdoutStderrCombined, combinedOutputBytes)
-	if err != nil {
-		if cmdCtx.Err() == context.DeadlineExceeded {
-			report.Err = fmt.Errorf("timed out after %s: %s", h.timeout, err)
-			return report
-		}
-		report.Err = err
-		return report
-	}
-
-	return report
-}

daemon/hooks/hook_type_mysql_lock_tables.go

@@ -1,145 +0,0 @@
-package hooks
-
-import (
-	"context"
-	"database/sql"
-	"fmt"
-	"strings"
-
-	sqldriver "database/sql/driver"
-
-	"github.com/go-sql-driver/mysql"
-
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-// https://dev.mysql.com/doc/mysql-backup-excerpt/5.7/en/backup-methods.html
-//
-// Making Backups Using a File System Snapshot:
-//
-//  If you are using a Veritas file system, you can make a backup like this:
-//
-//    	From a client program, execute FLUSH TABLES WITH READ LOCK.
-//    	From another shell, execute mount vxfs snapshot.
-//    	From the first client, execute UNLOCK TABLES.
-//    	Copy files from the snapshot.
-//    	Unmount the snapshot.
-//
-//	Similar snapshot capabilities may be available in other file systems, such as LVM or ZFS.
-type MySQLLockTables struct {
-	errIsFatal  bool
-	connector   sqldriver.Connector
-	filesystems Filter
-}
-
-type myLockTablesStateKey int
-
-const (
-	myLockTablesConnection myLockTablesStateKey = 1 + iota
-)
-
-func MyLockTablesFromConfig(in *config.HookMySQLLockTables) (*MySQLLockTables, error) {
-	conf, err := mysql.ParseDSN(in.DSN)
-	if err != nil {
-		return nil, errors.Wrap(err, "`dsn` invalid")
-	}
-	cn, err := mysql.NewConnector(conf)
-	if err != nil {
-		return nil, errors.Wrap(err, "`connect` invalid")
-	}
-
-	filesystems, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
-	if err != nil {
-		return nil, errors.Wrap(err, "`filesystems` invalid")
-	}
-
-	return &MySQLLockTables{
-		in.ErrIsFatal,
-		cn,
-		filesystems,
-	}, nil
-}
-
-func (h *MySQLLockTables) ErrIsFatal() bool    { return h.errIsFatal }
-func (h *MySQLLockTables) Filesystems() Filter { return h.filesystems }
-func (h *MySQLLockTables) String() string      { return "MySQL FLUSH TABLES WITH READ LOCK" }
-
-type MyLockTablesReport struct {
-	What string
-	Err  error
-}
-
-func (r *MyLockTablesReport) HadError() bool { return r.Err != nil }
-func (r *MyLockTablesReport) Error() string  { return r.String() }
-func (r *MyLockTablesReport) String() string {
-	var s strings.Builder
-	s.WriteString(r.What)
-	if r.Err != nil {
-		fmt.Fprintf(&s, ": %s", r.Err)
-	}
-	return s.String()
-}
-
-func (h *MySQLLockTables) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
-	fs, ok := extra[EnvFS]
-	if !ok {
-		panic(extra)
-	}
-	dp, err := zfs.NewDatasetPath(fs)
-	if err != nil {
-		panic(err)
-	}
-	if pass, err := h.filesystems.Filter(dp); err != nil {
-		return &MyLockTablesReport{What: "filesystem filter", Err: err}
-	} else if !pass {
-		getLogger(ctx).Debug("filesystem does not match filter, skipping")
-		return &MyLockTablesReport{What: "filesystem filter skipped this filesystem", Err: nil}
-	}
-
-	switch edge {
-	case Pre:
-		err := h.doRunPre(ctx, dp, dryRun, state)
-		return &MyLockTablesReport{"FLUSH TABLES WITH READ LOCK", err}
-	case Post:
-		err := h.doRunPost(ctx, dp, dryRun, state)
-		return &MyLockTablesReport{"UNLOCK TABLES", err}
-	}
-	return &MyLockTablesReport{What: "skipped this edge", Err: nil}
-}
-
-func (h *MySQLLockTables) doRunPre(ctx context.Context, fs *zfs.DatasetPath, dry bool, state map[interface{}]interface{}) (err error) {
-	db := sql.OpenDB(h.connector)
-	defer func(err *error) {
-		if *err != nil {
-			db.Close()
-		}
-	}(&err)
-
-	getLogger(ctx).Debug("do FLUSH TABLES WITH READ LOCK")
-	_, err = db.ExecContext(ctx, "FLUSH TABLES WITH READ LOCK")
-	if err != nil {
-		return
-	}
-
-	state[myLockTablesConnection] = db
-
-	return nil
-}
-
-func (h *MySQLLockTables) doRunPost(ctx context.Context, fs *zfs.DatasetPath, dry bool, state map[interface{}]interface{}) error {
-
-	db := state[myLockTablesConnection].(*sql.DB)
-	defer db.Close()
-
-	getLogger(ctx).Debug("do UNLOCK TABLES")
-	_, err := db.ExecContext(ctx, "UNLOCK TABLES")
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

daemon/hooks/hook_type_postgres_checkpoint.go

@@ -1,98 +0,0 @@
-package hooks
-
-import (
-	"context"
-	"database/sql"
-	"fmt"
-	"math"
-	"time"
-
-	"github.com/lib/pq"
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type PgChkptHook struct {
-	errIsFatal  bool
-	connector   *pq.Connector
-	filesystems Filter
-}
-
-func PgChkptHookFromConfig(in *config.HookPostgresCheckpoint) (*PgChkptHook, error) {
-	filesystems, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
-	if err != nil {
-		return nil, errors.Wrap(err, "`filesystems` invalid")
-	}
-	cn, err := pq.NewConnector(in.DSN)
-	if err != nil {
-		return nil, errors.Wrap(err, "`dsn` invalid")
-	}
-
-	return &PgChkptHook{
-		in.ErrIsFatal,
-		cn,
-		filesystems,
-	}, nil
-}
-
-func (h *PgChkptHook) ErrIsFatal() bool    { return h.errIsFatal }
-func (h *PgChkptHook) Filesystems() Filter { return h.filesystems }
-func (h *PgChkptHook) String() string      { return "postgres checkpoint" }
-
-type PgChkptHookReport struct{ Err error }
-
-func (r *PgChkptHookReport) HadError() bool { return r.Err != nil }
-func (r *PgChkptHookReport) Error() string  { return r.Err.Error() }
-func (r *PgChkptHookReport) String() string {
-	if r.Err != nil {
-		return fmt.Sprintf("postgres CHECKPOINT failed: %s", r.Err)
-	} else {
-		return "postgres CHECKPOINT completed"
-	}
-}
-
-func (h *PgChkptHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
-	if edge != Pre {
-		return &PgChkptHookReport{nil}
-	}
-	fs, ok := extra[EnvFS]
-	if !ok {
-		panic(extra)
-	}
-	dp, err := zfs.NewDatasetPath(fs)
-	if err != nil {
-		panic(err)
-	}
-	err = h.doRunPre(ctx, dp, dryRun)
-	return &PgChkptHookReport{err}
-}
-
-func (h *PgChkptHook) doRunPre(ctx context.Context, fs *zfs.DatasetPath, dry bool) error {
-
-	if pass, err := h.filesystems.Filter(fs); err != nil || !pass {
-		getLogger(ctx).Debug("filesystem does not match filter, skipping")
-		return err
-	}
-
-	db := sql.OpenDB(h.connector)
-	defer db.Close()
-	dl, ok := ctx.Deadline()
-	if ok {
-		timeout := uint64(math.Floor(time.Until(dl).Seconds() * 1000)) // TODO go1.13 milliseconds
-		getLogger(ctx).WithField("statement_timeout", timeout).Debug("setting statement timeout for CHECKPOINT")
-		_, err := db.ExecContext(ctx, "SET statement_timeout TO ?", timeout)
-		if err != nil {
-			return err
-		}
-	}
-	if dry {
-		getLogger(ctx).Debug("dry-run - use ping instead of CHECKPOINT")
-		return db.PingContext(ctx)
-	}
-	getLogger(ctx).Info("execute CHECKPOINT command")
-	_, err := db.ExecContext(ctx, "CHECKPOINT")
-	return err
-}

daemon/hooks/hooks_test.go

@@ -1,484 +0,0 @@
-package hooks_test
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"os"
-	"regexp"
-	"testing"
-	"text/template"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/hooks"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type comparisonAssertionFunc func(require.TestingT, interface{}, interface{}, ...interface{})
-type valueAssertionFunc func(require.TestingT, interface{}, ...interface{})
-
-type expectStep struct {
-	ExpectedEdge hooks.Edge
-	ExpectStatus hooks.StepStatus
-	OutputTest   valueAssertionFunc
-	ErrorTest    valueAssertionFunc
-}
-
-type testCase struct {
-	Name           string
-	Config         []string
-	IsSlow         bool
-	SuppressOutput bool
-
-	ExpectCallbackSkipped bool
-	ExpectHadFatalErr     bool
-	ExpectHadError        bool
-	ExpectStepReports     []expectStep
-}
-
-func curryLeft(f comparisonAssertionFunc, expected interface{}) valueAssertionFunc {
-	return curry(f, expected, false)
-}
-
-func curryRight(f comparisonAssertionFunc, expected interface{}) valueAssertionFunc {
-	return curry(f, expected, true)
-}
-
-func curry(f comparisonAssertionFunc, expected interface{}, right bool) (ret valueAssertionFunc) {
-	ret = func(t require.TestingT, s interface{}, v ...interface{}) {
-		var x interface{}
-		var y interface{}
-		if right {
-			x = s
-			y = expected
-		} else {
-			x = expected
-			y = s
-		}
-
-		if len(v) > 0 {
-			f(t, x, y, v)
-		} else {
-			f(t, x, y)
-		}
-	}
-	return
-}
-
-func TestHooks(t *testing.T) {
-	testFSName := "testpool/testdataset"
-	testSnapshotName := "testsnap"
-
-	tmpl, err := template.New("TestHooks").Parse(`
-jobs:
-- name: TestHooks
-  type: snap
-  filesystems: {"<": true}
-  snapshotting:
-    type: periodic
-    interval: 1m
-    prefix: zrepl_snapjob_
-    hooks:
-    {{- template "List" . }}
-  pruning:
-    keep:
-      - type: last_n
-        count: 10
-`)
-	if err != nil {
-		panic(err)
-	}
-
-	regexpTest := func(s string) valueAssertionFunc {
-		return curryLeft(require.Regexp, regexp.MustCompile(s))
-	}
-
-	containsTest := func(s string) valueAssertionFunc {
-		return curryRight(require.Contains, s)
-	}
-
-	testTable := []testCase{
-		testCase{
-			Name: "no_hooks",
-			ExpectStepReports: []expectStep{
-				{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
-			},
-		},
-		testCase{
-			Name:           "timeout",
-			IsSlow:         true,
-			ExpectHadError: true,
-			Config:         []string{`{type: command, path: {{.WorkDir}}/test/test-timeout.sh, timeout: 2s}`},
-			ExpectStepReports: []expectStep{
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepErr,
-					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s ZREPL_TIMEOUT=2", testFSName, testSnapshotName)),
-					ErrorTest:    regexpTest(`timed out after 2(.\d+)?s`),
-				},
-				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepSkippedDueToPreErr,
-				},
-			},
-		},
-		testCase{
-			Name:           "check_env",
-			Config:         []string{`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`},
-			ExpectHadError: false,
-			ExpectStepReports: []expectStep{
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
-				},
-				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
-				},
-			},
-		},
-
-		testCase{
-			Name:                  "nonfatal_pre_error_continues",
-			ExpectCallbackSkipped: false,
-			ExpectHadError:        true,
-			Config:                []string{`{type: command, path: {{.WorkDir}}/test/test-error.sh}`},
-			ExpectStepReports: []expectStep{
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepErr,
-					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
-					ErrorTest:    regexpTest("^command hook invocation.*exit status 1$"),
-				},
-				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepSkippedDueToPreErr, // post-edge is not executed for failing pre-edge
-				},
-			},
-		},
-
-		testCase{
-			Name:                  "pre_error_fatal_skips_subsequent_pre_edges_and_callback_and_its_post_edge_and_post_edges",
-			ExpectCallbackSkipped: true,
-			ExpectHadFatalErr:     true,
-			ExpectHadError:        true,
-			Config: []string{
-				`{type: command, path: {{.WorkDir}}/test/test-error.sh, err_is_fatal: true}`,
-				`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`,
-			},
-			ExpectStepReports: []expectStep{
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepErr,
-					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
-					ErrorTest:    regexpTest("^command hook invocation.*exit status 1$"),
-				},
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepSkippedDueToFatalErr,
-				},
-				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepSkippedDueToFatalErr},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepSkippedDueToFatalErr,
-				},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepSkippedDueToFatalErr,
-				},
-			},
-		},
-
-		testCase{
-			Name:              "post_error_fails_are_ignored_even_if_fatal",
-			ExpectHadFatalErr: false, // only occurs during Post, so it's not a fatal error
-			ExpectHadError:    true,
-			Config: []string{
-				`{type: command, path: {{.WorkDir}}/test/test-post-error.sh, err_is_fatal: true}`,
-				`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`,
-			},
-			ExpectStepReports: []expectStep{
-				expectStep{
-					// No-action run of test-post-error.sh
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   require.Empty,
-				},
-				expectStep{
-					// Pre run of test-report-env.sh
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
-				},
-				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
-				},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepErr,
-					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR post_testing %s@%s", testFSName, testSnapshotName)),
-					ErrorTest:    regexpTest("^command hook invocation.*exit status 1$"),
-				},
-			},
-		},
-
-		testCase{
-			Name:   "cleanup_check_env",
-			Config: []string{`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`},
-			ExpectStepReports: []expectStep{
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
-				},
-				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
-				},
-			},
-		},
-
-		testCase{
-			Name:              "pre_error_cancels_post",
-			Config:            []string{`{type: command, path: {{.WorkDir}}/test/test-pre-error-post-ok.sh}`},
-			ExpectHadError:    true,
-			ExpectHadFatalErr: false,
-			ExpectStepReports: []expectStep{
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepErr,
-					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
-					ErrorTest:    regexpTest("^command hook invocation.*exit status 1$"),
-				},
-				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepSkippedDueToPreErr,
-				},
-			},
-		},
-
-		testCase{
-			Name: "pre_error_does_not_cancel_other_posts_but_itself",
-			Config: []string{
-				`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`,
-				`{type: command, path: {{.WorkDir}}/test/test-pre-error-post-ok.sh}`,
-			},
-			ExpectHadError:    true,
-			ExpectHadFatalErr: false,
-			ExpectStepReports: []expectStep{
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
-				},
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepErr,
-					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
-					ErrorTest:    regexpTest("^command hook invocation.*exit status 1$"),
-				},
-				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepSkippedDueToPreErr,
-				},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
-				},
-			},
-		},
-
-		testCase{
-			Name:              "exceed_buffer_limit",
-			SuppressOutput:    true,
-			Config:            []string{`{type: command, path: {{.WorkDir}}/test/test-large-stdout.sh}`},
-			ExpectHadError:    false,
-			ExpectHadFatalErr: false,
-			ExpectStepReports: []expectStep{
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepOk,
-					OutputTest: func(t require.TestingT, s interface{}, v ...interface{}) {
-						require.Len(t, s, 1<<20)
-					},
-				},
-				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
-				expectStep{
-					// No-action run of above hook
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   require.Empty,
-				},
-			},
-		},
-
-		/*
-			Following not intended to test functionality of
-			filter package. Only to demonstrate that hook
-			filters are being applied. The following should
-			result in NO hooks running. If it does run, a
-			fatal hooks.RunReport will be returned.
-		*/
-		testCase{
-			Name: "exclude_all_filesystems",
-			Config: []string{
-				`{type: command, path: {{.WorkDir}}/test/test-error.sh, err_is_fatal: true, filesystems: {"<": false}}`,
-			},
-			ExpectStepReports: []expectStep{
-				expectStep{
-					ExpectedEdge: hooks.Callback,
-					ExpectStatus: hooks.StepOk,
-				},
-			},
-		},
-	}
-
-	parseHookConfig := func(t *testing.T, in string) *config.Config {
-		t.Helper()
-		conf, err := config.ParseConfigBytes([]byte(in))
-		require.NoError(t, err)
-		require.NotNil(t, conf)
-		return conf
-	}
-
-	fillHooks := func(tt *testCase) string {
-		// make hook path absolute
-		cwd, err := os.Getwd()
-		if err != nil {
-			panic("os.Getwd() failed")
-		}
-		var hooksTmpl string = "\n"
-		for _, l := range tt.Config {
-			hooksTmpl += fmt.Sprintf("    - %s\n", l)
-		}
-		tmpl.New("List").Parse(hooksTmpl)
-
-		var outBytes bytes.Buffer
-		data := struct {
-			WorkDir string
-		}{
-			WorkDir: cwd,
-		}
-		if err := tmpl.Execute(&outBytes, data); err != nil {
-			panic(err)
-		}
-
-		return outBytes.String()
-	}
-
-	var c *config.Config
-	fs, err := zfs.NewDatasetPath(testFSName)
-	require.NoError(t, err)
-
-	log := logger.NewTestLogger(t)
-
-	var cbReached bool
-	cb := hooks.NewCallbackHookForFilesystem("testcallback", fs, func(_ context.Context) error {
-		cbReached = true
-		return nil
-	})
-
-	hookEnvExtra := hooks.Env{
-		hooks.EnvFS:       fs.ToString(),
-		hooks.EnvSnapshot: testSnapshotName,
-	}
-
-	for _, tt := range testTable {
-		if testing.Short() && tt.IsSlow {
-			continue
-		}
-
-		t.Run(tt.Name, func(t *testing.T) {
-			c = parseHookConfig(t, fillHooks(&tt))
-			snp := c.Jobs[0].Ret.(*config.SnapJob).Snapshotting.Ret.(*config.SnapshottingPeriodic)
-			hookList, err := hooks.ListFromConfig(&snp.Hooks)
-			require.NoError(t, err)
-
-			filteredHooks, err := hookList.CopyFilteredForFilesystem(fs)
-			require.NoError(t, err)
-			plan, err := hooks.NewPlan(&filteredHooks, hooks.PhaseTesting, cb, hookEnvExtra)
-			require.NoError(t, err)
-			t.Logf("REPORT PRE EXECUTION:\n%s", plan.Report())
-
-			cbReached = false
-
-			ctx := context.Background()
-			if testing.Verbose() && !tt.SuppressOutput {
-				ctx = hooks.WithLogger(ctx, log)
-			}
-			plan.Run(ctx, false)
-			report := plan.Report()
-
-			t.Logf("REPORT POST EXECUTION:\n%s", report)
-
-			/*
-			 * TEST ASSERTIONS
-			 */
-
-			t.Logf("len(runReports)=%v", len(report))
-			t.Logf("len(tt.ExpectStepReports)=%v", len(tt.ExpectStepReports))
-			require.Equal(t, len(tt.ExpectStepReports), len(report), "ExpectStepReports must be same length as expected number of hook runs, excluding possible Callback")
-
-			// Check if callback ran, when required
-			if tt.ExpectCallbackSkipped {
-				require.False(t, cbReached, "callback ran but should not have run")
-			} else {
-				require.True(t, cbReached, "callback should have run but did not")
-			}
-
-			// Check if a fatal run error occurred and was expected
-			require.Equal(t, tt.ExpectHadFatalErr, report.HadFatalError(), "non-matching HadFatalError")
-			require.Equal(t, tt.ExpectHadError, report.HadError(), "non-matching HadError")
-
-			if tt.ExpectHadFatalErr {
-				require.True(t, tt.ExpectHadError, "ExpectHadFatalErr implies ExpectHadError")
-			}
-			if !tt.ExpectHadError {
-				require.False(t, tt.ExpectHadFatalErr, "!ExpectHadError implies !ExpectHadFatalErr")
-			}
-
-			// Iterate through each expected hook run
-			for i, hook := range tt.ExpectStepReports {
-				t.Logf("expecting report conforming to %v", hook)
-
-				exp, act := hook.ExpectStatus, report[i].Status
-				require.Equal(t, exp, act, "%s != %s", exp, act)
-
-				// Check for required ExpectedEdge
-				require.NotZero(t, hook.ExpectedEdge, "each hook must have an ExpectedEdge")
-				require.Equal(t, hook.ExpectedEdge, report[i].Edge,
-					"incorrect edge: expected %q, actual %q", hook.ExpectedEdge.String(), report[i].Edge.String(),
-				)
-
-				// Check for expected output
-				if hook.OutputTest != nil {
-					require.IsType(t, (*hooks.CommandHookReport)(nil), report[i].Report)
-					chr := report[i].Report.(*hooks.CommandHookReport)
-					hook.OutputTest(t, string(chr.CapturedStdoutStderrCombined))
-				}
-
-				// Check for expected errors
-				if hook.ErrorTest != nil {
-					hook.ErrorTest(t, string(report[i].Report.Error()))
-				}
-			}
-
-		})
-	}
-}

daemon/hooks/stepstatus_enumer.go

@@ -1,77 +0,0 @@
-// Code generated by "enumer -type=StepStatus -trimprefix=Step"; DO NOT EDIT.
-
-//
-package hooks
-
-import (
-	"fmt"
-)
-
-const (
-	_StepStatusName_0 = "PendingExec"
-	_StepStatusName_1 = "Ok"
-	_StepStatusName_2 = "Err"
-	_StepStatusName_3 = "SkippedDueToFatalErr"
-	_StepStatusName_4 = "SkippedDueToPreErr"
-)
-
-var (
-	_StepStatusIndex_0 = [...]uint8{0, 7, 11}
-	_StepStatusIndex_1 = [...]uint8{0, 2}
-	_StepStatusIndex_2 = [...]uint8{0, 3}
-	_StepStatusIndex_3 = [...]uint8{0, 20}
-	_StepStatusIndex_4 = [...]uint8{0, 18}
-)
-
-func (i StepStatus) String() string {
-	switch {
-	case 1 <= i && i <= 2:
-		i -= 1
-		return _StepStatusName_0[_StepStatusIndex_0[i]:_StepStatusIndex_0[i+1]]
-	case i == 4:
-		return _StepStatusName_1
-	case i == 8:
-		return _StepStatusName_2
-	case i == 16:
-		return _StepStatusName_3
-	case i == 32:
-		return _StepStatusName_4
-	default:
-		return fmt.Sprintf("StepStatus(%d)", i)
-	}
-}
-
-var _StepStatusValues = []StepStatus{1, 2, 4, 8, 16, 32}
-
-var _StepStatusNameToValueMap = map[string]StepStatus{
-	_StepStatusName_0[0:7]:  1,
-	_StepStatusName_0[7:11]: 2,
-	_StepStatusName_1[0:2]:  4,
-	_StepStatusName_2[0:3]:  8,
-	_StepStatusName_3[0:20]: 16,
-	_StepStatusName_4[0:18]: 32,
-}
-
-// StepStatusString retrieves an enum value from the enum constants string name.
-// Throws an error if the param is not part of the enum.
-func StepStatusString(s string) (StepStatus, error) {
-	if val, ok := _StepStatusNameToValueMap[s]; ok {
-		return val, nil
-	}
-	return 0, fmt.Errorf("%s does not belong to StepStatus values", s)
-}
-
-// StepStatusValues returns all values of the enum
-func StepStatusValues() []StepStatus {
-	return _StepStatusValues
-}
-
-// IsAStepStatus returns "true" if the value is listed in the enum definition. "false" otherwise
-func (i StepStatus) IsAStepStatus() bool {
-	for _, v := range _StepStatusValues {
-		if i == v {
-			return true
-		}
-	}
-	return false
-}

daemon/hooks/test/test-error.sh

@@ -1,5 +0,0 @@
-#!/bin/sh -eu
-
->&2 echo "TEST ERROR $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
-
-exit 1

daemon/hooks/test/test-large-stdout.sh

@@ -1,18 +0,0 @@
-#!/bin/sh -eu
-
-# Exceed default hook log size of 1<<20 bytes by double
-# The Go test should fail if buffer size exceeds 1MB
-
-pre_testing() {
-	ln=0
-	while :; do printf '%06d: 012345678901234567890123456789012345678901234567890123456789012345678901\n' "$ln"; ln="$(($ln + 1))"; done \
-	| head -c$(( 2 << 20 ))
-}
-
-case "$ZREPL_HOOKTYPE" in
-	pre_testing)
-		"$ZREPL_HOOKTYPE";;
-	*)
-		# Not handled by this script
-		exit 0;;
-esac

daemon/hooks/test/test-post-error.sh

@@ -1,15 +0,0 @@
-#!/bin/sh -eu
-
-post_testing() {
-	>&2 echo "TEST ERROR $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
-
-	exit 1
-}
-
-case "$ZREPL_HOOKTYPE" in
-	post_testing)
-		"$ZREPL_HOOKTYPE";;
-	*)
-		# Not handled by this script
-		exit 0;;
-esac

daemon/hooks/test/test-pre-error-post-ok.sh

@@ -1,11 +0,0 @@
-#!/bin/sh -eu
-
-if [ "$ZREPL_HOOKTYPE" = "pre_testing" ]; then
-	>&2 echo "TEST ERROR $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
-	exit 1
-elif [ "$ZREPL_HOOKTYPE" = "post_testing" ]; then
-	echo "TEST $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
-else
-	printf "Unknown hook type: %s" "$ZREPL_HOOKTYPE" 
-	exit 255
-fi

daemon/hooks/test/test-report-env.sh

@@ -1,3 +0,0 @@
-#!/bin/sh -eu
-
-echo "TEST $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME ${ZREPL_DRYRUN:+(dry run)}"

daemon/hooks/test/test-timeout.sh

@@ -1,5 +0,0 @@
-#!/bin/sh -eu
-
-echo "TEST $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME ZREPL_TIMEOUT=$ZREPL_TIMEOUT"
-
-exec sleep $(($ZREPL_TIMEOUT + 1))

daemon/job/active.go

@@ -1,491 +0,0 @@
-package job
-
-import (
-	"context"
-	"fmt"
-	"sync"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/daemon/job/reset"
-	"github.com/zrepl/zrepl/daemon/job/wakeup"
-	"github.com/zrepl/zrepl/daemon/logging"
-	"github.com/zrepl/zrepl/daemon/pruner"
-	"github.com/zrepl/zrepl/daemon/snapper"
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/replication"
-	"github.com/zrepl/zrepl/replication/driver"
-	"github.com/zrepl/zrepl/replication/logic"
-	"github.com/zrepl/zrepl/replication/report"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/transport"
-	"github.com/zrepl/zrepl/transport/fromconfig"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type ActiveSide struct {
-	mode      activeMode
-	name      endpoint.JobID
-	connecter transport.Connecter
-
-	prunerFactory *pruner.PrunerFactory
-
-	promRepStateSecs    *prometheus.HistogramVec // labels: state
-	promPruneSecs       *prometheus.HistogramVec // labels: prune_side
-	promBytesReplicated *prometheus.CounterVec   // labels: filesystem
-
-	tasksMtx sync.Mutex
-	tasks    activeSideTasks
-}
-
-//go:generate enumer -type=ActiveSideState
-type ActiveSideState int
-
-const (
-	ActiveSideReplicating ActiveSideState = 1 << iota
-	ActiveSidePruneSender
-	ActiveSidePruneReceiver
-	ActiveSideDone // also errors
-)
-
-type activeSideTasks struct {
-	state ActiveSideState
-
-	// valid for state ActiveSideReplicating, ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
-	replicationReport driver.ReportFunc
-	replicationCancel context.CancelFunc
-
-	// valid for state ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
-	prunerSender, prunerReceiver *pruner.Pruner
-
-	// valid for state ActiveSidePruneReceiver, ActiveSideDone
-	prunerSenderCancel, prunerReceiverCancel context.CancelFunc
-}
-
-func (a *ActiveSide) updateTasks(u func(*activeSideTasks)) activeSideTasks {
-	a.tasksMtx.Lock()
-	defer a.tasksMtx.Unlock()
-	copy := a.tasks
-	if u == nil {
-		return copy
-	}
-	u(&copy)
-	a.tasks = copy
-	return copy
-}
-
-type activeMode interface {
-	ConnectEndpoints(rpcLoggers rpc.Loggers, connecter transport.Connecter)
-	DisconnectEndpoints()
-	SenderReceiver() (logic.Sender, logic.Receiver)
-	Type() Type
-	PlannerPolicy() logic.PlannerPolicy
-	RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{})
-	SnapperReport() *snapper.Report
-	ResetConnectBackoff()
-}
-
-type modePush struct {
-	setupMtx      sync.Mutex
-	sender        *endpoint.Sender
-	receiver      *rpc.Client
-	senderConfig  *endpoint.SenderConfig
-	plannerPolicy *logic.PlannerPolicy
-	snapper       *snapper.PeriodicOrManual
-}
-
-func (m *modePush) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Connecter) {
-	m.setupMtx.Lock()
-	defer m.setupMtx.Unlock()
-	if m.receiver != nil || m.sender != nil {
-		panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
-	}
-	m.sender = endpoint.NewSender(*m.senderConfig)
-	m.receiver = rpc.NewClient(connecter, loggers)
-}
-
-func (m *modePush) DisconnectEndpoints() {
-	m.setupMtx.Lock()
-	defer m.setupMtx.Unlock()
-	m.receiver.Close()
-	m.sender = nil
-	m.receiver = nil
-}
-
-func (m *modePush) SenderReceiver() (logic.Sender, logic.Receiver) {
-	m.setupMtx.Lock()
-	defer m.setupMtx.Unlock()
-	return m.sender, m.receiver
-}
-
-func (m *modePush) Type() Type { return TypePush }
-
-func (m *modePush) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
-
-func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
-	m.snapper.Run(ctx, wakeUpCommon)
-}
-
-func (m *modePush) SnapperReport() *snapper.Report {
-	return m.snapper.Report()
-}
-
-func (m *modePush) ResetConnectBackoff() {
-	m.setupMtx.Lock()
-	defer m.setupMtx.Unlock()
-	if m.receiver != nil {
-		m.receiver.ResetConnectBackoff()
-	}
-}
-
-func modePushFromConfig(g *config.Global, in *config.PushJob, jobID endpoint.JobID) (*modePush, error) {
-	m := &modePush{}
-
-	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannnot build filesystem filter")
-	}
-
-	m.senderConfig = &endpoint.SenderConfig{
-		FSF:     fsf,
-		Encrypt: &zfs.NilBool{B: in.Send.Encrypted},
-		JobID:   jobID,
-	}
-	m.plannerPolicy = &logic.PlannerPolicy{
-		EncryptedSend: logic.TriFromBool(in.Send.Encrypted),
-	}
-
-	if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
-		return nil, errors.Wrap(err, "cannot build snapper")
-	}
-
-	return m, nil
-}
-
-type modePull struct {
-	setupMtx       sync.Mutex
-	receiver       *endpoint.Receiver
-	receiverConfig endpoint.ReceiverConfig
-	sender         *rpc.Client
-	rootFS         *zfs.DatasetPath
-	plannerPolicy  *logic.PlannerPolicy
-	interval       config.PositiveDurationOrManual
-}
-
-func (m *modePull) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Connecter) {
-	m.setupMtx.Lock()
-	defer m.setupMtx.Unlock()
-	if m.receiver != nil || m.sender != nil {
-		panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
-	}
-	m.receiver = endpoint.NewReceiver(m.receiverConfig)
-	m.sender = rpc.NewClient(connecter, loggers)
-}
-
-func (m *modePull) DisconnectEndpoints() {
-	m.setupMtx.Lock()
-	defer m.setupMtx.Unlock()
-	m.sender.Close()
-	m.sender = nil
-	m.receiver = nil
-}
-
-func (m *modePull) SenderReceiver() (logic.Sender, logic.Receiver) {
-	m.setupMtx.Lock()
-	defer m.setupMtx.Unlock()
-	return m.sender, m.receiver
-}
-
-func (*modePull) Type() Type { return TypePull }
-
-func (m *modePull) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
-
-func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
-	if m.interval.Manual {
-		GetLogger(ctx).Info("manual pull configured, periodic pull disabled")
-		// "waiting for wakeups" is printed in common ActiveSide.do
-		return
-	}
-	t := time.NewTicker(m.interval.Interval)
-	defer t.Stop()
-	for {
-		select {
-		case <-t.C:
-			select {
-			case wakeUpCommon <- struct{}{}:
-			default:
-				GetLogger(ctx).
-					WithField("pull_interval", m.interval).
-					Warn("pull job took longer than pull interval")
-				wakeUpCommon <- struct{}{} // block anyways, to queue up the wakeup
-			}
-		case <-ctx.Done():
-			return
-		}
-	}
-}
-
-func (m *modePull) SnapperReport() *snapper.Report {
-	return nil
-}
-
-func (m *modePull) ResetConnectBackoff() {
-	m.setupMtx.Lock()
-	defer m.setupMtx.Unlock()
-	if m.sender != nil {
-		m.sender.ResetConnectBackoff()
-	}
-}
-
-func modePullFromConfig(g *config.Global, in *config.PullJob, jobID endpoint.JobID) (m *modePull, err error) {
-	m = &modePull{}
-	m.interval = in.Interval
-
-	m.rootFS, err = zfs.NewDatasetPath(in.RootFS)
-	if err != nil {
-		return nil, errors.New("RootFS is not a valid zfs filesystem path")
-	}
-	if m.rootFS.Length() <= 0 {
-		return nil, errors.New("RootFS must not be empty") // duplicates error check of receiver
-	}
-
-	m.plannerPolicy = &logic.PlannerPolicy{
-		EncryptedSend: logic.DontCare,
-	}
-
-	m.receiverConfig = endpoint.ReceiverConfig{
-		JobID:                      jobID,
-		RootWithoutClientComponent: m.rootFS,
-		AppendClientIdentity:       false, // !
-		UpdateLastReceivedHold:     true,
-	}
-	if err := m.receiverConfig.Validate(); err != nil {
-		return nil, errors.Wrap(err, "cannot build receiver config")
-	}
-
-	return m, nil
-}
-
-func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}) (j *ActiveSide, err error) {
-
-	j = &ActiveSide{}
-	j.name, err = endpoint.MakeJobID(in.Name)
-	if err != nil {
-		return nil, errors.Wrap(err, "invalid job name")
-	}
-
-	switch v := configJob.(type) {
-	case *config.PushJob:
-		j.mode, err = modePushFromConfig(g, v, j.name) // shadow
-	case *config.PullJob:
-		j.mode, err = modePullFromConfig(g, v, j.name) // shadow
-	default:
-		panic(fmt.Sprintf("implementation error: unknown job type %T", v))
-	}
-	if err != nil {
-		return nil, err // no wrapping required
-	}
-
-	j.promRepStateSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace:   "zrepl",
-		Subsystem:   "replication",
-		Name:        "state_time",
-		Help:        "seconds spent during replication",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
-	}, []string{"state"})
-	j.promBytesReplicated = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Namespace:   "zrepl",
-		Subsystem:   "replication",
-		Name:        "bytes_replicated",
-		Help:        "number of bytes replicated from sender to receiver per filesystem",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
-	}, []string{"filesystem"})
-
-	j.connecter, err = fromconfig.ConnecterFromConfig(g, in.Connect)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot build client")
-	}
-
-	j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace:   "zrepl",
-		Subsystem:   "pruning",
-		Name:        "time",
-		Help:        "seconds spent in pruner",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
-	}, []string{"prune_side"})
-	j.prunerFactory, err = pruner.NewPrunerFactory(in.Pruning, j.promPruneSecs)
-	if err != nil {
-		return nil, err
-	}
-
-	return j, nil
-}
-
-func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) {
-	registerer.MustRegister(j.promRepStateSecs)
-	registerer.MustRegister(j.promPruneSecs)
-	registerer.MustRegister(j.promBytesReplicated)
-}
-
-func (j *ActiveSide) Name() string { return j.name.String() }
-
-type ActiveSideStatus struct {
-	Replication                    *report.Report
-	PruningSender, PruningReceiver *pruner.Report
-	Snapshotting                   *snapper.Report
-}
-
-func (j *ActiveSide) Status() *Status {
-	tasks := j.updateTasks(nil)
-
-	s := &ActiveSideStatus{}
-	t := j.mode.Type()
-	if tasks.replicationReport != nil {
-		s.Replication = tasks.replicationReport()
-	}
-	if tasks.prunerSender != nil {
-		s.PruningSender = tasks.prunerSender.Report()
-	}
-	if tasks.prunerReceiver != nil {
-		s.PruningReceiver = tasks.prunerReceiver.Report()
-	}
-	s.Snapshotting = j.mode.SnapperReport()
-	return &Status{Type: t, JobSpecific: s}
-}
-
-func (j *ActiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
-	pull, ok := j.mode.(*modePull)
-	if !ok {
-		_ = j.mode.(*modePush) // make sure we didn't introduce a new job type
-		return nil, false
-	}
-	return pull.rootFS.Copy(), true
-}
-
-func (j *ActiveSide) SenderConfig() *endpoint.SenderConfig {
-	push, ok := j.mode.(*modePush)
-	if !ok {
-		_ = j.mode.(*modePull) // make sure we didn't introduce a new job type
-		return nil
-	}
-	return push.senderConfig
-}
-
-func (j *ActiveSide) Run(ctx context.Context) {
-	log := GetLogger(ctx)
-	ctx = logging.WithSubsystemLoggers(ctx, log)
-
-	defer log.Info("job exiting")
-
-	periodicDone := make(chan struct{})
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-	go j.mode.RunPeriodic(ctx, periodicDone)
-
-	invocationCount := 0
-outer:
-	for {
-		log.Info("wait for wakeups")
-		select {
-		case <-ctx.Done():
-			log.WithError(ctx.Err()).Info("context")
-			break outer
-
-		case <-wakeup.Wait(ctx):
-			j.mode.ResetConnectBackoff()
-		case <-periodicDone:
-		}
-		invocationCount++
-		invLog := log.WithField("invocation", invocationCount)
-		j.do(WithLogger(ctx, invLog))
-	}
-}
-
-func (j *ActiveSide) do(ctx context.Context) {
-
-	log := GetLogger(ctx)
-	ctx = logging.WithSubsystemLoggers(ctx, log)
-	loggers := rpc.GetLoggersOrPanic(ctx) // filled by WithSubsystemLoggers
-	j.mode.ConnectEndpoints(loggers, j.connecter)
-	defer j.mode.DisconnectEndpoints()
-
-	// allow cancellation of an invocation (this function)
-	ctx, cancelThisRun := context.WithCancel(ctx)
-	defer cancelThisRun()
-	go func() {
-		select {
-		case <-reset.Wait(ctx):
-			log.Info("reset received, cancelling current invocation")
-			cancelThisRun()
-		case <-ctx.Done():
-		}
-	}()
-
-	sender, receiver := j.mode.SenderReceiver()
-
-	{
-		select {
-		case <-ctx.Done():
-			return
-		default:
-		}
-		ctx, repCancel := context.WithCancel(ctx)
-		var repWait driver.WaitFunc
-		j.updateTasks(func(tasks *activeSideTasks) {
-			// reset it
-			*tasks = activeSideTasks{}
-			tasks.replicationCancel = repCancel
-			tasks.replicationReport, repWait = replication.Do(
-				ctx, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver, j.mode.PlannerPolicy()),
-			)
-			tasks.state = ActiveSideReplicating
-		})
-		log.Info("start replication")
-		repWait(true) // wait blocking
-		repCancel()   // always cancel to free up context resources
-	}
-
-	{
-		select {
-		case <-ctx.Done():
-			return
-		default:
-		}
-		ctx, senderCancel := context.WithCancel(ctx)
-		tasks := j.updateTasks(func(tasks *activeSideTasks) {
-			tasks.prunerSender = j.prunerFactory.BuildSenderPruner(ctx, sender, sender)
-			tasks.prunerSenderCancel = senderCancel
-			tasks.state = ActiveSidePruneSender
-		})
-		log.Info("start pruning sender")
-		tasks.prunerSender.Prune()
-		log.Info("finished pruning sender")
-		senderCancel()
-	}
-	{
-		select {
-		case <-ctx.Done():
-			return
-		default:
-		}
-		ctx, receiverCancel := context.WithCancel(ctx)
-		tasks := j.updateTasks(func(tasks *activeSideTasks) {
-			tasks.prunerReceiver = j.prunerFactory.BuildReceiverPruner(ctx, receiver, sender)
-			tasks.prunerReceiverCancel = receiverCancel
-			tasks.state = ActiveSidePruneReceiver
-		})
-		log.Info("start pruning receiver")
-		tasks.prunerReceiver.Prune()
-		log.Info("finished pruning receiver")
-		receiverCancel()
-	}
-
-	j.updateTasks(func(tasks *activeSideTasks) {
-		tasks.state = ActiveSideDone
-	})
-
-}

daemon/job/activesidestate_enumer.go

@@ -1,67 +0,0 @@
-// Code generated by "enumer -type=ActiveSideState"; DO NOT EDIT.
-
-//
-package job
-
-import (
-	"fmt"
-)
-
-const (
-	_ActiveSideStateName_0 = "ActiveSideReplicatingActiveSidePruneSender"
-	_ActiveSideStateName_1 = "ActiveSidePruneReceiver"
-	_ActiveSideStateName_2 = "ActiveSideDone"
-)
-
-var (
-	_ActiveSideStateIndex_0 = [...]uint8{0, 21, 42}
-	_ActiveSideStateIndex_1 = [...]uint8{0, 23}
-	_ActiveSideStateIndex_2 = [...]uint8{0, 14}
-)
-
-func (i ActiveSideState) String() string {
-	switch {
-	case 1 <= i && i <= 2:
-		i -= 1
-		return _ActiveSideStateName_0[_ActiveSideStateIndex_0[i]:_ActiveSideStateIndex_0[i+1]]
-	case i == 4:
-		return _ActiveSideStateName_1
-	case i == 8:
-		return _ActiveSideStateName_2
-	default:
-		return fmt.Sprintf("ActiveSideState(%d)", i)
-	}
-}
-
-var _ActiveSideStateValues = []ActiveSideState{1, 2, 4, 8}
-
-var _ActiveSideStateNameToValueMap = map[string]ActiveSideState{
-	_ActiveSideStateName_0[0:21]:  1,
-	_ActiveSideStateName_0[21:42]: 2,
-	_ActiveSideStateName_1[0:23]:  4,
-	_ActiveSideStateName_2[0:14]:  8,
-}
-
-// ActiveSideStateString retrieves an enum value from the enum constants string name.
-// Throws an error if the param is not part of the enum.
-func ActiveSideStateString(s string) (ActiveSideState, error) {
-	if val, ok := _ActiveSideStateNameToValueMap[s]; ok {
-		return val, nil
-	}
-	return 0, fmt.Errorf("%s does not belong to ActiveSideState values", s)
-}
-
-// ActiveSideStateValues returns all values of the enum
-func ActiveSideStateValues() []ActiveSideState {
-	return _ActiveSideStateValues
-}
-
-// IsAActiveSideState returns "true" if the value is listed in the enum definition. "false" otherwise
-func (i ActiveSideState) IsAActiveSideState() bool {
-	for _, v := range _ActiveSideStateValues {
-		if i == v {
-			return true
-		}
-	}
-	return false
-}

daemon/job/build_jobs.go

@@ -1,109 +0,0 @@
-package job
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/config"
-)
-
-func JobsFromConfig(c *config.Config) ([]Job, error) {
-	js := make([]Job, len(c.Jobs))
-	for i := range c.Jobs {
-		j, err := buildJob(c.Global, c.Jobs[i])
-		if err != nil {
-			return nil, err
-		}
-		if j == nil || j.Name() == "" {
-			panic(fmt.Sprintf("implementation error: job builder returned nil job type %T", c.Jobs[i].Ret))
-		}
-		js[i] = j
-	}
-
-	// receiving-side root filesystems must not overlap
-	{
-		rfss := make([]string, 0, len(js))
-		for _, j := range js {
-			jrfs, ok := j.OwnedDatasetSubtreeRoot()
-			if !ok {
-				continue
-			}
-			rfss = append(rfss, jrfs.ToString())
-		}
-		if err := validateReceivingSidesDoNotOverlap(rfss); err != nil {
-			return nil, err
-		}
-	}
-
-	return js, nil
-}
-
-func buildJob(c *config.Global, in config.JobEnum) (j Job, err error) {
-	cannotBuildJob := func(e error, name string) (Job, error) {
-		return nil, errors.Wrapf(e, "cannot build job %q", name)
-	}
-	// FIXME prettify this
-	switch v := in.Ret.(type) {
-	case *config.SinkJob:
-		j, err = passiveSideFromConfig(c, &v.PassiveJob, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-	case *config.SourceJob:
-		j, err = passiveSideFromConfig(c, &v.PassiveJob, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-	case *config.SnapJob:
-		j, err = snapJobFromConfig(c, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-	case *config.PushJob:
-		j, err = activeSide(c, &v.ActiveJob, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-	case *config.PullJob:
-		j, err = activeSide(c, &v.ActiveJob, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-	default:
-		panic(fmt.Sprintf("implementation error: unknown job type %T", v))
-	}
-	return j, nil
-
-}
-
-func validateReceivingSidesDoNotOverlap(receivingRootFSs []string) error {
-	if len(receivingRootFSs) == 0 {
-		return nil
-	}
-	rfss := make([]string, len(receivingRootFSs))
-	copy(rfss, receivingRootFSs)
-	sort.Slice(rfss, func(i, j int) bool {
-		return strings.Compare(rfss[i], rfss[j]) == -1
-	})
-	// add tailing slash because of hierarchy-simulation
-	// rootfs/ is not root of rootfs2/
-	for i := 0; i < len(rfss); i++ {
-		rfss[i] += "/"
-	}
-	// idea:
-	//   no path in rfss must be prefix of another
-	//
-	// rfss is now lexicographically sorted, which means that
-	// if i is prefix of j, i < j (in lexicographical order)
-	// thus,
-	// if any i is prefix of i+n (n >= 1), there is overlap
-	for i := 0; i < len(rfss)-1; i++ {
-		if strings.HasPrefix(rfss[i+1], rfss[i]) {
-			return fmt.Errorf("receiving jobs with overlapping root filesystems are forbidden")
-		}
-	}
-	return nil
-}

daemon/job/build_jobs_test.go

@@ -1,110 +0,0 @@
-package job
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/zrepl/zrepl/config"
-)
-
-func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
-	type testCase struct {
-		err   bool
-		input []string
-	}
-	tcs := []testCase{
-		{false, nil},
-		{false, []string{}},
-		{false, []string{""}}, // not our job to determine valid paths
-		{false, []string{"a"}},
-		{false, []string{"some/path"}},
-		{false, []string{"zroot/sink1", "zroot/sink2", "zroot/sink3"}},
-		{false, []string{"zroot/foo", "zroot/foobar"}},
-		{true, []string{"zroot/b", "zroot/b"}},
-		{true, []string{"zroot/foo", "zroot/foo/bar", "zroot/baz"}},
-		{false, []string{"a/x", "b/x"}},
-		{false, []string{"a", "b"}},
-		{true, []string{"a", "a"}},
-		{true, []string{"a/x/y", "a/x"}},
-		{true, []string{"a/x", "a/x/y"}},
-		{true, []string{"a/x", "b/x", "a/x/y"}},
-		{true, []string{"a", "a/b", "a/c", "a/b"}},
-		{true, []string{"a/b", "a/c", "a/b", "a/d", "a/c"}},
-	}
-
-	for _, tc := range tcs {
-		t.Logf("input: %v", tc.input)
-		err := validateReceivingSidesDoNotOverlap(tc.input)
-		if tc.err {
-			assert.Error(t, err)
-		} else {
-			assert.NoError(t, err)
-		}
-	}
-}
-
-func TestJobIDErrorHandling(t *testing.T) {
-	tmpl := `
-jobs:
-- name: %s
-  type: push
-  connect:
-    type: local
-    listener_name: foo
-    client_identity: bar
-  filesystems: {"<": true}
-  snapshotting:
-    type: manual
-  pruning:
-    keep_sender:
-    - type: last_n
-      count: 10
-    keep_receiver:
-    - type: last_n
-      count: 10
-`
-	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
-
-	type Case struct {
-		jobName string
-		valid   bool
-	}
-	cases := []Case{
-		{"validjobname", true},
-		{"valid with spaces", true},
-		{"invalid\twith\ttabs", false},
-		{"invalid#withdelimiter", false},
-		{"invalid@withdelimiter", false},
-		{"withnewline\\nmiddle", false},
-		{"withnewline\\n", false},
-		{"withslash/", false},
-		{"withslash/inthemiddle", false},
-		{"/", false},
-	}
-
-	for i := range cases {
-		t.Run(cases[i].jobName, func(t *testing.T) {
-			c := cases[i]
-
-			conf, err := config.ParseConfigBytes([]byte(fill(c.jobName)))
-			require.NoError(t, err, "not expecting yaml-config to know about job ids")
-			require.NotNil(t, conf)
-			jobs, err := JobsFromConfig(conf)
-
-			if c.valid {
-				assert.NoError(t, err)
-				require.Len(t, jobs, 1)
-				assert.Equal(t, c.jobName, jobs[0].Name())
-			} else {
-				t.Logf("error: %s", err)
-				assert.Error(t, err)
-				assert.Nil(t, jobs)
-			}
-
-		})
-	}
-
-}

daemon/job/job.go

@@ -1,120 +0,0 @@
-package job
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type Logger = logger.Logger
-
-type contextKey int
-
-const (
-	contextKeyLog contextKey = iota
-)
-
-func GetLogger(ctx context.Context) Logger {
-	if l, ok := ctx.Value(contextKeyLog).(Logger); ok {
-		return l
-	}
-	return logger.NewNullLogger()
-}
-
-func WithLogger(ctx context.Context, l Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLog, l)
-}
-
-type Job interface {
-	Name() string
-	Run(ctx context.Context)
-	Status() *Status
-	RegisterMetrics(registerer prometheus.Registerer)
-	// Jobs that return a subtree of the dataset hierarchy
-	// must return the root of that subtree as rfs and ok = true
-	OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool)
-	SenderConfig() *endpoint.SenderConfig
-}
-
-type Type string
-
-const (
-	TypeInternal Type = "internal"
-	TypeSnap     Type = "snap"
-	TypePush     Type = "push"
-	TypeSink     Type = "sink"
-	TypePull     Type = "pull"
-	TypeSource   Type = "source"
-)
-
-type Status struct {
-	Type        Type
-	JobSpecific interface{}
-}
-
-func (s *Status) MarshalJSON() ([]byte, error) {
-	typeJson, err := json.Marshal(s.Type)
-	if err != nil {
-		return nil, err
-	}
-	jobJSON, err := json.Marshal(s.JobSpecific)
-	if err != nil {
-		return nil, err
-	}
-	m := map[string]json.RawMessage{
-		"type":         typeJson,
-		string(s.Type): jobJSON,
-	}
-	return json.Marshal(m)
-}
-
-func (s *Status) UnmarshalJSON(in []byte) (err error) {
-	var m map[string]json.RawMessage
-	if err := json.Unmarshal(in, &m); err != nil {
-		return err
-	}
-	tJSON, ok := m["type"]
-	if !ok {
-		return fmt.Errorf("field 'type' not found")
-	}
-	if err := json.Unmarshal(tJSON, &s.Type); err != nil {
-		return err
-	}
-	key := string(s.Type)
-	jobJSON, ok := m[key]
-	if !ok {
-		return fmt.Errorf("field '%s', not found", key)
-	}
-	switch s.Type {
-	case TypeSnap:
-		var st SnapJobStatus
-		err = json.Unmarshal(jobJSON, &st)
-		s.JobSpecific = &st
-
-	case TypePull:
-		fallthrough
-	case TypePush:
-		var st ActiveSideStatus
-		err = json.Unmarshal(jobJSON, &st)
-		s.JobSpecific = &st
-
-	case TypeSource:
-		fallthrough
-	case TypeSink:
-		var st PassiveStatus
-		err = json.Unmarshal(jobJSON, &st)
-		s.JobSpecific = &st
-
-	case TypeInternal:
-		// internal jobs do not report specifics
-	default:
-		err = fmt.Errorf("unknown job type '%s'", key)
-	}
-	return err
-}

daemon/job/passive.go

@@ -1,225 +0,0 @@
-package job
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/golang/protobuf/proto"
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/daemon/logging"
-	"github.com/zrepl/zrepl/daemon/snapper"
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/transport"
-	"github.com/zrepl/zrepl/transport/fromconfig"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type PassiveSide struct {
-	mode   passiveMode
-	name   endpoint.JobID
-	listen transport.AuthenticatedListenerFactory
-}
-
-type passiveMode interface {
-	Handler() rpc.Handler
-	RunPeriodic(ctx context.Context)
-	SnapperReport() *snapper.Report // may be nil
-	Type() Type
-}
-
-type modeSink struct {
-	receiverConfig endpoint.ReceiverConfig
-}
-
-func (m *modeSink) Type() Type { return TypeSink }
-
-func (m *modeSink) Handler() rpc.Handler {
-	return endpoint.NewReceiver(m.receiverConfig)
-}
-
-func (m *modeSink) RunPeriodic(_ context.Context)  {}
-func (m *modeSink) SnapperReport() *snapper.Report { return nil }
-
-func modeSinkFromConfig(g *config.Global, in *config.SinkJob, jobID endpoint.JobID) (m *modeSink, err error) {
-	m = &modeSink{}
-
-	rootDataset, err := zfs.NewDatasetPath(in.RootFS)
-	if err != nil {
-		return nil, errors.New("root dataset is not a valid zfs filesystem path")
-	}
-
-	m.receiverConfig = endpoint.ReceiverConfig{
-		JobID:                      jobID,
-		RootWithoutClientComponent: rootDataset,
-		AppendClientIdentity:       true, // !
-		UpdateLastReceivedHold:     true,
-	}
-	if err := m.receiverConfig.Validate(); err != nil {
-		return nil, errors.Wrap(err, "cannot build receiver config")
-	}
-
-	return m, nil
-}
-
-type modeSource struct {
-	senderConfig *endpoint.SenderConfig
-	snapper      *snapper.PeriodicOrManual
-}
-
-func modeSourceFromConfig(g *config.Global, in *config.SourceJob, jobID endpoint.JobID) (m *modeSource, err error) {
-	// FIXME exact dedup of modePush
-	m = &modeSource{}
-	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannnot build filesystem filter")
-	}
-	m.senderConfig = &endpoint.SenderConfig{
-		FSF:     fsf,
-		Encrypt: &zfs.NilBool{B: in.Send.Encrypted},
-		JobID:   jobID,
-	}
-
-	if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
-		return nil, errors.Wrap(err, "cannot build snapper")
-	}
-
-	return m, nil
-}
-
-func (m *modeSource) Type() Type { return TypeSource }
-
-func (m *modeSource) Handler() rpc.Handler {
-	return endpoint.NewSender(*m.senderConfig)
-}
-
-func (m *modeSource) RunPeriodic(ctx context.Context) {
-	m.snapper.Run(ctx, nil)
-}
-
-func (m *modeSource) SnapperReport() *snapper.Report {
-	return m.snapper.Report()
-}
-
-func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, configJob interface{}) (s *PassiveSide, err error) {
-
-	s = &PassiveSide{}
-
-	s.name, err = endpoint.MakeJobID(in.Name)
-	if err != nil {
-		return nil, errors.Wrap(err, "invalid job name")
-	}
-
-	switch v := configJob.(type) {
-	case *config.SinkJob:
-		s.mode, err = modeSinkFromConfig(g, v, s.name) // shadow
-	case *config.SourceJob:
-		s.mode, err = modeSourceFromConfig(g, v, s.name) // shadow
-	}
-	if err != nil {
-		return nil, err // no wrapping necessary
-	}
-
-	if s.listen, err = fromconfig.ListenerFactoryFromConfig(g, in.Serve); err != nil {
-		return nil, errors.Wrap(err, "cannot build listener factory")
-	}
-
-	return s, nil
-}
-
-func (j *PassiveSide) Name() string { return j.name.String() }
-
-type PassiveStatus struct {
-	Snapper *snapper.Report
-}
-
-func (s *PassiveSide) Status() *Status {
-	st := &PassiveStatus{
-		Snapper: s.mode.SnapperReport(),
-	}
-	return &Status{Type: s.mode.Type(), JobSpecific: st}
-}
-
-func (j *PassiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
-	sink, ok := j.mode.(*modeSink)
-	if !ok {
-		_ = j.mode.(*modeSource) // make sure we didn't introduce a new job type
-		return nil, false
-	}
-	return sink.receiverConfig.RootWithoutClientComponent.Copy(), true
-}
-
-func (j *PassiveSide) SenderConfig() *endpoint.SenderConfig {
-	source, ok := j.mode.(*modeSource)
-	if !ok {
-		_ = j.mode.(*modeSink) // make sure we didn't introduce a new job type
-		return nil
-	}
-	return source.senderConfig
-}
-
-func (*PassiveSide) RegisterMetrics(registerer prometheus.Registerer) {}
-
-func (j *PassiveSide) Run(ctx context.Context) {
-
-	log := GetLogger(ctx)
-	defer log.Info("job exiting")
-	ctx = logging.WithSubsystemLoggers(ctx, log)
-	{
-		ctx, cancel := context.WithCancel(ctx) // shadowing
-		defer cancel()
-		go j.mode.RunPeriodic(ctx)
-	}
-
-	handler := j.mode.Handler()
-	if handler == nil {
-		panic(fmt.Sprintf("implementation error: j.mode.Handler() returned nil: %#v", j))
-	}
-
-	ctxInterceptor := func(handlerCtx context.Context) context.Context {
-		reqLog := log.WithField(logging.ReqIDField, rpc.MustGetRequestID(handlerCtx))
-		return logging.WithSubsystemLoggers(handlerCtx, reqLog)
-	}
-
-	// ctxInterceptor already injected request loggers (with request id)
-	pre := func(ctx context.Context, method string, req interface{}) {
-		l := endpoint.GetLogger(ctx).WithField("method", method).WithField("reqT", fmt.Sprintf("%T", req))
-		if p, ok := req.(proto.Message); ok {
-			l = l.WithField("req", proto.CompactTextString(p))
-		}
-		ci, ok := ctx.Value(endpoint.ClientIdentityKey).(string)
-		if !ok {
-			// like endpoint, we must assume that rpc.Server sets it
-			panic(method)
-		}
-		l = l.WithField("clientIdentity", ci)
-		l.Debug("incoming")
-	}
-	post := func(ctx context.Context, res interface{}, err error) {
-		l := endpoint.GetLogger(ctx).
-			WithField("responseT", fmt.Sprintf("%T", res)).
-			WithField("errT", fmt.Sprintf("%T", err))
-		if s, ok := res.(fmt.Stringer); ok {
-			l = l.WithField("response", s.String())
-		}
-		if err != nil {
-			l = l.WithError(err)
-		}
-		l.Debug("handler done")
-	}
-
-	rpcLoggers := rpc.GetLoggersOrPanic(ctx) // WithSubsystemLoggers above
-	server := rpc.NewServer(handler, rpcLoggers, ctxInterceptor, pre, post)
-
-	listener, err := j.listen()
-	if err != nil {
-		log.WithError(err).Error("cannot listen")
-		return
-	}
-
-	server.Serve(ctx, listener)
-}

daemon/job/reset/reset.go

@@ -1,35 +0,0 @@
-package reset
-
-import (
-	"context"
-	"errors"
-)
-
-type contextKey int
-
-const contextKeyReset contextKey = iota
-
-func Wait(ctx context.Context) <-chan struct{} {
-	wc, ok := ctx.Value(contextKeyReset).(chan struct{})
-	if !ok {
-		wc = make(chan struct{})
-	}
-	return wc
-}
-
-type Func func() error
-
-var AlreadyReset = errors.New("already reset")
-
-func Context(ctx context.Context) (context.Context, Func) {
-	wc := make(chan struct{})
-	wuf := func() error {
-		select {
-		case wc <- struct{}{}:
-			return nil
-		default:
-			return AlreadyReset
-		}
-	}
-	return context.WithValue(ctx, contextKeyReset, wc), wuf
-}

daemon/job/snapjob.go

@@ -1,176 +0,0 @@
-package job
-
-import (
-	"context"
-	"sort"
-
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/daemon/job/wakeup"
-	"github.com/zrepl/zrepl/daemon/logging"
-	"github.com/zrepl/zrepl/daemon/pruner"
-	"github.com/zrepl/zrepl/daemon/snapper"
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/replication/logic/pdu"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type SnapJob struct {
-	name     endpoint.JobID
-	fsfilter zfs.DatasetFilter
-	snapper  *snapper.PeriodicOrManual
-
-	prunerFactory *pruner.LocalPrunerFactory
-
-	promPruneSecs *prometheus.HistogramVec // labels: prune_side
-
-	pruner *pruner.Pruner
-}
-
-func (j *SnapJob) Name() string { return j.name.String() }
-
-func (j *SnapJob) Type() Type { return TypeSnap }
-
-func snapJobFromConfig(g *config.Global, in *config.SnapJob) (j *SnapJob, err error) {
-	j = &SnapJob{}
-	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot build filesystem filter")
-	}
-	j.fsfilter = fsf
-
-	if j.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
-		return nil, errors.Wrap(err, "cannot build snapper")
-	}
-	j.name, err = endpoint.MakeJobID(in.Name)
-	if err != nil {
-		return nil, errors.Wrap(err, "invalid job name")
-	}
-	j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace:   "zrepl",
-		Subsystem:   "pruning",
-		Name:        "time",
-		Help:        "seconds spent in pruner",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
-	}, []string{"prune_side"})
-	j.prunerFactory, err = pruner.NewLocalPrunerFactory(in.Pruning, j.promPruneSecs)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot build snapjob pruning rules")
-	}
-	return j, nil
-}
-
-func (j *SnapJob) RegisterMetrics(registerer prometheus.Registerer) {
-	registerer.MustRegister(j.promPruneSecs)
-}
-
-type SnapJobStatus struct {
-	Pruning      *pruner.Report
-	Snapshotting *snapper.Report // may be nil
-}
-
-func (j *SnapJob) Status() *Status {
-	s := &SnapJobStatus{}
-	t := j.Type()
-	if j.pruner != nil {
-		s.Pruning = j.pruner.Report()
-	}
-	s.Snapshotting = j.snapper.Report()
-	return &Status{Type: t, JobSpecific: s}
-}
-
-func (j *SnapJob) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
-	return nil, false
-}
-
-func (j *SnapJob) SenderConfig() *endpoint.SenderConfig { return nil }
-
-func (j *SnapJob) Run(ctx context.Context) {
-	log := GetLogger(ctx)
-	ctx = logging.WithSubsystemLoggers(ctx, log)
-
-	defer log.Info("job exiting")
-
-	periodicDone := make(chan struct{})
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-	go j.snapper.Run(ctx, periodicDone)
-
-	invocationCount := 0
-outer:
-	for {
-		log.Info("wait for wakeups")
-		select {
-		case <-ctx.Done():
-			log.WithError(ctx.Err()).Info("context")
-			break outer
-
-		case <-wakeup.Wait(ctx):
-		case <-periodicDone:
-		}
-		invocationCount++
-		invLog := log.WithField("invocation", invocationCount)
-		j.doPrune(WithLogger(ctx, invLog))
-	}
-}
-
-// Adaptor that implements pruner.History around a pruner.Target.
-// The ReplicationCursor method is Get-op only and always returns
-// the filesystem's most recent version's GUID.
-//
-// TODO:
-// This is a work-around for the current package daemon/pruner
-// and package pruning.Snapshot limitation: they require the
-//  `Replicated` getter method be present, but obviously,
-// a local job like SnapJob can't deliver on that.
-// But the pruner.Pruner gives up on an FS if no replication
-// cursor is present, which is why this pruner returns the
-// most recent filesystem version.
-type alwaysUpToDateReplicationCursorHistory struct {
-	// the Target passed as Target to BuildLocalPruner
-	target pruner.Target
-}
-
-var _ pruner.History = (*alwaysUpToDateReplicationCursorHistory)(nil)
-
-func (h alwaysUpToDateReplicationCursorHistory) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
-	fsvReq := &pdu.ListFilesystemVersionsReq{
-		Filesystem: req.GetFilesystem(),
-	}
-	res, err := h.target.ListFilesystemVersions(ctx, fsvReq)
-	if err != nil {
-		return nil, err
-	}
-	fsvs := res.GetVersions()
-	if len(fsvs) <= 0 {
-		return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Notexist{Notexist: true}}, nil
-	}
-	// always return must recent version
-	sort.Slice(fsvs, func(i, j int) bool {
-		return fsvs[i].CreateTXG < fsvs[j].CreateTXG
-	})
-	mostRecent := fsvs[len(fsvs)-1]
-	return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Guid{Guid: mostRecent.GetGuid()}}, nil
-}
-
-func (h alwaysUpToDateReplicationCursorHistory) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error) {
-	return h.target.ListFilesystems(ctx, req)
-}
-
-func (j *SnapJob) doPrune(ctx context.Context) {
-	log := GetLogger(ctx)
-	ctx = logging.WithSubsystemLoggers(ctx, log)
-	sender := endpoint.NewSender(endpoint.SenderConfig{
-		JobID: j.name,
-		FSF:   j.fsfilter,
-		// FIXME encryption setting is irrelevant for SnapJob because the endpoint is only used as pruner.Target
-		Encrypt: &zfs.NilBool{B: true},
-	})
-	j.pruner = j.prunerFactory.BuildLocalPruner(ctx, sender, alwaysUpToDateReplicationCursorHistory{sender})
-	log.Info("start pruning")
-	j.pruner.Prune()
-	log.Info("finished pruning")
-}

daemon/job/wakeup/wakeup.go

@@ -1,35 +0,0 @@
-package wakeup
-
-import (
-	"context"
-	"errors"
-)
-
-type contextKey int
-
-const contextKeyWakeup contextKey = iota
-
-func Wait(ctx context.Context) <-chan struct{} {
-	wc, ok := ctx.Value(contextKeyWakeup).(chan struct{})
-	if !ok {
-		wc = make(chan struct{})
-	}
-	return wc
-}
-
-type Func func() error
-
-var AlreadyWokenUp = errors.New("already woken up")
-
-func Context(ctx context.Context) (context.Context, Func) {
-	wc := make(chan struct{})
-	wuf := func() error {
-		select {
-		case wc <- struct{}{}:
-			return nil
-		default:
-			return AlreadyWokenUp
-		}
-	}
-	return context.WithValue(ctx, contextKeyWakeup, wc), wuf
-}

daemon/logging/build_logging.go

@@ -1,239 +0,0 @@
-package logging
-
-import (
-	"context"
-	"crypto/tls"
-	"crypto/x509"
-	"log/syslog"
-	"os"
-
-	"github.com/mattn/go-isatty"
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/hooks"
-	"github.com/zrepl/zrepl/daemon/pruner"
-	"github.com/zrepl/zrepl/daemon/snapper"
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/replication/driver"
-	"github.com/zrepl/zrepl/replication/logic"
-	"github.com/zrepl/zrepl/rpc"
-	"github.com/zrepl/zrepl/rpc/transportmux"
-	"github.com/zrepl/zrepl/tlsconf"
-	"github.com/zrepl/zrepl/transport"
-)
-
-func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error) {
-
-	outlets := logger.NewOutlets()
-
-	if len(in) == 0 {
-		// Default config
-		out := WriterOutlet{&HumanFormatter{}, os.Stdout}
-		outlets.Add(out, logger.Warn)
-		return outlets, nil
-	}
-
-	var syslogOutlets, stdoutOutlets int
-	for lei, le := range in {
-
-		outlet, minLevel, err := ParseOutlet(le)
-		if err != nil {
-			return nil, errors.Wrapf(err, "cannot parse outlet #%d", lei)
-		}
-		var _ logger.Outlet = WriterOutlet{}
-		var _ logger.Outlet = &SyslogOutlet{}
-		switch outlet.(type) {
-		case *SyslogOutlet:
-			syslogOutlets++
-		case WriterOutlet:
-			stdoutOutlets++
-		}
-
-		outlets.Add(outlet, minLevel)
-
-	}
-
-	if syslogOutlets > 1 {
-		return nil, errors.Errorf("can only define one 'syslog' outlet")
-	}
-	if stdoutOutlets > 1 {
-		return nil, errors.Errorf("can only define one 'stdout' outlet")
-	}
-
-	return outlets, nil
-
-}
-
-type Subsystem string
-
-const (
-	SubsysReplication  Subsystem = "repl"
-	SubsyEndpoint      Subsystem = "endpoint"
-	SubsysPruning      Subsystem = "pruning"
-	SubsysSnapshot     Subsystem = "snapshot"
-	SubsysHooks        Subsystem = "hook"
-	SubsysTransport    Subsystem = "transport"
-	SubsysTransportMux Subsystem = "transportmux"
-	SubsysRPC          Subsystem = "rpc"
-	SubsysRPCControl   Subsystem = "rpc.ctrl"
-	SubsysRPCData      Subsystem = "rpc.data"
-)
-
-func WithSubsystemLoggers(ctx context.Context, log logger.Logger) context.Context {
-	ctx = logic.WithLogger(ctx, log.WithField(SubsysField, SubsysReplication))
-	ctx = driver.WithLogger(ctx, log.WithField(SubsysField, SubsysReplication))
-	ctx = endpoint.WithLogger(ctx, log.WithField(SubsysField, SubsyEndpoint))
-	ctx = pruner.WithLogger(ctx, log.WithField(SubsysField, SubsysPruning))
-	ctx = snapper.WithLogger(ctx, log.WithField(SubsysField, SubsysSnapshot))
-	ctx = hooks.WithLogger(ctx, log.WithField(SubsysField, SubsysHooks))
-	ctx = transport.WithLogger(ctx, log.WithField(SubsysField, SubsysTransport))
-	ctx = transportmux.WithLogger(ctx, log.WithField(SubsysField, SubsysTransportMux))
-	ctx = rpc.WithLoggers(ctx,
-		rpc.Loggers{
-			General: log.WithField(SubsysField, SubsysRPC),
-			Control: log.WithField(SubsysField, SubsysRPCControl),
-			Data:    log.WithField(SubsysField, SubsysRPCData),
-		},
-		// rpc.Loggers{
-		// 	General: logger.NewNullLogger(),
-		// 	Control: logger.NewNullLogger(),
-		// 	Data:    logger.NewNullLogger(),
-		// },
-	)
-	return ctx
-}
-
-func LogSubsystem(log logger.Logger, subsys Subsystem) logger.Logger {
-	return log.ReplaceField(SubsysField, subsys)
-}
-
-func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
-	var is string
-	switch j := i.(type) {
-	case string:
-		is = j
-	default:
-		return nil, errors.Errorf("invalid log format: wrong type: %T", i)
-	}
-
-	switch is {
-	case "human":
-		return &HumanFormatter{}, nil
-	case "logfmt":
-		return &LogfmtFormatter{}, nil
-	case "json":
-		return &JSONFormatter{}, nil
-	default:
-		return nil, errors.Errorf("invalid log format: '%s'", is)
-	}
-
-}
-
-func ParseOutlet(in config.LoggingOutletEnum) (o logger.Outlet, level logger.Level, err error) {
-
-	parseCommon := func(common config.LoggingOutletCommon) (logger.Level, EntryFormatter, error) {
-		if common.Level == "" || common.Format == "" {
-			return 0, nil, errors.Errorf("must specify 'level' and 'format' field")
-		}
-
-		minLevel, err := logger.ParseLevel(common.Level)
-		if err != nil {
-			return 0, nil, errors.Wrap(err, "cannot parse 'level' field")
-		}
-		formatter, err := parseLogFormat(common.Format)
-		if err != nil {
-			return 0, nil, errors.Wrap(err, "cannot parse 'formatter' field")
-		}
-		return minLevel, formatter, nil
-	}
-
-	var f EntryFormatter
-
-	switch v := in.Ret.(type) {
-	case *config.StdoutLoggingOutlet:
-		level, f, err = parseCommon(v.LoggingOutletCommon)
-		if err != nil {
-			break
-		}
-		o, err = parseStdoutOutlet(v, f)
-	case *config.TCPLoggingOutlet:
-		level, f, err = parseCommon(v.LoggingOutletCommon)
-		if err != nil {
-			break
-		}
-		o, err = parseTCPOutlet(v, f)
-	case *config.SyslogLoggingOutlet:
-		level, f, err = parseCommon(v.LoggingOutletCommon)
-		if err != nil {
-			break
-		}
-		o, err = parseSyslogOutlet(v, f)
-	default:
-		panic(v)
-	}
-	return o, level, err
-}
-
-func parseStdoutOutlet(in *config.StdoutLoggingOutlet, formatter EntryFormatter) (WriterOutlet, error) {
-	flags := MetadataAll
-	writer := os.Stdout
-	if !isatty.IsTerminal(writer.Fd()) && !in.Time {
-		flags &= ^MetadataTime
-	}
-	if isatty.IsTerminal(writer.Fd()) && !in.Color {
-		flags &= ^MetadataColor
-	}
-
-	formatter.SetMetadataFlags(flags)
-	return WriterOutlet{
-		formatter,
-		os.Stdout,
-	}, nil
-}
-
-func parseTCPOutlet(in *config.TCPLoggingOutlet, formatter EntryFormatter) (out *TCPOutlet, err error) {
-	var tlsConfig *tls.Config
-	if in.TLS != nil {
-		tlsConfig, err = func(m *config.TCPLoggingOutletTLS, host string) (*tls.Config, error) {
-			clientCert, err := tls.LoadX509KeyPair(m.Cert, m.Key)
-			if err != nil {
-				return nil, errors.Wrap(err, "cannot load client cert")
-			}
-
-			var rootCAs *x509.CertPool
-			if m.CA == "" {
-				if rootCAs, err = x509.SystemCertPool(); err != nil {
-					return nil, errors.Wrap(err, "cannot open system cert pool")
-				}
-			} else {
-				rootCAs, err = tlsconf.ParseCAFile(m.CA)
-				if err != nil {
-					return nil, errors.Wrap(err, "cannot parse CA cert")
-				}
-			}
-			if rootCAs == nil {
-				panic("invariant violated")
-			}
-
-			return tlsconf.ClientAuthClient(host, rootCAs, clientCert)
-		}(in.TLS, in.Address)
-		if err != nil {
-			return nil, errors.New("cannot not parse TLS config in field 'tls'")
-		}
-	}
-
-	formatter.SetMetadataFlags(MetadataAll)
-	return NewTCPOutlet(formatter, in.Net, in.Address, tlsConfig, in.RetryInterval), nil
-
-}
-
-func parseSyslogOutlet(in *config.SyslogLoggingOutlet, formatter EntryFormatter) (out *SyslogOutlet, err error) {
-	out = &SyslogOutlet{}
-	out.Formatter = formatter
-	out.Formatter.SetMetadataFlags(MetadataNone)
-	out.Facility = syslog.Priority(*in.Facility)
-	out.RetryInterval = in.RetryInterval
-	return out, nil
-}

daemon/logging/logging_formatters.go

@@ -1,221 +0,0 @@
-package logging
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"time"
-
-	"github.com/fatih/color"
-	"github.com/go-logfmt/logfmt"
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/logger"
-)
-
-const (
-	FieldLevel   = "level"
-	FieldMessage = "msg"
-	FieldTime    = "time"
-)
-
-const (
-	JobField    string = "job"
-	SubsysField string = "subsystem"
-	ReqIDField  string = "reqid"
-)
-
-type MetadataFlags int64
-
-const (
-	MetadataTime MetadataFlags = 1 << iota
-	MetadataLevel
-	MetadataColor
-
-	MetadataNone MetadataFlags = 0
-	MetadataAll  MetadataFlags = ^0
-)
-
-type NoFormatter struct{}
-
-func (f NoFormatter) SetMetadataFlags(flags MetadataFlags) {}
-
-func (f NoFormatter) Format(e *logger.Entry) ([]byte, error) {
-	return []byte(e.Message), nil
-}
-
-type HumanFormatter struct {
-	metadataFlags MetadataFlags
-	ignoreFields  map[string]bool
-}
-
-const HumanFormatterDateFormat = time.RFC3339
-
-func (f *HumanFormatter) SetMetadataFlags(flags MetadataFlags) {
-	f.metadataFlags = flags
-}
-
-func (f *HumanFormatter) SetIgnoreFields(ignore []string) {
-	if ignore == nil {
-		f.ignoreFields = nil
-		return
-	}
-	f.ignoreFields = make(map[string]bool, len(ignore))
-
-	for _, field := range ignore {
-		f.ignoreFields[field] = true
-	}
-}
-
-func (f *HumanFormatter) ignored(field string) bool {
-	return f.ignoreFields != nil && f.ignoreFields[field]
-}
-
-func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
-
-	var line bytes.Buffer
-	col := color.New()
-	if f.metadataFlags&MetadataColor != 0 {
-		col = e.Color()
-	}
-
-	if f.metadataFlags&MetadataTime != 0 {
-		fmt.Fprintf(&line, "%s ", e.Time.Format(HumanFormatterDateFormat))
-	}
-	if f.metadataFlags&MetadataLevel != 0 {
-		fmt.Fprintf(&line, "[%s]", col.Sprint(e.Level.Short()))
-	}
-
-	prefixFields := []string{JobField, SubsysField, ReqIDField}
-	prefixed := make(map[string]bool, len(prefixFields)+2)
-	for _, field := range prefixFields {
-		val, ok := e.Fields[field]
-		if !ok {
-			continue
-		}
-		if !f.ignored(field) {
-			fmt.Fprintf(&line, "[%s]", col.Sprint(val))
-			prefixed[field] = true
-		}
-	}
-
-	if line.Len() > 0 {
-		fmt.Fprint(&line, ": ")
-	}
-	fmt.Fprint(&line, e.Message)
-
-	if len(e.Fields)-len(prefixed) > 0 {
-		for field, value := range e.Fields {
-			if prefixed[field] || f.ignored(field) {
-				continue
-			}
-			fmt.Fprintf(&line, " %s=%q", col.Sprint(field), fmt.Sprint(value))
-		}
-	}
-
-	return line.Bytes(), nil
-}
-
-type JSONFormatter struct {
-	metadataFlags MetadataFlags
-}
-
-func (f *JSONFormatter) SetMetadataFlags(flags MetadataFlags) {
-	f.metadataFlags = flags
-}
-
-func (f *JSONFormatter) Format(e *logger.Entry) ([]byte, error) {
-	data := make(logger.Fields, len(e.Fields)+3)
-	for k, v := range e.Fields {
-		switch v := v.(type) {
-		case error:
-			// Otherwise errors are ignored by `encoding/json`
-			// https://github.com/sirupsen/logrus/issues/137
-			data[k] = v.Error()
-		default:
-			_, err := json.Marshal(v)
-			if err != nil {
-				return nil, errors.Errorf("field is not JSON encodable: %s", k)
-			}
-			data[k] = v
-		}
-	}
-
-	data[FieldMessage] = e.Message
-	data[FieldTime] = e.Time.Format(time.RFC3339)
-	data[FieldLevel] = e.Level
-
-	return json.Marshal(data)
-
-}
-
-type LogfmtFormatter struct {
-	metadataFlags MetadataFlags
-}
-
-func (f *LogfmtFormatter) SetMetadataFlags(flags MetadataFlags) {
-	f.metadataFlags = flags
-}
-
-func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
-	var buf bytes.Buffer
-	enc := logfmt.NewEncoder(&buf)
-
-	if f.metadataFlags&MetadataTime != 0 {
-		err := enc.EncodeKeyval(FieldTime, e.Time)
-		if err != nil {
-			return nil, errors.Wrap(err, "logfmt: encode time")
-		}
-	}
-	if f.metadataFlags&MetadataLevel != 0 {
-		err := enc.EncodeKeyval(FieldLevel, e.Level)
-		if err != nil {
-			return nil, errors.Wrap(err, "logfmt: encode level")
-		}
-	}
-
-	// at least try and put job and task in front
-	prefixed := make(map[string]bool, 2)
-	prefix := []string{JobField, SubsysField}
-	for _, pf := range prefix {
-		v, ok := e.Fields[pf]
-		if !ok {
-			break
-		}
-		if err := logfmtTryEncodeKeyval(enc, pf, v); err != nil {
-			return nil, err // unlikely
-		}
-		prefixed[pf] = true
-	}
-
-	err := enc.EncodeKeyval(FieldMessage, e.Message)
-	if err != nil {
-		return nil, errors.Wrap(err, "logfmt: encode message")
-	}
-	for k, v := range e.Fields {
-		if !prefixed[k] {
-			if err := logfmtTryEncodeKeyval(enc, k, v); err != nil {
-				return nil, err
-			}
-		}
-	}
-
-	return buf.Bytes(), nil
-}
-
-func logfmtTryEncodeKeyval(enc *logfmt.Encoder, field, value interface{}) error {
-
-	err := enc.EncodeKeyval(field, value)
-	switch err {
-	case nil: // ok
-		return nil
-	case logfmt.ErrUnsupportedValueType:
-		err := enc.EncodeKeyval(field, fmt.Sprintf("<%T>", value))
-		if err != nil {
-			return errors.Wrap(err, "cannot encode unsuuported value type Go type")
-		}
-		return nil
-	}
-	return errors.Wrapf(err, "cannot encode field '%s'", field)
-
-}

daemon/logging/logging_outlets.go

@@ -1,174 +0,0 @@
-package logging
-
-import (
-	"bytes"
-	"context"
-	"crypto/tls"
-	"io"
-	"log/syslog"
-	"net"
-	"time"
-
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/logger"
-)
-
-type EntryFormatter interface {
-	SetMetadataFlags(flags MetadataFlags)
-	Format(e *logger.Entry) ([]byte, error)
-}
-
-type WriterOutlet struct {
-	formatter EntryFormatter
-	writer    io.Writer
-}
-
-func (h WriterOutlet) WriteEntry(entry logger.Entry) error {
-	bytes, err := h.formatter.Format(&entry)
-	if err != nil {
-		return err
-	}
-	_, err = h.writer.Write(bytes)
-	if err != nil {
-		return err
-	}
-	_, err = h.writer.Write([]byte("\n"))
-	return err
-}
-
-type TCPOutlet struct {
-	formatter EntryFormatter
-	// Specifies how much time must pass between a connection error and a reconnection attempt
-	// Log entries written to the outlet during this time interval are silently dropped.
-	connect   func(ctx context.Context) (net.Conn, error)
-	entryChan chan *bytes.Buffer
-}
-
-func NewTCPOutlet(formatter EntryFormatter, network, address string, tlsConfig *tls.Config, retryInterval time.Duration) *TCPOutlet {
-
-	connect := func(ctx context.Context) (conn net.Conn, err error) {
-		deadl, ok := ctx.Deadline()
-		if !ok {
-			deadl = time.Time{}
-		}
-		dialer := net.Dialer{
-			Deadline: deadl,
-		}
-		if tlsConfig != nil {
-			conn, err = tls.DialWithDialer(&dialer, network, address, tlsConfig)
-		} else {
-			conn, err = dialer.DialContext(ctx, network, address)
-		}
-		return
-	}
-
-	entryChan := make(chan *bytes.Buffer, 1) // allow one message in flight while previos is in io.Copy()
-
-	o := &TCPOutlet{
-		formatter: formatter,
-		connect:   connect,
-		entryChan: entryChan,
-	}
-
-	go o.outLoop(retryInterval)
-
-	return o
-}
-
-// FIXME: use this method
-func (h *TCPOutlet) Close() {
-	close(h.entryChan)
-}
-
-func (h *TCPOutlet) outLoop(retryInterval time.Duration) {
-
-	var retry time.Time
-	var conn net.Conn
-	for msg := range h.entryChan {
-		var err error
-		for conn == nil {
-			time.Sleep(time.Until(retry))
-			ctx, cancel := context.WithDeadline(context.TODO(), time.Now().Add(retryInterval))
-			conn, err = h.connect(ctx)
-			cancel()
-			if err != nil {
-				retry = time.Now().Add(retryInterval)
-				conn = nil
-			}
-		}
-		err = conn.SetWriteDeadline(time.Now().Add(retryInterval))
-		if err == nil {
-			_, err = io.Copy(conn, msg)
-		}
-		if err != nil {
-			retry = time.Now().Add(retryInterval)
-			conn.Close()
-			conn = nil
-		}
-	}
-}
-
-func (h *TCPOutlet) WriteEntry(e logger.Entry) error {
-
-	ebytes, err := h.formatter.Format(&e)
-	if err != nil {
-		return err
-	}
-
-	buf := new(bytes.Buffer)
-	buf.Write(ebytes)
-	buf.WriteString("\n")
-
-	select {
-	case h.entryChan <- buf:
-		return nil
-	default:
-		return errors.New("connection broken or not fast enough")
-	}
-}
-
-type SyslogOutlet struct {
-	Formatter          EntryFormatter
-	RetryInterval      time.Duration
-	Facility           syslog.Priority
-	writer             *syslog.Writer
-	lastConnectAttempt time.Time
-}
-
-func (o *SyslogOutlet) WriteEntry(entry logger.Entry) error {
-
-	bytes, err := o.Formatter.Format(&entry)
-	if err != nil {
-		return err
-	}
-
-	s := string(bytes)
-
-	if o.writer == nil {
-		now := time.Now()
-		if now.Sub(o.lastConnectAttempt) < o.RetryInterval {
-			return nil // not an error toward logger
-		}
-		o.writer, err = syslog.New(o.Facility, "zrepl")
-		o.lastConnectAttempt = time.Now()
-		if err != nil {
-			o.writer = nil
-			return err
-		}
-	}
-
-	switch entry.Level {
-	case logger.Debug:
-		return o.writer.Debug(s)
-	case logger.Info:
-		return o.writer.Info(s)
-	case logger.Warn:
-		return o.writer.Warning(s)
-	case logger.Error:
-		return o.writer.Err(s)
-	default:
-		return o.writer.Err(s) // write as error as reaching this case is in fact an error
-	}
-
-}

daemon/main.go

@@ -1,16 +0,0 @@
-package daemon
-
-import (
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/logger"
-)
-
-type Logger = logger.Logger
-
-var DaemonCmd = &cli.Subcommand{
-	Use:   "daemon",
-	Short: "run the zrepl daemon",
-	Run: func(subcommand *cli.Subcommand, args []string) error {
-		return Run(subcommand.Config())
-	},
-}

daemon/nethelpers/helpers.go

@@ -1,50 +0,0 @@
-package nethelpers
-
-import (
-	"net"
-	"os"
-	"path/filepath"
-
-	"github.com/pkg/errors"
-)
-
-func PreparePrivateSockpath(sockpath string) error {
-	sockdir := filepath.Dir(sockpath)
-	sdstat, err := os.Stat(sockdir)
-	if err != nil {
-		return errors.Wrapf(err, "cannot stat(2) '%s'", sockdir)
-	}
-	if !sdstat.IsDir() {
-		return errors.Errorf("not a directory: %s", sockdir)
-	}
-	p := sdstat.Mode().Perm()
-	if p&0007 != 0 {
-		return errors.Errorf("socket directory must not be world-accessible: %s (permissions are %#o)", sockdir, p)
-	}
-
-	// Maybe things have not been cleaned up before
-	s, err := os.Stat(sockpath)
-	if os.IsNotExist(err) {
-		return nil
-	}
-	if err != nil {
-		return errors.Wrapf(err, "cannot stat(2) '%s'", sockpath)
-	}
-	if s.Mode()&os.ModeSocket == 0 {
-		return errors.Errorf("unexpected file type at path '%s'", sockpath)
-	}
-	err = os.Remove(sockpath)
-	if err != nil {
-		return errors.Wrapf(err, "cannot remove presumably stale socket '%s'", sockpath)
-	}
-	return nil
-}
-
-func ListenUnixPrivate(sockaddr *net.UnixAddr) (*net.UnixListener, error) {
-
-	if err := PreparePrivateSockpath(sockaddr.Name); err != nil {
-		return nil, err
-	}
-
-	return net.ListenUnix("unix", sockaddr)
-}

daemon/pprof.go

@@ -1,88 +0,0 @@
-package daemon
-
-import (
-	"net/http"
-	// FIXME: importing this package has the side-effect of poisoning the http.DefaultServeMux
-	// FIXME: with the /debug/pprof endpoints
-	"context"
-	"net"
-	"net/http/pprof"
-
-	"github.com/zrepl/zrepl/daemon/job"
-)
-
-type pprofServer struct {
-	cc       chan PprofServerControlMsg
-	listener net.Listener
-}
-
-type PprofServerControlMsg struct {
-	// Whether the server should listen for requests on the given address
-	Run bool
-	// Must be set if Run is true, undefined otherwise
-	HttpListenAddress string
-}
-
-func NewPProfServer(ctx context.Context) *pprofServer {
-
-	s := &pprofServer{
-		cc: make(chan PprofServerControlMsg),
-	}
-
-	go s.controlLoop(ctx)
-	return s
-}
-
-func (s *pprofServer) controlLoop(ctx context.Context) {
-outer:
-	for {
-
-		var msg PprofServerControlMsg
-		select {
-		case <-ctx.Done():
-			if s.listener != nil {
-				s.listener.Close()
-			}
-			break outer
-		case msg = <-s.cc:
-			// proceed
-		}
-
-		var err error
-		if msg.Run && s.listener == nil {
-
-			s.listener, err = net.Listen("tcp", msg.HttpListenAddress)
-			if err != nil {
-				s.listener = nil
-				continue
-			}
-
-			// FIXME: because net/http/pprof does not provide a mux,
-			mux := http.NewServeMux()
-			mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
-			mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
-			mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
-			mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
-			mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
-			go func() {
-				err := http.Serve(s.listener, mux)
-				if ctx.Err() != nil {
-					return
-				} else if err != nil {
-					job.GetLogger(ctx).WithError(err).Error("pprof server serve error")
-				}
-			}()
-			continue
-		}
-
-		if !msg.Run && s.listener != nil {
-			s.listener.Close()
-			s.listener = nil
-			continue
-		}
-	}
-}
-
-func (s *pprofServer) Control(msg PprofServerControlMsg) {
-	s.cc <- msg
-}

daemon/prometheus.go

@@ -1,101 +0,0 @@
-package daemon
-
-import (
-	"context"
-	"net"
-	"net/http"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/rpc/dataconn/frameconn"
-	"github.com/zrepl/zrepl/util/tcpsock"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type prometheusJob struct {
-	listen   string
-	freeBind bool
-}
-
-func newPrometheusJobFromConfig(in *config.PrometheusMonitoring) (*prometheusJob, error) {
-	if _, _, err := net.SplitHostPort(in.Listen); err != nil {
-		return nil, err
-	}
-	return &prometheusJob{in.Listen, in.ListenFreeBind}, nil
-}
-
-var prom struct {
-	taskLogEntries *prometheus.CounterVec
-}
-
-func init() {
-	prom.taskLogEntries = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "zrepl",
-		Subsystem: "daemon",
-		Name:      "log_entries",
-		Help:      "number of log entries per job task and level",
-	}, []string{"zrepl_job", "level"})
-	prometheus.MustRegister(prom.taskLogEntries)
-}
-
-func (j *prometheusJob) Name() string { return jobNamePrometheus }
-
-func (j *prometheusJob) Status() *job.Status { return &job.Status{Type: job.TypeInternal} }
-
-func (j *prometheusJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
-
-func (j *prometheusJob) SenderConfig() *endpoint.SenderConfig { return nil }
-
-func (j *prometheusJob) RegisterMetrics(registerer prometheus.Registerer) {}
-
-func (j *prometheusJob) Run(ctx context.Context) {
-
-	if err := zfs.PrometheusRegister(prometheus.DefaultRegisterer); err != nil {
-		panic(err)
-	}
-
-	if err := frameconn.PrometheusRegister(prometheus.DefaultRegisterer); err != nil {
-		panic(err)
-	}
-
-	log := job.GetLogger(ctx)
-
-	l, err := tcpsock.Listen(j.listen, j.freeBind)
-	if err != nil {
-		log.WithError(err).Error("cannot listen")
-		return
-	}
-	go func() {
-		<-ctx.Done()
-		l.Close()
-	}()
-
-	mux := http.NewServeMux()
-	mux.Handle("/metrics", promhttp.Handler())
-
-	err = http.Serve(l, mux)
-	if err != nil && ctx.Err() == nil {
-		log.WithError(err).Error("error while serving")
-	}
-
-}
-
-type prometheusJobOutlet struct {
-	jobName string
-}
-
-var _ logger.Outlet = prometheusJobOutlet{}
-
-func newPrometheusLogOutlet(jobName string) prometheusJobOutlet {
-	return prometheusJobOutlet{jobName}
-}
-
-func (o prometheusJobOutlet) WriteEntry(entry logger.Entry) error {
-	prom.taskLogEntries.WithLabelValues(o.jobName, entry.Level.String()).Inc()
-	return nil
-}

daemon/pruner/pruner.go

@@ -1,579 +0,0 @@
-package pruner
-
-import (
-	"context"
-	"fmt"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/pruning"
-	"github.com/zrepl/zrepl/replication/logic/pdu"
-	"github.com/zrepl/zrepl/util/envconst"
-)
-
-// Try to keep it compatible with gitub.com/zrepl/zrepl/endpoint.Endpoint
-type History interface {
-	ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error)
-	ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
-}
-
-// Try to keep it compatible with gitub.com/zrepl/zrepl/endpoint.Endpoint
-type Target interface {
-	ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
-	ListFilesystemVersions(ctx context.Context, req *pdu.ListFilesystemVersionsReq) (*pdu.ListFilesystemVersionsRes, error)
-	DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error)
-}
-
-type Logger = logger.Logger
-
-type contextKey int
-
-const contextKeyLogger contextKey = 0
-
-func WithLogger(ctx context.Context, log Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLogger, log)
-}
-
-func GetLogger(ctx context.Context) Logger {
-	if l, ok := ctx.Value(contextKeyLogger).(Logger); ok {
-		return l
-	}
-	return logger.NewNullLogger()
-}
-
-type args struct {
-	ctx                            context.Context
-	target                         Target
-	receiver                       History
-	rules                          []pruning.KeepRule
-	retryWait                      time.Duration
-	considerSnapAtCursorReplicated bool
-	promPruneSecs                  prometheus.Observer
-}
-
-type Pruner struct {
-	args args
-
-	mtx sync.RWMutex
-
-	state State
-
-	// State PlanErr
-	err error
-
-	// State Exec
-	execQueue *execQueue
-}
-
-type PrunerFactory struct {
-	senderRules                    []pruning.KeepRule
-	receiverRules                  []pruning.KeepRule
-	retryWait                      time.Duration
-	considerSnapAtCursorReplicated bool
-	promPruneSecs                  *prometheus.HistogramVec
-}
-
-type LocalPrunerFactory struct {
-	keepRules     []pruning.KeepRule
-	retryWait     time.Duration
-	promPruneSecs *prometheus.HistogramVec
-}
-
-func NewLocalPrunerFactory(in config.PruningLocal, promPruneSecs *prometheus.HistogramVec) (*LocalPrunerFactory, error) {
-	rules, err := pruning.RulesFromConfig(in.Keep)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot build pruning rules")
-	}
-	for _, r := range in.Keep {
-		if _, ok := r.Ret.(*config.PruneKeepNotReplicated); ok {
-			// rule NotReplicated  for a local pruner doesn't make sense
-			// because no replication happens with that job type
-			return nil, fmt.Errorf("single-site pruner cannot support `not_replicated` keep rule")
-		}
-	}
-	f := &LocalPrunerFactory{
-		keepRules:     rules,
-		retryWait:     envconst.Duration("ZREPL_PRUNER_RETRY_INTERVAL", 10*time.Second),
-		promPruneSecs: promPruneSecs,
-	}
-	return f, nil
-}
-
-func NewPrunerFactory(in config.PruningSenderReceiver, promPruneSecs *prometheus.HistogramVec) (*PrunerFactory, error) {
-	keepRulesReceiver, err := pruning.RulesFromConfig(in.KeepReceiver)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot build receiver pruning rules")
-	}
-
-	keepRulesSender, err := pruning.RulesFromConfig(in.KeepSender)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot build sender pruning rules")
-	}
-
-	considerSnapAtCursorReplicated := false
-	for _, r := range in.KeepSender {
-		knr, ok := r.Ret.(*config.PruneKeepNotReplicated)
-		if !ok {
-			continue
-		}
-		considerSnapAtCursorReplicated = considerSnapAtCursorReplicated || !knr.KeepSnapshotAtCursor
-	}
-	f := &PrunerFactory{
-		senderRules:                    keepRulesSender,
-		receiverRules:                  keepRulesReceiver,
-		retryWait:                      envconst.Duration("ZREPL_PRUNER_RETRY_INTERVAL", 10*time.Second),
-		considerSnapAtCursorReplicated: considerSnapAtCursorReplicated,
-		promPruneSecs:                  promPruneSecs,
-	}
-	return f, nil
-}
-
-func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, receiver History) *Pruner {
-	p := &Pruner{
-		args: args{
-			WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "sender")),
-			target,
-			receiver,
-			f.senderRules,
-			f.retryWait,
-			f.considerSnapAtCursorReplicated,
-			f.promPruneSecs.WithLabelValues("sender"),
-		},
-		state: Plan,
-	}
-	return p
-}
-
-func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target, receiver History) *Pruner {
-	p := &Pruner{
-		args: args{
-			WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "receiver")),
-			target,
-			receiver,
-			f.receiverRules,
-			f.retryWait,
-			false, // senseless here anyways
-			f.promPruneSecs.WithLabelValues("receiver"),
-		},
-		state: Plan,
-	}
-	return p
-}
-
-func (f *LocalPrunerFactory) BuildLocalPruner(ctx context.Context, target Target, receiver History) *Pruner {
-	p := &Pruner{
-		args: args{
-			ctx,
-			target,
-			receiver,
-			f.keepRules,
-			f.retryWait,
-			false, // considerSnapAtCursorReplicated is not relevant for local pruning
-			f.promPruneSecs.WithLabelValues("local"),
-		},
-		state: Plan,
-	}
-	return p
-}
-
-//go:generate enumer -type=State
-type State int
-
-const (
-	Plan State = 1 << iota
-	PlanErr
-	Exec
-	ExecErr
-	Done
-)
-
-type updater func(func(*Pruner))
-
-func (p *Pruner) Prune() {
-	p.prune(p.args)
-}
-
-func (p *Pruner) prune(args args) {
-	u := func(f func(*Pruner)) {
-		p.mtx.Lock()
-		defer p.mtx.Unlock()
-		f(p)
-	}
-	// TODO support automatic retries
-	// It is advisable to merge this code with package replication/driver before
-	// That will likely require re-modelling struct fs like replication/driver.attempt,
-	// including figuring out how to resume a plan after being interrupted by network errors
-	// The non-retrying code in this package should move straight to replication/logic.
-	doOneAttempt(&args, u)
-}
-
-type Report struct {
-	State              string
-	Error              string
-	Pending, Completed []FSReport
-}
-
-type FSReport struct {
-	Filesystem                string
-	SnapshotList, DestroyList []SnapshotReport
-	SkipReason                FSSkipReason
-	LastError                 string
-}
-
-type SnapshotReport struct {
-	Name       string
-	Replicated bool
-	Date       time.Time
-}
-
-func (p *Pruner) Report() *Report {
-	p.mtx.Lock()
-	defer p.mtx.Unlock()
-
-	r := Report{State: p.state.String()}
-
-	if p.err != nil {
-		r.Error = p.err.Error()
-	}
-
-	if p.execQueue != nil {
-		r.Pending, r.Completed = p.execQueue.Report()
-	}
-
-	return &r
-}
-
-func (p *Pruner) State() State {
-	p.mtx.Lock()
-	defer p.mtx.Unlock()
-	return p.state
-}
-
-type fs struct {
-	path string
-
-	// permanent error during planning
-	planErr        error
-	planErrContext string
-
-	// if != "", the fs was skipped for planning and the field
-	// contains the reason
-	skipReason FSSkipReason
-
-	// snapshots presented by target
-	// (type snapshot)
-	snaps []pruning.Snapshot
-	// destroy list returned by pruning.PruneSnapshots(snaps)
-	// (type snapshot)
-	destroyList []pruning.Snapshot
-
-	mtx sync.RWMutex
-
-	// only during Exec state, also used by execQueue
-	execErrLast error
-}
-
-type FSSkipReason string
-
-const (
-	NotSkipped                   = ""
-	SkipPlaceholder              = "filesystem is placeholder"
-	SkipNoCorrespondenceOnSender = "filesystem has no correspondence on sender"
-)
-
-func (r FSSkipReason) NotSkipped() bool {
-	return r == NotSkipped
-}
-
-func (f *fs) Report() FSReport {
-	f.mtx.Lock()
-	defer f.mtx.Unlock()
-
-	r := FSReport{}
-	r.Filesystem = f.path
-	r.SkipReason = f.skipReason
-	if !r.SkipReason.NotSkipped() {
-		return r
-	}
-
-	if f.planErr != nil {
-		r.LastError = f.planErr.Error()
-	} else if f.execErrLast != nil {
-		r.LastError = f.execErrLast.Error()
-	}
-
-	r.SnapshotList = make([]SnapshotReport, len(f.snaps))
-	for i, snap := range f.snaps {
-		r.SnapshotList[i] = snap.(snapshot).Report()
-	}
-
-	r.DestroyList = make([]SnapshotReport, len(f.destroyList))
-	for i, snap := range f.destroyList {
-		r.DestroyList[i] = snap.(snapshot).Report()
-	}
-
-	return r
-}
-
-type snapshot struct {
-	replicated bool
-	date       time.Time
-	fsv        *pdu.FilesystemVersion
-}
-
-func (s snapshot) Report() SnapshotReport {
-	return SnapshotReport{
-		Name:       s.Name(),
-		Replicated: s.Replicated(),
-		Date:       s.Date(),
-	}
-}
-
-var _ pruning.Snapshot = snapshot{}
-
-func (s snapshot) Name() string { return s.fsv.Name }
-
-func (s snapshot) Replicated() bool { return s.replicated }
-
-func (s snapshot) Date() time.Time { return s.date }
-
-func doOneAttempt(a *args, u updater) {
-
-	ctx, target, receiver := a.ctx, a.target, a.receiver
-
-	sfssres, err := receiver.ListFilesystems(ctx, &pdu.ListFilesystemReq{})
-	if err != nil {
-		u(func(p *Pruner) {
-			p.state = PlanErr
-			p.err = err
-		})
-		return
-	}
-	sfss := make(map[string]*pdu.Filesystem)
-	for _, sfs := range sfssres.GetFilesystems() {
-		sfss[sfs.GetPath()] = sfs
-	}
-
-	tfssres, err := target.ListFilesystems(ctx, &pdu.ListFilesystemReq{})
-	if err != nil {
-		u(func(p *Pruner) {
-			p.state = PlanErr
-			p.err = err
-		})
-		return
-	}
-	tfss := tfssres.GetFilesystems()
-
-	pfss := make([]*fs, len(tfss))
-tfss_loop:
-	for i, tfs := range tfss {
-
-		l := GetLogger(ctx).WithField("fs", tfs.Path)
-		l.Debug("plan filesystem")
-
-		pfs := &fs{
-			path: tfs.Path,
-		}
-		pfss[i] = pfs
-
-		if tfs.GetIsPlaceholder() {
-			pfs.skipReason = SkipPlaceholder
-			l.WithField("skip_reason", pfs.skipReason).Debug("skipping filesystem")
-			continue
-		} else if sfs := sfss[tfs.GetPath()]; sfs == nil {
-			pfs.skipReason = SkipNoCorrespondenceOnSender
-			l.WithField("skip_reason", pfs.skipReason).WithField("sfs", sfs.GetPath()).Debug("skipping filesystem")
-			continue
-		}
-
-		pfsPlanErrAndLog := func(err error, message string) {
-			t := fmt.Sprintf("%T", err)
-			pfs.planErr = err
-			pfs.planErrContext = message
-			l.WithField("orig_err_type", t).WithError(err).Error(fmt.Sprintf("%s: plan error, skipping filesystem", message))
-		}
-
-		tfsvsres, err := target.ListFilesystemVersions(ctx, &pdu.ListFilesystemVersionsReq{Filesystem: tfs.Path})
-		if err != nil {
-			pfsPlanErrAndLog(err, "cannot list filesystem versions")
-			continue tfss_loop
-		}
-		tfsvs := tfsvsres.GetVersions()
-		// no progress here since we could run in a live-lock (must have used target AND receiver before progress)
-
-		pfs.snaps = make([]pruning.Snapshot, 0, len(tfsvs))
-
-		rcReq := &pdu.ReplicationCursorReq{
-			Filesystem: tfs.Path,
-		}
-		rc, err := receiver.ReplicationCursor(ctx, rcReq)
-		if err != nil {
-			pfsPlanErrAndLog(err, "cannot get replication cursor bookmark")
-			continue tfss_loop
-		}
-		if rc.GetNotexist() {
-			err := errors.New("replication cursor bookmark does not exist (one successful replication is required before pruning works)")
-			pfsPlanErrAndLog(err, "")
-			continue tfss_loop
-		}
-
-		// scan from older to newer, all snapshots older than cursor are interpreted as replicated
-		sort.Slice(tfsvs, func(i, j int) bool {
-			return tfsvs[i].CreateTXG < tfsvs[j].CreateTXG
-		})
-
-		haveCursorSnapshot := false
-		for _, tfsv := range tfsvs {
-			if tfsv.Type != pdu.FilesystemVersion_Snapshot {
-				continue
-			}
-			if tfsv.Guid == rc.GetGuid() {
-				haveCursorSnapshot = true
-			}
-		}
-		preCursor := haveCursorSnapshot
-		for _, tfsv := range tfsvs {
-			if tfsv.Type != pdu.FilesystemVersion_Snapshot {
-				continue
-			}
-			creation, err := tfsv.CreationAsTime()
-			if err != nil {
-				err := fmt.Errorf("%s: %s", tfsv.RelName(), err)
-				pfsPlanErrAndLog(err, "fs version with invalid creation date")
-				continue tfss_loop
-			}
-			// note that we cannot use CreateTXG because target and receiver could be on different pools
-			atCursor := tfsv.Guid == rc.GetGuid()
-			preCursor = preCursor && !atCursor
-			pfs.snaps = append(pfs.snaps, snapshot{
-				replicated: preCursor || (a.considerSnapAtCursorReplicated && atCursor),
-				date:       creation,
-				fsv:        tfsv,
-			})
-		}
-		if preCursor {
-			pfsPlanErrAndLog(fmt.Errorf("replication cursor not found in prune target filesystem versions"), "")
-			continue tfss_loop
-		}
-
-		// Apply prune rules
-		pfs.destroyList = pruning.PruneSnapshots(pfs.snaps, a.rules)
-	}
-
-	u(func(pruner *Pruner) {
-		pruner.execQueue = newExecQueue(len(pfss))
-		for _, pfs := range pfss {
-			pruner.execQueue.Put(pfs, nil, false)
-		}
-		pruner.state = Exec
-	})
-
-	for {
-		var pfs *fs
-		u(func(pruner *Pruner) {
-			pfs = pruner.execQueue.Pop()
-		})
-		if pfs == nil {
-			break
-		}
-		doOneAttemptExec(a, u, pfs)
-	}
-
-	var rep *Report
-	{
-		// must not hold lock for report
-		var pruner *Pruner
-		u(func(p *Pruner) {
-			pruner = p
-		})
-		rep = pruner.Report()
-	}
-	u(func(p *Pruner) {
-		if len(rep.Pending) > 0 {
-			panic("queue should not have pending items at this point")
-		}
-		hadErr := false
-		for _, fsr := range rep.Completed {
-			hadErr = hadErr || fsr.SkipReason.NotSkipped() && fsr.LastError != ""
-		}
-		if hadErr {
-			p.state = ExecErr
-		} else {
-			p.state = Done
-		}
-	})
-
-}
-
-// attempts to exec pfs, puts it back into the queue with the result
-func doOneAttemptExec(a *args, u updater, pfs *fs) {
-
-	destroyList := make([]*pdu.FilesystemVersion, len(pfs.destroyList))
-	for i := range destroyList {
-		destroyList[i] = pfs.destroyList[i].(snapshot).fsv
-		GetLogger(a.ctx).
-			WithField("fs", pfs.path).
-			WithField("destroy_snap", destroyList[i].Name).
-			Debug("policy destroys snapshot")
-	}
-	req := pdu.DestroySnapshotsReq{
-		Filesystem: pfs.path,
-		Snapshots:  destroyList,
-	}
-	GetLogger(a.ctx).WithField("fs", pfs.path).Debug("destroying snapshots")
-	res, err := a.target.DestroySnapshots(a.ctx, &req)
-	if err != nil {
-		u(func(pruner *Pruner) {
-			pruner.execQueue.Put(pfs, err, false)
-		})
-		return
-	}
-	// check if all snapshots were destroyed
-	destroyResults := make(map[string]*pdu.DestroySnapshotRes)
-	for _, fsres := range res.Results {
-		destroyResults[fsres.Snapshot.Name] = fsres
-	}
-	err = nil
-	destroyFails := make([]*pdu.DestroySnapshotRes, 0)
-	for _, reqDestroy := range destroyList {
-		res, ok := destroyResults[reqDestroy.Name]
-		if !ok {
-			err = fmt.Errorf("missing destroy-result for %s", reqDestroy.RelName())
-			break
-		} else if res.Error != "" {
-			destroyFails = append(destroyFails, res)
-		}
-	}
-	if err == nil && len(destroyFails) > 0 {
-		names := make([]string, len(destroyFails))
-		pairs := make([]string, len(destroyFails))
-		allSame := true
-		lastMsg := destroyFails[0].Error
-		for i := 0; i < len(destroyFails); i++ {
-			allSame = allSame && destroyFails[i].Error == lastMsg
-			relname := destroyFails[i].Snapshot.RelName()
-			names[i] = relname
-			pairs[i] = fmt.Sprintf("(%s: %s)", relname, destroyFails[i].Error)
-		}
-		if allSame {
-			err = fmt.Errorf("destroys failed %s: %s",
-				strings.Join(names, ", "), lastMsg)
-		} else {
-			err = fmt.Errorf("destroys failed: %s", strings.Join(pairs, ", "))
-		}
-	}
-	u(func(pruner *Pruner) {
-		pruner.execQueue.Put(pfs, err, err == nil)
-	})
-	if err != nil {
-		GetLogger(a.ctx).WithError(err).Error("target could not destroy snapshots")
-		return
-	}
-}

daemon/pruner/pruner_queue.go

@@ -1,82 +0,0 @@
-package pruner
-
-import (
-	"sort"
-	"strings"
-	"sync"
-)
-
-type execQueue struct {
-	mtx                sync.Mutex
-	pending, completed []*fs
-}
-
-func newExecQueue(cap int) *execQueue {
-	q := execQueue{
-		pending:   make([]*fs, 0, cap),
-		completed: make([]*fs, 0, cap),
-	}
-	return &q
-}
-
-func (q *execQueue) Report() (pending, completed []FSReport) {
-	q.mtx.Lock()
-	defer q.mtx.Unlock()
-
-	pending = make([]FSReport, len(q.pending))
-	for i, fs := range q.pending {
-		pending[i] = fs.Report()
-	}
-	completed = make([]FSReport, len(q.completed))
-	for i, fs := range q.completed {
-		completed[i] = fs.Report()
-	}
-
-	return pending, completed
-}
-
-func (q *execQueue) HasCompletedFSWithErrors() bool {
-	q.mtx.Lock()
-	defer q.mtx.Unlock()
-	for _, fs := range q.completed {
-		if fs.execErrLast != nil {
-			return true
-		}
-	}
-	return false
-}
-
-func (q *execQueue) Pop() *fs {
-	if len(q.pending) == 0 {
-		return nil
-	}
-	fs := q.pending[0]
-	q.pending = q.pending[1:]
-	return fs
-}
-
-func (q *execQueue) Put(fs *fs, err error, done bool) {
-	fs.mtx.Lock()
-	fs.execErrLast = err
-	if done || err != nil {
-		fs.mtx.Unlock()
-		q.mtx.Lock()
-		q.completed = append(q.completed, fs)
-		q.mtx.Unlock()
-		return
-	}
-	fs.mtx.Unlock()
-
-	q.mtx.Lock()
-	// inefficient priority q
-	q.pending = append(q.pending, fs)
-	sort.SliceStable(q.pending, func(i, j int) bool {
-		q.pending[i].mtx.Lock()
-		defer q.pending[i].mtx.Unlock()
-		q.pending[j].mtx.Lock()
-		defer q.pending[j].mtx.Unlock()
-		return strings.Compare(q.pending[i].path, q.pending[j].path) == -1
-	})
-	q.mtx.Unlock()
-
-}

daemon/pruner/state_enumer.go

@@ -1,72 +0,0 @@
-// Code generated by "enumer -type=State"; DO NOT EDIT.
-
-//
-package pruner
-
-import (
-	"fmt"
-)
-
-const (
-	_StateName_0 = "PlanPlanErr"
-	_StateName_1 = "Exec"
-	_StateName_2 = "ExecErr"
-	_StateName_3 = "Done"
-)
-
-var (
-	_StateIndex_0 = [...]uint8{0, 4, 11}
-	_StateIndex_1 = [...]uint8{0, 4}
-	_StateIndex_2 = [...]uint8{0, 7}
-	_StateIndex_3 = [...]uint8{0, 4}
-)
-
-func (i State) String() string {
-	switch {
-	case 1 <= i && i <= 2:
-		i -= 1
-		return _StateName_0[_StateIndex_0[i]:_StateIndex_0[i+1]]
-	case i == 4:
-		return _StateName_1
-	case i == 8:
-		return _StateName_2
-	case i == 16:
-		return _StateName_3
-	default:
-		return fmt.Sprintf("State(%d)", i)
-	}
-}
-
-var _StateValues = []State{1, 2, 4, 8, 16}
-
-var _StateNameToValueMap = map[string]State{
-	_StateName_0[0:4]:  1,
-	_StateName_0[4:11]: 2,
-	_StateName_1[0:4]:  4,
-	_StateName_2[0:7]:  8,
-	_StateName_3[0:4]:  16,
-}
-
-// StateString retrieves an enum value from the enum constants string name.
-// Throws an error if the param is not part of the enum.
-func StateString(s string) (State, error) {
-	if val, ok := _StateNameToValueMap[s]; ok {
-		return val, nil
-	}
-	return 0, fmt.Errorf("%s does not belong to State values", s)
-}
-
-// StateValues returns all values of the enum
-func StateValues() []State {
-	return _StateValues
-}
-
-// IsAState returns "true" if the value is listed in the enum definition. "false" otherwise
-func (i State) IsAState() bool {
-	for _, v := range _StateValues {
-		if i == v {
-			return true
-		}
-	}
-	return false
-}

daemon/snapper/snapper.go

@@ -1,510 +0,0 @@
-package snapper
-
-import (
-	"context"
-	"fmt"
-	"sort"
-	"sync"
-	"time"
-
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/daemon/hooks"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/util/envconst"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-//go:generate stringer -type=SnapState
-type SnapState uint
-
-const (
-	SnapPending SnapState = 1 << iota
-	SnapStarted
-	SnapDone
-	SnapError
-)
-
-// All fields protected by Snapper.mtx
-type snapProgress struct {
-	state SnapState
-
-	// SnapStarted, SnapDone, SnapError
-	name     string
-	startAt  time.Time
-	hookPlan *hooks.Plan
-
-	// SnapDone
-	doneAt time.Time
-
-	// SnapErr TODO disambiguate state
-	runResults hooks.PlanReport
-}
-
-type args struct {
-	ctx            context.Context
-	log            Logger
-	prefix         string
-	interval       time.Duration
-	fsf            *filters.DatasetMapFilter
-	snapshotsTaken chan<- struct{}
-	hooks          *hooks.List
-	dryRun         bool
-}
-
-type Snapper struct {
-	args args
-
-	mtx   sync.Mutex
-	state State
-
-	// set in state Plan, used in Waiting
-	lastInvocation time.Time
-
-	// valid for state Snapshotting
-	plan map[*zfs.DatasetPath]*snapProgress
-
-	// valid for state SyncUp and Waiting
-	sleepUntil time.Time
-
-	// valid for state Err
-	err error
-}
-
-//go:generate stringer -type=State
-type State uint
-
-const (
-	SyncUp State = 1 << iota
-	SyncUpErrWait
-	Planning
-	Snapshotting
-	Waiting
-	ErrorWait
-	Stopped
-)
-
-func (s State) sf() state {
-	m := map[State]state{
-		SyncUp:        syncUp,
-		SyncUpErrWait: wait,
-		Planning:      plan,
-		Snapshotting:  snapshot,
-		Waiting:       wait,
-		ErrorWait:     wait,
-		Stopped:       nil,
-	}
-	return m[s]
-}
-
-type updater func(u func(*Snapper)) State
-type state func(a args, u updater) state
-
-type contextKey int
-
-const (
-	contextKeyLog contextKey = 0
-)
-
-type Logger = logger.Logger
-
-func WithLogger(ctx context.Context, log Logger) context.Context {
-	return context.WithValue(ctx, contextKeyLog, log)
-}
-
-func getLogger(ctx context.Context) Logger {
-	if log, ok := ctx.Value(contextKeyLog).(Logger); ok {
-		return log
-	}
-	return logger.NewNullLogger()
-}
-
-func PeriodicFromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in *config.SnapshottingPeriodic) (*Snapper, error) {
-	if in.Prefix == "" {
-		return nil, errors.New("prefix must not be empty")
-	}
-	if in.Interval <= 0 {
-		return nil, errors.New("interval must be positive")
-	}
-
-	hookList, err := hooks.ListFromConfig(&in.Hooks)
-	if err != nil {
-		return nil, errors.Wrap(err, "hook config error")
-	}
-
-	args := args{
-		prefix:   in.Prefix,
-		interval: in.Interval,
-		fsf:      fsf,
-		hooks:    hookList,
-		// ctx and log is set in Run()
-	}
-
-	return &Snapper{state: SyncUp, args: args}, nil
-}
-
-func (s *Snapper) Run(ctx context.Context, snapshotsTaken chan<- struct{}) {
-
-	getLogger(ctx).Debug("start")
-	defer getLogger(ctx).Debug("stop")
-
-	s.args.snapshotsTaken = snapshotsTaken
-	s.args.ctx = ctx
-	s.args.log = getLogger(ctx)
-	s.args.dryRun = false // for future expansion
-
-	u := func(u func(*Snapper)) State {
-		s.mtx.Lock()
-		defer s.mtx.Unlock()
-		if u != nil {
-			u(s)
-		}
-		return s.state
-	}
-
-	var st state = syncUp
-
-	for st != nil {
-		pre := u(nil)
-		st = st(s.args, u)
-		post := u(nil)
-		getLogger(ctx).
-			WithField("transition", fmt.Sprintf("%s=>%s", pre, post)).
-			Debug("state transition")
-
-	}
-
-}
-
-func onErr(err error, u updater) state {
-	return u(func(s *Snapper) {
-		s.err = err
-		preState := s.state
-		switch s.state {
-		case SyncUp:
-			s.state = SyncUpErrWait
-		case Planning:
-			fallthrough
-		case Snapshotting:
-			s.state = ErrorWait
-		}
-		s.args.log.WithError(err).WithField("pre_state", preState).WithField("post_state", s.state).Error("snapshotting error")
-	}).sf()
-}
-
-func onMainCtxDone(ctx context.Context, u updater) state {
-	return u(func(s *Snapper) {
-		s.err = ctx.Err()
-		s.state = Stopped
-	}).sf()
-}
-
-func syncUp(a args, u updater) state {
-	u(func(snapper *Snapper) {
-		snapper.lastInvocation = time.Now()
-	})
-	fss, err := listFSes(a.ctx, a.fsf)
-	if err != nil {
-		return onErr(err, u)
-	}
-	syncPoint, err := findSyncPoint(a.log, fss, a.prefix, a.interval)
-	if err != nil {
-		return onErr(err, u)
-	}
-	u(func(s *Snapper) {
-		s.sleepUntil = syncPoint
-	})
-	t := time.NewTimer(time.Until(syncPoint))
-	defer t.Stop()
-	select {
-	case <-t.C:
-		return u(func(s *Snapper) {
-			s.state = Planning
-		}).sf()
-	case <-a.ctx.Done():
-		return onMainCtxDone(a.ctx, u)
-	}
-}
-
-func plan(a args, u updater) state {
-	u(func(snapper *Snapper) {
-		snapper.lastInvocation = time.Now()
-	})
-	fss, err := listFSes(a.ctx, a.fsf)
-	if err != nil {
-		return onErr(err, u)
-	}
-
-	plan := make(map[*zfs.DatasetPath]*snapProgress, len(fss))
-	for _, fs := range fss {
-		plan[fs] = &snapProgress{state: SnapPending}
-	}
-	return u(func(s *Snapper) {
-		s.state = Snapshotting
-		s.plan = plan
-		s.err = nil
-	}).sf()
-}
-
-func snapshot(a args, u updater) state {
-
-	var plan map[*zfs.DatasetPath]*snapProgress
-	u(func(snapper *Snapper) {
-		plan = snapper.plan
-	})
-
-	hookMatchCount := make(map[hooks.Hook]int, len(*a.hooks))
-	for _, h := range *a.hooks {
-		hookMatchCount[h] = 0
-	}
-
-	anyFsHadErr := false
-	// TODO channel programs -> allow a little jitter?
-	for fs, progress := range plan {
-		suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
-		snapname := fmt.Sprintf("%s%s", a.prefix, suffix)
-
-		l := a.log.
-			WithField("fs", fs.ToString()).
-			WithField("snap", snapname)
-
-		hookEnvExtra := hooks.Env{
-			hooks.EnvFS:       fs.ToString(),
-			hooks.EnvSnapshot: snapname,
-		}
-
-		jobCallback := hooks.NewCallbackHookForFilesystem("snapshot", fs, func(_ context.Context) (err error) {
-			l.Debug("create snapshot")
-			err = zfs.ZFSSnapshot(fs, snapname, false) // TODO propagagte context to ZFSSnapshot
-			if err != nil {
-				l.WithError(err).Error("cannot create snapshot")
-			}
-			return
-		})
-
-		fsHadErr := false
-		var planReport hooks.PlanReport
-		var plan *hooks.Plan
-		{
-			filteredHooks, err := a.hooks.CopyFilteredForFilesystem(fs)
-			if err != nil {
-				l.WithError(err).Error("unexpected filter error")
-				fsHadErr = true
-				goto updateFSState
-			}
-			// account for running hooks
-			for _, h := range filteredHooks {
-				hookMatchCount[h] = hookMatchCount[h] + 1
-			}
-
-			var planErr error
-			plan, planErr = hooks.NewPlan(&filteredHooks, hooks.PhaseSnapshot, jobCallback, hookEnvExtra)
-			if planErr != nil {
-				fsHadErr = true
-				l.WithError(planErr).Error("cannot create job hook plan")
-				goto updateFSState
-			}
-		}
-		u(func(snapper *Snapper) {
-			progress.name = snapname
-			progress.startAt = time.Now()
-			progress.hookPlan = plan
-			progress.state = SnapStarted
-		})
-		{
-			l := hooks.GetLogger(a.ctx).WithField("fs", fs.ToString()).WithField("snap", snapname)
-			l.WithField("report", plan.Report().String()).Debug("begin run job plan")
-			plan.Run(hooks.WithLogger(a.ctx, l), a.dryRun)
-			planReport = plan.Report()
-			fsHadErr = planReport.HadError() // not just fatal errors
-			if fsHadErr {
-				l.WithField("report", planReport.String()).Error("end run job plan with error")
-			} else {
-				l.WithField("report", planReport.String()).Info("end run job plan successful")
-			}
-		}
-
-	updateFSState:
-		anyFsHadErr = anyFsHadErr || fsHadErr
-		u(func(snapper *Snapper) {
-			progress.doneAt = time.Now()
-			progress.state = SnapDone
-			if fsHadErr {
-				progress.state = SnapError
-			}
-			progress.runResults = planReport
-		})
-	}
-
-	select {
-	case a.snapshotsTaken <- struct{}{}:
-	default:
-		if a.snapshotsTaken != nil {
-			a.log.Warn("callback channel is full, discarding snapshot update event")
-		}
-	}
-
-	for h, mc := range hookMatchCount {
-		if mc == 0 {
-			hookIdx := -1
-			for idx, ah := range *a.hooks {
-				if ah == h {
-					hookIdx = idx
-					break
-				}
-			}
-			a.log.WithField("hook", h.String()).WithField("hook_number", hookIdx+1).Warn("hook did not match any snapshotted filesystems")
-		}
-	}
-
-	return u(func(snapper *Snapper) {
-		if anyFsHadErr {
-			snapper.state = ErrorWait
-			snapper.err = errors.New("one or more snapshots could not be created, check logs for details")
-		} else {
-			snapper.state = Waiting
-			snapper.err = nil
-		}
-	}).sf()
-}
-
-func wait(a args, u updater) state {
-	var sleepUntil time.Time
-	u(func(snapper *Snapper) {
-		lastTick := snapper.lastInvocation
-		snapper.sleepUntil = lastTick.Add(a.interval)
-		sleepUntil = snapper.sleepUntil
-		log := a.log.WithField("sleep_until", sleepUntil).WithField("duration", a.interval)
-		logFunc := log.Debug
-		if snapper.state == ErrorWait || snapper.state == SyncUpErrWait {
-			logFunc = log.Error
-		}
-		logFunc("enter wait-state after error")
-	})
-
-	t := time.NewTimer(time.Until(sleepUntil))
-	defer t.Stop()
-
-	select {
-	case <-t.C:
-		return u(func(snapper *Snapper) {
-			snapper.state = Planning
-		}).sf()
-	case <-a.ctx.Done():
-		return onMainCtxDone(a.ctx, u)
-	}
-}
-
-func listFSes(ctx context.Context, mf *filters.DatasetMapFilter) (fss []*zfs.DatasetPath, err error) {
-	return zfs.ZFSListMapping(ctx, mf)
-}
-
-var syncUpWarnNoSnapshotUntilSyncupMinDuration = envconst.Duration("ZREPL_SNAPPER_SYNCUP_WARN_MIN_DURATION", 1*time.Second)
-
-// see docs/snapshotting.rst
-func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval time.Duration) (syncPoint time.Time, err error) {
-
-	const (
-		prioHasVersions int = iota
-		prioNoVersions
-	)
-
-	type snapTime struct {
-		ds   *zfs.DatasetPath
-		prio int // lower is higher
-		time time.Time
-	}
-
-	if len(fss) == 0 {
-		return time.Now(), nil
-	}
-
-	snaptimes := make([]snapTime, 0, len(fss))
-	hardErrs := 0
-
-	now := time.Now()
-
-	log.Debug("examine filesystem state to find sync point")
-	for _, d := range fss {
-		l := log.WithField("fs", d.ToString())
-		syncPoint, err := findSyncPointFSNextOptimalSnapshotTime(l, now, interval, prefix, d)
-		if err == findSyncPointFSNoFilesystemVersionsErr {
-			snaptimes = append(snaptimes, snapTime{
-				ds:   d,
-				prio: prioNoVersions,
-				time: now,
-			})
-		} else if err != nil {
-			hardErrs++
-			l.WithError(err).Error("cannot determine optimal sync point for this filesystem")
-		} else {
-			l.WithField("syncPoint", syncPoint).Debug("found optimal sync point for this filesystem")
-			snaptimes = append(snaptimes, snapTime{
-				ds:   d,
-				prio: prioHasVersions,
-				time: syncPoint,
-			})
-		}
-	}
-
-	if hardErrs == len(fss) {
-		return time.Time{}, fmt.Errorf("hard errors in determining sync point for every matching filesystem")
-	}
-
-	if len(snaptimes) == 0 {
-		panic("implementation error: loop must either inc hardErrs or add result to snaptimes")
-	}
-
-	// sort ascending by (prio,time)
-	// => those filesystems with versions win over those without any
-	sort.Slice(snaptimes, func(i, j int) bool {
-		if snaptimes[i].prio == snaptimes[j].prio {
-			return snaptimes[i].time.Before(snaptimes[j].time)
-		}
-		return snaptimes[i].prio < snaptimes[j].prio
-	})
-
-	winnerSyncPoint := snaptimes[0].time
-	l := log.WithField("syncPoint", winnerSyncPoint.String())
-	l.Info("determined sync point")
-	if winnerSyncPoint.Sub(now) > syncUpWarnNoSnapshotUntilSyncupMinDuration {
-		for _, st := range snaptimes {
-			if st.prio == prioNoVersions {
-				l.WithField("fs", st.ds.ToString()).Warn("filesystem will not be snapshotted until sync point")
-			}
-		}
-	}
-
-	return snaptimes[0].time, nil
-
-}
-
-var findSyncPointFSNoFilesystemVersionsErr = fmt.Errorf("no filesystem versions")
-
-func findSyncPointFSNextOptimalSnapshotTime(l Logger, now time.Time, interval time.Duration, prefix string, d *zfs.DatasetPath) (time.Time, error) {
-
-	fsvs, err := zfs.ZFSListFilesystemVersions(d, filters.NewTypedPrefixFilter(prefix, zfs.Snapshot))
-	if err != nil {
-		return time.Time{}, errors.Wrap(err, "list filesystem versions")
-	}
-	if len(fsvs) <= 0 {
-		return time.Time{}, findSyncPointFSNoFilesystemVersionsErr
-	}
-
-	// Sort versions by creation
-	sort.SliceStable(fsvs, func(i, j int) bool {
-		return fsvs[i].CreateTXG < fsvs[j].CreateTXG
-	})
-
-	latest := fsvs[len(fsvs)-1]
-	l.WithField("creation", latest.Creation).Debug("found latest snapshot")
-
-	since := now.Sub(latest.Creation)
-	if since < 0 {
-		return time.Time{}, fmt.Errorf("snapshot %q is from the future: creation=%q now=%q", latest.ToAbsPath(d), latest.Creation, now)
-	}
-
-	return latest.Creation.Add(interval), nil
-}

daemon/snapper/snapper_all.go

@@ -1,48 +0,0 @@
-package snapper
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/filters"
-)
-
-// FIXME: properly abstract snapshotting:
-//   - split up things that trigger snapshotting from the mechanism
-//     - timer-based trigger (periodic)
-//     - call from control socket (manual)
-//     - mixed modes?
-//   - support a `zrepl snapshot JOBNAME` subcommand for config.SnapshottingManual
-type PeriodicOrManual struct {
-	s *Snapper
-}
-
-func (s *PeriodicOrManual) Run(ctx context.Context, wakeUpCommon chan<- struct{}) {
-	if s.s != nil {
-		s.s.Run(ctx, wakeUpCommon)
-	}
-}
-
-// Returns nil if manual
-func (s *PeriodicOrManual) Report() *Report {
-	if s.s != nil {
-		return s.s.Report()
-	}
-	return nil
-}
-
-func FromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in config.SnapshottingEnum) (*PeriodicOrManual, error) {
-	switch v := in.Ret.(type) {
-	case *config.SnapshottingPeriodic:
-		snapper, err := PeriodicFromConfig(g, fsf, v)
-		if err != nil {
-			return nil, err
-		}
-		return &PeriodicOrManual{snapper}, nil
-	case *config.SnapshottingManual:
-		return &PeriodicOrManual{}, nil
-	default:
-		return nil, fmt.Errorf("unknown snapshotting type %T", v)
-	}
-}

daemon/snapper/snapper_report.go

@@ -1,118 +0,0 @@
-package snapper
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-	"time"
-
-	"github.com/zrepl/zrepl/daemon/hooks"
-)
-
-type Report struct {
-	State State
-	// valid in state SyncUp and Waiting
-	SleepUntil time.Time
-	// valid in state Err
-	Error string
-	// valid in state Snapshotting
-	Progress []*ReportFilesystem
-}
-
-type ReportFilesystem struct {
-	Path  string
-	State SnapState
-
-	// Valid in SnapStarted and later
-	SnapName      string
-	StartAt       time.Time
-	Hooks         string
-	HooksHadError bool
-
-	// Valid in SnapDone | SnapError
-	DoneAt time.Time
-}
-
-func errOrEmptyString(e error) string {
-	if e != nil {
-		return e.Error()
-	}
-	return ""
-}
-
-func (s *Snapper) Report() *Report {
-	s.mtx.Lock()
-	defer s.mtx.Unlock()
-
-	pReps := make([]*ReportFilesystem, 0, len(s.plan))
-	for fs, p := range s.plan {
-		var hooksStr string
-		var hooksHadError bool
-		if p.hookPlan != nil {
-			hr := p.hookPlan.Report()
-			// FIXME: technically this belongs into client
-			// but we can't serialize hooks.Step ATM
-			rightPad := func(str string, length int, pad string) string {
-				if len(str) > length {
-					return str[:length]
-				}
-				return str + strings.Repeat(pad, length-len(str))
-			}
-			hooksHadError = hr.HadError()
-			rows := make([][]string, len(hr))
-			const numCols = 4
-			lens := make([]int, numCols)
-			for i, e := range hr {
-				rows[i] = make([]string, numCols)
-				rows[i][0] = fmt.Sprintf("%d", i+1)
-				rows[i][1] = e.Status.String()
-				runTime := "..."
-				if e.Status != hooks.StepPending {
-					runTime = e.End.Sub(e.Begin).Round(time.Millisecond).String()
-				}
-				rows[i][2] = runTime
-				rows[i][3] = ""
-				if e.Report != nil {
-					rows[i][3] = e.Report.String()
-				}
-				for j, col := range lens {
-					if len(rows[i][j]) > col {
-						lens[j] = len(rows[i][j])
-					}
-				}
-			}
-			rowsFlat := make([]string, len(hr))
-			for i, r := range rows {
-				colsPadded := make([]string, len(r))
-				for j, c := range r[:len(r)-1] {
-					colsPadded[j] = rightPad(c, lens[j], " ")
-				}
-				colsPadded[len(r)-1] = r[len(r)-1]
-				rowsFlat[i] = strings.Join(colsPadded, " ")
-			}
-			hooksStr = strings.Join(rowsFlat, "\n")
-		}
-		pReps = append(pReps, &ReportFilesystem{
-			Path:          fs.ToString(),
-			State:         p.state,
-			SnapName:      p.name,
-			StartAt:       p.startAt,
-			DoneAt:        p.doneAt,
-			Hooks:         hooksStr,
-			HooksHadError: hooksHadError,
-		})
-	}
-
-	sort.Slice(pReps, func(i, j int) bool {
-		return strings.Compare(pReps[i].Path, pReps[j].Path) == -1
-	})
-
-	r := &Report{
-		State:      s.state,
-		SleepUntil: s.sleepUntil,
-		Error:      errOrEmptyString(s.err),
-		Progress:   pReps,
-	}
-
-	return r
-}

daemon/snapper/snapstate_string.go

@@ -1,39 +0,0 @@
-// Code generated by "stringer -type=SnapState"; DO NOT EDIT.
-
-package snapper
-
-import "strconv"
-
-func _() {
-	// An "invalid array index" compiler error signifies that the constant values have changed.
-	// Re-run the stringer command to generate them again.
-	var x [1]struct{}
-	_ = x[SnapPending-1]
-	_ = x[SnapStarted-2]
-	_ = x[SnapDone-4]
-	_ = x[SnapError-8]
-}
-
-const (
-	_SnapState_name_0 = "SnapPendingSnapStarted"
-	_SnapState_name_1 = "SnapDone"
-	_SnapState_name_2 = "SnapError"
-)
-
-var (
-	_SnapState_index_0 = [...]uint8{0, 11, 22}
-)
-
-func (i SnapState) String() string {
-	switch {
-	case 1 <= i && i <= 2:
-		i -= 1
-		return _SnapState_name_0[_SnapState_index_0[i]:_SnapState_index_0[i+1]]
-	case i == 4:
-		return _SnapState_name_1
-	case i == 8:
-		return _SnapState_name_2
-	default:
-		return "SnapState(" + strconv.FormatInt(int64(i), 10) + ")"
-	}
-}