prototype resumable send & recv

.circleci/config.yml

@@ -1,381 +0,0 @@
-version: 2.1
-
-commands:
-  setup-home-local-bin:
-    steps:
-      - run:
-          shell: /bin/bash -euo pipefail
-          command: |
-            mkdir -p "$HOME/.local/bin"
-            line='export PATH="$HOME/.local/bin:$PATH"'
-            if grep "$line" $BASH_ENV >/dev/null; then
-              echo "$line" >> $BASH_ENV
-            fi
-
-  invoke-lazy-sh:
-    parameters:
-      subcommand:
-        type: string
-    steps:
-      - run:
-          environment:
-            TERM: xterm
-          command: ./lazy.sh <<parameters.subcommand>>
-
-  apt-update-and-install-common-deps:
-    steps:
-      - run: sudo apt-get update
-      - run: sudo apt-get install -y gawk make
-      # CircleCI doesn't update its cimg/go images.
-      # So, need to update manually to get up-to-date trust chains.
-      # The need for this was required for cimg/go:1.12, but let's future proof this here and now.
-      - run: sudo apt-get install -y git ca-certificates
-
-  restore-cache-gomod:
-    steps:
-      - restore_cache:
-          key: go-mod-v4-{{ checksum "go.sum" }}
-  save-cache-gomod:
-    steps:
-      - save_cache:
-          key: go-mod-v4-{{ checksum "go.sum" }}
-          paths:
-            - "/go/pkg/mod"
-
-  install-godep:
-    steps:
-      - apt-update-and-install-common-deps
-      - invoke-lazy-sh:
-          subcommand: godep
-
-  install-docdep:
-    steps:
-      - apt-update-and-install-common-deps
-      - run: sudo apt install python3 python3-pip libgirepository1.0-dev
-      - invoke-lazy-sh:
-          subcommand: docdep
-
-  download-and-install-minio-client:
-    steps:
-      - setup-home-local-bin
-      - restore_cache:
-          key: minio-client-v2
-      - run:
-          shell: /bin/bash -eo pipefail
-          command: |
-            if which mc; then exit 0; fi
-            sudo curl -sSL https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2020-08-20T00-23-01Z \
-              -o "$HOME/.local/bin/mc"
-            sudo chmod +x "$HOME/.local/bin/mc"
-      - save_cache:
-          key: minio-client-v2
-          paths:
-            - "$HOME/.local/bin/mc"
-
-  upload-minio:
-    parameters:
-      src:
-        type: string
-      dst:
-        type: string
-    steps:
-      - run:
-          shell: /bin/bash -eo pipefail
-          when: always
-          command: |
-            if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
-              echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
-              exit 0
-            fi
-            set -u # from now on
-
-            mc config host add --api s3v4 zrepl-minio https://minio.cschwarz.com ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
-
-            # keep in sync with set-github-minio-status
-            jobprefix=zrepl-ci-artifacts/${CIRCLE_SHA1}-pipeline-<<pipeline.number>>/${CIRCLE_JOB}
-
-            # Upload artifacts
-            mkdir -p ./artifacts
-            mc cp -r <<parameters.src>> "zrepl-minio/$jobprefix/<<parameters.dst>>"
-
-  set-github-minio-status:
-    parameters:
-      context:
-        type: string
-      description:
-        type: string
-      minio-dst:
-        type: string
-    steps:
-      - run:
-          shell: /bin/bash -eo pipefail
-          command: |
-            if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
-              echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
-              exit 0
-            fi
-            set -u # from now on
-
-            # keep in sync with with upload-minio command
-            jobprefix=zrepl-ci-artifacts/${CIRCLE_SHA1}-pipeline-<<pipeline.number>>/${CIRCLE_JOB}
-            # Push Artifact Link to GitHub
-            REPO="zrepl/zrepl"
-            COMMIT="${CIRCLE_SHA1}"
-            JOB_NAME="${CIRCLE_JOB}"
-            CONTEXT="<<parameters.context>>"
-            DESCRIPTION="<<parameters.description>>"
-            TARGETURL=https://minio.cschwarz.com/minio/"$jobprefix"/"<<parameters.minio-dst>>"
-            curl "https://api.github.com/repos/$REPO/statuses/$COMMIT" \
-              -H "Content-Type: application/json" \
-              -H "Authorization: token $GITHUB_COMMIT_STATUS_TOKEN" \
-              -X POST \
-              -d '{"context":"'"$CONTEXT"'", "state": "success", "description":"'"$DESCRIPTION"'", "target_url":"'"$TARGETURL"'"}'
-
-
-  trigger-pipeline:
-    parameters:
-      body_no_shell_subst:
-        type: string
-    steps:
-      - run: |
-          curl -X POST https://circleci.com/api/v2/project/github/zrepl/zrepl/pipeline \
-            -H 'Content-Type: application/json' \
-            -H 'Accept: application/json' \
-            -H "Circle-Token: $ZREPL_BOT_CIRCLE_TOKEN" \
-            --data '<<parameters.body_no_shell_subst>>'
-
-parameters:
-  do_ci:
-    type: boolean
-    default: true
-
-  do_release:
-    type: boolean
-    default: false
-
-  release_docker_baseimage_tag:
-    type: string
-    default: "1.19"
-
-workflows:
-  version: 2
-
-  ci:
-    when: << pipeline.parameters.do_ci >>
-    jobs:
-      - quickcheck-docs
-      - quickcheck-go: &quickcheck-go-smoketest
-          name: quickcheck-go-amd64-linux-1.19
-          goversion: &latest-go-release "1.19"
-          goos: linux
-          goarch: amd64
-      - test-go-on-latest-go-release:
-          goversion: *latest-go-release
-      - quickcheck-go:
-          requires:
-            - quickcheck-go-amd64-linux-1.19 #quickcheck-go-smoketest.name
-          matrix: &quickcheck-go-matrix
-            alias: quickcheck-go-matrix
-            parameters:
-              goversion: [*latest-go-release, "1.18"]
-              goos: ["linux", "freebsd"]
-              goarch: ["amd64", "arm64"]
-            exclude:
-              # don't re-do quickcheck-go-smoketest
-              - goversion: *latest-go-release
-                goos: linux
-                goarch: amd64
-
-  release:
-    when: << pipeline.parameters.do_release >>
-    jobs:
-      - release-build
-      - release-deb:
-          requires:
-            - release-build
-      - release-rpm:
-          requires:
-            - release-build
-      - release-upload:
-          requires:
-            - release-build
-            - release-deb
-            - release-rpm
-
-  periodic:
-    triggers:
-      - schedule:
-          cron: "00 17 * * *"
-          filters:
-            branches:
-              only:
-                - master
-                - stable
-                - problame/circleci-build
-    jobs:
-      - periodic-full-pipeline-run
-
-  zrepl.github.io:
-    jobs:
-      - publish-zrepl-github-io:
-          filters:
-            branches:
-              only:
-                - stable
-
-jobs:
-  quickcheck-docs:
-    docker:
-      - image: cimg/base:2020.08
-    steps:
-      - checkout
-      - install-docdep
-      - run: make docs
-
-      - download-and-install-minio-client
-      - upload-minio:
-          src: artifacts
-          dst: ""
-      - set-github-minio-status:
-          context: artifacts/${CIRCLE_JOB}
-          description: artifacts of CI job ${CIRCLE_JOB}
-          minio-dst: ""
-
-  quickcheck-go:
-    parameters:
-      goversion:
-        type: string
-      goos:
-        type: string
-      goarch:
-        type: string
-    docker:
-      - image: cimg/go:<<parameters.goversion>>
-    environment:
-      GOOS: <<parameters.goos>>
-      GOARCH: <<parameters.goarch>>
-
-    steps:
-      - checkout
-
-      - install-godep
-
-      - restore-cache-gomod
-      - run: go mod download
-      - run: cd build && go mod download
-      - save-cache-gomod
-
-      - run: make formatcheck
-      - run: make generate-platform-test-list
-      - run: make zrepl-bin test-platform-bin
-      - run: make vet
-      - run: make lint
-
-      - download-and-install-minio-client
-      - run: rm -f artifacts/generate-platform-test-list
-      - store_artifacts:
-          path: artifacts
-      - upload-minio:
-          src: artifacts
-          dst: ""
-      - set-github-minio-status:
-          context: artifacts/${CIRCLE_JOB}
-          description: artifacts of CI job ${CIRCLE_JOB}
-          minio-dst: ""
-
-  test-go-on-latest-go-release:
-    parameters:
-      goversion:
-        type: string
-    docker:
-      - image: cimg/go:<<parameters.goversion>>
-    steps:
-      - checkout
-      - restore-cache-gomod
-      - run: make test-go
-      # don't save-cache-gomod here, test-go doesn't pull all the dependencies
-
-  release-build:
-    machine:
-      image: ubuntu-2004:202201-02
-    steps:
-      - checkout
-      - run: make release-docker RELEASE_DOCKER_BASEIMAGE_TAG=<<pipeline.parameters.release_docker_baseimage_tag>>
-      - persist_to_workspace:
-          root: .
-          paths: [.]
-  release-deb:
-    machine:
-      image: ubuntu-2004:202201-02
-    steps:
-      - attach_workspace:
-          at: .
-      - run: make debs-docker
-      - persist_to_workspace:
-          root: .
-          paths:
-            - "artifacts/*.deb"
-
-  release-rpm:
-    machine:
-      image: ubuntu-2004:202201-02
-    steps:
-      - attach_workspace:
-          at: .
-      - run: make rpms-docker
-      - persist_to_workspace:
-          root: .
-          paths:
-            - "artifacts/*.rpm"
-
-  release-upload:
-    docker:
-      - image: cimg/base:2020.08
-    steps:
-      - attach_workspace:
-          at: .
-      - store_artifacts:
-          path: artifacts
-      - download-and-install-minio-client
-      - upload-minio:
-          src: artifacts
-          dst: ""
-      - set-github-minio-status:
-          context: artifacts/release
-          description: CI-generated release artifacts
-          minio-dst: ""
-
-  periodic-full-pipeline-run:
-    docker:
-      - image: cimg/base:2020.08
-    steps:
-      - trigger-pipeline:
-          body_no_shell_subst: '{"branch":"<<pipeline.git.branch>>", "parameters": { "do_ci": true, "do_release": true }}'
-
-  publish-zrepl-github-io:
-    docker:
-      - image: cimg/python:3.7
-    steps:
-      - checkout
-      - invoke-lazy-sh:
-          subcommand: docdep
-      - run:
-          command: |
-            git config --global user.email "zreplbot@cschwarz.com"
-            git config --global user.name "zrepl-github-io-ci"
-
-      # https://circleci.com/docs/2.0/add-ssh-key/#adding-multiple-keys-with-blank-hostnames
-      - run: ssh-add -D
-      # the default circleci ssh config only additional ssh keys for Host !github.com
-      - run:
-          command: |
-            cat > ~/.ssh/config \<<EOF
-            Host *
-              IdentityFile /home/circleci/.ssh/id_rsa_458e62c517f6c480e40452126ce47421
-            EOF
-      - add_ssh_keys:
-          fingerprints:
-            # deploy key for zrepl.github.io
-            - "45:8e:62:c5:17:f6:c4:80:e4:04:52:12:6c:e4:74:21"
-
-      - run: bash -x docs/publish.sh -c -a

.circleci/trigger_debian_binary_packaging_workflow.bash

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-COMMIT="$1"
-GO_VERSION="$2"
-
-curl -v -X POST https://api.github.com/repos/zrepl/debian-binary-packaging/dispatches \
-    -H 'Accept: application/vnd.github.v3+json' \
-    -H "Authorization: token $GITHUB_ACCESS_TOKEN" \
-    --data '{"event_type": "push", "client_payload": { "zrepl_main_repo_commit": "'"$COMMIT"'", "go_version": "'"$GO_VERSION"'" }}'

.github/FUNDING.yml

@@ -1,5 +0,0 @@
-github: problame
-patreon: zrepl
-liberapay: zrepl
-custom: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96
-ko_fi: problame

.gitignore

@@ -1,13 +1,3 @@
-# Build
-artifacts/
-
-# Golang
-vendor/
-
-# IDEs
-.idea/
-.vscode/
-
 # Compiled Object files, Static and Dynamic libs (Shared Objects)
 *.o
 *.a

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "docs/themes/docdock"]
+	path = docs/themes/docdock
+	url = git@github.com:zrepl/hugo-theme-docdock.git
+

.golangci.yml

@@ -1,16 +0,0 @@
-linters:
-  enable:
-  - goimports
-
-issues:
-  exclude-rules:
-    - path: _test\.go
-      linters:
-        - errcheck
-    # Disable staticcheck 'Empty body in an if or else branch' as it's useful
-    # to put a comment into an empty else-clause that explains why whatever
-    # is done in the if-caluse is not necessary if the condition is false.
-    - linters:
-        - staticcheck
-      text: "SA9003:"
-

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2017-2018 Christian Schwarz
+Copyright (c) 2017 Christian Schwarz
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,350 +0,0 @@
-.PHONY: generate build test vet cover release docs docs-clean clean format lint platformtest
-.PHONY: release bins-all release-noarch
-.DEFAULT_GOAL := zrepl-bin
-
-ARTIFACTDIR := artifacts
-
-ifdef ZREPL_VERSION
-    _ZREPL_VERSION := $(ZREPL_VERSION)
-endif
-ifndef _ZREPL_VERSION
-    _ZREPL_VERSION := $(shell git describe --always --dirty 2>/dev/null || echo "ZREPL_BUILD_INVALID_VERSION" )
-    ifeq ($(_ZREPL_VERSION),ZREPL_BUILD_INVALID_VERSION) # can't use .SHELLSTATUS because Debian Stretch is still on gmake 4.1
-        $(error cannot infer variable ZREPL_VERSION using git and variable is not overriden by make invocation)
-    endif
-endif
-
-GO := go
-GOOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOOS"')
-GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"')
-GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"')
-GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"')
-GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
-GO_ENV_VARS := GO111MODULE=on
-GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
-GO_MOD_READONLY := -mod=readonly
-GO_EXTRA_BUILDFLAGS :=
-GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
-GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
-GOLANGCI_LINT := golangci-lint
-GOCOVMERGE := gocovmerge
-RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.19
-RELEASE_DOCKER_BASEIMAGE ?= golang:$(RELEASE_DOCKER_BASEIMAGE_TAG)
-
-ifneq ($(GOARM),)
-	ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM)
-else
-	ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)
-endif
-
-.PHONY: printvars
-printvars:
-	@echo GOOS=$(GOOS)
-	@echo GOARCH=$(GOARCH)
-	@echo GOARM=$(GOARM)
-
-
-##################### PRODUCING A RELEASE #############
-.PHONY: release wrapup-and-checksum check-git-clean sign clean
-
-release: clean
-	# no cross-platform support for target test
-	$(MAKE) test-go
-	$(MAKE) bins-all
-	$(MAKE) noarch
-	$(MAKE) wrapup-and-checksum
-	$(MAKE) check-git-clean
-ifeq (SIGN, 1)
-	$(MAKE) sign
-endif
-	@echo "ZREPL RELEASE ARTIFACTS AVAILABLE IN artifacts/release"
-
-release-docker: $(ARTIFACTDIR)
-	sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > artifacts/release-docker.Dockerfile
-	docker build -t zrepl_release --pull -f artifacts/release-docker.Dockerfile .
-	docker run --rm -i -v $(CURDIR):/src -u $$(id -u):$$(id -g) \
-		zrepl_release \
-		make release GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
-
-debs-docker:
-	$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=deb
-rpms-docker:
-	$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=rpm
-_debs_or_rpms_docker: # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
-	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=amd64
-	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=arm64
-	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=arm GOARM=7
-	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=386
-
-rpm: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
-	$(eval _ZREPL_RPM_VERSION := $(subst -,.,$(_ZREPL_VERSION)))
-	$(eval _ZREPL_RPM_TOPDIR_ABS := $(CURDIR)/$(ARTIFACTDIR)/rpmbuild)
-	rm -rf "$(_ZREPL_RPM_TOPDIR_ABS)"
-	mkdir "$(_ZREPL_RPM_TOPDIR_ABS)"
-	mkdir -p "$(_ZREPL_RPM_TOPDIR_ABS)"/{SPECS,RPMS,BUILD,BUILDROOT}
-	sed "s/^Version:.*/Version:          $(_ZREPL_RPM_VERSION)/g" \
-	    packaging/rpm/zrepl.spec >  $(_ZREPL_RPM_TOPDIR_ABS)/SPECS/zrepl.spec
-
-	# see /usr/lib/rpm/platform
-ifeq ($(GOARCH),amd64)
-	$(eval _ZREPL_RPMBUILD_TARGET := x86_64)
-else ifeq ($(GOARCH), 386)
-	$(eval _ZREPL_RPMBUILD_TARGET := i386)
-else ifeq ($(GOARCH), arm64)
-	$(eval _ZREPL_RPMBUILD_TARGET := aarch64)
-else ifeq ($(GOARCH), arm)
-	$(eval _ZREPL_RPMBUILD_TARGET := armv7hl)
-else
-	$(eval _ZREPL_RPMBUILD_TARGET := $(GOARCH))
-endif
-	rpmbuild \
-		--build-in-place \
-		--define "_sourcedir $(CURDIR)" \
-		--define "_topdir $(_ZREPL_RPM_TOPDIR_ABS)" \
-		--define "_zrepl_binary_filename zrepl-$(ZREPL_TARGET_TUPLE)" \
-		--target $(_ZREPL_RPMBUILD_TARGET) \
-		-bb "$(_ZREPL_RPM_TOPDIR_ABS)"/SPECS/zrepl.spec
-	cp "$(_ZREPL_RPM_TOPDIR_ABS)"/RPMS/$(_ZREPL_RPMBUILD_TARGET)/zrepl-$(_ZREPL_RPM_VERSION)*.rpm $(ARTIFACTDIR)/
-
-rpm-docker:
-	docker build -t zrepl_rpm_pkg --pull -f packaging/rpm/Dockerfile .
-	docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
-		zrepl_rpm_pkg \
-		make rpm GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
-
-deb: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
-
-	cp packaging/deb/debian/changelog.template packaging/deb/debian/changelog
-	sed -i 's/DATE_DASH_R_OUTPUT/$(shell date -R)/' packaging/deb/debian/changelog
-	VERSION="$(subst -,.,$(_ZREPL_VERSION))"; \
-		export VERSION="$${VERSION#v}"; \
-		sed -i 's/VERSION/'"$$VERSION"'/' packaging/deb/debian/changelog
-
-ifeq ($(GOARCH), arm)
-	$(eval  DEB_HOST_ARCH := armhf)
-else ifeq ($(GOARCH), 386)
-	$(eval DEB_HOST_ARCH := i386)
-else
-	$(eval DEB_HOST_ARCH := $(GOARCH))
-endif
-
-	export ZREPL_DPKG_ZREPL_BINARY_FILENAME=zrepl-$(ZREPL_TARGET_TUPLE); \
-		dpkg-buildpackage -b --no-sign --host-arch $(DEB_HOST_ARCH)
-	cp ../*.deb artifacts/
-
-deb-docker:
-	docker build -t zrepl_debian_pkg --pull -f packaging/deb/Dockerfile .
-	# Use a small open file limit to make fakeroot work. If we don't
-	# specify it, docker daemon will use its file limit. I don't know
-	# what changed (Docker, its systemd service, its Go version). But I
-	# observed fakeroot iterating close(i) up to i > 1000000, which costs
-	# a good amount of CPU time and makes the build slow.
-	docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
-		--ulimit nofile=1024:1024 \
-		zrepl_debian_pkg \
-		make deb GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
-
-# expects `release` target to have run before
-NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
-wrapup-and-checksum:
-	rm -f $(NOARCH_TARBALL)
-	tar --mtime='1970-01-01' --sort=name \
-		--transform 's/$(ARTIFACTDIR)/zrepl-$(_ZREPL_VERSION)-noarch/' \
-		--transform 's#dist#zrepl-$(_ZREPL_VERSION)-noarch/dist#' \
-		--transform 's#config/samples#zrepl-$(_ZREPL_VERSION)-noarch/config#' \
-		-acf $(NOARCH_TARBALL) \
-		$(ARTIFACTDIR)/docs/html \
-		$(ARTIFACTDIR)/bash_completion \
-		$(ARTIFACTDIR)/_zrepl.zsh_completion \
-		$(ARTIFACTDIR)/go_env.txt \
-		dist \
-		config/samples
-	rm -rf "$(ARTIFACTDIR)/release"
-	mkdir -p "$(ARTIFACTDIR)/release"
-	cp -l $(ARTIFACTDIR)/zrepl-* \
-		$(ARTIFACTDIR)/platformtest-* \
-		"$(ARTIFACTDIR)/release"
-	cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
-
-check-git-clean:
-	@# note that we use ZREPL_VERSION and not _ZREPL_VERSION because we want to detect the override
-	@if git describe --always --dirty 2>/dev/null | grep dirty >/dev/null; then \
-        echo '[INFO] either git reports checkout is dirty or git is not installed or this is not a git checkout'; \
-		if [ "$(ZREPL_VERSION)" = "" ]; then \
-			echo '[WARN] git checkout is dirty and make variable ZREPL_VERSION was not used to override'; \
-			git status; \
-			echo "git diff:";  \
-			git diff | cat; \
-			exit 1; \
-		fi; \
-	fi;
-
-tag-release:
-	test -n "$(ZREPL_TAG_VERSION)" || exit 1
-	git tag -u E27CA5FC -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)"
-
-sign:
-	gpg -u "89BC 5D89 C845 568B F578  B306 CDBD 8EC8 E27C A5FC" \
-		--armor \
-		--detach-sign $(ARTIFACTDIR)/release/sha512sum.txt
-
-clean: docs-clean
-	rm -rf "$(ARTIFACTDIR)"
-
-##################### BINARIES #####################
-.PHONY: bins-all lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin generate-platform-test-list
-
-BINS_ALL_TARGETS := zrepl-bin test-platform-bin vet lint
-GO_SUPPORTS_ILLUMOS := $(shell $(GO) version | gawk -F '.' '/^go version /{split($$0, comps, " "); split(comps[3], v, "."); if (v[1] == "go1" && v[2] >= 13) { print "illumos"; } else { print "noillumos"; }}')
-bins-all:
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=amd64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=386
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=arm	  GOARM=7
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=arm64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=amd64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm     GOARM=7
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=386
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=darwin    GOARCH=amd64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=solaris   GOARCH=amd64
-ifeq ($(GO_SUPPORTS_ILLUMOS), illumos)
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=illumos   GOARCH=amd64
-else ifeq ($(GO_SUPPORTS_ILLUMOS), noillumos)
-	@echo "SKIPPING ILLUMOS BUILD BECAUSE GO VERSION DOESN'T SUPPORT IT"
-else
-	@echo "CANNOT DETERMINE WHETHER GO VERSION SUPPORTS GOOS=illumos"; exit 1
-endif
-
-lint:
-	$(GO_ENV_VARS) $(GOLANGCI_LINT) run ./...
-
-vet:
-	$(GO_ENV_VARS) $(GO) vet $(GO_BUILDFLAGS) ./...
-
-test-go: $(ARTIFACTDIR)
-	rm -f "$(ARTIFACTDIR)/gotest.cover"
-ifeq ($(COVER),1)
-	$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
-		-coverpkg github.com/zrepl/zrepl/... \
-		-covermode atomic \
-		-coverprofile "$(ARTIFACTDIR)/gotest.cover" \
-		./...
-else
-	$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
-		./...
-endif
-
-zrepl-bin:
-	$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(ZREPL_TARGET_TUPLE)"
-
-generate-platform-test-list:
-	$(GO_BUILD) -o $(ARTIFACTDIR)/generate-platform-test-list ./platformtest/tests/gen
-
-COVER_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-cover-$(ZREPL_TARGET_TUPLE)
-cover-platform-bin:
-	$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
-		-c -o "$(COVER_PLATFORM_BIN_PATH)" \
-		-covermode=atomic -cover -coverpkg github.com/zrepl/zrepl/... \
-		./platformtest/harness
-cover-platform:
-	# do not track dependency on cover-platform-bin to allow build of binary outside of test VM
-	export _TEST_PLATFORM_CMD="$(COVER_PLATFORM_BIN_PATH) \
-		-test.coverprofile \"$(ARTIFACTDIR)/platformtest.cover\" \
-		-test.v \
-		__DEVEL--i-heard-you-like-tests"; \
-		$(MAKE) _test-or-cover-platform-impl
-
-TEST_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)
-test-platform-bin:
-	$(GO_BUILD) -o "$(TEST_PLATFORM_BIN_PATH)" ./platformtest/harness
-test-platform:
-	export _TEST_PLATFORM_CMD="\"$(TEST_PLATFORM_BIN_PATH)\""; \
-		$(MAKE) _test-or-cover-platform-impl
-
-ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
-ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
-ZREPL_PLATFORMTEST_MOUNTPOINT := /tmp/zreplplatformtest.pool
-ZREPL_PLATFORMTEST_ZFS_LOG := /tmp/zreplplatformtest.zfs.log
-# ZREPL_PLATFORMTEST_STOP_AND_KEEP := -failure.stop-and-keep-pool
-ZREPL_PLATFORMTEST_ARGS :=
-_test-or-cover-platform-impl: $(ARTIFACTDIR)
-ifndef _TEST_PLATFORM_CMD
-	$(error _TEST_PLATFORM_CMD is undefined, caller 'cover-platform' or 'test-platform' should have defined it)
-endif
-	rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
-	rm -f "$(ARTIFACTDIR)/platformtest.cover"
-	platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
-		$(_TEST_PLATFORM_CMD) \
-		-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
-		-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
-		-mountpoint "$(ZREPL_PLATFORMTEST_MOUNTPOINT)" \
-		$(ZREPL_PLATFORMTEST_STOP_AND_KEEP) \
-		$(ZREPL_PLATFORMTEST_ARGS)
-
-cover-merge: $(ARTIFACTDIR)
-	$(GOCOVMERGE) $(ARTIFACTDIR)/platformtest.cover $(ARTIFACTDIR)/gotest.cover > $(ARTIFACTDIR)/merged.cover
-cover-html: cover-merge
-	$(GO) tool cover -html "$(ARTIFACTDIR)/merged.cover" -o "$(ARTIFACTDIR)/merged.cover.html"
-
-cover-full:
-	test "$$(id -u)" = "0" || echo "MUST RUN AS ROOT" 1>&2
-	$(MAKE) test-go COVER=1
-	$(MAKE) cover-platform-bin
-	$(MAKE) cover-platform
-	$(MAKE) cover-html
-
-##################### DEV TARGETS #####################
-# not part of the build, must do that manually
-.PHONY: generate formatcheck format
-
-generate: generate-platform-test-list
-	protoc -I=replication/logic/pdu --go_out=replication/logic/pdu --go-grpc_out=replication/logic/pdu replication/logic/pdu/pdu.proto
-	protoc -I=rpc/grpcclientidentity/example --go_out=rpc/grpcclientidentity/example/pdu --go-grpc_out=rpc/grpcclientidentity/example/pdu rpc/grpcclientidentity/example/grpcauth.proto
-	$(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./...
-
-GOIMPORTS := goimports -srcdir . -local 'github.com/zrepl/zrepl'
-FINDSRCFILES := find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go'
-
-formatcheck:
-	@# goimports doesn't have a knob to exit with non-zero status code if formatting is needed
-	@# see https://go-review.googlesource.com/c/tools/+/237378
-	@ affectedfiles=$$($(GOIMPORTS) -l $(shell $(FINDSRCFILES)) | tee /dev/stderr | wc -l); test "$$affectedfiles" = 0
-
-format:
-	@ $(GOIMPORTS) -w -d $(shell  $(FINDSRCFILES))
-
-##################### NOARCH #####################
-.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
-
-
-$(ARTIFACTDIR):
-	mkdir -p "$@"
-$(ARTIFACTDIR)/docs: $(ARTIFACTDIR)
-	mkdir -p "$@"
-
-noarch: $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs
-	# pass
-
-$(ARTIFACTDIR)/bash_completion:
-	$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
-	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion bash "$@"
-
-$(ARTIFACTDIR)/_zrepl.zsh_completion:
-	$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
-	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion zsh "$@"
-
-$(ARTIFACTDIR)/go_env.txt:
-	$(GO_ENV_VARS) $(GO) env > $@
-	$(GO) version >> $@
-
-docs: $(ARTIFACTDIR)/docs
-	# https://www.sphinx-doc.org/en/master/man/sphinx-build.html
-	$(MAKE) -C docs \
-		html \
-		BUILDDIR=../artifacts/docs \
-		SPHINXOPTS="-W --keep-going -n"
-
-docs-clean:
-	$(MAKE) -C docs \
-		clean \
-		BUILDDIR=../artifacts/docs

README.md

@@ -1,135 +1,2 @@
-[![GitHub license](https://img.shields.io/github/license/zrepl/zrepl.svg)](https://github.com/zrepl/zrepl/blob/master/LICENSE)
-[![Language: Go](https://img.shields.io/badge/lang-Go-6ad7e5.svg)](https://golang.org/)
-[![User Docs](https://img.shields.io/badge/docs-web-blue.svg)](https://zrepl.github.io)
-[![Support me on Patreon](https://img.shields.io/badge/dynamic/json?color=yellow&label=Patreon&query=data.attributes.patron_count&url=https%3A%2F%2Fwww.patreon.com%2Fapi%2Fcampaigns%2F3095079)](https://patreon.com/zrepl)
-[![Donate via GitHub Sponsors](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&style=flat&color=yellow)](https://github.com/sponsors/problame)
-[![Donate via Liberapay](https://img.shields.io/liberapay/patrons/zrepl.svg?logo=liberapay)](https://liberapay.com/zrepl/donate)
-[![Donate via PayPal](https://img.shields.io/badge/donate-paypal-yellow.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
-[![Twitter](https://img.shields.io/twitter/url/https/github.com/zrepl/zrepl.svg?style=social)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl)
-[![Chat](https://img.shields.io/badge/chat-matrix-blue.svg)](https://matrix.to/#/#zrepl:matrix.org)
-
 # zrepl
-zrepl is a one-stop ZFS backup & replication solution.
-
-## User Documentation
-
-**User Documentation** can be found at [zrepl.github.io](https://zrepl.github.io).
-
-## Bug Reports
-
-1. If the issue is reproducible, enable debug logging, reproduce and capture the log.
-2. Open an issue on GitHub, with logs pasted as GitHub gists / inline.
-
-## Feature Requests
-
-1. Does your feature request require default values / some kind of configuration?
-   If so, think of an expressive configuration example.
-2. Think of at least one use case that generalizes from your concrete application.
-3. Open an issue on GitHub with example conf & use case attached.
-4. **Optional**: [Post a bounty](https://www.bountysource.com/teams/zrepl) on the issue, or [contact Christian Schwarz](https://cschwarz.com) for contract work.
-
-The above does not apply if you already implemented everything.
-Check out the *Coding Workflow* section below for details.
-
-## Building, Releasing, Downstream-Packaging
-
-This section provides an overview of the zrepl build & release process.
-Check out `docs/installation/compile-from-source.rst` for build-from-source instructions.
-
-### Overview
-
-zrepl is written in [Go](https://golang.org) and uses [Go modules](https://github.com/golang/go/wiki/Modules) to manage dependencies.
-The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
-
-Install **build dependencies** using  `./lazy.sh devsetup`.
-`lazy.sh` uses `python3-pip` to fetch the build dependencies for the docs - you might want to use a [venv](https://docs.python.org/3/library/venv.html).
-If you just want to install the Go dependencies, run `./lazy.sh godep`.
-
-The **test suite** is split into pure **Go tests** (`make test-go`) and **platform tests** that interact with ZFS and thus generally **require root privileges** (`sudo make test-platform`).
-Platform tests run on their own pool with the name `zreplplatformtest`, which is created using the file vdev in `/tmp`.
-
-For a full **code coverage** profile, run `make test-go COVER=1 && sudo make test-platform && make cover-merge`.
-An HTML report can be generated using `make cover-html`.
-
-**Code generation** is triggered by `make generate`. Generated code is committed to the source tree.
-
-### Build & Release Process
-
-**The `Makefile` is catering to the needs of developers & CI, not distro packagers**.
-It provides phony targets for
-* local development (building, running tests, etc)
-* building a release in Docker (used by the CI & release management)
-* building .deb and .rpm packages out of the release artifacts.
-
-**Build tooling & dependencies** are documented as code in `lazy.sh`.
-Go dependencies are then fetched by the go command and pip dependencies are pinned through a `requirements.txt`.
-
-**We use CircleCI for continuous integration**.
-There are two workflows:
-
-* `ci` runs for every commit / branch / tag pushed to GitHub.
-  It is supposed to run very fast (<5min and provides quick feedback to developers).
-  It runs formatting checks, lints and tests on the most important OSes / architectures.
-  Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
-
-* `release` runs
-  * on manual triggers through the CircleCI API (in order to produce a release)
-  * periodically on `master`
-  Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
-
-**Releases** are issued via Git tags + GitHub Releases feature.
-The procedure to issue a release is as follows:
-* Issue the source release:
-  * Git tag the release on the `master` branch.
-  * Push the tag.
-  * Run `./docs/publish.sh` to re-build & push zrepl.github.io.
-* Issue the official binary release:
-  * Run the `release` pipeline (triggered via CircleCI API)
-  * Download the artifacts to the release manager's machine.
-  * Create a GitHub release, edit the changelog, upload all the release artifacts, including .rpm and .deb files.
-  * Issue the GitHub release.
-  * Add the .rpm and .deb files to the official zrepl repos, publish those.
-
-**Official binary releases are not re-built when Go receives an update. If the Go update is critical to zrepl (e.g. a Go security update that affects zrepl), we'd issue a new source release**.
-The rationale for this is that whereas distros provide a mechanism for this (`$zrepl_source_release-$distro_package_revision`), GitHub Releases doesn't which means we'd need to update the existing GitHub release's assets, which nobody would notice (no RSS feed updates, etc.).
-Downstream packagers can read the changelog to determine whether they want to push that minor release into their distro or simply skip it.
-
-### Additional Notes to Distro Package Maintainers
-
-* Run the platform tests (Docs -> Usage -> Platform Tests) **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
-* Ship a default config that adheres to your distro's `hier` and logging system.
-* Ship a service manager file and _please_ try to upstream it to this repository.
-  * `dist/systemd` contains a Systemd unit template.
-* Ship other material provided in `./dist`, e.g. in `/usr/share/zrepl/`.
-* Have a look at the `Makefile`'s `ZREPL_VERSION` variable and how it passed to Go's `ldFlags`.
-  This is how `zrepl version` knows what version number to show.
-  Your build system should set the `ldFlags` flags appropriately and add a prefix or suffix that indicates that the given zrepl binary is a distro build, not an official one.
-* Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
-
-
-## Contributing Code
-
-* Open an issue when starting to hack on a new feature
-* Commits should reference the issue they are related to
-* Docs improvements not documenting new features do not require an issue.
-
-### Breaking Changes
-
-Backward-incompatible changes must be documented in the git commit message and are listed in `docs/changelog.rst`.
-
-### Glossary & Naming Inconsistencies
-
-In ZFS, *dataset* refers to the objects *filesystem*, *ZVOL* and *snapshot*. <br />
-However, we need a word for *filesystem* & *ZVOL* but not a snapshot, bookmark, etc.
-
-Toward the user, the following terminology is used:
-
-* **filesystem**: a ZFS filesystem or a ZVOL
-* **filesystem version**: a ZFS snapshot or a bookmark
-
-Sadly, the zrepl implementation is inconsistent in its use of these words:
-variables and types are often named *dataset* when they in fact refer to a *filesystem*.
-
-There will not be a big refactoring (an attempt was made, but it's destroying too much history without much gain).
-
-However, new contributions & patches should fix naming without further notice in the commit message.
+ZFS dataset replication. Not a shell script.

build.Dockerfile

@@ -1,30 +0,0 @@
-FROM !SUBSTITUTED_BY_MAKEFILE
-
-RUN apt-get update && apt-get install -y \
-    python3-pip \
-    unzip \
-    gawk
-
-ADD build.installprotoc.bash ./
-RUN bash build.installprotoc.bash
-
-ADD lazy.sh /tmp/lazy.sh
-ADD docs/requirements.txt /tmp/requirements.txt
-ENV ZREPL_LAZY_DOCS_REQPATH=/tmp/requirements.txt
-RUN /tmp/lazy.sh docdep
-
-# prepare volume mount of git checkout to /zrepl
-RUN mkdir -p /src/github.com/zrepl/zrepl
-RUN mkdir -p /.cache && chmod -R 0777 /.cache
-
-# $GOPATH is /go
-# Go 1.12 doesn't use modules within GOPATH, but 1.13 and later do
-# => store source outside of GOPATH
-WORKDIR /src
-
-# Install build tools (e.g. protobuf generator, stringer) into $GOPATH/bin
-ADD build/ /tmp/build
-RUN /tmp/lazy.sh godep
-
-RUN chmod -R 0777 /go
-

build.installprotoc.bash

@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-MACH=$(uname -m)
-MACH="${MACH/aarch64/aarch_64}"
-
-VERSION=3.6.1
-FILENAME=protoc-"$VERSION"-linux-"$MACH".zip
-
-if [ -e "$FILENAME" ]; then
-    echo "$FILENAME" already exists 1>&2
-    exit 1
-fi
-
-wget https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
-
-stat "$FILENAME"
-
-sha256sum -c --ignore-missing <<EOF
-6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807  protoc-3.6.1-linux-x86_64.zip
-af8e5aaaf39ddec62ec8dd2be1b8d9602c6da66564883a16393ade5f71170922  protoc-3.6.1-linux-aarch_64.zip
-EOF
-
-unzip -d /usr "$FILENAME"

build/go.mod

@@ -1,13 +0,0 @@
-module github.com/zrepl/zrepl/build
-
-go 1.12
-
-require (
-	github.com/alvaroloes/enumer v1.1.1
-	github.com/golangci/golangci-lint v1.50.1
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad
-	golang.org/x/tools v0.2.0
-	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 // indirect
-	google.golang.org/protobuf v1.28.0
-)

build/tools.go

@@ -1,15 +0,0 @@
-//go:build tools
-// +build tools
-
-package main
-
-// the lines are parsed by lazy.sh, do not edit
-import (
-	_ "github.com/alvaroloes/enumer"
-	_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
-	_ "github.com/wadey/gocovmerge"
-	_ "golang.org/x/tools/cmd/goimports"
-	_ "golang.org/x/tools/cmd/stringer"
-	_ "google.golang.org/grpc/cmd/protoc-gen-go-grpc"
-	_ "google.golang.org/protobuf/cmd/protoc-gen-go"
-)

cli/cli.go

@@ -1,155 +0,0 @@
-package cli
-
-import (
-	"context"
-	"fmt"
-	"os"
-
-	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
-
-	"github.com/zrepl/zrepl/daemon/logging/trace"
-
-	"github.com/zrepl/zrepl/config"
-)
-
-var rootArgs struct {
-	configPath string
-}
-
-var rootCmd = &cobra.Command{
-	Use:   "zrepl",
-	Short: "One-stop ZFS replication solution",
-}
-
-func init() {
-	rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
-}
-
-var genCompletionCmd = &cobra.Command{
-	Use:   "gencompletion",
-	Short: "generate shell auto-completions",
-}
-
-type completionCmdInfo struct {
-	genFunc func(outpath string) error
-	help    string
-}
-
-var completionCmdMap = map[string]completionCmdInfo{
-	"zsh": {
-		rootCmd.GenZshCompletionFile,
-		"  save to file `_zrepl` in your zsh's $fpath",
-	},
-	"bash": {
-		rootCmd.GenBashCompletionFile,
-		"  save to a path and source that path in your .bashrc",
-	},
-}
-
-func init() {
-	for sh, info := range completionCmdMap {
-		sh, info := sh, info
-		genCompletionCmd.AddCommand(&cobra.Command{
-			Use:     fmt.Sprintf("%s path/to/out/file", sh),
-			Short:   fmt.Sprintf("generate %s completions", sh),
-			Example: info.help,
-			Run: func(cmd *cobra.Command, args []string) {
-				if len(args) != 1 {
-					fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
-					err := cmd.Usage()
-					if err != nil {
-						panic(err)
-					}
-					os.Exit(1)
-				}
-				if err := info.genFunc(args[0]); err != nil {
-					fmt.Fprintf(os.Stderr, "error generating %s completion: %s", sh, err)
-					os.Exit(1)
-				}
-			},
-		})
-	}
-	rootCmd.AddCommand(genCompletionCmd)
-}
-
-type Subcommand struct {
-	Use              string
-	Short            string
-	Example          string
-	NoRequireConfig  bool
-	Run              func(ctx context.Context, subcommand *Subcommand, args []string) error
-	SetupFlags       func(f *pflag.FlagSet)
-	SetupSubcommands func() []*Subcommand
-
-	config    *config.Config
-	configErr error
-}
-
-func (s *Subcommand) ConfigParsingError() error {
-	return s.configErr
-}
-
-func (s *Subcommand) Config() *config.Config {
-	if !s.NoRequireConfig && s.config == nil {
-		panic("command that requires config is running and has no config set")
-	}
-	return s.config
-}
-
-func (s *Subcommand) run(cmd *cobra.Command, args []string) {
-	s.tryParseConfig()
-	ctx := context.Background()
-	endTask := trace.WithTaskFromStackUpdateCtx(&ctx)
-	defer endTask()
-	err := s.Run(ctx, s, args)
-	endTask()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s\n", err)
-		os.Exit(1)
-	}
-}
-
-func (s *Subcommand) tryParseConfig() {
-	config, err := config.ParseConfig(rootArgs.configPath)
-	s.configErr = err
-	if err != nil {
-		if s.NoRequireConfig {
-			// doesn't matter
-			return
-		} else {
-			fmt.Fprintf(os.Stderr, "could not parse config: %s\n", err)
-			os.Exit(1)
-		}
-	}
-	s.config = config
-}
-
-func AddSubcommand(s *Subcommand) {
-	addSubcommandToCobraCmd(rootCmd, s)
-}
-
-func addSubcommandToCobraCmd(c *cobra.Command, s *Subcommand) {
-	cmd := cobra.Command{
-		Use:     s.Use,
-		Short:   s.Short,
-		Example: s.Example,
-	}
-	if s.SetupSubcommands == nil {
-		cmd.Run = s.run
-	} else {
-		for _, sub := range s.SetupSubcommands() {
-			addSubcommandToCobraCmd(&cmd, sub)
-		}
-	}
-	if s.SetupFlags != nil {
-		s.SetupFlags(cmd.Flags())
-	}
-	c.AddCommand(&cmd)
-}
-
-func Run() {
-	if err := rootCmd.Execute(); err != nil {
-		os.Exit(1)
-	}
-}

client/configcheck.go

@@ -1,131 +0,0 @@
-package client
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"os"
-
-	"github.com/kr/pretty"
-	"github.com/pkg/errors"
-	"github.com/spf13/pflag"
-	"github.com/zrepl/yaml-config"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/daemon/logging"
-	"github.com/zrepl/zrepl/logger"
-)
-
-var configcheckArgs struct {
-	format        string
-	what          string
-	skipCertCheck bool
-}
-
-var ConfigcheckCmd = &cli.Subcommand{
-	Use:   "configcheck",
-	Short: "check if config can be parsed without errors",
-	SetupFlags: func(f *pflag.FlagSet) {
-		f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]")
-		f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]")
-		f.BoolVar(&configcheckArgs.skipCertCheck, "skip-cert-check", false, "skip checking cert files")
-	},
-	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
-		formatMap := map[string]func(interface{}){
-			"": func(i interface{}) {},
-			"pretty": func(i interface{}) {
-				if _, err := pretty.Println(i); err != nil {
-					panic(err)
-				}
-			},
-			"json": func(i interface{}) {
-				if err := json.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
-					panic(err)
-				}
-			},
-			"yaml": func(i interface{}) {
-				if err := yaml.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
-					panic(err)
-				}
-			},
-		}
-
-		formatter, ok := formatMap[configcheckArgs.format]
-		if !ok {
-			return fmt.Errorf("unsupported --format %q", configcheckArgs.format)
-		}
-
-		var hadErr bool
-
-		parseFlags := config.ParseFlagsNone
-
-		if configcheckArgs.skipCertCheck {
-			parseFlags |= config.ParseFlagsNoCertCheck
-		}
-
-		// further: try to build jobs
-		confJobs, err := job.JobsFromConfig(subcommand.Config(), parseFlags)
-
-		if err != nil {
-			err := errors.Wrap(err, "cannot build jobs from config")
-			if configcheckArgs.what == "jobs" {
-				return err
-			} else {
-				fmt.Fprintf(os.Stderr, "%s\n", err)
-				confJobs = nil
-				hadErr = true
-			}
-		}
-
-		// further: try to build logging outlets
-		outlets, err := logging.OutletsFromConfig(*subcommand.Config().Global.Logging)
-		if err != nil {
-			err := errors.Wrap(err, "cannot build logging from config")
-			if configcheckArgs.what == "logging" {
-				return err
-			} else {
-				fmt.Fprintf(os.Stderr, "%s\n", err)
-				outlets = nil
-				hadErr = true
-			}
-		}
-
-		whatMap := map[string]func(){
-			"all": func() {
-				o := struct {
-					config  *config.Config
-					jobs    []job.Job
-					logging *logger.Outlets
-				}{
-					subcommand.Config(),
-					confJobs,
-					outlets,
-				}
-				formatter(o)
-			},
-			"config": func() {
-				formatter(subcommand.Config())
-			},
-			"jobs": func() {
-				formatter(confJobs)
-			},
-			"logging": func() {
-				formatter(outlets)
-			},
-		}
-
-		wf, ok := whatMap[configcheckArgs.what]
-		if !ok {
-			return fmt.Errorf("unsupported --format %q", configcheckArgs.what)
-		}
-		wf()
-
-		if hadErr {
-			return fmt.Errorf("config parsing failed")
-		} else {
-			return nil
-		}
-	},
-}

client/jsonclient.go

@@ -1,49 +0,0 @@
-package client
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"io"
-	"net"
-	"net/http"
-
-	"github.com/pkg/errors"
-)
-
-func controlHttpClient(sockpath string) (client http.Client, err error) {
-	return http.Client{
-		Transport: &http.Transport{
-			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
-				return net.Dial("unix", sockpath)
-			},
-		},
-	}, nil
-}
-
-func jsonRequestResponse(c http.Client, endpoint string, req interface{}, res interface{}) error {
-	var buf bytes.Buffer
-	encodeErr := json.NewEncoder(&buf).Encode(req)
-	if encodeErr != nil {
-		return encodeErr
-	}
-
-	resp, err := c.Post("http://unix"+endpoint, "application/json", &buf)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		var msg bytes.Buffer
-		_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
-		return errors.Errorf("%s", msg.String())
-	}
-
-	decodeError := json.NewDecoder(resp.Body).Decode(&res)
-	if decodeError != nil {
-		return decodeError
-	}
-
-	return nil
-}

client/migrate.go

@@ -1,268 +0,0 @@
-package client
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/fatih/color"
-	"github.com/kr/pretty"
-	"github.com/pkg/errors"
-	"github.com/spf13/pflag"
-
-	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/zfs"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-)
-
-var (
-	MigrateCmd = &cli.Subcommand{
-		Use:   "migrate",
-		Short: "perform migration of the on-disk / zfs properties",
-		SetupSubcommands: func() []*cli.Subcommand {
-			return migrations
-		},
-	}
-)
-
-var migrations = []*cli.Subcommand{
-	&cli.Subcommand{
-		Use: "0.0.X:0.1:placeholder",
-		Run: doMigratePlaceholder0_1,
-		SetupFlags: func(f *pflag.FlagSet) {
-			f.BoolVar(&migratePlaceholder0_1Args.dryRun, "dry-run", false, "dry run")
-		},
-	},
-	&cli.Subcommand{
-		Use: "replication-cursor:v1-v2",
-		Run: doMigrateReplicationCursor,
-		SetupFlags: func(f *pflag.FlagSet) {
-			f.BoolVar(&migrateReplicationCursorArgs.dryRun, "dry-run", false, "dry run")
-		},
-	},
-}
-
-var migratePlaceholder0_1Args struct {
-	dryRun bool
-}
-
-func doMigratePlaceholder0_1(ctx context.Context, sc *cli.Subcommand, args []string) error {
-	if len(args) != 0 {
-		return fmt.Errorf("migration does not take arguments, got %v", args)
-	}
-
-	cfg := sc.Config()
-
-	allFSS, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
-	if err != nil {
-		return errors.Wrap(err, "cannot list filesystems")
-	}
-
-	type workItem struct {
-		jobName string
-		rootFS  *zfs.DatasetPath
-		fss     []*zfs.DatasetPath
-	}
-	var wis []workItem
-	for i, j := range cfg.Jobs {
-		var rfsS string
-		switch job := j.Ret.(type) {
-		case *config.SinkJob:
-			rfsS = job.RootFS
-		case *config.PullJob:
-			rfsS = job.RootFS
-		default:
-			fmt.Printf("ignoring job %q (%d/%d, type %T)\n", j.Name(), i, len(cfg.Jobs), j.Ret)
-			continue
-		}
-		rfs, err := zfs.NewDatasetPath(rfsS)
-		if err != nil {
-			return errors.Wrapf(err, "root fs for job %q is not a valid dataset path", j.Name())
-		}
-		var fss []*zfs.DatasetPath
-		for _, fs := range allFSS {
-			if fs.HasPrefix(rfs) {
-				fss = append(fss, fs)
-			}
-		}
-		wis = append(wis, workItem{j.Name(), rfs, fss})
-	}
-
-	for _, wi := range wis {
-		fmt.Printf("job %q => migrate filesystems below root_fs %q\n", wi.jobName, wi.rootFS.ToString())
-		if len(wi.fss) == 0 {
-			fmt.Printf("\tno filesystems\n")
-			continue
-		}
-		for _, fs := range wi.fss {
-			fmt.Printf("\t%q ... ", fs.ToString())
-			r, err := zfs.ZFSMigrateHashBasedPlaceholderToCurrent(ctx, fs, migratePlaceholder0_1Args.dryRun)
-			if err != nil {
-				fmt.Printf("error: %s\n", err)
-			} else if !r.NeedsModification {
-				fmt.Printf("unchanged (placeholder=%v)\n", r.OriginalState.IsPlaceholder)
-			} else {
-				fmt.Printf("migrate (placeholder=%v) (old value = %q)\n",
-					r.OriginalState.IsPlaceholder, r.OriginalState.RawLocalPropertyValue)
-			}
-		}
-	}
-
-	return nil
-}
-
-var migrateReplicationCursorArgs struct {
-	dryRun bool
-}
-
-var bold = color.New(color.Bold)
-var succ = color.New(color.FgGreen)
-var fail = color.New(color.FgRed)
-
-var migrateReplicationCursorSkipSentinel = fmt.Errorf("skipping this filesystem")
-
-func doMigrateReplicationCursor(ctx context.Context, sc *cli.Subcommand, args []string) error {
-	if len(args) != 0 {
-		return fmt.Errorf("migration does not take arguments, got %v", args)
-	}
-
-	cfg := sc.Config()
-	jobs, err := job.JobsFromConfig(cfg, config.ParseFlagsNone)
-	if err != nil {
-		fmt.Printf("cannot parse config:\n%s\n\n", err)
-		fmt.Printf("NOTE: this migration was released together with a change in job name requirements.\n")
-		return fmt.Errorf("exiting migration after error")
-	}
-
-	v1cursorJobs := make([]job.Job, 0, len(cfg.Jobs))
-	for i, j := range cfg.Jobs {
-		if jobs[i].Name() != j.Name() {
-			panic("implementation error")
-		}
-		switch j.Ret.(type) {
-		case *config.PushJob:
-			v1cursorJobs = append(v1cursorJobs, jobs[i])
-		case *config.SourceJob:
-			v1cursorJobs = append(v1cursorJobs, jobs[i])
-		default:
-			fmt.Printf("ignoring job %q (%d/%d, type %T), not supposed to create v1 replication cursors\n", j.Name(), i, len(cfg.Jobs), j.Ret)
-			continue
-		}
-	}
-
-	// scan all filesystems for v1 replication cursors
-
-	fss, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
-	if err != nil {
-		return errors.Wrap(err, "list filesystems")
-	}
-
-	var hadError bool
-	for _, fs := range fss {
-
-		bold.Printf("INSPECT FILESYSTEM %q\n", fs.ToString())
-
-		err := doMigrateReplicationCursorFS(ctx, v1cursorJobs, fs)
-		if err == migrateReplicationCursorSkipSentinel {
-			bold.Printf("FILESYSTEM SKIPPED\n")
-		} else if err != nil {
-			hadError = true
-			fail.Printf("MIGRATION FAILED: %s\n", err)
-		} else {
-			succ.Printf("FILESYSTEM %q COMPLETE\n", fs.ToString())
-		}
-	}
-
-	if hadError {
-		fail.Printf("\n\none or more filesystems could not be migrated, please inspect output and or re-run migration")
-		return errors.Errorf("")
-	}
-	return nil
-}
-
-func doMigrateReplicationCursorFS(ctx context.Context, v1CursorJobs []job.Job, fs *zfs.DatasetPath) error {
-
-	var owningJob job.Job = nil
-	for _, job := range v1CursorJobs {
-		conf := job.SenderConfig()
-		if conf == nil {
-			continue
-		}
-		pass, err := conf.FSF.Filter(fs)
-		if err != nil {
-			return errors.Wrapf(err, "filesystem filter error in job %q for fs %q", job.Name(), fs.ToString())
-		}
-		if !pass {
-			continue
-		}
-		if owningJob != nil {
-			return errors.Errorf("jobs %q and %q both match %q\ncannot attribute replication cursor to either one", owningJob.Name(), job.Name(), fs)
-		}
-		owningJob = job
-	}
-	if owningJob == nil {
-		fmt.Printf("no job's Filesystems filter matches\n")
-		return migrateReplicationCursorSkipSentinel
-	}
-	fmt.Printf("identified owning job %q\n", owningJob.Name())
-
-	bookmarks, err := zfs.ZFSListFilesystemVersions(ctx, fs, zfs.ListFilesystemVersionsOptions{
-		Types: zfs.Bookmarks,
-	})
-	if err != nil {
-		return errors.Wrapf(err, "list filesystem versions of %q", fs.ToString())
-	}
-
-	var oldCursor *zfs.FilesystemVersion
-	for i, fsv := range bookmarks {
-		_, _, err := endpoint.ParseReplicationCursorBookmarkName(fsv.ToAbsPath(fs))
-		if err != endpoint.ErrV1ReplicationCursor {
-			continue
-		}
-
-		if oldCursor != nil {
-			fmt.Printf("unexpected v1 replication cursor candidate: %q", fsv.ToAbsPath(fs))
-			return errors.Wrap(err, "multiple filesystem versions identified as v1 replication cursors")
-		}
-
-		oldCursor = &bookmarks[i]
-
-	}
-
-	if oldCursor == nil {
-		bold.Printf("no v1 replication cursor found for filesystem %q\n", fs.ToString())
-		return migrateReplicationCursorSkipSentinel
-	}
-
-	fmt.Printf("found v1 replication cursor:\n%s\n", pretty.Sprint(oldCursor))
-
-	mostRecentNew, err := endpoint.GetMostRecentReplicationCursorOfJob(ctx, fs.ToString(), owningJob.SenderConfig().JobID)
-	if err != nil {
-		return errors.Wrapf(err, "get most recent v2 replication cursor")
-	}
-
-	if mostRecentNew == nil {
-		return errors.Errorf("no v2 replication cursor found for job %q on filesystem %q", owningJob.SenderConfig().JobID, fs.ToString())
-	}
-
-	fmt.Printf("most recent v2 replication cursor:\n%#v", oldCursor)
-
-	if !(mostRecentNew.CreateTXG >= oldCursor.CreateTXG) {
-		return errors.Errorf("v1 replication cursor createtxg is higher than v2 cursor's, skipping this filesystem")
-	}
-
-	fmt.Printf("determined that v2 cursor is bookmark of same or newer version than v1 cursor\n")
-	fmt.Printf("destroying v1 cursor %q\n", oldCursor.ToAbsPath(fs))
-
-	if migrateReplicationCursorArgs.dryRun {
-		succ.Printf("DRY RUN\n")
-	} else {
-		if err := zfs.ZFSDestroyFilesystemVersion(ctx, fs, oldCursor); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

client/migrate_test.go

@@ -1,16 +0,0 @@
-package client
-
-import "testing"
-
-func TestMigrationsUnambiguousNames(t *testing.T) {
-	names := make(map[string]bool)
-	for _, mig := range migrations {
-		if _, ok := names[mig.Use]; ok {
-			t.Errorf("duplicate migration name %q", mig.Use)
-			t.FailNow()
-			return
-		} else {
-			names[mig.Use] = true
-		}
-	}
-}

client/pprof.go

@@ -1,10 +0,0 @@
-package client
-
-import "github.com/zrepl/zrepl/cli"
-
-var PprofCmd = &cli.Subcommand{
-	Use: "pprof",
-	SetupSubcommands: func() []*cli.Subcommand {
-		return []*cli.Subcommand{PprofListenCmd, pprofActivityTraceCmd}
-	},
-}

client/pprof_activity_trace.go

@@ -1,44 +0,0 @@
-package client
-
-import (
-	"context"
-	"io"
-	"log"
-	"os"
-
-	"golang.org/x/net/websocket"
-
-	"github.com/zrepl/zrepl/cli"
-)
-
-var pprofActivityTraceCmd = &cli.Subcommand{
-	Use:   "activity-trace ZREPL_PPROF_HOST:ZREPL_PPROF_PORT",
-	Short: "attach to zrepl daemon with activated pprof listener and dump an activity-trace to stdout",
-	Run:   runPProfActivityTrace,
-}
-
-func runPProfActivityTrace(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
-	log := log.New(os.Stderr, "", 0)
-
-	die := func() {
-		log.Printf("exiting after error")
-		os.Exit(1)
-	}
-
-	if len(args) != 1 {
-		log.Printf("exactly one positional argument is required")
-		die()
-	}
-
-	url := "ws://" + args[0] + "/debug/zrepl/activity-trace" // FIXME dont' repeat that
-
-	log.Printf("attaching to activity trace stream %s", url)
-	ws, err := websocket.Dial(url, "", url)
-	if err != nil {
-		log.Printf("error: %s", err)
-		die()
-	}
-
-	_, err = io.Copy(os.Stdout, ws)
-	return err
-}

client/pprof_listen.go

@@ -1,68 +0,0 @@
-package client
-
-import (
-	"context"
-	"errors"
-	"log"
-	"os"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon"
-)
-
-var pprofListenCmd struct {
-	daemon.PprofServerControlMsg
-}
-
-var PprofListenCmd = &cli.Subcommand{
-	Use:   "listen off | [on TCP_LISTEN_ADDRESS]",
-	Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
-	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
-		if len(args) < 1 {
-			goto enargs
-		}
-		switch args[0] {
-		case "on":
-			pprofListenCmd.Run = true
-			if len(args) != 2 {
-				return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
-			}
-			pprofListenCmd.HttpListenAddress = args[1]
-		case "off":
-			if len(args) != 1 {
-				goto enargs
-			}
-			pprofListenCmd.Run = false
-		}
-
-		RunPProf(subcommand.Config())
-		return nil
-	enargs:
-		return errors.New("invalid number of positional arguments")
-
-	},
-}
-
-func RunPProf(conf *config.Config) {
-	log := log.New(os.Stderr, "", 0)
-
-	die := func() {
-		log.Printf("exiting after error")
-		os.Exit(1)
-	}
-
-	log.Printf("connecting to zrepl daemon")
-
-	httpc, err := controlHttpClient(conf.Global.Control.SockPath)
-	if err != nil {
-		log.Printf("error creating http client: %s", err)
-		die()
-	}
-	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofListenCmd.PprofServerControlMsg, struct{}{})
-	if err != nil {
-		log.Printf("error sending control message: %s", err)
-		die()
-	}
-	log.Printf("finished")
-}

client/signal.go

@@ -1,42 +0,0 @@
-package client
-
-import (
-	"context"
-
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon"
-)
-
-var SignalCmd = &cli.Subcommand{
-	Use:   "signal [wakeup|reset] JOB",
-	Short: "wake up a job from wait state or abort its current invocation",
-	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
-		return runSignalCmd(subcommand.Config(), args)
-	},
-}
-
-func runSignalCmd(config *config.Config, args []string) error {
-	if len(args) != 2 {
-		return errors.Errorf("Expected 2 arguments: [wakeup|reset] JOB")
-	}
-
-	httpc, err := controlHttpClient(config.Global.Control.SockPath)
-	if err != nil {
-		return err
-	}
-
-	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointSignal,
-		struct {
-			Name string
-			Op   string
-		}{
-			Name: args[1],
-			Op:   args[0],
-		},
-		struct{}{},
-	)
-	return err
-}

client/status/client/client.go

@@ -1,111 +0,0 @@
-package client
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"io"
-	"net"
-	"net/http"
-
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/daemon"
-)
-
-type Client struct {
-	h *http.Client
-}
-
-func New(network, addr string) (*Client, error) {
-	httpc, err := makeControlHttpClient(func(_ context.Context) (net.Conn, error) { return net.Dial(network, addr) })
-	if err != nil {
-		return nil, err
-	}
-	return &Client{httpc}, nil
-}
-
-func (c *Client) Status() (s daemon.Status, _ error) {
-	err := jsonRequestResponse(c.h, daemon.ControlJobEndpointStatus,
-		struct{}{},
-		&s,
-	)
-	return s, err
-}
-
-func (c *Client) StatusRaw() ([]byte, error) {
-	var r json.RawMessage
-	err := jsonRequestResponse(c.h, daemon.ControlJobEndpointStatus, struct{}{}, &r)
-	if err != nil {
-		return nil, err
-	}
-	return r, nil
-}
-
-func (c *Client) signal(job, sig string) error {
-	return jsonRequestResponse(c.h, daemon.ControlJobEndpointSignal,
-		struct {
-			Name string
-			Op   string
-		}{
-			Name: job,
-			Op:   sig,
-		},
-		struct{}{},
-	)
-}
-
-func (c *Client) SignalWakeup(job string) error {
-	return c.signal(job, "wakeup")
-}
-
-func (c *Client) SignalReset(job string) error {
-	return c.signal(job, "reset")
-}
-
-func makeControlHttpClient(dialfunc func(context.Context) (net.Conn, error)) (client *http.Client, err error) {
-	return &http.Client{
-		Transport: &http.Transport{
-			DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
-				return dialfunc(ctx)
-			},
-		},
-	}, nil
-}
-
-func jsonRequestResponse(c *http.Client, endpoint string, req interface{}, res interface{}) error {
-	var buf bytes.Buffer
-	encodeErr := json.NewEncoder(&buf).Encode(req)
-	if encodeErr != nil {
-		return encodeErr
-	}
-
-	hreq, err := http.NewRequest("POST", "http://unix"+endpoint, &buf)
-	if err != nil {
-		return err
-	}
-	hreq.Header.Set("Content-Type", "application/json")
-	// Prevent EOF errors when client request frequency and server keepalive close are at the same time.
-	// Found this by watching http.Server.ConnState changes, then found
-	// https://stackoverflow.com/questions/17714494/golang-http-request-results-in-eof-errors-when-making-multiple-requests-successi
-	// Note: The issue seems even more prounounced with local TCP sockets than unix domain sockets. So, I used that for debugging.
-	hreq.Close = true
-	resp, err := c.Do(hreq)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		var msg bytes.Buffer
-		_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
-		return errors.Errorf("%s", msg.String())
-	}
-
-	decodeError := json.NewDecoder(resp.Body).Decode(&res)
-	if decodeError != nil {
-		return decodeError
-	}
-
-	return nil
-}

client/status/status.go

@@ -1,97 +0,0 @@
-package status
-
-import (
-	"context"
-	"os"
-	"time"
-
-	"github.com/mattn/go-isatty"
-	"github.com/pkg/errors"
-	"github.com/spf13/pflag"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/client/status/client"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon"
-	"github.com/zrepl/zrepl/util/choices"
-)
-
-type Client interface {
-	Status() (daemon.Status, error)
-	StatusRaw() ([]byte, error)
-	SignalWakeup(job string) error
-	SignalReset(job string) error
-}
-
-type statusFlags struct {
-	Mode  choices.Choices
-	Job   string
-	Delay time.Duration
-}
-
-var statusv2Flags statusFlags
-
-type statusv2Mode int
-
-const (
-	StatusV2ModeInteractive statusv2Mode = 1 + iota
-	StatusV2ModeDump
-	StatusV2ModeRaw
-	StatusV2ModeLegacy
-)
-
-var Subcommand = &cli.Subcommand{
-	Use:   "status",
-	Short: "retrieve & display daemon status information",
-	SetupFlags: func(f *pflag.FlagSet) {
-		statusv2Flags.Mode.Init(
-			"interactive", StatusV2ModeInteractive,
-			"dump", StatusV2ModeDump,
-			"raw", StatusV2ModeRaw,
-			"legacy", StatusV2ModeLegacy,
-		)
-		statusv2Flags.Mode.SetTypeString("mode")
-		statusv2Flags.Mode.SetDefaultValue(StatusV2ModeInteractive)
-		f.Var(&statusv2Flags.Mode, "mode", statusv2Flags.Mode.Usage())
-		f.StringVar(&statusv2Flags.Job, "job", "", "only show specified job (works in \"dump\" and \"interactive\" mode)")
-		f.DurationVarP(&statusv2Flags.Delay, "delay", "d", 1*time.Second, "use -d 3s for 3 seconds delay (minimum delay is 1s)")
-	},
-	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
-		return runStatusV2Command(ctx, subcommand.Config(), args)
-	},
-}
-
-func runStatusV2Command(ctx context.Context, config *config.Config, args []string) error {
-
-	c, err := client.New("unix", config.Global.Control.SockPath)
-	if err != nil {
-		return errors.Wrapf(err, "connect to daemon socket at %q", config.Global.Control.SockPath)
-	}
-
-	mode := statusv2Flags.Mode.Value().(statusv2Mode)
-
-	if !isatty.IsTerminal(os.Stdout.Fd()) && mode != StatusV2ModeDump && mode != StatusV2ModeRaw {
-		dumpmode, err := statusv2Flags.Mode.InputForChoice(StatusV2ModeDump)
-		if err != nil {
-			panic(err)
-		}
-		rawmode, err := statusv2Flags.Mode.InputForChoice(StatusV2ModeRaw)
-		if err != nil {
-			panic(err)
-		}
-		return errors.Errorf("error: stdout is not a tty, please use --mode %s or --mode %s", dumpmode, rawmode)
-	}
-
-	switch mode {
-	case StatusV2ModeInteractive:
-		return interactive(c, statusv2Flags)
-	case StatusV2ModeDump:
-		return dump(c, statusv2Flags.Job)
-	case StatusV2ModeRaw:
-		return raw(c)
-	case StatusV2ModeLegacy:
-		return legacy(c, statusv2Flags)
-	default:
-		panic("unreachable")
-	}
-}

client/status/status_dump.go

@@ -1,70 +0,0 @@
-package status
-
-import (
-	"fmt"
-	"os"
-	"strings"
-
-	"github.com/gdamore/tcell"
-	"github.com/mattn/go-isatty"
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/client/status/viewmodel"
-)
-
-func dump(c Client, job string) error {
-	s, err := c.Status()
-	if err != nil {
-		return err
-	}
-
-	if job != "" {
-		if _, ok := s.Jobs[job]; !ok {
-			return errors.Errorf("job %q not found", job)
-		}
-	}
-
-	width := (1 << 31) - 1
-	wrap := false
-	hline := strings.Repeat("-", 80)
-	if isatty.IsTerminal(os.Stdout.Fd()) {
-		wrap = true
-		screen, err := tcell.NewScreen()
-		if err != nil {
-			return errors.Wrap(err, "get terminal dimensions")
-		}
-		if err := screen.Init(); err != nil {
-			return errors.Wrap(err, "init screen")
-		}
-		width, _ = screen.Size()
-		screen.Fini()
-		hline = strings.Repeat("-", width)
-	}
-
-	m := viewmodel.New()
-	params := viewmodel.Params{
-		Report:                  s.Jobs,
-		ReportFetchError:        nil,
-		SelectedJob:             nil,
-		FSFilter:                func(s string) bool { return true },
-		DetailViewWidth:         width,
-		DetailViewWrap:          wrap,
-		ShortKeybindingOverview: "",
-	}
-	m.Update(params)
-	for _, j := range m.Jobs() {
-		if job != "" && j.Name() != job {
-			continue
-		}
-		params.SelectedJob = j
-		m.Update(params)
-		fmt.Println(m.SelectedJob().FullDescription())
-		if job != "" {
-			return nil
-		} else {
-			fmt.Println(hline)
-		}
-	}
-
-	return nil
-}

client/status/status_interactive.go

@@ -1,347 +0,0 @@
-package status
-
-import (
-	"fmt"
-	"regexp"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/gdamore/tcell/v2"
-	tview "gitlab.com/tslocum/cview"
-
-	"github.com/zrepl/zrepl/client/status/viewmodel"
-)
-
-func interactive(c Client, flag statusFlags) error {
-
-	// Set this so we don't overwrite the default terminal colors
-	// See https://github.com/rivo/tview/blob/master/styles.go
-	tview.Styles.PrimitiveBackgroundColor = tcell.ColorDefault
-	tview.Styles.ContrastBackgroundColor = tcell.ColorDefault
-	tview.Styles.PrimaryTextColor = tcell.ColorDefault
-	tview.Styles.BorderColor = tcell.ColorDefault
-	app := tview.NewApplication()
-
-	jobDetailSplit := tview.NewFlex()
-	jobMenu := tview.NewTreeView()
-	jobMenuRoot := tview.NewTreeNode("jobs")
-	jobMenuRoot.SetSelectable(true)
-	jobMenu.SetRoot(jobMenuRoot)
-	jobMenu.SetCurrentNode(jobMenuRoot)
-	jobMenu.SetSelectedTextColor(tcell.ColorGreen)
-	jobTextDetail := tview.NewTextView()
-	jobTextDetail.SetWrap(false)
-
-	jobMenu.SetBorder(true)
-	jobTextDetail.SetBorder(true)
-
-	toolbarSplit := tview.NewFlex()
-	toolbarSplit.SetDirection(tview.FlexRow)
-	inputBarContainer := tview.NewFlex()
-	fsFilterInput := tview.NewInputField()
-	fsFilterInput.SetBorder(false)
-	fsFilterInput.SetFieldBackgroundColor(tcell.ColorDefault)
-	inputBarLabel := tview.NewTextView()
-	inputBarLabel.SetText("[::b]FILTER ")
-	inputBarLabel.SetDynamicColors(true)
-	inputBarContainer.AddItem(inputBarLabel, 7, 1, false)
-	inputBarContainer.AddItem(fsFilterInput, 0, 10, false)
-	toolbarSplit.AddItem(inputBarContainer, 1, 0, false)
-	toolbarSplit.AddItem(jobDetailSplit, 0, 10, false)
-
-	bottombar := tview.NewFlex()
-	bottombar.SetDirection(tview.FlexColumn)
-	bottombarDateView := tview.NewTextView()
-	bottombar.AddItem(bottombarDateView, len(time.Now().String()), 0, false)
-	bottomBarStatus := tview.NewTextView()
-	bottomBarStatus.SetDynamicColors(true)
-	bottomBarStatus.SetTextAlign(tview.AlignRight)
-	bottombar.AddItem(bottomBarStatus, 0, 10, false)
-	toolbarSplit.AddItem(bottombar, 1, 0, false)
-
-	tabbableWithJobMenu := []tview.Primitive{jobMenu, jobTextDetail, fsFilterInput}
-	tabbableWithoutJobMenu := []tview.Primitive{jobTextDetail, fsFilterInput}
-	var tabbable []tview.Primitive
-	tabbableActiveIndex := 0
-	tabbableRedraw := func() {
-		if len(tabbable) == 0 {
-			app.SetFocus(nil)
-			return
-		}
-		if tabbableActiveIndex >= len(tabbable) {
-			app.SetFocus(tabbable[0])
-			return
-		}
-		app.SetFocus(tabbable[tabbableActiveIndex])
-	}
-	tabbableCycle := func() {
-		if len(tabbable) == 0 {
-			return
-		}
-		tabbableActiveIndex = (tabbableActiveIndex + 1) % len(tabbable)
-		app.SetFocus(tabbable[tabbableActiveIndex])
-		tabbableRedraw()
-	}
-
-	jobMenuVisisble := false
-	reconfigureJobDetailSplit := func(setJobMenuVisible bool) {
-		if jobMenuVisisble == setJobMenuVisible {
-			return
-		}
-		jobMenuVisisble = setJobMenuVisible
-		if setJobMenuVisible {
-			jobDetailSplit.RemoveItem(jobTextDetail)
-			jobDetailSplit.AddItem(jobMenu, 0, 1, true)
-			jobDetailSplit.AddItem(jobTextDetail, 0, 5, false)
-			tabbable = tabbableWithJobMenu
-		} else {
-			jobDetailSplit.RemoveItem(jobMenu)
-			tabbable = tabbableWithoutJobMenu
-		}
-		tabbableRedraw()
-	}
-
-	showModal := func(m *tview.Modal, modalDoneFunc func(idx int, label string)) {
-		preModalFocus := app.GetFocus()
-		m.SetDoneFunc(func(idx int, label string) {
-			if modalDoneFunc != nil {
-				modalDoneFunc(idx, label)
-			}
-			app.SetRoot(toolbarSplit, true)
-			app.SetFocus(preModalFocus)
-			app.Draw()
-		})
-		app.SetRoot(m, true)
-		app.Draw()
-	}
-
-	app.SetRoot(toolbarSplit, true)
-	// initial focus
-	tabbableActiveIndex = len(tabbable)
-	tabbableCycle()
-	reconfigureJobDetailSplit(true)
-
-	m := viewmodel.New()
-	params := &viewmodel.Params{
-		Report:                  nil,
-		SelectedJob:             nil,
-		FSFilter:                func(_ string) bool { return true },
-		DetailViewWidth:         100,
-		DetailViewWrap:          false,
-		ShortKeybindingOverview: "[::b]Q[::-] quit  [::b]<TAB>[::-] switch panes [::b]W[::-] wrap lines  [::b]Shift+M[::-] toggle navbar  [::b]Shift+S[::-] signal job [::b]</>[::-] filter filesystems",
-	}
-	paramsMtx := &sync.Mutex{}
-	var redraw func()
-	viewmodelupdate := func(cb func(*viewmodel.Params)) {
-		paramsMtx.Lock()
-		defer paramsMtx.Unlock()
-		cb(params)
-		m.Update(*params)
-	}
-	redraw = func() {
-		jobs := m.Jobs()
-		if flag.Job != "" {
-			job_found := false
-			for _, job := range jobs {
-				if strings.Compare(flag.Job, job.Name()) == 0 {
-					jobs = []*viewmodel.Job{job}
-					job_found = true
-					break
-				}
-			}
-			if !job_found {
-				jobs = nil
-			}
-		}
-		redrawJobsList := false
-		var selectedJobN *tview.TreeNode
-		if len(jobMenuRoot.GetChildren()) == len(jobs) {
-			for i, jobN := range jobMenuRoot.GetChildren() {
-				if jobN.GetReference().(*viewmodel.Job) != jobs[i] {
-					redrawJobsList = true
-					break
-				}
-				if jobN.GetReference().(*viewmodel.Job) == m.SelectedJob() {
-					selectedJobN = jobN
-				}
-			}
-		} else {
-			redrawJobsList = true
-		}
-		if redrawJobsList {
-			selectedJobN = nil
-			children := make([]*tview.TreeNode, len(jobs))
-			for i := range jobs {
-				jobN := tview.NewTreeNode(jobs[i].JobTreeTitle())
-				jobN.SetReference(jobs[i])
-				jobN.SetSelectable(true)
-				children[i] = jobN
-				jobN.SetSelectedFunc(func() {
-					viewmodelupdate(func(p *viewmodel.Params) {
-						p.SelectedJob = jobN.GetReference().(*viewmodel.Job)
-					})
-				})
-				if jobs[i] == m.SelectedJob() {
-					selectedJobN = jobN
-				}
-			}
-			jobMenuRoot.SetChildren(children)
-		}
-
-		if selectedJobN != nil && jobMenu.GetCurrentNode() != selectedJobN {
-			jobMenu.SetCurrentNode(selectedJobN)
-		} else if selectedJobN == nil {
-			// select something, otherwise selection breaks (likely bug in tview)
-			jobMenu.SetCurrentNode(jobMenuRoot)
-		}
-
-		if selJ := m.SelectedJob(); selJ != nil {
-			jobTextDetail.SetText(selJ.FullDescription())
-		} else {
-			jobTextDetail.SetText("please select a job")
-		}
-
-		bottombardatestring := m.DateString()
-		bottombarDateView.SetText(bottombardatestring)
-		bottombar.ResizeItem(bottombarDateView, len(bottombardatestring), 0)
-
-		bottomBarStatus.SetText(m.BottomBarStatus())
-
-		app.Draw()
-
-	}
-
-	go func() {
-		defer func() {
-			if err := recover(); err != nil {
-				app.Suspend(func() {
-					panic(err)
-				})
-			}
-		}()
-		for {
-			st, err := c.Status()
-			viewmodelupdate(func(p *viewmodel.Params) {
-				p.Report = st.Jobs
-				p.ReportFetchError = err
-			})
-			app.QueueUpdateDraw(redraw)
-
-			time.Sleep(flag.Delay)
-		}
-	}()
-
-	jobMenu.SetChangedFunc(func(jobN *tview.TreeNode) {
-		viewmodelupdate(func(p *viewmodel.Params) {
-			p.SelectedJob, _ = jobN.GetReference().(*viewmodel.Job)
-		})
-		redraw()
-		jobTextDetail.ScrollToBeginning()
-	})
-	jobMenu.SetSelectedFunc(func(jobN *tview.TreeNode) {
-		app.SetFocus(jobTextDetail)
-	})
-
-	app.SetBeforeDrawFunc(func(screen tcell.Screen) bool {
-		viewmodelupdate(func(p *viewmodel.Params) {
-			_, _, p.DetailViewWidth, _ = jobTextDetail.GetInnerRect()
-		})
-		return false
-	})
-
-	app.SetInputCapture(func(e *tcell.EventKey) *tcell.EventKey {
-		if e.Key() == tcell.KeyTab {
-			tabbableCycle()
-			return nil
-		}
-
-		if e.Key() == tcell.KeyRune && app.GetFocus() == fsFilterInput {
-			return e
-		}
-
-		if e.Key() == tcell.KeyRune && e.Rune() == '/' {
-			if app.GetFocus() != fsFilterInput {
-				app.SetFocus(fsFilterInput)
-			}
-			return e
-		}
-
-		if e.Key() == tcell.KeyRune && e.Rune() == 'M' {
-			reconfigureJobDetailSplit(!jobMenuVisisble)
-			return nil
-		}
-
-		if e.Key() == tcell.KeyRune && e.Rune() == 'q' {
-			app.Stop()
-		}
-
-		if e.Key() == tcell.KeyRune && e.Rune() == 'S' {
-			job, ok := jobMenu.GetCurrentNode().GetReference().(*viewmodel.Job)
-			if !ok {
-				return nil
-			}
-			signals := []string{"wakeup", "reset"}
-			clientFuncs := []func(job string) error{c.SignalWakeup, c.SignalReset}
-			sigMod := tview.NewModal()
-			sigMod.SetBackgroundColor(tcell.ColorDefault)
-			sigMod.SetBorder(true)
-			sigMod.GetForm().SetButtonTextColorFocused(tcell.ColorGreen)
-			sigMod.AddButtons(signals)
-			sigMod.SetText(fmt.Sprintf("Send a signal to job %q", job.Name()))
-			showModal(sigMod, func(idx int, _ string) {
-				go func() {
-					if idx == -1 {
-						return
-					}
-					err := clientFuncs[idx](job.Name())
-					if err != nil {
-						app.QueueUpdate(func() {
-							me := tview.NewModal()
-							me.SetText(fmt.Sprintf("signal error: %s", err))
-							me.AddButtons([]string{"Close"})
-							showModal(me, nil)
-						})
-					}
-				}()
-			})
-		}
-
-		return e
-	})
-
-	fsFilterInput.SetChangedFunc(func(searchterm string) {
-		viewmodelupdate(func(p *viewmodel.Params) {
-			p.FSFilter = func(fs string) bool {
-				r, err := regexp.Compile(searchterm)
-				if err != nil {
-					return true
-				}
-				return r.MatchString(fs)
-			}
-		})
-		redraw()
-		jobTextDetail.ScrollToBeginning()
-	})
-	fsFilterInput.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
-		if event.Key() == tcell.KeyEnter {
-			app.SetFocus(jobTextDetail)
-			return nil
-		}
-		return event
-	})
-
-	jobTextDetail.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey {
-		if event.Key() == tcell.KeyRune && event.Rune() == 'w' {
-			// toggle wrapping
-			viewmodelupdate(func(p *viewmodel.Params) {
-				p.DetailViewWrap = !p.DetailViewWrap
-			})
-			redraw()
-			return nil
-		}
-		return event
-	})
-
-	return app.Run()
-}

client/status/status_legacy.go

@@ -1,128 +0,0 @@
-package status
-
-import (
-	"fmt"
-	"os"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/gdamore/tcell/v2"
-	"github.com/mattn/go-isatty"
-	"github.com/pkg/errors"
-	tview "gitlab.com/tslocum/cview"
-
-	"github.com/zrepl/zrepl/client/status/viewmodel"
-)
-
-func legacy(c Client, flag statusFlags) error {
-
-	// Set this so we don't overwrite the default terminal colors
-	// See https://github.com/rivo/tview/blob/master/styles.go
-	tview.Styles.PrimitiveBackgroundColor = tcell.ColorDefault
-	tview.Styles.ContrastBackgroundColor = tcell.ColorDefault
-	tview.Styles.PrimaryTextColor = tcell.ColorDefault
-	tview.Styles.BorderColor = tcell.ColorDefault
-	app := tview.NewApplication()
-
-	textView := tview.NewTextView()
-	textView.SetWrap(true)
-	textView.SetScrollable(true) // so that it allows us to set scroll position
-	textView.SetScrollBarVisibility(tview.ScrollBarNever)
-
-	app.SetRoot(textView, true)
-
-	width := (1 << 31) - 1
-	wrap := false
-	if isatty.IsTerminal(os.Stdout.Fd()) {
-		wrap = true
-		screen, err := tcell.NewScreen()
-		if err != nil {
-			return errors.Wrap(err, "get terminal dimensions")
-		}
-		if err := screen.Init(); err != nil {
-			return errors.Wrap(err, "init screen")
-		}
-		width, _ = screen.Size()
-		screen.Fini()
-	}
-
-	paramsMtx := &sync.Mutex{}
-	params := viewmodel.Params{
-		Report:                  nil,
-		ReportFetchError:        nil,
-		SelectedJob:             nil,
-		FSFilter:                func(s string) bool { return true },
-		DetailViewWidth:         width,
-		DetailViewWrap:          wrap,
-		ShortKeybindingOverview: "",
-	}
-
-	redraw := func() {
-		textView.Clear()
-
-		paramsMtx.Lock()
-		defer paramsMtx.Unlock()
-
-		if params.ReportFetchError != nil {
-			fmt.Fprintln(textView, params.ReportFetchError.Error())
-		} else if params.Report != nil {
-			m := viewmodel.New()
-			m.Update(params)
-			for _, j := range m.Jobs() {
-				if flag.Job != "" && j.Name() != flag.Job {
-					continue
-				}
-				params.SelectedJob = j
-				m.Update(params)
-				fmt.Fprintln(textView, m.SelectedJob().FullDescription())
-				if flag.Job != "" {
-					break
-				} else {
-					hline := strings.Repeat("-", params.DetailViewWidth)
-					fmt.Fprintln(textView, hline)
-				}
-			}
-		} else {
-			fmt.Fprintln(textView, "waiting for request results")
-		}
-		textView.ScrollToBeginning()
-	}
-
-	app.SetBeforeDrawFunc(func(screen tcell.Screen) bool {
-		// sync resizes to `params`
-		paramsMtx.Lock()
-		_, _, newWidth, _ := textView.GetInnerRect()
-		if newWidth != params.DetailViewWidth {
-			params.DetailViewWidth = newWidth
-			app.QueueUpdateDraw(redraw)
-		}
-		paramsMtx.Unlock()
-
-		textView.ScrollToBeginning() // has the effect of inhibiting user scrolls
-		return false
-	})
-
-	go func() {
-		defer func() {
-			if err := recover(); err != nil {
-				app.Suspend(func() {
-					panic(err)
-				})
-			}
-		}()
-		for {
-			st, err := c.Status()
-			paramsMtx.Lock()
-			params.Report = st.Jobs
-			params.ReportFetchError = err
-			paramsMtx.Unlock()
-
-			app.QueueUpdateDraw(redraw)
-
-			time.Sleep(flag.Delay)
-		}
-	}()
-
-	return app.Run()
-}

client/status/status_raw.go

@@ -1,18 +0,0 @@
-package status
-
-import (
-	"bytes"
-	"io"
-	"os"
-)
-
-func raw(c Client) error {
-	b, err := c.StatusRaw()
-	if err != nil {
-		return err
-	}
-	if _, err := io.Copy(os.Stdout, bytes.NewReader(b)); err != nil {
-		return err
-	}
-	return nil
-}

client/status/viewmodel/bytecountbinary.go

@@ -1,26 +0,0 @@
-package viewmodel
-
-import (
-	"fmt"
-	"math"
-)
-
-func ByteCountBinaryUint(b uint64) string {
-	if b > math.MaxInt64 {
-		panic(b)
-	}
-	return ByteCountBinary(int64(b))
-}
-
-func ByteCountBinary(b int64) string {
-	const unit = 1024
-	if b < unit {
-		return fmt.Sprintf("%d B", b)
-	}
-	div, exp := unit, 0
-	for n := b / unit; n >= unit; n /= unit {
-		div *= unit
-		exp++
-	}
-	return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
-}

client/status/viewmodel/bytesprogresshistory.go

@@ -1,49 +0,0 @@
-package viewmodel
-
-import "time"
-
-type byteProgressMeasurement struct {
-	time time.Time
-	val  uint64
-}
-
-type bytesProgressHistory struct {
-	last        *byteProgressMeasurement // pointer as poor man's optional
-	changeCount int
-	lastChange  time.Time
-}
-
-func (p *bytesProgressHistory) Update(currentVal uint64) (bytesPerSecondAvg int64, changeCount int) {
-
-	if p.last == nil {
-		p.last = &byteProgressMeasurement{
-			time: time.Now(),
-			val:  currentVal,
-		}
-		return 0, 0
-	}
-
-	if p.last.val != currentVal {
-		p.changeCount++
-		p.lastChange = time.Now()
-	}
-
-	if time.Since(p.lastChange) > 3*time.Second {
-		p.last = nil
-		return 0, 0
-	}
-
-	var deltaV int64
-	if currentVal >= p.last.val {
-		deltaV = int64(currentVal - p.last.val)
-	} else {
-		deltaV = -int64(p.last.val - currentVal)
-	}
-	deltaT := time.Since(p.last.time)
-	rate := float64(deltaV) / deltaT.Seconds()
-
-	p.last.time = time.Now()
-	p.last.val = currentVal
-
-	return int64(rate), p.changeCount
-}

client/status/viewmodel/render.go

@@ -1,656 +0,0 @@
-package viewmodel
-
-import (
-	"fmt"
-	"math"
-	"sort"
-	"strings"
-	"time"
-
-	"github.com/go-playground/validator/v10"
-	yaml "github.com/zrepl/yaml-config"
-
-	"github.com/zrepl/zrepl/client/status/viewmodel/stringbuilder"
-	"github.com/zrepl/zrepl/daemon"
-	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/daemon/pruner"
-	"github.com/zrepl/zrepl/daemon/snapper"
-	"github.com/zrepl/zrepl/replication/report"
-)
-
-type M struct {
-	jobs            map[string]*Job
-	jobsList        []*Job
-	selectedJob     *Job
-	dateString      string
-	bottomBarStatus string
-}
-
-type Job struct {
-	// long-lived
-	name         string
-	byteProgress *bytesProgressHistory
-
-	lastStatus      *job.Status
-	fulldescription string
-}
-
-func New() *M {
-	return &M{
-		jobs:        make(map[string]*Job),
-		jobsList:    make([]*Job, 0),
-		selectedJob: nil,
-	}
-}
-
-type FilterFunc func(string) bool
-
-type Params struct {
-	Report                  map[string]*job.Status
-	ReportFetchError        error
-	SelectedJob             *Job
-	FSFilter                FilterFunc `validate:"required"`
-	DetailViewWidth         int        `validate:"gte=1"`
-	DetailViewWrap          bool
-	ShortKeybindingOverview string
-}
-
-var validate = validator.New()
-
-func (m *M) Update(p Params) {
-
-	if err := validate.Struct(p); err != nil {
-		panic(err)
-	}
-
-	if p.ReportFetchError != nil {
-		m.bottomBarStatus = fmt.Sprintf("[red::]status fetch: %s", p.ReportFetchError)
-	} else {
-		m.bottomBarStatus = p.ShortKeybindingOverview
-		for jobname, st := range p.Report {
-			// TODO handle job renames & deletions
-			j, ok := m.jobs[jobname]
-			if !ok {
-				j = &Job{
-					name:         jobname,
-					byteProgress: &bytesProgressHistory{},
-				}
-				m.jobs[jobname] = j
-				m.jobsList = append(m.jobsList, j)
-			}
-			j.lastStatus = st
-		}
-	}
-
-	// filter out internal jobs
-	var jobsList []*Job
-	for _, j := range m.jobsList {
-		if daemon.IsInternalJobName(j.name) {
-			continue
-		}
-		jobsList = append(jobsList, j)
-	}
-	m.jobsList = jobsList
-
-	// determinism!
-	sort.Slice(m.jobsList, func(i, j int) bool {
-		return strings.Compare(m.jobsList[i].name, m.jobsList[j].name) < 0
-	})
-
-	// try to not lose the selected job
-	m.selectedJob = nil
-	for _, j := range m.jobsList {
-		j.updateFullDescription(p)
-		if j == p.SelectedJob {
-			m.selectedJob = j
-		}
-	}
-
-	m.dateString = time.Now().Format(time.RFC3339)
-
-}
-
-func (m *M) BottomBarStatus() string { return m.bottomBarStatus }
-
-func (m *M) Jobs() []*Job { return m.jobsList }
-
-// may be nil
-func (m *M) SelectedJob() *Job { return m.selectedJob }
-
-func (m *M) DateString() string { return m.dateString }
-
-func (j *Job) updateFullDescription(p Params) {
-	width := p.DetailViewWidth
-	if !p.DetailViewWrap {
-		width = 10000000 // FIXME
-	}
-	b := stringbuilder.New(stringbuilder.Config{
-		IndentMultiplier: 3,
-		Width:            width,
-	})
-	drawJob(b, j.name, j.lastStatus, j.byteProgress, p.FSFilter)
-	j.fulldescription = b.String()
-}
-
-func (j *Job) JobTreeTitle() string {
-	return j.name
-}
-
-func (j *Job) FullDescription() string {
-	return j.fulldescription
-}
-
-func (j *Job) Name() string {
-	return j.name
-}
-
-func drawJob(t *stringbuilder.B, name string, v *job.Status, history *bytesProgressHistory, fsfilter FilterFunc) {
-
-	t.Printf("Job: %s\n", name)
-	t.Printf("Type: %s\n\n", v.Type)
-
-	if v.Type == job.TypePush || v.Type == job.TypePull {
-		activeStatus, ok := v.JobSpecific.(*job.ActiveSideStatus)
-		if !ok || activeStatus == nil {
-			t.Printf("ActiveSideStatus is null")
-			t.Newline()
-			return
-		}
-
-		t.Printf("Replication:")
-		t.AddIndentAndNewline(1)
-		renderReplicationReport(t, activeStatus.Replication, history, fsfilter)
-		t.AddIndentAndNewline(-1)
-
-		t.Printf("Pruning Sender:")
-		t.AddIndentAndNewline(1)
-		renderPrunerReport(t, activeStatus.PruningSender, fsfilter)
-		t.AddIndentAndNewline(-1)
-
-		t.Printf("Pruning Receiver:")
-		t.AddIndentAndNewline(1)
-		renderPrunerReport(t, activeStatus.PruningReceiver, fsfilter)
-		t.AddIndentAndNewline(-1)
-
-		if v.Type == job.TypePush {
-			t.Printf("Snapshotting:")
-			t.AddIndentAndNewline(1)
-			renderSnapperReport(t, activeStatus.Snapshotting, fsfilter)
-			t.AddIndentAndNewline(-1)
-		}
-
-	} else if v.Type == job.TypeSnap {
-		snapStatus, ok := v.JobSpecific.(*job.SnapJobStatus)
-		if !ok || snapStatus == nil {
-			t.Printf("SnapJobStatus is null")
-			t.Newline()
-			return
-		}
-		t.Printf("Pruning snapshots:")
-		t.AddIndentAndNewline(1)
-		renderPrunerReport(t, snapStatus.Pruning, fsfilter)
-		t.AddIndentAndNewline(-1)
-		t.Printf("Snapshotting:")
-		t.AddIndentAndNewline(1)
-		renderSnapperReport(t, snapStatus.Snapshotting, fsfilter)
-		t.AddIndentAndNewline(-1)
-	} else if v.Type == job.TypeSource {
-
-		st := v.JobSpecific.(*job.PassiveStatus)
-		t.Printf("Snapshotting:\n")
-		t.AddIndent(1)
-		renderSnapperReport(t, st.Snapper, fsfilter)
-		t.AddIndentAndNewline(-1)
-
-	} else {
-		t.Printf("No status representation for job type '%s', dumping as YAML", v.Type)
-		t.Newline()
-		asYaml, err := yaml.Marshal(v.JobSpecific)
-		if err != nil {
-			t.Printf("Error marshaling status to YAML: %s", err)
-			t.Newline()
-			return
-		}
-		t.Write(string(asYaml))
-		t.Newline()
-	}
-}
-
-func printFilesystemStatus(t *stringbuilder.B, rep *report.FilesystemReport, maxFS int) {
-
-	expected, replicated, containsInvalidSizeEstimates := rep.BytesSum()
-	sizeEstimationImpreciseNotice := ""
-	if containsInvalidSizeEstimates {
-		sizeEstimationImpreciseNotice = " (some steps lack size estimation)"
-	}
-	if rep.CurrentStep < len(rep.Steps) && rep.Steps[rep.CurrentStep].Info.BytesExpected == 0 {
-		sizeEstimationImpreciseNotice = " (step lacks size estimation)"
-	}
-
-	userVisisbleCurrentStep, userVisibleTotalSteps := rep.CurrentStep, len(rep.Steps)
-	// `.CurrentStep` is == len(rep.Steps) if all steps are done.
-	// Until then, it's an index into .Steps that starts at 0.
-	// For the user, we want it to start at 1.
-	if rep.CurrentStep >= len(rep.Steps) {
-		// rep.CurrentStep is what we want to show.
-		// We check for >= and not == for robustness.
-	} else {
-		// We're not done yet, so, make step count start at 1
-		// (The `.State` is included in the output, indicating we're not done yet)
-		userVisisbleCurrentStep = rep.CurrentStep + 1
-	}
-	status := fmt.Sprintf("%s (step %d/%d, %s/%s)%s",
-		strings.ToUpper(string(rep.State)),
-		userVisisbleCurrentStep, userVisibleTotalSteps,
-		ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected),
-		sizeEstimationImpreciseNotice,
-	)
-
-	activeIndicator := " "
-	if rep.BlockedOn == report.FsBlockedOnNothing &&
-		(rep.State == report.FilesystemPlanning || rep.State == report.FilesystemStepping) {
-		activeIndicator = "*"
-	}
-	t.AddIndent(1)
-	t.Printf("%s %s %s ",
-		activeIndicator,
-		stringbuilder.RightPad(rep.Info.Name, maxFS, " "),
-		status)
-
-	next := ""
-	if err := rep.Error(); err != nil {
-		next = err.Err
-	} else if rep.State != report.FilesystemDone {
-		if nextStep := rep.NextStep(); nextStep != nil {
-			if nextStep.IsIncremental() {
-				next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
-			} else {
-				next = fmt.Sprintf("next: full send %s", nextStep.Info.To)
-			}
-			attribs := []string{}
-
-			if nextStep.Info.Resumed {
-				attribs = append(attribs, "resumed")
-			}
-
-			if len(attribs) > 0 {
-				next += fmt.Sprintf(" (%s)", strings.Join(attribs, ", "))
-			}
-		} else {
-			next = "" // individual FSes may still be in planning state
-		}
-
-	}
-	t.Printf("%s", next)
-
-	t.AddIndent(-1)
-	t.Newline()
-}
-
-func renderReplicationReport(t *stringbuilder.B, rep *report.Report, history *bytesProgressHistory, fsfilter FilterFunc) {
-	if rep == nil {
-		t.Printf("...\n")
-		return
-	}
-
-	if rep.WaitReconnectError != nil {
-		t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: %s", rep.WaitReconnectError)
-		t.Newline()
-	}
-	if !rep.WaitReconnectSince.IsZero() {
-		delta := time.Until(rep.WaitReconnectUntil).Round(time.Second)
-		if rep.WaitReconnectUntil.IsZero() || delta > 0 {
-			var until string
-			if rep.WaitReconnectUntil.IsZero() {
-				until = "waiting indefinitely"
-			} else {
-				until = fmt.Sprintf("hard fail in %s @ %s", delta, rep.WaitReconnectUntil)
-			}
-			t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnecting with exponential backoff (since %s) (%s)",
-				rep.WaitReconnectSince, until)
-		} else {
-			t.PrintfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnects reached hard-fail timeout @ %s", rep.WaitReconnectUntil)
-		}
-		t.Newline()
-	}
-
-	// TODO visualize more than the latest attempt by folding all attempts into one
-	if len(rep.Attempts) == 0 {
-		t.Printf("no attempts made yet")
-		return
-	} else {
-		t.Printf("Attempt #%d", len(rep.Attempts))
-		if len(rep.Attempts) > 1 {
-			t.Printf(". Previous attempts failed with the following statuses:")
-			t.AddIndentAndNewline(1)
-			for i, a := range rep.Attempts[:len(rep.Attempts)-1] {
-				t.PrintfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)\n", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt))
-			}
-			t.AddIndentAndNewline(-1)
-		} else {
-			t.Newline()
-		}
-	}
-
-	latest := rep.Attempts[len(rep.Attempts)-1]
-	sort.Slice(latest.Filesystems, func(i, j int) bool {
-		return latest.Filesystems[i].Info.Name < latest.Filesystems[j].Info.Name
-	})
-
-	// apply filter
-	filtered := make([]*report.FilesystemReport, 0, len(latest.Filesystems))
-	for _, fs := range latest.Filesystems {
-		if !fsfilter(fs.Info.Name) {
-			continue
-		}
-		filtered = append(filtered, fs)
-	}
-	latest.Filesystems = filtered
-
-	t.Printf("Status: %s", latest.State)
-	t.Newline()
-	if !latest.FinishAt.IsZero() {
-		t.Printf("Last Run: %s (lasted %s)\n", latest.FinishAt.Round(time.Second), latest.FinishAt.Sub(latest.StartAt).Round(time.Second))
-	} else {
-		t.Printf("Started: %s (lasting %s)\n", latest.StartAt.Round(time.Second), time.Since(latest.StartAt).Round(time.Second))
-	}
-
-	if latest.State == report.AttemptPlanningError {
-		t.Printf("Problem: ")
-		t.PrintfDrawIndentedAndWrappedIfMultiline("%s", latest.PlanError)
-		t.Newline()
-	} else if latest.State == report.AttemptFanOutError {
-		t.Printf("Problem: one or more of the filesystems encountered errors")
-		t.Newline()
-	}
-
-	if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
-		// Draw global progress bar
-		// Progress: [---------------]
-		expected, replicated, containsInvalidSizeEstimates := latest.BytesSum()
-		rate, changeCount := history.Update(replicated)
-		eta := time.Duration(0)
-		if rate > 0 {
-			eta = time.Duration((float64(expected)-float64(replicated))/float64(rate)) * time.Second
-		}
-
-		t.Write("Progress: ")
-		t.DrawBar(50, replicated, expected, changeCount)
-		t.Write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected), ByteCountBinary(rate)))
-		if eta != 0 {
-			t.Write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
-		}
-		t.Newline()
-		if containsInvalidSizeEstimates {
-			t.Write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
-			t.Newline()
-		}
-
-		if len(latest.Filesystems) == 0 {
-			t.Write("NOTE: no filesystems were considered for replication!")
-			t.Newline()
-		}
-
-		var maxFSLen int
-		for _, fs := range latest.Filesystems {
-			if len(fs.Info.Name) > maxFSLen {
-				maxFSLen = len(fs.Info.Name)
-			}
-		}
-		for _, fs := range latest.Filesystems {
-			printFilesystemStatus(t, fs, maxFSLen)
-		}
-
-	}
-}
-
-func humanizeDuration(duration time.Duration) string {
-	days := int64(duration.Hours() / 24)
-	hours := int64(math.Mod(duration.Hours(), 24))
-	minutes := int64(math.Mod(duration.Minutes(), 60))
-	seconds := int64(math.Mod(duration.Seconds(), 60))
-
-	var parts []string
-
-	force := false
-	chunks := []int64{days, hours, minutes, seconds}
-	for i, chunk := range chunks {
-		if force || chunk > 0 {
-			padding := 0
-			if force {
-				padding = 2
-			}
-			parts = append(parts, fmt.Sprintf("%*d%c", padding, chunk, "dhms"[i]))
-			force = true
-		}
-	}
-
-	return strings.Join(parts, " ")
-}
-
-func renderPrunerReport(t *stringbuilder.B, r *pruner.Report, fsfilter FilterFunc) {
-	if r == nil {
-		t.Printf("...\n")
-		return
-	}
-
-	state, err := pruner.StateString(r.State)
-	if err != nil {
-		t.Printf("Status: %q (parse error: %q)\n", r.State, err)
-		return
-	}
-
-	t.Printf("Status: %s", state)
-	t.Newline()
-
-	if r.Error != "" {
-		t.Printf("Error: %s\n", r.Error)
-	}
-
-	type commonFS struct {
-		*pruner.FSReport
-		completed bool
-	}
-	all := make([]commonFS, 0, len(r.Pending)+len(r.Completed))
-	for i := range r.Pending {
-		all = append(all, commonFS{&r.Pending[i], false})
-	}
-	for i := range r.Completed {
-		all = append(all, commonFS{&r.Completed[i], true})
-	}
-
-	// filter all
-	filtered := make([]commonFS, 0, len(all))
-	for _, fs := range all {
-		if fsfilter(fs.FSReport.Filesystem) {
-			filtered = append(filtered, fs)
-		}
-	}
-	all = filtered
-
-	switch state {
-	case pruner.Plan:
-		fallthrough
-	case pruner.PlanErr:
-		return
-	}
-
-	if len(all) == 0 {
-		t.Printf("nothing to do\n")
-		return
-	}
-
-	var totalDestroyCount, completedDestroyCount int
-	var maxFSname int
-	for _, fs := range all {
-		totalDestroyCount += len(fs.DestroyList)
-		if fs.completed {
-			completedDestroyCount += len(fs.DestroyList)
-		}
-		if maxFSname < len(fs.Filesystem) {
-			maxFSname = len(fs.Filesystem)
-		}
-	}
-
-	// global progress bar
-	progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
-	t.Write("Progress: ")
-	t.Write("[")
-	t.Write(stringbuilder.Times("=", progress))
-	t.Write(">")
-	t.Write(stringbuilder.Times("-", 80-progress))
-	t.Write("]")
-	t.Printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
-	t.Newline()
-
-	sort.SliceStable(all, func(i, j int) bool {
-		return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
-	})
-
-	// Draw a table-like representation of 'all'
-	for _, fs := range all {
-		t.Write(stringbuilder.RightPad(fs.Filesystem, maxFSname, " "))
-		t.Write(" ")
-		if !fs.SkipReason.NotSkipped() {
-			t.Printf("skipped: %s\n", fs.SkipReason)
-			continue
-		}
-		if fs.LastError != "" {
-			if strings.ContainsAny(fs.LastError, "\r\n") {
-				t.Printf("ERROR:")
-				t.PrintfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError)
-			} else {
-				t.PrintfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError)
-			}
-			t.Newline()
-			continue
-		}
-
-		pruneRuleActionStr := fmt.Sprintf("(destroy %d of %d snapshots)",
-			len(fs.DestroyList), len(fs.SnapshotList))
-
-		if fs.completed {
-			t.Printf("Completed  %s\n", pruneRuleActionStr)
-			continue
-		}
-
-		t.Write("Pending    ") // whitespace is padding 10
-		if len(fs.DestroyList) == 1 {
-			t.Write(fs.DestroyList[0].Name)
-		} else {
-			t.Write(pruneRuleActionStr)
-		}
-		t.Newline()
-	}
-
-}
-
-func renderSnapperReport(t *stringbuilder.B, r *snapper.Report, fsfilter FilterFunc) {
-	if r == nil {
-		t.Printf("<no snapshotting report available>\n")
-		return
-	}
-	t.Printf("Type: %s\n", r.Type)
-	if r.Periodic != nil {
-		renderSnapperReportPeriodic(t, r.Periodic, fsfilter)
-	} else if r.Cron != nil {
-		renderSnapperReportCron(t, r.Cron, fsfilter)
-	} else {
-		t.Printf("<no details available>")
-	}
-}
-
-func renderSnapperReportPeriodic(t *stringbuilder.B, r *snapper.PeriodicReport, fsfilter FilterFunc) {
-	t.Printf("Status: %s", r.State)
-	t.Newline()
-
-	if r.Error != "" {
-		t.Printf("Error: %s\n", r.Error)
-	}
-	if !r.SleepUntil.IsZero() {
-		t.Printf("Sleep until: %s\n", r.SleepUntil)
-	}
-
-	renderSnapperPlanReportFilesystem(t, r.Progress, fsfilter)
-}
-
-func renderSnapperReportCron(t *stringbuilder.B, r *snapper.CronReport, fsfilter FilterFunc) {
-	t.Printf("State: %s\n", r.State)
-
-	now := time.Now()
-	if r.WakeupTime.After(now) {
-		t.Printf("Sleep until: %s (%s remaining)\n", r.WakeupTime, r.WakeupTime.Sub(now).Round(time.Second))
-	} else {
-		t.Printf("Started: %s (lasting %s)\n", r.WakeupTime, now.Sub(r.WakeupTime).Round(time.Second))
-	}
-
-	renderSnapperPlanReportFilesystem(t, r.Progress, fsfilter)
-}
-
-func renderSnapperPlanReportFilesystem(t *stringbuilder.B, fss []*snapper.ReportFilesystem, fsfilter FilterFunc) {
-	sort.Slice(fss, func(i, j int) bool {
-		return strings.Compare(fss[i].Path, fss[j].Path) == -1
-	})
-
-	dur := func(d time.Duration) string {
-		return d.Round(100 * time.Millisecond).String()
-	}
-
-	type row struct {
-		path, state, duration, remainder, hookReport string
-	}
-	var widths struct {
-		path, state, duration int
-	}
-	rows := make([]*row, 0, len(fss))
-	for _, fs := range fss {
-		if !fsfilter(fs.Path) {
-			continue
-		}
-		r := &row{
-			path:  fs.Path,
-			state: fs.State.String(),
-		}
-		if fs.HooksHadError {
-			r.hookReport = fs.Hooks // FIXME render here, not in daemon
-		}
-		switch fs.State {
-		case snapper.SnapPending:
-			r.duration = "..."
-			r.remainder = ""
-		case snapper.SnapStarted:
-			r.duration = dur(time.Since(fs.StartAt))
-			r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
-		case snapper.SnapDone:
-			fallthrough
-		case snapper.SnapError:
-			r.duration = dur(fs.DoneAt.Sub(fs.StartAt))
-			r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
-		}
-		rows = append(rows, r)
-		if len(r.path) > widths.path {
-			widths.path = len(r.path)
-		}
-		if len(r.state) > widths.state {
-			widths.state = len(r.state)
-		}
-		if len(r.duration) > widths.duration {
-			widths.duration = len(r.duration)
-		}
-	}
-
-	for _, r := range rows {
-		path := stringbuilder.RightPad(r.path, widths.path, " ")
-		state := stringbuilder.RightPad(r.state, widths.state, " ")
-		duration := stringbuilder.RightPad(r.duration, widths.duration, " ")
-		t.Printf("%s %s %s", path, state, duration)
-		t.PrintfDrawIndentedAndWrappedIfMultiline(" %s", r.remainder)
-		if r.hookReport != "" {
-			t.AddIndent(1)
-			t.Newline()
-			t.Printf("%s", r.hookReport)
-			t.AddIndent(-1)
-		}
-		t.Newline()
-	}
-}

client/status/viewmodel/stringbuilder/stringbuilder.go

@@ -1,119 +0,0 @@
-package stringbuilder
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/go-playground/validator/v10"
-)
-
-type B struct {
-	// const
-	indentMultiplier int
-
-	// mut
-	sb     *strings.Builder
-	indent int
-	width  int
-	x, y   int
-}
-
-type Config struct {
-	IndentMultiplier int `validate:"gte=1"`
-	Width            int `validate:"gte=1"`
-}
-
-var validate = validator.New()
-
-func New(config Config) *B {
-
-	if err := validate.Struct(config); err != nil {
-		panic(err)
-	}
-
-	return &B{sb: &strings.Builder{}, width: config.Width, indentMultiplier: config.IndentMultiplier}
-}
-
-func (b *B) String() string { return b.sb.String() }
-
-func (w *B) Newline() {
-	w.Write("\n")
-}
-
-func (w *B) PrintfDrawIndentedAndWrappedIfMultiline(format string, args ...interface{}) {
-	whole := fmt.Sprintf(format, args...)
-	if strings.ContainsAny(whole, "\n\r") {
-		w.AddIndent(1)
-		defer w.AddIndent(-1)
-	}
-	w.Write(whole)
-}
-
-func (w *B) Printf(format string, args ...interface{}) {
-	whole := fmt.Sprintf(format, args...)
-	w.Write(whole)
-}
-
-func (t *B) AddIndent(delta int) {
-	t.indent += delta * t.indentMultiplier
-}
-
-func (t *B) AddIndentAndNewline(delta int) {
-	t.indent += delta * t.indentMultiplier
-	t.Write("\n")
-}
-
-func (w *B) Write(s string) {
-	for _, c := range s {
-		if c == '\n' {
-			fmt.Fprint(w.sb, "\n")
-			w.x = 0
-			fmt.Fprint(w.sb, Times(" ", w.indent-w.x))
-			w.x = w.indent
-			w.y++
-			continue
-		}
-		if w.x >= w.width {
-			fmt.Fprint(w.sb, "\n")
-			w.x = 0
-			fmt.Fprint(w.sb, Times(" ", w.indent-w.x))
-			w.x = w.indent
-		}
-		fmt.Fprintf(w.sb, "%c", c)
-		w.x++
-	}
-}
-
-func Times(str string, n int) (out string) {
-	for i := 0; i < n; i++ {
-		out += str
-	}
-	return
-}
-
-func RightPad(str string, length int, pad string) string {
-	if len(str) > length {
-		return str[:length]
-	}
-	return str + strings.Repeat(pad, length-len(str))
-}
-
-// changeCount = 0 indicates stall / no progress
-func (w *B) DrawBar(length int, bytes, totalBytes uint64, changeCount int) {
-	const arrowPositions = `>\|/`
-	var completedLength int
-	if totalBytes > 0 {
-		completedLength = int(uint64(length) * bytes / totalBytes)
-		if completedLength > length {
-			completedLength = length
-		}
-	} else if totalBytes == bytes {
-		completedLength = length
-	}
-
-	w.Write("[")
-	w.Write(Times("=", completedLength))
-	w.Write(string(arrowPositions[changeCount%len(arrowPositions)]))
-	w.Write(Times("-", length-completedLength))
-	w.Write("]")
-}

client/stdinserver.go

@@ -1,51 +0,0 @@
-package client
-
-import (
-	"os"
-
-	"github.com/problame/go-netssh"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-
-	"context"
-	"errors"
-	"log"
-	"path"
-)
-
-var StdinserverCmd = &cli.Subcommand{
-	Use:   "stdinserver CLIENT_IDENTITY",
-	Short: "stdinserver transport mode (started from authorized_keys file as forced command)",
-	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
-		return runStdinserver(subcommand.Config(), args)
-	},
-}
-
-func runStdinserver(config *config.Config, args []string) error {
-
-	// NOTE: the netssh proxying protocol requires exiting with non-zero status if anything goes wrong
-	defer os.Exit(1)
-
-	log := log.New(os.Stderr, "", log.LUTC|log.Ldate|log.Ltime)
-
-	if len(args) != 1 || args[0] == "" {
-		err := errors.New("must specify client_identity as positional argument")
-		return err
-	}
-
-	identity := args[0]
-	unixaddr := path.Join(config.Global.Serve.StdinServer.SockDir, identity)
-
-	log.Printf("proxying client identity '%s' to zrepl daemon '%s'", identity, unixaddr)
-
-	ctx := netssh.ContextWithLog(context.TODO(), log)
-
-	err := netssh.Proxy(ctx, unixaddr)
-	if err == nil {
-		log.Print("proxying finished successfully, exiting with status 0")
-		os.Exit(0)
-	}
-	log.Printf("error proxying: %s", err)
-	return nil
-}

client/testcmd.go

@@ -1,220 +0,0 @@
-package client
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"os"
-
-	"github.com/pkg/errors"
-	"github.com/spf13/pflag"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-var TestCmd = &cli.Subcommand{
-	Use: "test",
-	SetupSubcommands: func() []*cli.Subcommand {
-		return []*cli.Subcommand{testFilter, testPlaceholder, testDecodeResumeToken}
-	},
-}
-
-var testFilterArgs struct {
-	job   string
-	all   bool
-	input string
-}
-
-var testFilter = &cli.Subcommand{
-	Use:   "filesystems --job JOB [--all | --input INPUT]",
-	Short: "test filesystems filter specified in push or source job",
-	SetupFlags: func(f *pflag.FlagSet) {
-		f.StringVar(&testFilterArgs.job, "job", "", "the name of the push or source job")
-		f.StringVar(&testFilterArgs.input, "input", "", "a filesystem name to test against the job's filters")
-		f.BoolVar(&testFilterArgs.all, "all", false, "test all local filesystems")
-	},
-	Run: runTestFilterCmd,
-}
-
-func runTestFilterCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
-
-	if testFilterArgs.job == "" {
-		return fmt.Errorf("must specify --job flag")
-	}
-	if !(testFilterArgs.all != (testFilterArgs.input != "")) { // xor
-		return fmt.Errorf("must set one: --all or --input")
-	}
-
-	conf := subcommand.Config()
-
-	var confFilter config.FilesystemsFilter
-	job, err := conf.Job(testFilterArgs.job)
-	if err != nil {
-		return err
-	}
-	switch j := job.Ret.(type) {
-	case *config.SourceJob:
-		confFilter = j.Filesystems
-	case *config.PushJob:
-		confFilter = j.Filesystems
-	case *config.SnapJob:
-		confFilter = j.Filesystems
-	default:
-		return fmt.Errorf("job type %T does not have filesystems filter", j)
-	}
-
-	f, err := filters.DatasetMapFilterFromConfig(confFilter)
-	if err != nil {
-		return fmt.Errorf("filter invalid: %s", err)
-	}
-
-	var fsnames []string
-	if testFilterArgs.input != "" {
-		fsnames = []string{testFilterArgs.input}
-	} else {
-		out, err := zfs.ZFSList(ctx, []string{"name"})
-		if err != nil {
-			return fmt.Errorf("could not list ZFS filesystems: %s", err)
-		}
-		for _, row := range out {
-
-			fsnames = append(fsnames, row[0])
-		}
-	}
-
-	fspaths := make([]*zfs.DatasetPath, len(fsnames))
-	for i, fsname := range fsnames {
-		path, err := zfs.NewDatasetPath(fsname)
-		if err != nil {
-			return err
-		}
-		fspaths[i] = path
-	}
-
-	hadFilterErr := false
-	for _, in := range fspaths {
-		var res string
-		var errStr string
-		pass, err := f.Filter(in)
-		if err != nil {
-			res = "ERROR"
-			errStr = err.Error()
-			hadFilterErr = true
-		} else if pass {
-			res = "ACCEPT"
-		} else {
-			res = "REJECT"
-		}
-		fmt.Printf("%s\t%s\t%s\n", res, in.ToString(), errStr)
-	}
-
-	if hadFilterErr {
-		return fmt.Errorf("filter errors occurred")
-	}
-	return nil
-}
-
-var testPlaceholderArgs struct {
-	ds  string
-	all bool
-}
-
-var testPlaceholder = &cli.Subcommand{
-	Use:   "placeholder [--all | --dataset DATASET]",
-	Short: fmt.Sprintf("list received placeholder filesystems (zfs property %q)", zfs.PlaceholderPropertyName),
-	Example: `
-	placeholder --all
-	placeholder --dataset path/to/sink/clientident/fs`,
-	NoRequireConfig: true,
-	SetupFlags: func(f *pflag.FlagSet) {
-		f.StringVar(&testPlaceholderArgs.ds, "dataset", "", "dataset path (not required to exist)")
-		f.BoolVar(&testPlaceholderArgs.all, "all", false, "list tab-separated placeholder status of all filesystems")
-	},
-	Run: runTestPlaceholder,
-}
-
-func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
-
-	var checkDPs []*zfs.DatasetPath
-	var datasetWasExplicitArgument bool
-
-	// all actions first
-	if testPlaceholderArgs.all {
-		datasetWasExplicitArgument = false
-		out, err := zfs.ZFSList(ctx, []string{"name"})
-		if err != nil {
-			return errors.Wrap(err, "could not list ZFS filesystems")
-		}
-		for _, row := range out {
-			dp, err := zfs.NewDatasetPath(row[0])
-			if err != nil {
-				panic(err)
-			}
-			checkDPs = append(checkDPs, dp)
-		}
-	} else {
-		datasetWasExplicitArgument = true
-		dp, err := zfs.NewDatasetPath(testPlaceholderArgs.ds)
-		if err != nil {
-			return err
-		}
-		if dp.Empty() {
-			return fmt.Errorf("must specify --dataset DATASET or --all")
-		}
-		checkDPs = append(checkDPs, dp)
-	}
-
-	fmt.Printf("IS_PLACEHOLDER\tDATASET\tzrepl:placeholder\n")
-	for _, dp := range checkDPs {
-		ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, dp)
-		if err != nil {
-			return errors.Wrap(err, "cannot get placeholder state")
-		}
-		if !ph.FSExists {
-			if datasetWasExplicitArgument {
-				return errors.Errorf("filesystem %q does not exist", ph.FS)
-			} else {
-				// got deleted between ZFSList and ZFSGetFilesystemPlaceholderState
-				continue
-			}
-		}
-		is := "yes"
-		if !ph.IsPlaceholder {
-			is = "no"
-		}
-		fmt.Printf("%s\t%s\t%s\n", is, dp.ToString(), ph.RawLocalPropertyValue)
-	}
-	return nil
-}
-
-var testDecodeResumeTokenArgs struct {
-	token string
-}
-
-var testDecodeResumeToken = &cli.Subcommand{
-	Use:   "decoderesumetoken --token TOKEN",
-	Short: "decode resume token",
-	SetupFlags: func(f *pflag.FlagSet) {
-		f.StringVar(&testDecodeResumeTokenArgs.token, "token", "", "the resume token obtained from the receive_resume_token property")
-	},
-	Run: runTestDecodeResumeTokenCmd,
-}
-
-func runTestDecodeResumeTokenCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
-	if testDecodeResumeTokenArgs.token == "" {
-		return fmt.Errorf("token argument must be specified")
-	}
-	token, err := zfs.ParseResumeToken(ctx, testDecodeResumeTokenArgs.token)
-	if err != nil {
-		return err
-	}
-	enc := json.NewEncoder(os.Stdout)
-	enc.SetIndent("", " ")
-	if err := enc.Encode(&token); err != nil {
-		panic(err)
-	}
-	return nil
-}

client/version.go

@@ -1,75 +0,0 @@
-package client
-
-import (
-	"context"
-	"fmt"
-	"os"
-
-	"github.com/spf13/pflag"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon"
-	"github.com/zrepl/zrepl/version"
-)
-
-var versionArgs struct {
-	Show      string
-	Config    *config.Config
-	ConfigErr error
-}
-
-var VersionCmd = &cli.Subcommand{
-	Use:             "version",
-	Short:           "print version of zrepl binary and running daemon",
-	NoRequireConfig: true,
-	SetupFlags: func(f *pflag.FlagSet) {
-		f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)")
-	},
-	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
-		versionArgs.Config = subcommand.Config()
-		versionArgs.ConfigErr = subcommand.ConfigParsingError()
-		return runVersionCmd()
-	},
-}
-
-func runVersionCmd() error {
-	args := versionArgs
-
-	if args.Show != "daemon" && args.Show != "client" && args.Show != "" {
-		return fmt.Errorf("show flag must be 'client' or 'server' or be left empty")
-	}
-
-	var clientVersion, daemonVersion *version.ZreplVersionInformation
-	if args.Show == "client" || args.Show == "" {
-		clientVersion = version.NewZreplVersionInformation()
-		fmt.Printf("client: %s\n", clientVersion.String())
-	}
-	if args.Show == "daemon" || args.Show == "" {
-
-		if args.ConfigErr != nil {
-			return fmt.Errorf("config parsing error: %s", args.ConfigErr)
-		}
-
-		httpc, err := controlHttpClient(args.Config.Global.Control.SockPath)
-		if err != nil {
-			return fmt.Errorf("server: error: %s\n", err)
-		}
-
-		var info version.ZreplVersionInformation
-		err = jsonRequestResponse(httpc, daemon.ControlJobEndpointVersion, "", &info)
-		if err != nil {
-			return fmt.Errorf("server: error: %s\n", err)
-		}
-		daemonVersion = &info
-		fmt.Printf("server: %s\n", daemonVersion.String())
-	}
-
-	if args.Show == "" {
-		if clientVersion.Version != daemonVersion.Version {
-			fmt.Fprintf(os.Stderr, "WARNING: client version != daemon version, restart zrepl daemon\n")
-		}
-	}
-
-	return nil
-}

client/zfsabstractions.go

@@ -1,147 +0,0 @@
-package client
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-
-	"github.com/spf13/pflag"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-var (
-	ZFSAbstractionsCmd = &cli.Subcommand{
-		Use:   "zfs-abstraction",
-		Short: "manage abstractions that zrepl builds on top of ZFS",
-		SetupSubcommands: func() []*cli.Subcommand {
-			return []*cli.Subcommand{
-				zabsCmdList,
-				zabsCmdReleaseAll,
-				zabsCmdReleaseStale,
-				zabsCmdCreate,
-			}
-		},
-	}
-)
-
-// a common set of CLI flags that map to the fields of an
-// endpoint.ListZFSHoldsAndBookmarksQuery
-type zabsFilterFlags struct {
-	Filesystems FilesystemsFilterFlag
-	Job         JobIDFlag
-	Types       AbstractionTypesFlag
-	Concurrency int64
-}
-
-// produce a query from the CLI flags
-func (f zabsFilterFlags) Query() (endpoint.ListZFSHoldsAndBookmarksQuery, error) {
-	q := endpoint.ListZFSHoldsAndBookmarksQuery{
-		FS:          f.Filesystems.FlagValue(),
-		What:        f.Types.FlagValue(),
-		JobID:       f.Job.FlagValue(),
-		Concurrency: f.Concurrency,
-	}
-	return q, q.Validate()
-}
-
-func (f *zabsFilterFlags) registerZabsFilterFlags(s *pflag.FlagSet, verb string) {
-	// Note: the default value is defined in the .FlagValue methods
-	s.Var(&f.Filesystems, "fs", fmt.Sprintf("only %s holds on the specified filesystem [default: all filesystems] [comma-separated list of <dataset-pattern>:<ok|!> pairs]", verb))
-	s.Var(&f.Job, "job", fmt.Sprintf("only %s holds created by the specified job [default: any job]", verb))
-
-	variants := make([]string, 0, len(endpoint.AbstractionTypesAll))
-	for v := range endpoint.AbstractionTypesAll {
-		variants = append(variants, string(v))
-	}
-	sort.Strings(variants)
-	variantsJoined := strings.Join(variants, "|")
-	s.Var(&f.Types, "type", fmt.Sprintf("only %s holds of the specified type [default: all] [comma-separated list of %s]", verb, variantsJoined))
-
-	s.Int64VarP(&f.Concurrency, "concurrency", "p", 1, "number of concurrently queried filesystems")
-}
-
-type JobIDFlag struct{ J *endpoint.JobID }
-
-func (f *JobIDFlag) Set(s string) error {
-	if len(s) == 0 {
-		*f = JobIDFlag{J: nil}
-		return nil
-	}
-
-	jobID, err := endpoint.MakeJobID(s)
-	if err != nil {
-		return err
-	}
-	*f = JobIDFlag{J: &jobID}
-	return nil
-}
-func (f JobIDFlag) Type() string               { return "job-ID" }
-func (f JobIDFlag) String() string             { return fmt.Sprint(f.J) }
-func (f JobIDFlag) FlagValue() *endpoint.JobID { return f.J }
-
-type AbstractionTypesFlag map[endpoint.AbstractionType]bool
-
-func (f *AbstractionTypesFlag) Set(s string) error {
-	ats, err := endpoint.AbstractionTypeSetFromStrings(strings.Split(s, ","))
-	if err != nil {
-		return err
-	}
-	*f = AbstractionTypesFlag(ats)
-	return nil
-}
-func (f AbstractionTypesFlag) Type() string { return "abstraction-type" }
-func (f AbstractionTypesFlag) String() string {
-	return endpoint.AbstractionTypeSet(f).String()
-}
-func (f AbstractionTypesFlag) FlagValue() map[endpoint.AbstractionType]bool {
-	if len(f) > 0 {
-		return f
-	}
-	return endpoint.AbstractionTypesAll
-}
-
-type FilesystemsFilterFlag struct {
-	F endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter
-}
-
-func (flag *FilesystemsFilterFlag) Set(s string) error {
-	mappings := strings.Split(s, ",")
-	if len(mappings) == 1 && !strings.Contains(mappings[0], ":") {
-		flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
-			FS: &mappings[0],
-		}
-		return nil
-	}
-
-	f := filters.NewDatasetMapFilter(len(mappings), true)
-	for _, m := range mappings {
-		thisMappingErr := fmt.Errorf("expecting comma-separated list of <dataset-pattern>:<ok|!> pairs, got %q", m)
-		lhsrhs := strings.SplitN(m, ":", 2)
-		if len(lhsrhs) != 2 {
-			return thisMappingErr
-		}
-		err := f.Add(lhsrhs[0], lhsrhs[1])
-		if err != nil {
-			return fmt.Errorf("%s: %s", thisMappingErr, err)
-		}
-	}
-	flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
-		Filter: f,
-	}
-	return nil
-}
-func (flag FilesystemsFilterFlag) Type() string { return "filesystem filter spec" }
-func (flag FilesystemsFilterFlag) String() string {
-	return fmt.Sprintf("%v", flag.F)
-}
-func (flag FilesystemsFilterFlag) FlagValue() endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter {
-	var z FilesystemsFilterFlag
-	if flag == z {
-		return endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{Filter: zfs.NoFilter()}
-	}
-	return flag.F
-}

client/zfsabstractions_create.go

@@ -1,14 +0,0 @@
-package client
-
-import "github.com/zrepl/zrepl/cli"
-
-var zabsCmdCreate = &cli.Subcommand{
-	Use:             "create",
-	NoRequireConfig: true,
-	Short:           `create zrepl ZFS abstractions (mostly useful for debugging & development, users should not need to use this command)`,
-	SetupSubcommands: func() []*cli.Subcommand {
-		return []*cli.Subcommand{
-			zabsCmdCreateStepHold,
-		}
-	},
-}

client/zfsabstractions_create_step_hold.go

@@ -1,58 +0,0 @@
-package client
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/pkg/errors"
-	"github.com/spf13/pflag"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-var zabsCreateStepHoldFlags struct {
-	target string
-	jobid  JobIDFlag
-}
-
-var zabsCmdCreateStepHold = &cli.Subcommand{
-	Use:             "step",
-	Run:             doZabsCreateStep,
-	NoRequireConfig: true,
-	Short:           `create a step hold or bookmark`,
-	SetupFlags: func(f *pflag.FlagSet) {
-		f.StringVarP(&zabsCreateStepHoldFlags.target, "target", "t", "", "snapshot to be held / bookmark to be held")
-		f.VarP(&zabsCreateStepHoldFlags.jobid, "jobid", "j", "jobid for which the hold is installed")
-	},
-}
-
-func doZabsCreateStep(ctx context.Context, sc *cli.Subcommand, args []string) error {
-	if len(args) > 0 {
-		return errors.New("subcommand takes no arguments")
-	}
-
-	f := &zabsCreateStepHoldFlags
-
-	fs, _, _, err := zfs.DecomposeVersionString(f.target)
-	if err != nil {
-		return errors.Wrapf(err, "%q invalid target", f.target)
-	}
-
-	if f.jobid.FlagValue() == nil {
-		return errors.Errorf("jobid must be set")
-	}
-
-	v, err := zfs.ZFSGetFilesystemVersion(ctx, f.target)
-	if err != nil {
-		return errors.Wrapf(err, "get info about target %q", f.target)
-	}
-
-	step, err := endpoint.HoldStep(ctx, fs, v, *f.jobid.FlagValue())
-	if err != nil {
-		return errors.Wrap(err, "create step hold")
-	}
-	fmt.Println(step.String())
-	return nil
-}

client/zfsabstractions_list.go

@@ -1,100 +0,0 @@
-package client
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"os"
-	"sync"
-
-	"github.com/fatih/color"
-	"github.com/pkg/errors"
-	"github.com/spf13/pflag"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/util/chainlock"
-)
-
-var zabsListFlags struct {
-	Filter zabsFilterFlags
-	Json   bool
-}
-
-var zabsCmdList = &cli.Subcommand{
-	Use:             "list",
-	Short:           `list zrepl ZFS abstractions`,
-	Run:             doZabsList,
-	NoRequireConfig: true,
-	SetupFlags: func(f *pflag.FlagSet) {
-		zabsListFlags.Filter.registerZabsFilterFlags(f, "list")
-		f.BoolVar(&zabsListFlags.Json, "json", false, "emit JSON")
-	},
-}
-
-func doZabsList(ctx context.Context, sc *cli.Subcommand, args []string) error {
-	var err error
-
-	if len(args) > 0 {
-		return errors.New("this subcommand takes no positional arguments")
-	}
-
-	q, err := zabsListFlags.Filter.Query()
-	if err != nil {
-		return errors.Wrap(err, "invalid filter specification on command line")
-	}
-
-	abstractions, errors, drainDone, err := endpoint.ListAbstractionsStreamed(ctx, q)
-	if err != nil {
-		return err // context clear by invocation of command
-	}
-	defer drainDone()
-
-	var line chainlock.L
-	var wg sync.WaitGroup
-	defer wg.Wait()
-
-	// print results
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		enc := json.NewEncoder(os.Stdout)
-		for a := range abstractions {
-			func() {
-				defer line.Lock().Unlock()
-				if zabsListFlags.Json {
-					enc.SetIndent("", "  ")
-					if err := enc.Encode(a); err != nil {
-						panic(err)
-					}
-					fmt.Println()
-				} else {
-					fmt.Println(a)
-				}
-			}()
-		}
-	}()
-
-	// print errors to stderr
-	errorColor := color.New(color.FgRed)
-	var errorsSlice []endpoint.ListAbstractionsError
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		for err := range errors {
-			func() {
-				defer line.Lock().Unlock()
-				errorsSlice = append(errorsSlice, err)
-				errorColor.Fprintf(os.Stderr, "%s\n", err)
-			}()
-		}
-	}()
-	wg.Wait()
-	if len(errorsSlice) > 0 {
-		errorColor.Add(color.Bold).Fprintf(os.Stderr, "there were errors in listing the abstractions")
-		return fmt.Errorf("")
-	} else {
-		return nil
-	}
-
-}

client/zfsabstractions_release.go

@@ -1,143 +0,0 @@
-package client
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"os"
-
-	"github.com/fatih/color"
-	"github.com/pkg/errors"
-	"github.com/spf13/pflag"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/endpoint"
-)
-
-// shared between release-all and release-step
-var zabsReleaseFlags struct {
-	Filter zabsFilterFlags
-	Json   bool
-	DryRun bool
-}
-
-func registerZabsReleaseFlags(s *pflag.FlagSet) {
-	zabsReleaseFlags.Filter.registerZabsFilterFlags(s, "release")
-	s.BoolVar(&zabsReleaseFlags.Json, "json", false, "emit json instead of pretty-printed")
-	s.BoolVar(&zabsReleaseFlags.DryRun, "dry-run", false, "do a dry-run")
-}
-
-var zabsCmdReleaseAll = &cli.Subcommand{
-	Use:             "release-all",
-	Run:             doZabsReleaseAll,
-	NoRequireConfig: true,
-	Short:           `(DANGEROUS) release ALL zrepl ZFS abstractions (mostly useful for uninstalling zrepl completely or for "de-zrepl-ing" a filesystem)`,
-	SetupFlags:      registerZabsReleaseFlags,
-}
-
-var zabsCmdReleaseStale = &cli.Subcommand{
-	Use:             "release-stale",
-	Run:             doZabsReleaseStale,
-	NoRequireConfig: true,
-	Short:           `release stale zrepl ZFS abstractions (useful if zrepl has a bug and does not do it by itself)`,
-	SetupFlags:      registerZabsReleaseFlags,
-}
-
-func doZabsReleaseAll(ctx context.Context, sc *cli.Subcommand, args []string) error {
-	var err error
-
-	if len(args) > 0 {
-		return errors.New("this subcommand takes no positional arguments")
-	}
-
-	q, err := zabsReleaseFlags.Filter.Query()
-	if err != nil {
-		return errors.Wrap(err, "invalid filter specification on command line")
-	}
-
-	abstractions, listErrors, err := endpoint.ListAbstractions(ctx, q)
-	if err != nil {
-		return err // context clear by invocation of command
-	}
-	if len(listErrors) > 0 {
-		color.New(color.FgRed).Fprintf(os.Stderr, "there were errors in listing the abstractions:\n%s\n", listErrors)
-		// proceed anyways with rest of abstractions
-	}
-
-	return doZabsRelease_Common(ctx, abstractions)
-}
-
-func doZabsReleaseStale(ctx context.Context, sc *cli.Subcommand, args []string) error {
-
-	var err error
-
-	if len(args) > 0 {
-		return errors.New("this subcommand takes no positional arguments")
-	}
-
-	q, err := zabsReleaseFlags.Filter.Query()
-	if err != nil {
-		return errors.Wrap(err, "invalid filter specification on command line")
-	}
-
-	stalenessInfo, err := endpoint.ListStale(ctx, q)
-	if err != nil {
-		return err // context clear by invocation of command
-	}
-
-	return doZabsRelease_Common(ctx, stalenessInfo.Stale)
-}
-
-func doZabsRelease_Common(ctx context.Context, destroy []endpoint.Abstraction) error {
-
-	if zabsReleaseFlags.DryRun {
-		if zabsReleaseFlags.Json {
-			m, err := json.MarshalIndent(destroy, "", "  ")
-			if err != nil {
-				panic(err)
-			}
-			if _, err := os.Stdout.Write(m); err != nil {
-				panic(err)
-			}
-			fmt.Println()
-		} else {
-			for _, a := range destroy {
-				fmt.Printf("would destroy %s\n", a)
-			}
-		}
-		return nil
-	}
-
-	outcome := endpoint.BatchDestroy(ctx, destroy)
-	hadErr := false
-
-	enc := json.NewEncoder(os.Stdout)
-	enc.SetIndent("", "  ")
-	colorErr := color.New(color.FgRed)
-	printfSuccess := color.New(color.FgGreen).FprintfFunc()
-	printfSection := color.New(color.Bold).FprintfFunc()
-
-	for res := range outcome {
-		hadErr = hadErr || res.DestroyErr != nil
-		if zabsReleaseFlags.Json {
-			err := enc.Encode(res)
-			if err != nil {
-				colorErr.Fprintf(os.Stderr, "cannot marshal there were errors in destroying the abstractions")
-			}
-		} else {
-			printfSection(os.Stdout, "destroy %s ...", res.Abstraction)
-			if res.DestroyErr != nil {
-				colorErr.Fprintf(os.Stdout, " failed:\n%s\n", res.DestroyErr)
-			} else {
-				printfSuccess(os.Stdout, " OK\n")
-			}
-		}
-	}
-
-	if hadErr {
-		colorErr.Add(color.Bold).Fprintf(os.Stderr, "there were errors in destroying the abstractions")
-		return fmt.Errorf("")
-	} else {
-		return nil
-	}
-}

cmd/autosnap.go

@@ -0,0 +1,91 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/jobrun"
+	"github.com/zrepl/zrepl/zfs"
+	"os"
+	"sync"
+	"time"
+)
+
+var AutosnapCmd = &cobra.Command{
+	Use:   "autosnap",
+	Short: "perform automatic snapshotting",
+	Run:   cmdAutosnap,
+}
+
+func init() {
+	RootCmd.AddCommand(AutosnapCmd)
+}
+
+func cmdAutosnap(cmd *cobra.Command, args []string) {
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		runner.Start()
+	}()
+
+	if len(args) < 1 {
+		log.Printf("must specify exactly one job as positional argument")
+		os.Exit(1)
+	}
+
+	snap, ok := conf.Autosnaps[args[0]]
+	if !ok {
+		log.Printf("could not find autosnap job: %s", args[0])
+		os.Exit(1)
+	}
+
+	job := jobrun.Job{
+		Name:           snap.JobName,
+		RepeatStrategy: snap.Interval,
+		RunFunc: func(log jobrun.Logger) error {
+			log.Printf("doing autosnap: %v", snap)
+			ctx := AutosnapContext{snap}
+			return doAutosnap(ctx, log)
+		},
+	}
+	runner.AddJob(job)
+
+	wg.Wait()
+
+}
+
+type AutosnapContext struct {
+	Autosnap *Autosnap
+}
+
+func doAutosnap(ctx AutosnapContext, log Logger) (err error) {
+
+	snap := ctx.Autosnap
+
+	filesystems, err := zfs.ZFSListMapping(snap.DatasetFilter)
+	if err != nil {
+		return fmt.Errorf("cannot filter datasets: %s", err)
+	}
+
+	suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
+	snapname := fmt.Sprintf("%s%s", snap.Prefix, suffix)
+
+	hadError := false
+
+	for _, fs := range filesystems { // optimization: use recursive snapshots / channel programs here
+		log.Printf("snapshotting filesystem %s@%s", fs, snapname)
+		err := zfs.ZFSSnapshot(fs, snapname, false)
+		if err != nil {
+			log.Printf("error snapshotting %s: %s", fs, err)
+			hadError = true
+		}
+	}
+
+	if hadError {
+		err = fmt.Errorf("errors occurred during autosnap, check logs for details")
+	}
+
+	return
+
+}

cmd/config.go

@@ -0,0 +1,730 @@
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"github.com/jinzhu/copier"
+	"github.com/mitchellh/mapstructure"
+	"github.com/zrepl/zrepl/jobrun"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/sshbytestream"
+	. "github.com/zrepl/zrepl/util"
+	"github.com/zrepl/zrepl/zfs"
+	yaml "gopkg.in/yaml.v2"
+	"io"
+	"io/ioutil"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+var (
+	JobSectionPush     string = "push"
+	JobSectionPull     string = "pull"
+	JobSectionPrune    string = "prune"
+	JobSectionAutosnap string = "autosnap"
+)
+
+type Remote struct {
+	Name      string
+	Transport Transport
+}
+
+type Transport interface {
+	Connect(rpcLog Logger) (rpc.RPCRequester, error)
+}
+type LocalTransport struct {
+	Handler rpc.RPCHandler
+}
+type SSHTransport struct {
+	Host                 string
+	User                 string
+	Port                 uint16
+	IdentityFile         string   `mapstructure:"identity_file"`
+	TransportOpenCommand []string `mapstructure:"transport_open_command"`
+	SSHCommand           string   `mapstructure:"ssh_command"`
+	Options              []string
+	ConnLogReadFile      string `mapstructure:"connlog_read_file"`
+	ConnLogWriteFile     string `mapstructure:"connlog_write_file"`
+}
+
+type Push struct {
+	JobName           string // for use with jobrun package
+	To                *Remote
+	Filter            zfs.DatasetFilter
+	InitialReplPolicy rpc.InitialReplPolicy
+	RepeatStrategy    jobrun.RepeatStrategy
+}
+type Pull struct {
+	JobName           string // for use with jobrun package
+	From              *Remote
+	Mapping           DatasetMapFilter
+	InitialReplPolicy rpc.InitialReplPolicy
+	RepeatStrategy    jobrun.RepeatStrategy
+}
+
+type Prune struct {
+	JobName         string // for use with jobrun package
+	DatasetFilter   zfs.DatasetFilter
+	SnapshotFilter  zfs.FilesystemVersionFilter
+	RetentionPolicy *RetentionGrid // TODO abstract interface to support future policies?
+}
+
+type Autosnap struct {
+	JobName       string // for use with jobrun package
+	Prefix        string
+	Interval      jobrun.RepeatStrategy
+	DatasetFilter zfs.DatasetFilter
+}
+
+type Config struct {
+	Remotes   map[string]*Remote
+	Pushs     map[string]*Push            // job name -> job
+	Pulls     map[string]*Pull            // job name -> job
+	Sinks     map[string]DatasetMapFilter // client identity -> mapping
+	PullACLs  map[string]DatasetMapFilter // client identity -> filter
+	Prunes    map[string]*Prune           // job name -> job
+	Autosnaps map[string]*Autosnap        // job name -> job
+}
+
+func ParseConfig(path string) (config Config, err error) {
+
+	c := make(map[string]interface{}, 0)
+
+	var bytes []byte
+
+	if bytes, err = ioutil.ReadFile(path); err != nil {
+		return
+	}
+
+	if err = yaml.Unmarshal(bytes, &c); err != nil {
+		return
+	}
+
+	return parseMain(c)
+}
+
+func parseMain(root map[string]interface{}) (c Config, err error) {
+	if c.Remotes, err = parseRemotes(root["remotes"]); err != nil {
+		return
+	}
+
+	remoteLookup := func(name string) (remote *Remote, err error) {
+		remote = c.Remotes[name]
+		if remote == nil {
+			err = fmt.Errorf("remote '%s' not defined", name)
+		}
+		return
+	}
+
+	if c.Pushs, err = parsePushs(root["pushs"], remoteLookup); err != nil {
+		return
+	}
+	if c.Pulls, err = parsePulls(root["pulls"], remoteLookup); err != nil {
+		return
+	}
+	if c.Sinks, err = parseSinks(root["sinks"]); err != nil {
+		return
+	}
+	if c.PullACLs, err = parsePullACLs(root["pull_acls"]); err != nil {
+		return
+	}
+	if c.Prunes, err = parsePrunes(root["prune"]); err != nil {
+		return
+	}
+	if c.Autosnaps, err = parseAutosnaps(root["autosnap"]); err != nil {
+		return
+	}
+
+	return
+}
+
+func fullJobName(section, name string) (full string, err error) {
+	if len(name) < 1 {
+		err = fmt.Errorf("job name not set")
+		return
+	}
+	full = fmt.Sprintf("%s.%s", section, name)
+	return
+}
+
+func parseRemotes(v interface{}) (remotes map[string]*Remote, err error) {
+
+	asMap := make(map[string]struct {
+		Transport map[string]interface{}
+	}, 0)
+	if err = mapstructure.Decode(v, &asMap); err != nil {
+		return
+	}
+
+	remotes = make(map[string]*Remote, len(asMap))
+	for name, p := range asMap {
+
+		if name == rpc.LOCAL_TRANSPORT_IDENTITY {
+			err = errors.New(fmt.Sprintf("remote name '%s' reserved for local pulls", rpc.LOCAL_TRANSPORT_IDENTITY))
+			return
+		}
+
+		var transport Transport
+		if transport, err = parseTransport(p.Transport); err != nil {
+			return
+		}
+		remotes[name] = &Remote{
+			Name:      name,
+			Transport: transport,
+		}
+	}
+
+	return
+}
+
+func parseTransport(it map[string]interface{}) (t Transport, err error) {
+
+	if len(it) != 1 {
+		err = errors.New("ambiguous transport type")
+		return
+	}
+
+	for key, val := range it {
+		switch key {
+		case "ssh":
+			t := SSHTransport{}
+			if err = mapstructure.Decode(val, &t); err != nil {
+				err = errors.New(fmt.Sprintf("could not parse ssh transport: %s", err))
+				return nil, err
+			}
+			return t, nil
+		default:
+			return nil, errors.New(fmt.Sprintf("unknown transport type '%s'\n", key))
+		}
+	}
+
+	return // unreachable
+
+}
+
+type remoteLookup func(name string) (*Remote, error)
+
+func parsePushs(v interface{}, rl remoteLookup) (p map[string]*Push, err error) {
+
+	asMap := make(map[string]struct {
+		To                string
+		Filter            map[string]string
+		InitialReplPolicy string
+		Repeat            map[string]string
+	}, 0)
+
+	if err = mapstructure.Decode(v, &asMap); err != nil {
+		return
+	}
+
+	p = make(map[string]*Push, len(asMap))
+
+	for name, e := range asMap {
+
+		var toRemote *Remote
+		if toRemote, err = rl(e.To); err != nil {
+			return
+		}
+		push := &Push{
+			To: toRemote,
+		}
+
+		if push.JobName, err = fullJobName(JobSectionPush, name); err != nil {
+			return
+		}
+		if push.Filter, err = parseDatasetMapFilter(e.Filter, true); err != nil {
+			return
+		}
+
+		if push.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, rpc.DEFAULT_INITIAL_REPL_POLICY); err != nil {
+			return
+		}
+
+		if push.RepeatStrategy, err = parseRepeatStrategy(e.Repeat); err != nil {
+			return
+		}
+
+		p[name] = push
+	}
+
+	return
+}
+
+func parsePulls(v interface{}, rl remoteLookup) (p map[string]*Pull, err error) {
+
+	asMap := make(map[string]struct {
+		From              string
+		Mapping           map[string]string
+		InitialReplPolicy string
+		Repeat            map[string]string
+	}, 0)
+
+	if err = mapstructure.Decode(v, &asMap); err != nil {
+		return
+	}
+
+	p = make(map[string]*Pull, len(asMap))
+
+	for name, e := range asMap {
+
+		if len(e.From) < 1 {
+			err = fmt.Errorf("source not set ('from' attribute is empty)")
+			return
+		}
+
+		var fromRemote *Remote
+
+		if e.From == rpc.LOCAL_TRANSPORT_IDENTITY {
+			fromRemote = &Remote{
+				Name:      rpc.LOCAL_TRANSPORT_IDENTITY,
+				Transport: LocalTransport{},
+			}
+		} else {
+			if fromRemote, err = rl(e.From); err != nil {
+				return
+			}
+		}
+
+		pull := &Pull{
+			From: fromRemote,
+		}
+		if pull.JobName, err = fullJobName(JobSectionPull, name); err != nil {
+			return
+		}
+		if pull.Mapping, err = parseDatasetMapFilter(e.Mapping, false); err != nil {
+			return
+		}
+		if pull.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, rpc.DEFAULT_INITIAL_REPL_POLICY); err != nil {
+			return
+		}
+		if pull.RepeatStrategy, err = parseRepeatStrategy(e.Repeat); err != nil {
+			return
+		}
+
+		p[name] = pull
+	}
+
+	return
+}
+
+func parseInitialReplPolicy(v interface{}, defaultPolicy rpc.InitialReplPolicy) (p rpc.InitialReplPolicy, err error) {
+	s, ok := v.(string)
+	if !ok {
+		goto err
+	}
+
+	switch {
+	case s == "":
+		p = defaultPolicy
+	case s == "most_recent":
+		p = rpc.InitialReplPolicyMostRecent
+	case s == "all":
+		p = rpc.InitialReplPolicyAll
+	default:
+		goto err
+	}
+
+	return
+
+err:
+	err = errors.New(fmt.Sprintf("expected InitialReplPolicy, got %#v", v))
+	return
+}
+
+func parseRepeatStrategy(r map[string]string) (s jobrun.RepeatStrategy, err error) {
+
+	if r == nil {
+		return jobrun.NoRepeatStrategy{}, nil
+	}
+
+	if repeatStr, ok := r["interval"]; ok {
+		d, err := parseDuration(repeatStr)
+		if err != nil {
+			return nil, err
+		}
+		s = &jobrun.PeriodicRepeatStrategy{d}
+		return s, err
+	} else {
+		return nil, fmt.Errorf("attribute 'interval' not found but required in repeat specification")
+	}
+
+}
+
+func expectList(v interface{}) (asList []interface{}, err error) {
+	var ok bool
+	if asList, ok = v.([]interface{}); !ok {
+		err = errors.New("expected list")
+	}
+	return
+}
+
+func parseSinks(v interface{}) (m map[string]DatasetMapFilter, err error) {
+	var asMap map[string]map[string]interface{}
+	if err = mapstructure.Decode(v, &asMap); err != nil {
+		return
+	}
+
+	m = make(map[string]DatasetMapFilter, len(asMap))
+
+	for identity, entry := range asMap {
+		parseSink := func() (mapping DatasetMapFilter, err error) {
+			mappingMap, ok := entry["mapping"]
+			if !ok {
+				err = fmt.Errorf("no mapping specified")
+				return
+			}
+			mapping, err = parseDatasetMapFilter(mappingMap, false)
+			return
+		}
+		mapping, sinkErr := parseSink()
+		if sinkErr != nil {
+			err = fmt.Errorf("cannot parse sink for identity '%s': %s", identity, sinkErr)
+			return
+		}
+		m[identity] = mapping
+	}
+	return
+}
+
+func parsePullACLs(v interface{}) (m map[string]DatasetMapFilter, err error) {
+	var asMap map[string]map[string]interface{}
+	if err = mapstructure.Decode(v, &asMap); err != nil {
+		return
+	}
+
+	m = make(map[string]DatasetMapFilter, len(asMap))
+
+	for identity, entry := range asMap {
+		parsePullACL := func() (filter DatasetMapFilter, err error) {
+			filterMap, ok := entry["filter"]
+			if !ok {
+				err = fmt.Errorf("no filter specified")
+				return
+			}
+			filter, err = parseDatasetMapFilter(filterMap, true)
+			return
+		}
+		filter, filterErr := parsePullACL()
+		if filterErr != nil {
+			err = fmt.Errorf("cannot parse pull-ACL for identity '%s': %s", identity, filterErr)
+			return
+		}
+		m[identity] = filter
+	}
+	return
+}
+
+func parseDatasetMapFilter(mi interface{}, filterOnly bool) (f DatasetMapFilter, err error) {
+
+	var m map[string]string
+	if err = mapstructure.Decode(mi, &m); err != nil {
+		err = fmt.Errorf("maps / filters must be specified as map[string]string: %s", err)
+		return
+	}
+
+	f = NewDatasetMapFilter(len(m), filterOnly)
+	for pathPattern, mapping := range m {
+		if err = f.Add(pathPattern, mapping); err != nil {
+			err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err)
+			return
+		}
+	}
+	return
+}
+
+func (t SSHTransport) Connect(rpcLog Logger) (r rpc.RPCRequester, err error) {
+	var stream io.ReadWriteCloser
+	var rpcTransport sshbytestream.SSHTransport
+	if err = copier.Copy(&rpcTransport, t); err != nil {
+		return
+	}
+	if stream, err = sshbytestream.Outgoing(rpcTransport); err != nil {
+		return
+	}
+	stream, err = NewReadWriteCloserLogger(stream, t.ConnLogReadFile, t.ConnLogWriteFile)
+	if err != nil {
+		return
+	}
+	return rpc.ConnectByteStreamRPC(stream, rpcLog)
+}
+
+func (t LocalTransport) Connect(rpcLog Logger) (r rpc.RPCRequester, err error) {
+	if t.Handler == nil {
+		panic("local transport with uninitialized handler")
+	}
+	return rpc.ConnectLocalRPC(t.Handler), nil
+}
+
+func (t *LocalTransport) SetHandler(handler rpc.RPCHandler) {
+	t.Handler = handler
+}
+
+func parsePrunes(m interface{}) (rets map[string]*Prune, err error) {
+
+	asList := make(map[string]map[string]interface{}, 0)
+	if err = mapstructure.Decode(m, &asList); err != nil {
+		return
+	}
+
+	rets = make(map[string]*Prune, len(asList))
+
+	for name, e := range asList {
+		var prune *Prune
+		if prune, err = parsePrune(e, name); err != nil {
+			err = fmt.Errorf("cannot parse prune job %s: %s", name, err)
+			return
+		}
+		rets[name] = prune
+	}
+
+	return
+}
+
+func parsePrune(e map[string]interface{}, name string) (prune *Prune, err error) {
+
+	// Only support grid policy for now
+	policyName, ok := e["policy"]
+	if !ok || policyName != "grid" {
+		err = fmt.Errorf("prune job with unimplemented policy '%s'", policyName)
+		return
+	}
+
+	var i struct {
+		Grid           string
+		DatasetFilter  map[string]string `mapstructure:"dataset_filter"`
+		SnapshotFilter map[string]string `mapstructure:"snapshot_filter"`
+	}
+
+	if err = mapstructure.Decode(e, &i); err != nil {
+		return
+	}
+
+	prune = &Prune{}
+
+	if prune.JobName, err = fullJobName(JobSectionPrune, name); err != nil {
+		return
+	}
+
+	// Parse grid policy
+	intervals, err := parseRetentionGridIntervalsString(i.Grid)
+	if err != nil {
+		err = fmt.Errorf("cannot parse retention grid: %s", err)
+		return
+	}
+	// Assert intervals are of increasing length (not necessarily required, but indicates config mistake)
+	lastDuration := time.Duration(0)
+	for i := range intervals {
+		if intervals[i].Length < lastDuration {
+			err = fmt.Errorf("retention grid interval length must be monotonically increasing:"+
+				"interval %d is shorter than %d", i+1, i)
+			return
+		} else {
+			lastDuration = intervals[i].Length
+		}
+	}
+	prune.RetentionPolicy = NewRetentionGrid(intervals)
+
+	// Parse filters
+	if prune.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil {
+		err = fmt.Errorf("cannot parse dataset filter: %s", err)
+		return
+	}
+	if prune.SnapshotFilter, err = parseSnapshotFilter(i.SnapshotFilter); err != nil {
+		err = fmt.Errorf("cannot parse snapshot filter: %s", err)
+		return
+	}
+
+	return
+}
+
+var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
+
+func parseDuration(e string) (d time.Duration, err error) {
+	comps := durationStringRegex.FindStringSubmatch(e)
+	if len(comps) != 3 {
+		err = fmt.Errorf("does not match regex: %s %#v", e, comps)
+		return
+	}
+
+	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
+	if err != nil {
+		return
+	}
+
+	var durationUnit time.Duration
+	switch comps[2] {
+	case "s":
+		durationUnit = time.Second
+	case "m":
+		durationUnit = time.Minute
+	case "h":
+		durationUnit = time.Hour
+	case "d":
+		durationUnit = 24 * time.Hour
+	case "w":
+		durationUnit = 24 * 7 * time.Hour
+	default:
+		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
+		return
+	}
+
+	d = time.Duration(durationFactor) * durationUnit
+	return
+
+}
+
+var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
+
+func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {
+
+	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
+	if comps == nil {
+		err = fmt.Errorf("retention string does not match expected format")
+		return
+	}
+
+	times, err := strconv.Atoi(comps[1])
+	if err != nil {
+		return nil, err
+	} else if times <= 0 {
+		return nil, fmt.Errorf("contains factor <= 0")
+	}
+
+	duration, err := parseDuration(comps[2])
+	if err != nil {
+		return nil, err
+	}
+
+	keepCount := 1
+	if comps[3] != "" {
+		// Decompose key=value, comma separated
+		// For now, only keep_count is supported
+		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
+		res := re.FindStringSubmatch(comps[4])
+		if res == nil || len(res) != 2 {
+			err = fmt.Errorf("interval parameter contains unknown parameters")
+			return
+		}
+		if res[1] == "all" {
+			keepCount = RetentionGridKeepCountAll
+		} else {
+			keepCount, err = strconv.Atoi(res[1])
+			if err != nil {
+				err = fmt.Errorf("cannot parse keep_count value")
+				return
+			}
+		}
+	}
+
+	intervals = make([]RetentionInterval, times)
+	for i := range intervals {
+		intervals[i] = RetentionInterval{
+			Length:    duration,
+			KeepCount: keepCount,
+		}
+	}
+
+	return
+
+}
+
+func parseRetentionGridIntervalsString(s string) (intervals []RetentionInterval, err error) {
+
+	ges := strings.Split(s, "|")
+	intervals = make([]RetentionInterval, 0, 7*len(ges))
+
+	for intervalIdx, e := range ges {
+		parsed, err := parseRetentionGridIntervalString(e)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
+		}
+		intervals = append(intervals, parsed...)
+	}
+
+	return
+}
+
+type prefixSnapshotFilter struct {
+	prefix string
+}
+
+func (f prefixSnapshotFilter) Filter(fsv zfs.FilesystemVersion) (accept bool, err error) {
+	return fsv.Type == zfs.Snapshot && strings.HasPrefix(fsv.Name, f.prefix), nil
+}
+
+func parseSnapshotFilter(fm map[string]string) (snapFilter zfs.FilesystemVersionFilter, err error) {
+	prefix, ok := fm["prefix"]
+	if !ok {
+		err = fmt.Errorf("unsupported snapshot filter")
+		return
+	}
+	snapFilter = prefixSnapshotFilter{prefix}
+	return
+}
+
+func parseAutosnaps(m interface{}) (snaps map[string]*Autosnap, err error) {
+
+	asMap := make(map[string]interface{}, 0)
+	if err = mapstructure.Decode(m, &asMap); err != nil {
+		return
+	}
+
+	snaps = make(map[string]*Autosnap, len(asMap))
+
+	for name, e := range asMap {
+		var snap *Autosnap
+		if snap, err = parseAutosnap(e, name); err != nil {
+			err = fmt.Errorf("cannot parse autonsap job %s: %s", name, err)
+			return
+		}
+		snaps[name] = snap
+	}
+
+	return
+
+}
+
+func parseAutosnap(m interface{}, name string) (a *Autosnap, err error) {
+
+	var i struct {
+		Prefix        string
+		Interval      string
+		DatasetFilter map[string]string `mapstructure:"dataset_filter"`
+	}
+
+	if err = mapstructure.Decode(m, &i); err != nil {
+		err = fmt.Errorf("structure unfit: %s", err)
+		return
+	}
+
+	a = &Autosnap{}
+
+	if a.JobName, err = fullJobName(JobSectionAutosnap, name); err != nil {
+		return
+	}
+
+	if len(i.Prefix) < 1 {
+		err = fmt.Errorf("prefix must not be empty")
+		return
+	}
+	a.Prefix = i.Prefix
+
+	var interval time.Duration
+	if interval, err = parseDuration(i.Interval); err != nil {
+		err = fmt.Errorf("cannot parse interval: %s", err)
+		return
+	}
+	a.Interval = &jobrun.PeriodicRepeatStrategy{interval}
+
+	if len(i.DatasetFilter) == 0 {
+		err = fmt.Errorf("dataset_filter not specified")
+		return
+	}
+	if a.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil {
+		err = fmt.Errorf("cannot parse dataset filter: %s", err)
+	}
+
+	return
+}

cmd/config_mapfilter.go

@@ -0,0 +1,162 @@
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type DatasetMapFilter struct {
+	entries []datasetMapFilterEntry
+
+	// if set, only valid filter entries can be added using Add()
+	// and Map() will always return an error
+	filterOnly bool
+}
+
+type datasetMapFilterEntry struct {
+	path *zfs.DatasetPath
+	// the mapping. since this datastructure acts as both mapping and filter
+	// we have to convert it to the desired rep dynamically
+	mapping      string
+	subtreeMatch bool
+}
+
+var NoMatchError error = errors.New("no match found in mapping")
+
+func NewDatasetMapFilter(capacity int, filterOnly bool) DatasetMapFilter {
+	return DatasetMapFilter{
+		entries: make([]datasetMapFilterEntry, 0, capacity),
+	}
+}
+
+func (m *DatasetMapFilter) Add(pathPattern, mapping string) (err error) {
+
+	if m.filterOnly {
+		if _, err = parseDatasetFilterResult(mapping); err != nil {
+			return
+		}
+	}
+
+	// assert path glob adheres to spec
+	const SUBTREE_PATTERN string = "<"
+	patternCount := strings.Count(pathPattern, SUBTREE_PATTERN)
+	switch {
+	case patternCount > 1:
+	case patternCount == 1 && !strings.HasSuffix(pathPattern, SUBTREE_PATTERN):
+		err = fmt.Errorf("pattern invalid: only one '<' at end of string allowed")
+		return
+	}
+
+	pathStr := strings.TrimSuffix(pathPattern, SUBTREE_PATTERN)
+	path, err := zfs.NewDatasetPath(pathStr)
+	if err != nil {
+		return fmt.Errorf("pattern is not a dataset path: %s", err)
+	}
+
+	entry := datasetMapFilterEntry{
+		path:         path,
+		mapping:      mapping,
+		subtreeMatch: patternCount > 0,
+	}
+	m.entries = append(m.entries, entry)
+	return
+
+}
+
+// find the most specific prefix mapping we have
+//
+// longer prefix wins over shorter prefix, direct wins over glob
+func (m DatasetMapFilter) mostSpecificPrefixMapping(path *zfs.DatasetPath) (idx int, found bool) {
+	lcp, lcp_entry_idx := -1, -1
+	direct_idx := -1
+	for e := range m.entries {
+		entry := m.entries[e]
+		ep := m.entries[e].path
+		lep := ep.Length()
+
+		switch {
+		case !entry.subtreeMatch && ep.Equal(path):
+			direct_idx = e
+			continue
+		case entry.subtreeMatch && path.HasPrefix(ep) && lep > lcp:
+			lcp = lep
+			lcp_entry_idx = e
+		default:
+			continue
+		}
+	}
+
+	if lcp_entry_idx >= 0 || direct_idx >= 0 {
+		found = true
+		switch {
+		case direct_idx >= 0:
+			idx = direct_idx
+		case lcp_entry_idx >= 0:
+			idx = lcp_entry_idx
+		}
+	}
+	return
+}
+
+func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error) {
+
+	if m.filterOnly {
+		err = fmt.Errorf("using a filter for mapping simply does not work")
+		return
+	}
+
+	mi, hasMapping := m.mostSpecificPrefixMapping(source)
+	if !hasMapping {
+		err = NoMatchError
+		return
+	}
+	me := m.entries[mi]
+
+	target, err = zfs.NewDatasetPath(me.mapping)
+	if err != nil {
+		err = fmt.Errorf("mapping target is not a dataset path: %s", err)
+		return
+	}
+	if m.entries[mi].subtreeMatch {
+		// strip common prefix
+		extendComps := source.Copy()
+		if me.path.Empty() {
+			// special case: trying to map the root => strip first component
+			extendComps.TrimNPrefixComps(1)
+		} else {
+			extendComps.TrimPrefix(me.path)
+		}
+		target.Extend(extendComps)
+	}
+	return
+}
+
+func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
+	mi, hasMapping := m.mostSpecificPrefixMapping(p)
+	if !hasMapping {
+		pass = false
+		return
+	}
+	me := m.entries[mi]
+	pass, err = parseDatasetFilterResult(me.mapping)
+	return
+}
+
+// Parse a dataset filter result
+func parseDatasetFilterResult(result string) (pass bool, err error) {
+	l := strings.ToLower(result)
+	switch strings.ToLower(l) {
+	case "ok":
+		pass = true
+		return
+	case "omit":
+		return
+	default:
+		err = fmt.Errorf("'%s' is not a valid filter result", result)
+		return
+	}
+	return
+}

cmd/config_test.go

@@ -0,0 +1,123 @@
+package cmd
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/zrepl/zrepl/util"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func TestSampleConfigFileIsParsedWithoutErrors(t *testing.T) {
+	_, err := ParseConfig("./sampleconf/zrepl.yml")
+	assert.Nil(t, err)
+}
+
+func TestParseRetentionGridStringParsing(t *testing.T) {
+
+	intervals, err := parseRetentionGridIntervalsString("2x10m(keep=2) | 1x1h | 3x1w")
+
+	assert.Nil(t, err)
+	assert.Len(t, intervals, 6)
+	proto := util.RetentionInterval{
+		KeepCount: 2,
+		Length:    10 * time.Minute,
+	}
+	assert.EqualValues(t, proto, intervals[0])
+	assert.EqualValues(t, proto, intervals[1])
+
+	proto.KeepCount = 1
+	proto.Length = 1 * time.Hour
+	assert.EqualValues(t, proto, intervals[2])
+
+	proto.Length = 7 * 24 * time.Hour
+	assert.EqualValues(t, proto, intervals[3])
+	assert.EqualValues(t, proto, intervals[4])
+	assert.EqualValues(t, proto, intervals[5])
+
+	intervals, err = parseRetentionGridIntervalsString("|")
+	assert.Error(t, err)
+	intervals, err = parseRetentionGridIntervalsString("2x10m")
+	assert.NoError(t, err)
+
+	intervals, err = parseRetentionGridIntervalsString("1x10m(keep=all)")
+	assert.NoError(t, err)
+	assert.Len(t, intervals, 1)
+	assert.EqualValues(t, util.RetentionGridKeepCountAll, intervals[0].KeepCount)
+
+}
+
+func TestDatasetMapFilter(t *testing.T) {
+
+	expectMapping := func(m map[string]string, from, to string) {
+		dmf, err := parseDatasetMapFilter(m, false)
+		if err != nil {
+			t.Logf("expect test map to be valid: %s", err)
+			t.FailNow()
+		}
+		fromPath, err := zfs.NewDatasetPath(from)
+		if err != nil {
+			t.Logf("expect test from path to be valid: %s", err)
+			t.FailNow()
+		}
+		toPath, err := zfs.NewDatasetPath(to)
+		if err != nil {
+			t.Logf("expect test to path to be valid: %s", err)
+			t.FailNow()
+		}
+
+		res, err := dmf.Map(fromPath)
+		t.Logf("%s => %s", fromPath.ToString(), res.ToString())
+		assert.Nil(t, err)
+		assert.True(t, res.Equal(toPath))
+	}
+
+	expectFilter := func(m map[string]string, path string, pass bool) {
+		dmf, err := parseDatasetMapFilter(m, true)
+		if err != nil {
+			t.Logf("expect test filter to be valid: %s", err)
+			t.FailNow()
+		}
+		p, err := zfs.NewDatasetPath(path)
+		if err != nil {
+			t.Logf("expect test path to be valid: %s", err)
+			t.FailNow()
+		}
+		res, err := dmf.Filter(p)
+		assert.Nil(t, err)
+		assert.Equal(t, pass, res)
+	}
+
+	map1 := map[string]string{
+		"a/b/c<": "root1",
+		"a/b<":   "root2",
+		"<":      "root3/b/c",
+		"q<":     "root4/1/2",
+	}
+
+	expectMapping(map1, "a/b/c", "root1")
+	expectMapping(map1, "a/b/c/d", "root1/d")
+	expectMapping(map1, "a/b/e", "root2/e")
+	expectMapping(map1, "a/b", "root2")
+	expectMapping(map1, "x", "root3/b/c")
+	expectMapping(map1, "x/y", "root3/b/c/y")
+	expectMapping(map1, "q", "root4/1/2")
+	expectMapping(map1, "q/r", "root4/1/2/r")
+
+	filter1 := map[string]string{
+		"<":    "omit",
+		"a<":   "ok",
+		"a/b<": "omit",
+	}
+
+	expectFilter(filter1, "b", false)
+	expectFilter(filter1, "a", true)
+	expectFilter(filter1, "a/d", true)
+	expectFilter(filter1, "a/b", false)
+	expectFilter(filter1, "a/b/c", false)
+
+	filter2 := map[string]string{}
+	expectFilter(filter2, "foo", false) // default to omit
+
+}

cmd/handler.go

@@ -0,0 +1,177 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/zfs"
+	"io"
+)
+
+type DatasetMapping interface {
+	Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error)
+}
+
+type Handler struct {
+	Logger          Logger
+	PullACL         zfs.DatasetFilter
+	SinkMappingFunc func(clientIdentity string) (mapping DatasetMapping, err error)
+}
+
+func (h Handler) HandleFilesystemRequest(r rpc.FilesystemRequest) (roots []*zfs.DatasetPath, err error) {
+
+	h.Logger.Printf("handling fsr: %#v", r)
+
+	h.Logger.Printf("using PullACL: %#v", h.PullACL)
+
+	if roots, err = zfs.ZFSListMapping(h.PullACL); err != nil {
+		h.Logger.Printf("handle fsr err: %v\n", err)
+		return
+	}
+
+	h.Logger.Printf("returning: %#v", roots)
+
+	return
+}
+
+func (h Handler) HandleFilesystemVersionsRequest(r rpc.FilesystemVersionsRequest) (versions []zfs.FilesystemVersion, err error) {
+
+	h.Logger.Printf("handling filesystem versions request: %#v", r)
+
+	// allowed to request that?
+	if h.pullACLCheck(r.Filesystem); err != nil {
+		return
+	}
+
+	// find our versions
+	if versions, err = zfs.ZFSListFilesystemVersions(r.Filesystem, nil); err != nil {
+		h.Logger.Printf("our versions error: %#v\n", err)
+		return
+	}
+
+	h.Logger.Printf("our versions: %#v\n", versions)
+	return
+
+}
+
+func (h Handler) HandleInitialTransferRequest(r rpc.InitialTransferRequest) (stream io.Reader, err error) {
+
+	h.Logger.Printf("handling initial transfer request: %#v", r)
+	if err = h.pullACLCheck(r.Filesystem); err != nil {
+		return
+	}
+
+	h.Logger.Printf("invoking zfs send")
+
+	if stream, err = zfs.ZFSSend(r.Filesystem, &r.FilesystemVersion, nil); err != nil {
+		h.Logger.Printf("error sending filesystem: %#v", err)
+	}
+
+	return
+
+}
+
+func (h Handler) HandleIncrementalTransferRequest(r rpc.IncrementalTransferRequest) (stream io.Reader, err error) {
+
+	h.Logger.Printf("handling incremental transfer request: %#v", r)
+	if err = h.pullACLCheck(r.Filesystem); err != nil {
+		return
+	}
+
+	h.Logger.Printf("invoking zfs send")
+
+	if stream, err = zfs.ZFSSend(r.Filesystem, &r.From, &r.To); err != nil {
+		h.Logger.Printf("error sending filesystem: %#v", err)
+	}
+
+	return
+
+}
+
+func (h Handler) HandleResumeTransferRequest(r rpc.ResumeTransferRequest) (stream io.Reader, err error) {
+	// decode receive_resume_token: zfs send -nvP <token>
+	// A) use exit code to determine if could send (exit != 0 means could not send)
+	// B) check ACL if the filesystem in toname field (it is a snapshot so strip its snapshot name first)
+	//    is allowed for the given client
+	/*
+
+			# zfs send -nvt will print nvlist contents on ZoL and FreeBSD, should be good enough
+			# will print regardless of whether it can send or not -> good for us
+			# need clean way to extract that...
+				# expect 'resume token contents:\nnvlist version: 0\n'
+				# parse everything after that is tab-indented as key value pairs separated by = sign
+				# => return dict
+
+			zfs send -nv -t 1-c6491acbe-c8-789c636064000310a500c4ec50360710e72765a526973030b0419460caa7a515a79630c001489e0d493ea9b224b5182451885d7f497e7a69660a0343f79b1b9a8a2b3db65b20c97382e5f312735319188a4bf28b12d353f5931293b34b0b8af5ab8a520b72f4d3f2f31d8a53f35220660300c1091dbe
+		resume token contents:
+		nvlist version: 0
+		        object = 0x6
+		        offset = 0x0
+		        bytes = 0x7100
+		        toguid = 0xb748a92129d8ec8b
+		        toname = storage/backups/zrepl/foo@send
+		cannot resume send: 'storage/backups/zrepl/foo@send' used in the initial send no longer exists
+
+		zfs send -nvt 1-ebbbbea7e-f0-789c636064000310a501c49c50360710a715e5e7a69766a6304041f79b1b9a8a2b3db62b00d9ec48eaf293b252934b181858a0ea30e4d3d28a534b18e00024cf86249f5459925a0ca44fc861d75f920f71c59c5fdf6f7b3eea32b24092e704cbe725e6a632301497e41725a6a7ea27252667971614eb5715a516e4e8a7e5e73b14a7e6a51881cd06005a222749
+		resume token contents:
+		nvlist version: 0
+		        fromguid = 0xb748a92129d8ec8b #NOTE the fromguid field which is only in this one, so don't hardcode
+		        object = 0x4
+		        offset = 0x0
+		        bytes = 0x1ec8
+		        toguid = 0x328ae249dbf7fa9c
+		        toname = storage/backups/zrepl/foo@send2
+		send from storage/backups/zrepl/foo@send to storage/backups/zrepl/foo@send2 estimated size is 1.02M
+
+	*/
+	panic("not implemented")
+	return nil, nil
+}
+
+func (h Handler) HandlePullMeRequest(r rpc.PullMeRequest, clientIdentity string, client rpc.RPCRequester) (err error) {
+
+	// Check if we have a sink for this request
+	// Use that mapping to do what happens in doPull
+
+	h.Logger.Printf("handling PullMeRequest: %#v", r)
+
+	var sinkMapping DatasetMapping
+	sinkMapping, err = h.SinkMappingFunc(clientIdentity)
+	if err != nil {
+		h.Logger.Printf("no sink mapping for client identity '%s', denying PullMeRequest", clientIdentity)
+		err = fmt.Errorf("no sink for client identity '%s'", clientIdentity)
+		return
+	}
+
+	h.Logger.Printf("doing pull...")
+
+	err = doPull(PullContext{
+		Remote:            client,
+		Log:               h.Logger,
+		Mapping:           sinkMapping,
+		InitialReplPolicy: r.InitialReplPolicy,
+	})
+	if err != nil {
+		h.Logger.Printf("PullMeRequest failed with error: %s", err)
+		return
+	}
+
+	h.Logger.Printf("finished handling PullMeRequest: %#v", r)
+
+	return
+}
+
+func (h Handler) pullACLCheck(p *zfs.DatasetPath) (err error) {
+	var allowed bool
+	allowed, err = h.PullACL.Filter(p)
+	if err != nil {
+		err = fmt.Errorf("error evaluating ACL: %s", err)
+		h.Logger.Printf(err.Error())
+		return
+	}
+	if !allowed {
+		err = fmt.Errorf("ACL prohibits access to %s", p.ToString())
+		h.Logger.Printf(err.Error())
+		return
+	}
+	return
+}

cmd/main.go

@@ -0,0 +1,104 @@
+// zrepl replicates ZFS filesystems & volumes between pools
+//
+// Code Organization
+//
+// The cmd package uses github.com/spf13/cobra for its CLI.
+//
+// It combines the other packages in the zrepl project to implement zrepl functionality.
+//
+// Each subcommand's code is in the corresponding *.go file.
+// All other *.go files contain code shared by the subcommands.
+package cmd
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/jobrun"
+	"golang.org/x/sys/unix"
+	"io"
+	golog "log"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+)
+
+type Logger interface {
+	Printf(format string, v ...interface{})
+}
+
+// global state / facilities
+var (
+	conf     Config
+	runner   *jobrun.JobRunner
+	logFlags int = golog.LUTC | golog.Ldate | golog.Ltime
+	logOut   io.Writer
+	log      Logger
+)
+
+var RootCmd = &cobra.Command{
+	Use:   "zrepl",
+	Short: "ZFS dataset replication",
+	Long: `Replicate ZFS filesystems & volumes between pools:
+
+  - push & pull mode
+  - automatic snapshot creation & pruning
+  - local / over the network
+  - ACLs instead of blank SSH access`,
+}
+
+var rootArgs struct {
+	configFile string
+	stderrFile string
+	httpPprof  string
+}
+
+func init() {
+	cobra.OnInitialize(initConfig)
+	RootCmd.PersistentFlags().StringVar(&rootArgs.configFile, "config", "", "config file path")
+	RootCmd.PersistentFlags().StringVar(&rootArgs.stderrFile, "stderrFile", "-", "redirect stderr to given path")
+	RootCmd.PersistentFlags().StringVar(&rootArgs.httpPprof, "debug.pprof.http", "", "run pprof http server on given port")
+}
+
+func initConfig() {
+
+	// Logging & stderr redirection
+	if rootArgs.stderrFile != "-" {
+		file, err := os.OpenFile(rootArgs.stderrFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
+		if err != nil {
+			return
+		}
+
+		if err = unix.Dup2(int(file.Fd()), int(os.Stderr.Fd())); err != nil {
+			file.WriteString(fmt.Sprintf("error redirecting stderr file %s: %s\n", rootArgs.stderrFile, err))
+			return
+		}
+		logOut = file
+	} else {
+		logOut = os.Stderr
+	}
+
+	log = golog.New(logOut, "", logFlags)
+
+	// CPU profiling
+	if rootArgs.httpPprof != "" {
+		go func() {
+			http.ListenAndServe(rootArgs.httpPprof, nil)
+		}()
+	}
+
+	// Config
+	if rootArgs.configFile == "" {
+		log.Printf("config file not set")
+		os.Exit(1)
+	}
+	var err error
+	if conf, err = ParseConfig(rootArgs.configFile); err != nil {
+		log.Printf("error parsing config: %s", err)
+		os.Exit(1)
+	}
+
+	jobrunLogger := golog.New(os.Stderr, "jobrun ", logFlags)
+	runner = jobrun.NewJobRunner(jobrunLogger)
+	return
+
+}

cmd/prune.go

@@ -0,0 +1,139 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/util"
+	"github.com/zrepl/zrepl/zfs"
+	"os"
+	"sort"
+	"time"
+)
+
+var pruneArgs struct {
+	job    string
+	dryRun bool
+}
+
+var PruneCmd = &cobra.Command{
+	Use:   "prune",
+	Short: "perform pruning",
+	Run:   cmdPrune,
+}
+
+func init() {
+	PruneCmd.Flags().StringVar(&pruneArgs.job, "job", "", "job to run")
+	PruneCmd.Flags().BoolVarP(&pruneArgs.dryRun, "dryrun", "n", false, "dry run")
+	RootCmd.AddCommand(PruneCmd)
+}
+
+func cmdPrune(cmd *cobra.Command, args []string) {
+
+	if len(args) < 1 {
+		log.Printf("must specify exactly one job as positional argument")
+		os.Exit(1)
+	}
+
+	job, ok := conf.Prunes[args[0]]
+	if !ok {
+		log.Printf("could not find prune job: %s", args[0])
+		os.Exit(1)
+	}
+
+	log.Printf("Beginning prune job:\n%s", job)
+	ctx := PruneContext{job, time.Now(), pruneArgs.dryRun}
+	err := doPrune(ctx, log)
+	if err != nil {
+		log.Printf("Prune job failed with error: %s", err)
+		os.Exit(1)
+	}
+
+}
+
+type PruneContext struct {
+	Prune  *Prune
+	Now    time.Time
+	DryRun bool
+}
+
+type retentionGridAdaptor struct {
+	zfs.FilesystemVersion
+}
+
+func (a retentionGridAdaptor) Date() time.Time {
+	return a.Creation
+}
+
+func (a retentionGridAdaptor) LessThan(b util.RetentionGridEntry) bool {
+	return a.CreateTXG < b.(retentionGridAdaptor).CreateTXG
+}
+
+func doPrune(ctx PruneContext, log Logger) error {
+
+	if ctx.DryRun {
+		log.Printf("doing dry run")
+	}
+
+	prune := ctx.Prune
+
+	// ZFSListSnapsFiltered --> todo can extend fsfilter or need new? Have already something per fs
+	// Dedicated snapshot object? Adaptor object to FilesystemVersion?
+
+	filesystems, err := zfs.ZFSListMapping(prune.DatasetFilter)
+	if err != nil {
+		return fmt.Errorf("error applying filesystem filter: %s", err)
+	}
+
+	for _, fs := range filesystems {
+
+		fsversions, err := zfs.ZFSListFilesystemVersions(fs, prune.SnapshotFilter)
+		if err != nil {
+			return fmt.Errorf("error listing filesytem versions of %s: %s", fs, err)
+		}
+
+		if len(fsversions) == 0 {
+			continue
+		}
+
+		adaptors := make([]util.RetentionGridEntry, len(fsversions))
+		for fsv := range fsversions {
+			adaptors[fsv] = retentionGridAdaptor{fsversions[fsv]}
+		}
+
+		sort.SliceStable(adaptors, func(i, j int) bool {
+			return adaptors[i].LessThan(adaptors[j])
+		})
+		ctx.Now = adaptors[len(adaptors)-1].Date()
+
+		describe := func(a retentionGridAdaptor) string {
+			timeSince := a.Creation.Sub(ctx.Now)
+			const day time.Duration = 24 * time.Hour
+			days := timeSince / day
+			remainder := timeSince % day
+			return fmt.Sprintf("%s@%dd%s from now", a.ToAbsPath(fs), days, remainder)
+		}
+
+		keep, remove := prune.RetentionPolicy.FitEntries(ctx.Now, adaptors)
+		for _, a := range remove {
+			r := a.(retentionGridAdaptor)
+			log.Printf("remove %s", describe(r))
+			// do echo what we'll do and exec zfs destroy if not dry run
+			// special handling for EBUSY (zfs hold)
+			// error handling for clones? just echo to cli, skip over, and exit with non-zero status code (we're idempotent)
+			if !ctx.DryRun {
+				err := zfs.ZFSDestroy(r.ToAbsPath(fs))
+				if err != nil {
+					// handle
+					log.Printf("error: %s", err)
+				}
+			}
+		}
+		for _, a := range keep {
+			r := a.(retentionGridAdaptor)
+			log.Printf("would keep %s", describe(r))
+		}
+	}
+
+	return nil
+
+}

cmd/replication.go

@@ -0,0 +1,587 @@
+package cmd
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/jobrun"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/util"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var runArgs struct {
+	job  string
+	once bool
+}
+
+var RunCmd = &cobra.Command{
+	Use:   "run",
+	Short: "run push & pull replication",
+	Run:   cmdRun,
+}
+
+var PushCmd = &cobra.Command{
+	Use:   "push",
+	Short: "run push job (first positional argument)",
+	Run:   cmdPush,
+}
+
+var PullCmd = &cobra.Command{
+	Use:   "pull",
+	Short: "run pull job (first positional argument)",
+	Run:   cmdPull,
+}
+
+func init() {
+	RootCmd.AddCommand(RunCmd)
+	RunCmd.Flags().BoolVar(&runArgs.once, "once", false, "run jobs only once, regardless of configured repeat behavior")
+	RunCmd.Flags().StringVar(&runArgs.job, "job", "", "run only the given job")
+
+	RootCmd.AddCommand(PushCmd)
+	RootCmd.AddCommand(PullCmd)
+}
+
+func cmdPush(cmd *cobra.Command, args []string) {
+
+	if len(args) != 1 {
+		log.Printf("must specify exactly one job as positional argument")
+		os.Exit(1)
+	}
+	job, ok := conf.Pushs[args[0]]
+	if !ok {
+		log.Printf("could not find push job %s", args[0])
+		os.Exit(1)
+	}
+	if err := jobPush(job, log); err != nil {
+		log.Printf("error doing push: %s", err)
+		os.Exit(1)
+	}
+
+}
+
+func cmdPull(cmd *cobra.Command, args []string) {
+
+	if len(args) != 1 {
+		log.Printf("must specify exactly one job as positional argument")
+		os.Exit(1)
+	}
+	job, ok := conf.Pulls[args[0]]
+	if !ok {
+		log.Printf("could not find pull job %s", args[0])
+		os.Exit(1)
+	}
+
+	if err := jobPull(job, log); err != nil {
+		log.Printf("error doing pull: %s", err)
+		os.Exit(1)
+	}
+
+}
+
+func cmdRun(cmd *cobra.Command, args []string) {
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		runner.Start()
+	}()
+
+	jobs := make([]jobrun.Job, len(conf.Pulls)+len(conf.Pushs))
+	i := 0
+	for _, pull := range conf.Pulls {
+		jobs[i] = jobrun.Job{
+			Name:           fmt.Sprintf("pull.%d", i),
+			RepeatStrategy: pull.RepeatStrategy,
+			RunFunc: func(log jobrun.Logger) error {
+				log.Printf("doing pull: %v", pull)
+				return jobPull(pull, log)
+			},
+		}
+		i++
+	}
+	for _, push := range conf.Pushs {
+		jobs[i] = jobrun.Job{
+			Name:           fmt.Sprintf("push.%d", i),
+			RepeatStrategy: push.RepeatStrategy,
+			RunFunc: func(log jobrun.Logger) error {
+				log.Printf("doing push: %v", push)
+				return jobPush(push, log)
+			},
+		}
+		i++
+	}
+
+	for _, j := range jobs {
+		if runArgs.once {
+			j.RepeatStrategy = jobrun.NoRepeatStrategy{}
+		}
+		if runArgs.job != "" {
+			if runArgs.job == j.Name {
+				runner.AddJob(j)
+				break
+			}
+			continue
+		}
+		runner.AddJob(j)
+	}
+
+	for {
+		select {
+		case job := <-runner.NotificationChan():
+			log.Printf("job %s reported error: %v\n", job.Name, job.LastError)
+		}
+	}
+
+	wg.Wait()
+
+}
+
+type localPullACL struct{}
+
+func (a localPullACL) Filter(p *zfs.DatasetPath) (pass bool, err error) {
+	return true, nil
+}
+
+func jobPull(pull *Pull, log jobrun.Logger) (err error) {
+
+	if lt, ok := pull.From.Transport.(LocalTransport); ok {
+
+		lt.SetHandler(Handler{
+			Logger: log,
+			// Allow access to any dataset since we control what mapping
+			// is passed to the pull routine.
+			// All local datasets will be passed to its Map() function,
+			// but only those for which a mapping exists will actually be pulled.
+			// We can pay this small performance penalty for now.
+			PullACL: localPullACL{},
+		})
+		pull.From.Transport = lt
+		log.Printf("fixing up local transport: %#v", pull.From.Transport)
+	}
+
+	var remote rpc.RPCRequester
+
+	if remote, err = pull.From.Transport.Connect(log); err != nil {
+		return
+	}
+
+	defer closeRPCWithTimeout(log, remote, time.Second*10, "")
+
+	return doPull(PullContext{remote, log, pull.Mapping, pull.InitialReplPolicy})
+}
+
+func jobPush(push *Push, log jobrun.Logger) (err error) {
+
+	if _, ok := push.To.Transport.(LocalTransport); ok {
+		panic("no support for local pushs")
+	}
+
+	var remote rpc.RPCRequester
+	if remote, err = push.To.Transport.Connect(log); err != nil {
+		return err
+	}
+
+	defer closeRPCWithTimeout(log, remote, time.Second*10, "")
+
+	log.Printf("building handler for PullMeRequest")
+	handler := Handler{
+		Logger:          log,
+		PullACL:         push.Filter,
+		SinkMappingFunc: nil, // no need for that in the handler for PullMe
+	}
+	log.Printf("handler: %#v", handler)
+
+	r := rpc.PullMeRequest{
+		InitialReplPolicy: push.InitialReplPolicy,
+	}
+	log.Printf("doing PullMeRequest: %#v", r)
+
+	if err = remote.PullMeRequest(r, handler); err != nil {
+		log.Printf("PullMeRequest failed: %s", err)
+		return
+	}
+
+	log.Printf("push job finished")
+	return
+
+}
+
+func closeRPCWithTimeout(log Logger, remote rpc.RPCRequester, timeout time.Duration, goodbye string) {
+	log.Printf("closing rpc connection")
+
+	ch := make(chan error)
+	go func() {
+		ch <- remote.CloseRequest(rpc.CloseRequest{goodbye})
+		close(ch)
+	}()
+
+	var err error
+	select {
+	case <-time.After(timeout):
+		err = fmt.Errorf("timeout exceeded (%s)", timeout)
+	case closeRequestErr := <-ch:
+		err = closeRequestErr
+	}
+
+	if err != nil {
+		log.Printf("error closing connection: %s", err)
+		err = remote.ForceClose()
+		if err != nil {
+			log.Printf("error force-closing connection: %s", err)
+		}
+	}
+	return
+}
+
+type PullContext struct {
+	Remote            rpc.RPCRequester
+	Log               Logger
+	Mapping           DatasetMapping
+	InitialReplPolicy rpc.InitialReplPolicy
+}
+
+func doPull(pull PullContext) (err error) {
+
+	remote := pull.Remote
+	log := pull.Log
+
+	log.Printf("requesting remote filesystem list")
+	fsr := rpc.FilesystemRequest{}
+	var remoteFilesystems []*zfs.DatasetPath
+	if remoteFilesystems, err = remote.FilesystemRequest(fsr); err != nil {
+		return
+	}
+
+	log.Printf("map remote filesystems to local paths and determine order for per-filesystem sync")
+	type RemoteLocalMapping struct {
+		Remote *zfs.DatasetPath
+		Local  *zfs.DatasetPath
+	}
+	replMapping := make(map[string]RemoteLocalMapping, len(remoteFilesystems))
+	localTraversal := zfs.NewDatasetPathForest()
+	for fs := range remoteFilesystems {
+		var err error
+		var localFs *zfs.DatasetPath
+		localFs, err = pull.Mapping.Map(remoteFilesystems[fs])
+		if err != nil {
+			if err != NoMatchError {
+				err := fmt.Errorf("error mapping %s: %s", remoteFilesystems[fs], err)
+				log.Printf("%s", err)
+				return err
+			}
+			continue
+		}
+		log.Printf("%s => %s", remoteFilesystems[fs].ToString(), localFs.ToString())
+		m := RemoteLocalMapping{remoteFilesystems[fs], localFs}
+		replMapping[m.Local.ToString()] = m
+		localTraversal.Add(m.Local)
+	}
+
+	log.Printf("build cache for already present local filesystem state")
+	localFilesystemState, err := zfs.ZFSListFilesystemState()
+	if err != nil {
+		log.Printf("error requesting local filesystem state: %s", err)
+		return err
+	}
+
+	log.Printf("start per-filesystem sync")
+	localTraversal.WalkTopDown(func(v zfs.DatasetPathVisit) bool {
+
+		if v.FilledIn {
+			if _, exists := localFilesystemState[v.Path.ToString()]; exists {
+				// No need to verify if this is a placeholder or not. It is sufficient
+				// to know we can add child filesystems to it
+				return true
+			}
+			log.Printf("creating placeholder filesystem %s", v.Path.ToString())
+			err = zfs.ZFSCreatePlaceholderFilesystem(v.Path)
+			if err != nil {
+				err = fmt.Errorf("aborting, cannot create placeholder filesystem %s: %s", v.Path, err)
+				return false
+			}
+			return true
+		}
+
+		m, ok := replMapping[v.Path.ToString()]
+		if !ok {
+			panic("internal inconsistency: replMapping should contain mapping for any path that was not filled in by WalkTopDown()")
+		}
+
+		log := func(format string, args ...interface{}) {
+			log.Printf("[%s => %s]: %s", m.Remote.ToString(), m.Local.ToString(), fmt.Sprintf(format, args...))
+		}
+
+		log("examing local filesystem state")
+		localState, localExists := localFilesystemState[m.Local.ToString()]
+		var versions []zfs.FilesystemVersion
+		switch {
+		case !localExists:
+			log("local filesystem does not exist")
+		case localState.Placeholder:
+			log("local filesystem is marked as placeholder")
+		case localState.ResumeToken != "":
+
+			log("local filesystem has receive_resume_token")
+			if false { // TODO Check if managed resume is disabled (explain information leakage in docs!)
+				log("managed resume tokens are disabled via zrepl config, assuming config change or external administrative action")
+				log("policy forbids aborting the partial recv automatically, skipping this filesystem")
+				log("for zrepl to resume replication for this dataset, administrators should finishing the recv manually or abort the recv via `zfs recv -A %s`", m.Local.ToString())
+				return true // TODO right choice to allow children? ... should be, since the fs exists...
+			}
+
+			log("decoding receive_resume_token")
+			// TODO, see handler comments
+			// TODO override logger with fromguid toguid
+
+			log("requesting resume of transfer")
+			r := rpc.ResumeTransferRequest{m.Remote, localState.ResumeToken}
+			stream, err := remote.ResumeTransferRequest(r)
+			if err != nil {
+
+				log("resume transfer request failed: %s", err)
+				rre, ok := err.(*rpc.ResumeTransferError)
+				if !ok {
+					log("skipping this filesystem, could be temporary issue")
+					return true // TODO right choice to allow children
+				}
+
+				// Determine if we should clear the resume token
+				clearAndUseSnaps := false
+				switch rre.Reason {
+				case rpc.ResumeTransferErrorReasonNotImplemented:
+					fallthrough
+				case rpc.ResumeTransferErrorReasonDisabled:
+					fallthrough
+				case rpc.ResumeTransferErrorReasonZFSErrorPermanent:
+					clearAndUseSnaps = true
+
+				case rpc.ResumeTransferErrorReasonZFSErrorMaybeTemporary:
+					fallthrough
+				default:
+					clearAndUseSnaps = false
+				}
+
+				if !clearAndUseSnaps {
+					log("skipping this filesystem, error identified as temporary")
+					return true // TODO right choice to allow children
+				}
+
+				log("clearing local receive_resume_token")
+				if err := zfs.ZFSRecvAbort(m.Local); err != nil {
+					log("error clearing receive_resume_token: %s", err)
+					return true // TODO right choice to allow children? the filesystem exists, so it should be ok
+				}
+				// TODO go back to top of function (put all this into separate function, then tail recursive call)
+				// TODO return this_function()
+			}
+
+			// TODO warning code duplication, see below. need to unify this
+			log("invoking zfs receive")
+			watcher := util.IOProgressWatcher{Reader: stream}
+			watcher.KickOff(1*time.Second, func(p util.IOProgress) {
+				log("progress on receive operation: %v bytes received", p.TotalRX)
+			})
+
+			recvArgs := []string{"-u"}
+			if localState.Placeholder {
+				log("receive with forced rollback to replace placeholder filesystem")
+				recvArgs = append(recvArgs, "-F")
+			}
+
+			if err = zfs.ZFSRecv(m.Local, &watcher, recvArgs...); err != nil {
+				log("error receiving stream: %s", err)
+				return false
+			}
+			log("finished receiving stream, %v bytes total", watcher.Progress().TotalRX)
+
+			// TODO further problem property handling code must be duplicated here...
+
+			// TODO tail recursive call to this function, we might not be dony syncing yet
+			// TODO return this_function()
+			return true
+
+		default:
+			log("local filesystem exists")
+			log("requesting local filesystem versions")
+			if versions, err = zfs.ZFSListFilesystemVersions(m.Local, nil); err != nil {
+				log("cannot get local filesystem versions: %s", err)
+				return false
+			}
+		}
+
+		log("requesting remote filesystem versions")
+		var theirVersions []zfs.FilesystemVersion
+		theirVersions, err = remote.FilesystemVersionsRequest(rpc.FilesystemVersionsRequest{
+			Filesystem: m.Remote,
+		})
+		if err != nil {
+			log("error requesting remote filesystem versions: %s", err)
+			log("stopping replication for all filesystems mapped as children of %s", m.Local.ToString())
+			return false
+		}
+
+		log("computing diff between remote and local filesystem versions")
+		diff := zfs.MakeFilesystemDiff(versions, theirVersions)
+		log("%s", diff)
+
+		if localState.Placeholder && diff.Conflict != zfs.ConflictAllRight {
+			panic("internal inconsistency: local placeholder implies ConflictAllRight")
+		}
+
+		switch diff.Conflict {
+		case zfs.ConflictAllRight:
+
+			log("performing initial sync, following policy: '%s'", pull.InitialReplPolicy)
+
+			if pull.InitialReplPolicy != rpc.InitialReplPolicyMostRecent {
+				panic(fmt.Sprintf("policy '%s' not implemented", pull.InitialReplPolicy))
+			}
+
+			snapsOnly := make([]zfs.FilesystemVersion, 0, len(diff.MRCAPathRight))
+			for s := range diff.MRCAPathRight {
+				if diff.MRCAPathRight[s].Type == zfs.Snapshot {
+					snapsOnly = append(snapsOnly, diff.MRCAPathRight[s])
+				}
+			}
+
+			if len(snapsOnly) < 1 {
+				log("cannot perform initial sync: no remote snapshots. stopping...")
+				return false
+			}
+
+			r := rpc.InitialTransferRequest{
+				Filesystem:        m.Remote,
+				FilesystemVersion: snapsOnly[len(snapsOnly)-1],
+			}
+
+			log("requesting snapshot stream for %s", r.FilesystemVersion)
+
+			var stream io.Reader
+			if stream, err = remote.InitialTransferRequest(r); err != nil {
+				log("error requesting initial transfer: %s", err)
+				return false
+			}
+			log("received initial transfer request response")
+
+			log("invoking zfs receive")
+			watcher := util.IOProgressWatcher{Reader: stream}
+			watcher.KickOff(1*time.Second, func(p util.IOProgress) {
+				log("progress on receive operation: %v bytes received", p.TotalRX)
+			})
+
+			recvArgs := []string{"-u"}
+			if localState.Placeholder {
+				log("receive with forced rollback to replace placeholder filesystem")
+				recvArgs = append(recvArgs, "-F")
+			}
+
+			if err = zfs.ZFSRecv(m.Local, &watcher, recvArgs...); err != nil {
+				log("error receiving stream: %s", err)
+				return false
+			}
+			log("finished receiving stream, %v bytes total", watcher.Progress().TotalRX)
+
+			log("configuring properties of received filesystem")
+			if err = zfs.ZFSSet(m.Local, "readonly", "on"); err != nil {
+
+			}
+
+			log("finished initial transfer")
+			return true
+
+		case zfs.ConflictIncremental:
+
+			if len(diff.IncrementalPath) < 2 {
+				log("remote and local are in sync")
+				return true
+			}
+
+			log("following incremental path from diff")
+			var pathRx uint64
+
+			for i := 0; i < len(diff.IncrementalPath)-1; i++ {
+
+				from, to := diff.IncrementalPath[i], diff.IncrementalPath[i+1]
+
+				log := func(format string, args ...interface{}) {
+					log("[%v/%v][%s => %s]: %s", i+1, len(diff.IncrementalPath)-1,
+						from.Name, to.Name, fmt.Sprintf(format, args...))
+				}
+
+				log("requesting incremental snapshot stream")
+				r := rpc.IncrementalTransferRequest{
+					Filesystem: m.Remote,
+					From:       from,
+					To:         to,
+				}
+				var stream io.Reader
+				if stream, err = remote.IncrementalTransferRequest(r); err != nil {
+					log("error requesting incremental snapshot stream: %s", err)
+					return false
+				}
+
+				log("invoking zfs receive")
+				watcher := util.IOProgressWatcher{Reader: stream}
+				watcher.KickOff(1*time.Second, func(p util.IOProgress) {
+					log("progress on receive operation: %v bytes received", p.TotalRX)
+				})
+
+				if err = zfs.ZFSRecv(m.Local, &watcher); err != nil {
+					log("error receiving stream: %s", err)
+					return false
+				}
+
+				totalRx := watcher.Progress().TotalRX
+				pathRx += totalRx
+				log("finished incremental transfer, %v bytes total", totalRx)
+
+			}
+
+			log("finished following incremental path, %v bytes total", pathRx)
+			return true
+
+		case zfs.ConflictNoCommonAncestor:
+
+			log("remote and local filesystem have snapshots, but no common one")
+			log("perform manual replication to establish a common snapshot history")
+			log("remote versions:")
+			for _, v := range diff.MRCAPathRight {
+				log(" %s (GUID %v)", v, v.Guid)
+			}
+			log("local versions:")
+			for _, v := range diff.MRCAPathLeft {
+				log(" %s (GUID %v)", v, v.Guid)
+			}
+			return false
+
+		case zfs.ConflictDiverged:
+
+			log("remote and local filesystem share a history but have diverged")
+			log("perform manual replication or delete snapshots on the receiving" +
+				"side  to establish an incremental replication parse")
+			log("remote-only versions:")
+			for _, v := range diff.MRCAPathRight {
+				log(" %s (GUID %v)", v, v.Guid)
+			}
+			log("local-only versions:")
+			for _, v := range diff.MRCAPathLeft {
+				log(" %s (GUID %v)", v, v.Guid)
+			}
+			return false
+
+		}
+
+		panic("implementation error: this should not be reached")
+		return false
+
+	})
+
+	return
+
+}

cmd/sampleconf/zrepl.yml

@@ -0,0 +1,109 @@
+remotes:
+  offsite_backups:
+    transport:
+      ssh:
+        host: 192.168.122.6
+        user: root
+        port: 22
+        identity_file: /etc/zrepl/identities/offsite_backups
+
+pushs:
+
+  offsite:
+    to: offsite_backups
+    filter: {
+      # like in pull_acls
+      "tank/var/db<":   ok,
+      "tank/usr/home<": ok,
+    }
+
+pulls:
+
+  offsite:
+    from: offsite_backups
+    mapping: {
+      # like in sinks
+    }
+
+  # local replication, only allowed in pull mode
+  # the from name 'local' is reserved for this purpose
+  homemirror:
+    from: local
+    repeat:
+      interval: 15m
+    mapping: {
+      "tank/usr/home":"mirrorpool/foo/bar"
+    }
+
+sinks:
+
+  db1:
+    mapping: {
+      # direct mapping
+      "ssdpool/var/db/postgresql9.6":"zroot/backups/db1/pg_data"
+    }
+
+  mirror1:
+    mapping: {
+      # "<" subtree wildcard matches the dataset left of < and all its children
+      "tank/foo/bar<":"zroot/backups/mirror1" 
+    }
+
+  mirror2:
+    # more specific path patterns win over less specific ones
+    # direct mappings win over subtree wildcards
+    # detailed rule precedence: check unit tests & docs for exact behavior
+    # TODO subcommand to test a mapping & filter
+    mapping: {
+      "tank<":                       "zroot/backups/mirror1/tank1",
+      "tank/cdn/root<":              "storage/cdn/root",
+      "tank/legacydb":               "legacypool/backups/legacydb",
+    }
+
+
+pull_acls:
+
+  office_backup:
+    filter: {
+      # valid filter results (right hand side): ok, omit
+      # default is to omit
+      # rule precedence is same as for mappings
+      "tank<":         omit,
+      "tank/usr/home": ok,
+    }
+
+
+prune:
+
+  clean_backups:
+    policy: grid
+    grid: 6x10m | 24x1h | 7x1d | 5 x 1w | 4 x 5w
+    dataset_filter: {
+      "tank/backups/legacyscript<": omit,
+      "tank/backups<":              ok,
+    }
+    snapshot_filter: {
+      prefix: zrepl_
+    }
+
+  hfbak_prune: # cleans up after hfbak autosnap job
+    policy: grid
+    grid: 1x1m(keep=all)
+    dataset_filter: {
+      "pool1<": ok
+    }
+    snapshot_filter: {
+      prefix: zrepl_hfbak_
+    }
+
+autosnap:
+
+  hfbak:
+    prefix: zrepl_hfbak_
+    interval: 1s
+    dataset_filter: {
+      "pool1<": ok
+    }
+    # prune: hfbak_prune
+    # future versions may inline the retention policy here, but for now,
+    # pruning has to be triggered manually (it's safe to run autosnap + prune in parallel)

cmd/stdinserver.go

@@ -0,0 +1,72 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/sshbytestream"
+	"io"
+	golog "log"
+	"os"
+)
+
+var StdinserverCmd = &cobra.Command{
+	Use:   "stdinserver CLIENT_IDENTITY",
+	Short: "start in stdin server mode (from authorized_keys file)",
+	Run:   cmdStdinServer,
+}
+
+func init() {
+	RootCmd.AddCommand(StdinserverCmd)
+}
+
+func cmdStdinServer(cmd *cobra.Command, args []string) {
+
+	var err error
+	defer func() {
+		if err != nil {
+			log.Printf("stdinserver exiting with error: %s", err)
+			os.Exit(1)
+		}
+	}()
+
+	if len(args) != 1 || args[0] == "" {
+		err = fmt.Errorf("must specify client identity as positional argument")
+		return
+	}
+	identity := args[0]
+
+	pullACL, ok := conf.PullACLs[identity]
+	if !ok {
+		err = fmt.Errorf("could not find PullACL for identity '%s'", identity)
+		return
+	}
+
+	var sshByteStream io.ReadWriteCloser
+	if sshByteStream, err = sshbytestream.Incoming(); err != nil {
+		return
+	}
+
+	sinkMapping := func(identity string) (m DatasetMapping, err error) {
+		sink, ok := conf.Sinks[identity]
+		if !ok {
+			return nil, fmt.Errorf("could not find sink for identity '%s'", identity)
+		}
+		return sink, nil
+	}
+
+	sinkLogger := golog.New(logOut, fmt.Sprintf("sink[%s] ", identity), logFlags)
+	handler := Handler{
+		Logger:          sinkLogger,
+		SinkMappingFunc: sinkMapping,
+		PullACL:         pullACL,
+	}
+
+	if err = rpc.ListenByteStreamRPC(sshByteStream, identity, handler, sinkLogger); err != nil {
+		log.Printf("listenbytestreamerror: %#v\n", err)
+		os.Exit(1)
+	}
+
+	return
+
+}

config/config.go

@@ -1,714 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"io/ioutil"
-	"log/syslog"
-	"os"
-	"reflect"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/robfig/cron/v3"
-	"github.com/zrepl/yaml-config"
-
-	"github.com/zrepl/zrepl/util/datasizeunit"
-	zfsprop "github.com/zrepl/zrepl/zfs/property"
-)
-
-type ParseFlags uint
-
-const (
-	ParseFlagsNone        ParseFlags = 0
-	ParseFlagsNoCertCheck ParseFlags = 1 << iota
-)
-
-type Config struct {
-	Jobs   []JobEnum `yaml:"jobs"`
-	Global *Global   `yaml:"global,optional,fromdefaults"`
-}
-
-func (c *Config) Job(name string) (*JobEnum, error) {
-	for _, j := range c.Jobs {
-		if j.Name() == name {
-			return &j, nil
-		}
-	}
-	return nil, fmt.Errorf("job %q not defined in config", name)
-}
-
-type JobEnum struct {
-	Ret interface{}
-}
-
-func (j JobEnum) Name() string {
-	var name string
-	switch v := j.Ret.(type) {
-	case *SnapJob:
-		name = v.Name
-	case *PushJob:
-		name = v.Name
-	case *SinkJob:
-		name = v.Name
-	case *PullJob:
-		name = v.Name
-	case *SourceJob:
-		name = v.Name
-	default:
-		panic(fmt.Sprintf("unknown job type %T", v))
-	}
-	return name
-}
-
-type ActiveJob struct {
-	Type               string                `yaml:"type"`
-	Name               string                `yaml:"name"`
-	Connect            ConnectEnum           `yaml:"connect"`
-	Pruning            PruningSenderReceiver `yaml:"pruning"`
-	Debug              JobDebugSettings      `yaml:"debug,optional"`
-	Replication        *Replication          `yaml:"replication,optional,fromdefaults"`
-	ConflictResolution *ConflictResolution   `yaml:"conflict_resolution,optional,fromdefaults"`
-}
-
-type ConflictResolution struct {
-	InitialReplication string `yaml:"initial_replication,optional,default=most_recent"`
-}
-
-type PassiveJob struct {
-	Type  string           `yaml:"type"`
-	Name  string           `yaml:"name"`
-	Serve ServeEnum        `yaml:"serve"`
-	Debug JobDebugSettings `yaml:"debug,optional"`
-}
-
-type SnapJob struct {
-	Type         string            `yaml:"type"`
-	Name         string            `yaml:"name"`
-	Pruning      PruningLocal      `yaml:"pruning"`
-	Debug        JobDebugSettings  `yaml:"debug,optional"`
-	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
-	Filesystems  FilesystemsFilter `yaml:"filesystems"`
-}
-
-type SendOptions struct {
-	Encrypted        bool `yaml:"encrypted,optional,default=false"`
-	Raw              bool `yaml:"raw,optional,default=false"`
-	SendProperties   bool `yaml:"send_properties,optional,default=false"`
-	BackupProperties bool `yaml:"backup_properties,optional,default=false"`
-	LargeBlocks      bool `yaml:"large_blocks,optional,default=false"`
-	Compressed       bool `yaml:"compressed,optional,default=false"`
-	EmbeddedData     bool `yaml:"embedded_data,optional,default=false"`
-	Saved            bool `yaml:"saved,optional,default=false"`
-
-	BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
-}
-
-type RecvOptions struct {
-	// Note: we cannot enforce encrypted recv as the ZFS cli doesn't provide a mechanism for it
-	// Encrypted bool `yaml:"may_encrypted"`
-	// Future:
-	// Reencrypt bool `yaml:"reencrypt"`
-
-	Properties *PropertyRecvOptions `yaml:"properties,fromdefaults"`
-
-	BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
-
-	Placeholder *PlaceholderRecvOptions `yaml:"placeholder,fromdefaults"`
-}
-
-var _ yaml.Unmarshaler = &datasizeunit.Bits{}
-
-type BandwidthLimit struct {
-	Max            datasizeunit.Bits `yaml:"max,default=-1 B"`
-	BucketCapacity datasizeunit.Bits `yaml:"bucket_capacity,default=128 KiB"`
-}
-
-type Replication struct {
-	Protection  *ReplicationOptionsProtection  `yaml:"protection,optional,fromdefaults"`
-	Concurrency *ReplicationOptionsConcurrency `yaml:"concurrency,optional,fromdefaults"`
-}
-
-type ReplicationOptionsProtection struct {
-	Initial     string `yaml:"initial,optional,default=guarantee_resumability"`
-	Incremental string `yaml:"incremental,optional,default=guarantee_resumability"`
-}
-
-type ReplicationOptionsConcurrency struct {
-	Steps         int `yaml:"steps,optional,default=1"`
-	SizeEstimates int `yaml:"size_estimates,optional,default=4"`
-}
-
-type PropertyRecvOptions struct {
-	Inherit  []zfsprop.Property          `yaml:"inherit,optional"`
-	Override map[zfsprop.Property]string `yaml:"override,optional"`
-}
-
-type PlaceholderRecvOptions struct {
-	Encryption string `yaml:"encryption,default=unspecified"`
-}
-
-type PushJob struct {
-	ActiveJob    `yaml:",inline"`
-	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
-	Filesystems  FilesystemsFilter `yaml:"filesystems"`
-	Send         *SendOptions      `yaml:"send,fromdefaults,optional"`
-}
-
-func (j *PushJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
-func (j *PushJob) GetSendOptions() *SendOptions      { return j.Send }
-
-type PullJob struct {
-	ActiveJob `yaml:",inline"`
-	RootFS    string                   `yaml:"root_fs"`
-	Interval  PositiveDurationOrManual `yaml:"interval"`
-	Recv      *RecvOptions             `yaml:"recv,fromdefaults,optional"`
-}
-
-func (j *PullJob) GetRootFS() string             { return j.RootFS }
-func (j *PullJob) GetAppendClientIdentity() bool { return false }
-func (j *PullJob) GetRecvOptions() *RecvOptions  { return j.Recv }
-
-type PositiveDurationOrManual struct {
-	Interval time.Duration
-	Manual   bool
-}
-
-var _ yaml.Unmarshaler = (*PositiveDurationOrManual)(nil)
-
-func (i *PositiveDurationOrManual) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	var s string
-	if err := u(&s, true); err != nil {
-		return err
-	}
-	switch s {
-	case "manual":
-		i.Manual = true
-		i.Interval = 0
-	case "":
-		return fmt.Errorf("value must not be empty")
-	default:
-		i.Manual = false
-		i.Interval, err = parsePositiveDuration(s)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-type SinkJob struct {
-	PassiveJob `yaml:",inline"`
-	RootFS     string       `yaml:"root_fs"`
-	Recv       *RecvOptions `yaml:"recv,optional,fromdefaults"`
-}
-
-func (j *SinkJob) GetRootFS() string             { return j.RootFS }
-func (j *SinkJob) GetAppendClientIdentity() bool { return true }
-func (j *SinkJob) GetRecvOptions() *RecvOptions  { return j.Recv }
-
-type SourceJob struct {
-	PassiveJob   `yaml:",inline"`
-	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
-	Filesystems  FilesystemsFilter `yaml:"filesystems"`
-	Send         *SendOptions      `yaml:"send,optional,fromdefaults"`
-}
-
-func (j *SourceJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
-func (j *SourceJob) GetSendOptions() *SendOptions      { return j.Send }
-
-type FilesystemsFilter map[string]bool
-
-type SnapshottingEnum struct {
-	Ret interface{}
-}
-
-type SnapshottingPeriodic struct {
-	Type            string            `yaml:"type"`
-	Prefix          string            `yaml:"prefix"`
-	Interval        *PositiveDuration `yaml:"interval"`
-	Hooks           HookList          `yaml:"hooks,optional"`
-	TimestampFormat string            `yaml:"timestamp_format,optional,default=dense"`
-}
-
-type CronSpec struct {
-	Schedule cron.Schedule
-}
-
-var _ yaml.Unmarshaler = &CronSpec{}
-
-func (s *CronSpec) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
-	var specString string
-	if err := unmarshal(&specString, false); err != nil {
-		return err
-	}
-
-	// Use standard cron format.
-	// Disable the various "descriptors" (@daily, etc)
-	// They are just aliases to "top of hour", "midnight", etc.
-	parser := cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.SecondOptional)
-
-	sched, err := parser.Parse(specString)
-	if err != nil {
-		return errors.Wrap(err, "cron syntax invalid")
-	}
-	s.Schedule = sched
-	return nil
-}
-
-type SnapshottingCron struct {
-	Type            string   `yaml:"type"`
-	Prefix          string   `yaml:"prefix"`
-	Cron            CronSpec `yaml:"cron"`
-	Hooks           HookList `yaml:"hooks,optional"`
-	TimestampFormat string   `yaml:"timestamp_format,optional,default=dense"`
-}
-
-type SnapshottingManual struct {
-	Type string `yaml:"type"`
-}
-
-type PruningSenderReceiver struct {
-	KeepSender   []PruningEnum `yaml:"keep_sender"`
-	KeepReceiver []PruningEnum `yaml:"keep_receiver"`
-}
-
-type PruningLocal struct {
-	Keep []PruningEnum `yaml:"keep"`
-}
-
-type LoggingOutletEnumList []LoggingOutletEnum
-
-func (l *LoggingOutletEnumList) SetDefault() {
-	def := `
-type: "stdout"
-time: true
-level: "warn"
-format: "human"
-`
-	s := &StdoutLoggingOutlet{}
-	err := yaml.UnmarshalStrict([]byte(def), &s)
-	if err != nil {
-		panic(err)
-	}
-	*l = []LoggingOutletEnum{LoggingOutletEnum{Ret: s}}
-}
-
-var _ yaml.Defaulter = &LoggingOutletEnumList{}
-
-type Global struct {
-	Logging    *LoggingOutletEnumList `yaml:"logging,optional,fromdefaults"`
-	Monitoring []MonitoringEnum       `yaml:"monitoring,optional"`
-	Control    *GlobalControl         `yaml:"control,optional,fromdefaults"`
-	Serve      *GlobalServe           `yaml:"serve,optional,fromdefaults"`
-}
-
-func Default(i interface{}) {
-	v := reflect.ValueOf(i)
-	if v.Kind() != reflect.Ptr {
-		panic(v)
-	}
-	y := `{}`
-	err := yaml.Unmarshal([]byte(y), v.Interface())
-	if err != nil {
-		panic(err)
-	}
-}
-
-type ConnectEnum struct {
-	Ret interface{}
-}
-
-type ConnectCommon struct {
-	Type string `yaml:"type"`
-}
-
-type TCPConnect struct {
-	ConnectCommon `yaml:",inline"`
-	Address       string        `yaml:"address,hostport"`
-	DialTimeout   time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
-}
-
-type TLSConnect struct {
-	ConnectCommon `yaml:",inline"`
-	Address       string        `yaml:"address,hostport"`
-	Ca            string        `yaml:"ca"`
-	Cert          string        `yaml:"cert"`
-	Key           string        `yaml:"key"`
-	ServerCN      string        `yaml:"server_cn"`
-	DialTimeout   time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
-}
-
-type SSHStdinserverConnect struct {
-	ConnectCommon        `yaml:",inline"`
-	Host                 string        `yaml:"host"`
-	User                 string        `yaml:"user"`
-	Port                 uint16        `yaml:"port"`
-	IdentityFile         string        `yaml:"identity_file"`
-	TransportOpenCommand []string      `yaml:"transport_open_command,optional"` //TODO unused
-	SSHCommand           string        `yaml:"ssh_command,optional"`            //TODO unused
-	Options              []string      `yaml:"options,optional"`
-	DialTimeout          time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
-}
-
-type LocalConnect struct {
-	ConnectCommon  `yaml:",inline"`
-	ListenerName   string        `yaml:"listener_name"`
-	ClientIdentity string        `yaml:"client_identity"`
-	DialTimeout    time.Duration `yaml:"dial_timeout,zeropositive,default=2s"`
-}
-
-type ServeEnum struct {
-	Ret interface{}
-}
-
-type ServeCommon struct {
-	Type string `yaml:"type"`
-}
-
-type TCPServe struct {
-	ServeCommon    `yaml:",inline"`
-	Listen         string            `yaml:"listen,hostport"`
-	ListenFreeBind bool              `yaml:"listen_freebind,default=false"`
-	Clients        map[string]string `yaml:"clients"`
-}
-
-type TLSServe struct {
-	ServeCommon      `yaml:",inline"`
-	Listen           string        `yaml:"listen,hostport"`
-	ListenFreeBind   bool          `yaml:"listen_freebind,default=false"`
-	Ca               string        `yaml:"ca"`
-	Cert             string        `yaml:"cert"`
-	Key              string        `yaml:"key"`
-	ClientCNs        []string      `yaml:"client_cns"`
-	HandshakeTimeout time.Duration `yaml:"handshake_timeout,zeropositive,default=10s"`
-}
-
-type StdinserverServer struct {
-	ServeCommon      `yaml:",inline"`
-	ClientIdentities []string `yaml:"client_identities"`
-}
-
-type LocalServe struct {
-	ServeCommon  `yaml:",inline"`
-	ListenerName string `yaml:"listener_name"`
-}
-
-type PruningEnum struct {
-	Ret interface{}
-}
-
-type PruneKeepNotReplicated struct {
-	Type                 string `yaml:"type"`
-	KeepSnapshotAtCursor bool   `yaml:"keep_snapshot_at_cursor,optional,default=true"`
-}
-
-type PruneKeepLastN struct {
-	Type  string `yaml:"type"`
-	Count int    `yaml:"count"`
-	Regex string `yaml:"regex,optional"`
-}
-
-type PruneKeepRegex struct { // FIXME rename to KeepRegex
-	Type   string `yaml:"type"`
-	Regex  string `yaml:"regex"`
-	Negate bool   `yaml:"negate,optional,default=false"`
-}
-
-type LoggingOutletEnum struct {
-	Ret interface{}
-}
-
-type LoggingOutletCommon struct {
-	Type   string `yaml:"type"`
-	Level  string `yaml:"level"`
-	Format string `yaml:"format"`
-}
-
-type StdoutLoggingOutlet struct {
-	LoggingOutletCommon `yaml:",inline"`
-	Time                bool `yaml:"time,default=true"`
-	Color               bool `yaml:"color,default=true"`
-}
-
-type SyslogLoggingOutlet struct {
-	LoggingOutletCommon `yaml:",inline"`
-	Facility            *SyslogFacility `yaml:"facility,optional,fromdefaults"`
-	RetryInterval       time.Duration   `yaml:"retry_interval,positive,default=10s"`
-}
-
-type TCPLoggingOutlet struct {
-	LoggingOutletCommon `yaml:",inline"`
-	Address             string               `yaml:"address,hostport"`
-	Net                 string               `yaml:"net,default=tcp"`
-	RetryInterval       time.Duration        `yaml:"retry_interval,positive,default=10s"`
-	TLS                 *TCPLoggingOutletTLS `yaml:"tls,optional"`
-}
-
-type TCPLoggingOutletTLS struct {
-	CA   string `yaml:"ca"`
-	Cert string `yaml:"cert"`
-	Key  string `yaml:"key"`
-}
-
-type MonitoringEnum struct {
-	Ret interface{}
-}
-
-type PrometheusMonitoring struct {
-	Type           string `yaml:"type"`
-	Listen         string `yaml:"listen,hostport"`
-	ListenFreeBind bool   `yaml:"listen_freebind,default=false"`
-}
-
-type SyslogFacility syslog.Priority
-
-func (f *SyslogFacility) SetDefault() {
-	*f = SyslogFacility(syslog.LOG_LOCAL0)
-}
-
-var _ yaml.Defaulter = (*SyslogFacility)(nil)
-
-type GlobalControl struct {
-	SockPath string `yaml:"sockpath,default=/var/run/zrepl/control"`
-}
-
-type GlobalServe struct {
-	StdinServer *GlobalStdinServer `yaml:"stdinserver,optional,fromdefaults"`
-}
-
-type GlobalStdinServer struct {
-	SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
-}
-
-type JobDebugSettings struct {
-	Conn *struct {
-		ReadDump  string `yaml:"read_dump"`
-		WriteDump string `yaml:"write_dump"`
-	} `yaml:"conn,optional"`
-	RPCLog bool `yaml:"rpc_log,optional,default=false"`
-}
-
-type HookList []HookEnum
-
-type HookEnum struct {
-	Ret interface{}
-}
-
-type HookCommand struct {
-	Path               string            `yaml:"path"`
-	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
-	Filesystems        FilesystemsFilter `yaml:"filesystems,optional,default={'<': true}"`
-	HookSettingsCommon `yaml:",inline"`
-}
-
-type HookPostgresCheckpoint struct {
-	HookSettingsCommon `yaml:",inline"`
-	DSN                string            `yaml:"dsn"`
-	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
-	Filesystems        FilesystemsFilter `yaml:"filesystems"` // required, user should not CHECKPOINT for every FS
-}
-
-type HookMySQLLockTables struct {
-	HookSettingsCommon `yaml:",inline"`
-	DSN                string            `yaml:"dsn"`
-	Timeout            time.Duration     `yaml:"timeout,optional,positive,default=30s"`
-	Filesystems        FilesystemsFilter `yaml:"filesystems"`
-}
-
-type HookSettingsCommon struct {
-	Type       string `yaml:"type"`
-	ErrIsFatal bool   `yaml:"err_is_fatal,optional,default=false"`
-}
-
-func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) {
-	var in struct {
-		Type string
-	}
-	if err := u(&in, true); err != nil {
-		return nil, err
-	}
-	if in.Type == "" {
-		return nil, &yaml.TypeError{Errors: []string{"must specify type"}}
-	}
-
-	v, ok := types[in.Type]
-	if !ok {
-		return nil, &yaml.TypeError{Errors: []string{fmt.Sprintf("invalid type name %q", in.Type)}}
-	}
-	if err := u(v, false); err != nil {
-		return nil, err
-	}
-	return v, nil
-}
-
-func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"snap":   &SnapJob{},
-		"push":   &PushJob{},
-		"sink":   &SinkJob{},
-		"pull":   &PullJob{},
-		"source": &SourceJob{},
-	})
-	return
-}
-
-func (t *ConnectEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"tcp":             &TCPConnect{},
-		"tls":             &TLSConnect{},
-		"ssh+stdinserver": &SSHStdinserverConnect{},
-		"local":           &LocalConnect{},
-	})
-	return
-}
-
-func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"tcp":         &TCPServe{},
-		"tls":         &TLSServe{},
-		"stdinserver": &StdinserverServer{},
-		"local":       &LocalServe{},
-	})
-	return
-}
-
-func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"not_replicated": &PruneKeepNotReplicated{},
-		"last_n":         &PruneKeepLastN{},
-		"grid":           &PruneGrid{},
-		"regex":          &PruneKeepRegex{},
-	})
-	return
-}
-
-func (t *SnapshottingEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"periodic": &SnapshottingPeriodic{},
-		"manual":   &SnapshottingManual{},
-		"cron":     &SnapshottingCron{},
-	})
-	return
-}
-
-func (t *LoggingOutletEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"stdout": &StdoutLoggingOutlet{},
-		"syslog": &SyslogLoggingOutlet{},
-		"tcp":    &TCPLoggingOutlet{},
-	})
-	return
-}
-
-func (t *MonitoringEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"prometheus": &PrometheusMonitoring{},
-	})
-	return
-}
-
-func (t *SyslogFacility) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	var s string
-	if err := u(&s, true); err != nil {
-		return err
-	}
-	var level syslog.Priority
-	switch s {
-	case "kern":
-		level = syslog.LOG_KERN
-	case "user":
-		level = syslog.LOG_USER
-	case "mail":
-		level = syslog.LOG_MAIL
-	case "daemon":
-		level = syslog.LOG_DAEMON
-	case "auth":
-		level = syslog.LOG_AUTH
-	case "syslog":
-		level = syslog.LOG_SYSLOG
-	case "lpr":
-		level = syslog.LOG_LPR
-	case "news":
-		level = syslog.LOG_NEWS
-	case "uucp":
-		level = syslog.LOG_UUCP
-	case "cron":
-		level = syslog.LOG_CRON
-	case "authpriv":
-		level = syslog.LOG_AUTHPRIV
-	case "ftp":
-		level = syslog.LOG_FTP
-	case "local0":
-		level = syslog.LOG_LOCAL0
-	case "local1":
-		level = syslog.LOG_LOCAL1
-	case "local2":
-		level = syslog.LOG_LOCAL2
-	case "local3":
-		level = syslog.LOG_LOCAL3
-	case "local4":
-		level = syslog.LOG_LOCAL4
-	case "local5":
-		level = syslog.LOG_LOCAL5
-	case "local6":
-		level = syslog.LOG_LOCAL6
-	case "local7":
-		level = syslog.LOG_LOCAL7
-	default:
-		return fmt.Errorf("invalid syslog level: %q", s)
-	}
-	*t = SyslogFacility(level)
-	return nil
-}
-
-func (t *HookEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	t.Ret, err = enumUnmarshal(u, map[string]interface{}{
-		"command":             &HookCommand{},
-		"postgres-checkpoint": &HookPostgresCheckpoint{},
-		"mysql-lock-tables":   &HookMySQLLockTables{},
-	})
-	return
-}
-
-var ConfigFileDefaultLocations = []string{
-	"/etc/zrepl/zrepl.yml",
-	"/usr/local/etc/zrepl/zrepl.yml",
-}
-
-func ParseConfig(path string) (i *Config, err error) {
-
-	if path == "" {
-		// Try default locations
-		for _, l := range ConfigFileDefaultLocations {
-			stat, statErr := os.Stat(l)
-			if statErr != nil {
-				continue
-			}
-			if !stat.Mode().IsRegular() {
-				err = errors.Errorf("file at default location is not a regular file: %s", l)
-				return
-			}
-			path = l
-			break
-		}
-	}
-
-	var bytes []byte
-
-	if bytes, err = ioutil.ReadFile(path); err != nil {
-		return
-	}
-
-	return ParseConfigBytes(bytes)
-}
-
-func ParseConfigBytes(bytes []byte) (*Config, error) {
-	var c *Config
-	if err := yaml.UnmarshalStrict(bytes, &c); err != nil {
-		return nil, err
-	}
-	if c == nil {
-		return nil, fmt.Errorf("config is empty or only consists of comments")
-	}
-	return c, nil
-}

config/config_dep_test.go

@@ -1,90 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-	"github.com/zrepl/yaml-config"
-)
-
-type A struct {
-	B  *B     `yaml:"b,optional,fromdefaults"`
-	A1 string `yaml:"a1,optional"`
-}
-
-type B struct {
-	C *C     `yaml:"c,optional,fromdefaults"`
-	D string `yaml:"d,default=ddd"`
-	E string `yaml:"e,optional"`
-}
-
-type C struct {
-	Q string `yaml:"q,optional"`
-	R string `yaml:"r,default=r"`
-}
-
-func TestDepFromDefaults(t *testing.T) {
-
-	type testcase struct {
-		name   string
-		yaml   string
-		expect *A
-	}
-
-	tcs := []testcase{
-		{
-			name: "empty",
-			yaml: `{}`,
-			expect: &A{
-				B: &B{
-					C: &C{
-						R: "r",
-					},
-					D: "ddd",
-				},
-			},
-		},
-		{
-			name: "a1 set",
-			yaml: `{"a1":"blah"}`,
-			expect: &A{
-				A1: "blah",
-				B: &B{
-					C: &C{
-						R: "r",
-					},
-					D: "ddd",
-				},
-			},
-		},
-		{
-			name: "D set",
-			yaml: `
-b:
-  d: 4d
-`,
-			expect: &A{
-				B: &B{
-					D: "4d",
-					C: &C{
-						R: "r",
-					},
-				},
-			},
-		},
-	}
-
-	for tci := range tcs {
-		t.Run(fmt.Sprintf("%d-%s", tci, tcs[tci].name), func(t *testing.T) {
-			tc := tcs[tci]
-
-			var a A
-			err := yaml.UnmarshalStrict([]byte(tc.yaml), &a)
-			require.NoError(t, err)
-
-			require.Equal(t, tc.expect, &a)
-		})
-	}
-
-}

config/config_duration.go

@@ -1,106 +0,0 @@
-package config
-
-import (
-	"errors"
-	"fmt"
-	"regexp"
-	"strconv"
-	"time"
-
-	"github.com/kr/pretty"
-	"github.com/zrepl/yaml-config"
-)
-
-type Duration struct{ d time.Duration }
-
-func (d Duration) Duration() time.Duration { return d.d }
-
-var _ yaml.Unmarshaler = &Duration{}
-
-func (d *Duration) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
-	var s string
-	err := unmarshal(&s, false)
-	if err != nil {
-		return err
-	}
-	d.d, err = parseDuration(s)
-	if err != nil {
-		d.d = 0
-		return &yaml.TypeError{Errors: []string{fmt.Sprintf("cannot parse value %q: %s", s, err)}}
-	}
-	return nil
-}
-
-type PositiveDuration struct{ d Duration }
-
-var _ yaml.Unmarshaler = &PositiveDuration{}
-
-func (d PositiveDuration) Duration() time.Duration { return d.d.Duration() }
-
-func (d *PositiveDuration) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
-	err := d.d.UnmarshalYAML(unmarshal)
-	if err != nil {
-		return err
-	}
-	if d.d.Duration() <= 0 {
-		return fmt.Errorf("duration must be positive, got %s", d.d.Duration())
-	}
-	return nil
-}
-
-func parsePositiveDuration(e string) (time.Duration, error) {
-	d, err := parseDuration(e)
-	if err != nil {
-		return d, err
-	}
-	if d <= 0 {
-		return 0, errors.New("duration must be positive integer")
-	}
-	return d, err
-}
-
-var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*([\+-]?\d+)\s*(|s|m|h|d|w)\s*$`)
-
-func parseDuration(e string) (d time.Duration, err error) {
-	comps := durationStringRegex.FindStringSubmatch(e)
-	if comps == nil {
-		err = fmt.Errorf("must match %s", durationStringRegex)
-		return
-	}
-	if len(comps) != 3 {
-		panic(pretty.Sprint(comps))
-	}
-
-	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
-	if err != nil {
-		return 0, err
-	}
-
-	var durationUnit time.Duration
-	switch comps[2] {
-	case "":
-		if durationFactor != 0 {
-			err = fmt.Errorf("missing time unit")
-			return
-		} else {
-			// It's the case where user specified '0'.
-			// We want to allow this, just like time.ParseDuration.
-		}
-	case "s":
-		durationUnit = time.Second
-	case "m":
-		durationUnit = time.Minute
-	case "h":
-		durationUnit = time.Hour
-	case "d":
-		durationUnit = 24 * time.Hour
-	case "w":
-		durationUnit = 24 * 7 * time.Hour
-	default:
-		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
-		return
-	}
-
-	d = time.Duration(durationFactor) * durationUnit
-	return
-}

config/config_global_test.go

@@ -1,115 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"log/syslog"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"github.com/zrepl/yaml-config"
-)
-
-func testValidGlobalSection(t *testing.T, s string) *Config {
-	jobdef := `
-jobs:
-- name: dummyjob
-  type: sink
-  serve:
-    type: tcp
-    listen: ":2342"
-    clients: {
-      "10.0.0.1":"foo"
-    }
-  root_fs: zroot/foo
-`
-	_, err := ParseConfigBytes([]byte(jobdef))
-	require.NoError(t, err)
-	return testValidConfig(t, s+jobdef)
-}
-
-func TestOutletTypes(t *testing.T) {
-	conf := testValidGlobalSection(t, `
-global:
-  logging:
-  - type: stdout
-    level: debug
-    format: human
-  - type: syslog
-    level: info
-    retry_interval: 20s
-    format: human
-  - type: tcp
-    level: debug
-    format: json
-    address: logserver.example.com:1234
-  - type: tcp
-    level: debug
-    format: json
-    address: encryptedlogserver.example.com:1234
-    retry_interval: 20s 
-    tls:
-      ca: /etc/zrepl/log/ca.crt
-      cert: /etc/zrepl/log/key.pem
-      key: /etc/zrepl/log/cert.pem
-`)
-	assert.Equal(t, 4, len(*conf.Global.Logging))
-	assert.NotNil(t, (*conf.Global.Logging)[3].Ret.(*TCPLoggingOutlet).TLS)
-}
-
-func TestDefaultLoggingOutlet(t *testing.T) {
-	conf := testValidGlobalSection(t, "")
-	assert.Equal(t, 1, len(*conf.Global.Logging))
-	o := (*conf.Global.Logging)[0].Ret.(*StdoutLoggingOutlet)
-	assert.Equal(t, "warn", o.Level)
-	assert.Equal(t, "human", o.Format)
-}
-
-func TestPrometheusMonitoring(t *testing.T) {
-	conf := testValidGlobalSection(t, `
-global:
-  monitoring:
-    - type: prometheus
-      listen: ':9811'
-`)
-	assert.Equal(t, ":9811", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
-}
-
-func TestSyslogLoggingOutletFacility(t *testing.T) {
-	type SyslogFacilityPriority struct {
-		Facility string
-		Priority syslog.Priority
-	}
-	syslogFacilitiesPriorities := []SyslogFacilityPriority{
-		{"", syslog.LOG_LOCAL0}, // default
-		{"kern", syslog.LOG_KERN}, {"daemon", syslog.LOG_DAEMON}, {"auth", syslog.LOG_AUTH},
-		{"syslog", syslog.LOG_SYSLOG}, {"lpr", syslog.LOG_LPR}, {"news", syslog.LOG_NEWS},
-		{"uucp", syslog.LOG_UUCP}, {"cron", syslog.LOG_CRON}, {"authpriv", syslog.LOG_AUTHPRIV},
-		{"ftp", syslog.LOG_FTP}, {"local0", syslog.LOG_LOCAL0}, {"local1", syslog.LOG_LOCAL1},
-		{"local2", syslog.LOG_LOCAL2}, {"local3", syslog.LOG_LOCAL3}, {"local4", syslog.LOG_LOCAL4},
-		{"local5", syslog.LOG_LOCAL5}, {"local6", syslog.LOG_LOCAL6}, {"local7", syslog.LOG_LOCAL7},
-	}
-
-	for _, sFP := range syslogFacilitiesPriorities {
-		logcfg := fmt.Sprintf(`
-global:
-  logging:
-  - type: syslog
-    level: info
-    format: human
-    facility: %s
-`, sFP.Facility)
-		conf := testValidGlobalSection(t, logcfg)
-		assert.Equal(t, 1, len(*conf.Global.Logging))
-		assert.True(t, SyslogFacility(sFP.Priority) == *(*conf.Global.Logging)[0].Ret.(*SyslogLoggingOutlet).Facility)
-	}
-}
-
-func TestLoggingOutletEnumList_SetDefaults(t *testing.T) {
-	e := &LoggingOutletEnumList{}
-	var i yaml.Defaulter = e
-	require.NotPanics(t, func() {
-		i.SetDefault()
-		assert.Equal(t, "warn", (*e)[0].Ret.(*StdoutLoggingOutlet).Level)
-	})
-}

config/config_jobid_test.go

@@ -1 +0,0 @@
-package config

config/config_minimal_test.go

@@ -1,40 +0,0 @@
-package config
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestConfigEmptyFails(t *testing.T) {
-	conf, err := testConfig(t, "\n")
-	assert.Nil(t, conf)
-	assert.Error(t, err)
-}
-
-func TestJobsOnlyWorks(t *testing.T) {
-	testValidConfig(t, `
-jobs:
-- name: push
-  type: push
-  # snapshot the filesystems matched by the left-hand-side of the mapping
-  # every 10m with zrepl_ as prefix
-  connect:
-    type: tcp
-    address: localhost:2342
-  filesystems: {
-    "pool1/var/db<": true,
-    "pool1/usr/home<": true,
-    "pool1/usr/home/paranoid": false, #don't backup paranoid user
-    "pool1/poudriere/ports<": false #don't backup the ports trees
-  }
-  snapshotting:
-    type: manual
-  pruning:
-    keep_sender:
-    - type: not_replicated
-    keep_receiver:
-    - type: last_n 
-      count: 1
-`)
-}

config/config_positiveintervalormanual_test.go

@@ -1,41 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/zrepl/yaml-config"
-)
-
-func TestPositiveDurationOrManual(t *testing.T) {
-	cases := []struct {
-		Comment, Input string
-		Result         *PositiveDurationOrManual
-	}{
-		{"empty is error", "", nil},
-		{"negative is error", "-1s", nil},
-		{"zero seconds is error", "0s", nil},
-		{"zero is error", "0", nil},
-		{"non-manual is error", "something", nil},
-		{"positive seconds works", "1s", &PositiveDurationOrManual{Manual: false, Interval: 1 * time.Second}},
-		{"manual works", "manual", &PositiveDurationOrManual{Manual: true, Interval: 0}},
-	}
-	for _, tc := range cases {
-		t.Run(tc.Comment, func(t *testing.T) {
-			var out struct {
-				FieldName PositiveDurationOrManual `yaml:"fieldname"`
-			}
-			input := fmt.Sprintf("\nfieldname: %s\n", tc.Input)
-			err := yaml.UnmarshalStrict([]byte(input), &out)
-			if tc.Result == nil {
-				assert.Error(t, err)
-				t.Logf("%#v", out)
-			} else {
-				assert.Equal(t, *tc.Result, out.FieldName)
-			}
-		})
-	}
-
-}

config/config_recv_test.go

@@ -1,134 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	zfsprop "github.com/zrepl/zrepl/zfs/property"
-)
-
-func TestRecvOptions(t *testing.T) {
-	tmpl := `
-jobs:
-- name: foo
-  type: pull
-  connect:
-    type: local
-    listener_name: foo
-    client_identity: bar
-  root_fs: "zreplplatformtest"
-  %s
-  interval: manual
-  pruning:
-    keep_sender:
-    - type: last_n
-      count: 10
-    keep_receiver:
-    - type: last_n
-      count: 10
-
-`
-
-	recv_properties_empty := `
-  recv:
-    properties:
-`
-
-	recv_inherit_empty := `
-  recv:
-    properties:
-      inherit:
-`
-
-	recv_inherit := `
-  recv:
-    properties:
-      inherit:
-        - testprop
-`
-
-	recv_override_empty := `
-  recv:
-    properties:
-      override:
-`
-
-	recv_override := `
-  recv:
-    properties:
-      override:
-        testprop2: "test123"
-`
-
-	recv_override_and_inherit := `
-  recv:
-    properties:
-      inherit:
-        - testprop
-      override:
-        testprop2: "test123"
-`
-
-	recv_empty := `
-  recv: {}
-`
-
-	recv_not_specified := `
-`
-
-	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
-
-	t.Run("recv_inherit_empty", func(t *testing.T) {
-		c := testValidConfig(t, fill(recv_inherit_empty))
-		assert.NotNil(t, c)
-	})
-
-	t.Run("recv_inherit", func(t *testing.T) {
-		c := testValidConfig(t, fill(recv_inherit))
-		inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
-		assert.NotEmpty(t, inherit)
-		assert.Contains(t, inherit, zfsprop.Property("testprop"))
-	})
-
-	t.Run("recv_override_empty", func(t *testing.T) {
-		c := testValidConfig(t, fill(recv_override_empty))
-		assert.NotNil(t, c)
-	})
-
-	t.Run("recv_override", func(t *testing.T) {
-		c := testValidConfig(t, fill(recv_override))
-		override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
-		require.Len(t, override, 1)
-		require.Equal(t, "test123", override["testprop2"])
-
-	})
-
-	t.Run("recv_override_and_inherit", func(t *testing.T) {
-		c := testValidConfig(t, fill(recv_override_and_inherit))
-		inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
-		override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
-		assert.NotEmpty(t, inherit)
-		assert.Contains(t, inherit, zfsprop.Property("testprop"))
-		assert.NotEmpty(t, override)
-		assert.Equal(t, "test123", override["testprop2"])
-	})
-
-	t.Run("recv_properties_empty", func(t *testing.T) {
-		c := testValidConfig(t, fill(recv_properties_empty))
-		assert.NotNil(t, c)
-	})
-
-	t.Run("recv_empty", func(t *testing.T) {
-		c := testValidConfig(t, fill(recv_empty))
-		assert.NotNil(t, c)
-	})
-
-	t.Run("send_not_specified", func(t *testing.T) {
-		c := testValidConfig(t, fill(recv_not_specified))
-		assert.NotNil(t, c)
-	})
-
-}

config/config_send_test.go

@@ -1,146 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestSendOptions(t *testing.T) {
-	tmpl := `
-jobs:
-- name: foo
-  type: push
-  connect:
-    type: local
-    listener_name: foo
-    client_identity: bar
-  filesystems: {"<": true}
-  %s
-  snapshotting:
-    type: manual
-  pruning:
-    keep_sender:
-    - type: last_n
-      count: 10
-    keep_receiver:
-    - type: last_n
-      count: 10
-`
-	encrypted_false := `
-  send:
-    encrypted: false
-`
-
-	encrypted_true := `
-  send:
-    encrypted: true
-`
-
-	raw_true := `
-  send:
-    raw: true
-
-`
-
-	raw_false := `
-  send:
-    raw: false
-
-`
-
-	raw_and_encrypted := `
-  send:
-    encrypted: true
-    raw: true
-`
-
-	properties_and_encrypted := `
-  send:
-    encrypted: true
-    send_properties: true
-`
-
-	properties_true := `
-  send:
-    send_properties: true
-`
-
-	properties_false := `
-  send:
-    send_properties: false
-`
-
-	send_empty := `
-  send: {}
-`
-
-	send_not_specified := `
-`
-
-	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
-
-	t.Run("encrypted_false", func(t *testing.T) {
-		c := testValidConfig(t, fill(encrypted_false))
-		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
-		assert.Equal(t, false, encrypted)
-	})
-	t.Run("encrypted_true", func(t *testing.T) {
-		c := testValidConfig(t, fill(encrypted_true))
-		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
-		assert.Equal(t, true, encrypted)
-	})
-
-	t.Run("send_empty", func(t *testing.T) {
-		c := testValidConfig(t, fill(send_empty))
-		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
-		assert.Equal(t, false, encrypted)
-	})
-
-	t.Run("properties_and_encrypted", func(t *testing.T) {
-		c := testValidConfig(t, fill(properties_and_encrypted))
-		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
-		properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
-		assert.Equal(t, true, encrypted)
-		assert.Equal(t, true, properties)
-	})
-
-	t.Run("properties_false", func(t *testing.T) {
-		c := testValidConfig(t, fill(properties_false))
-		properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
-		assert.Equal(t, false, properties)
-	})
-
-	t.Run("properties_true", func(t *testing.T) {
-		c := testValidConfig(t, fill(properties_true))
-		properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
-		assert.Equal(t, true, properties)
-	})
-
-	t.Run("raw_true", func(t *testing.T) {
-		c := testValidConfig(t, fill(raw_true))
-		raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
-		assert.Equal(t, true, raw)
-	})
-
-	t.Run("raw_false", func(t *testing.T) {
-		c := testValidConfig(t, fill(raw_false))
-		raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
-		assert.Equal(t, false, raw)
-	})
-
-	t.Run("raw_and_encrypted", func(t *testing.T) {
-		c := testValidConfig(t, fill(raw_and_encrypted))
-		raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
-		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
-		assert.Equal(t, true, raw)
-		assert.Equal(t, true, encrypted)
-	})
-
-	t.Run("send_not_specified", func(t *testing.T) {
-		c := testValidConfig(t, fill(send_not_specified))
-		assert.NotNil(t, c)
-	})
-
-}

config/config_snapshotting_test.go

@@ -1,176 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestSnapshotting(t *testing.T) {
-	tmpl := `
-jobs:
-- name: foo
-  type: push
-  connect:
-    type: local
-    listener_name: foo
-    client_identity: bar
-  filesystems: {"<": true}
-  %s
-  pruning:
-    keep_sender:
-    - type: last_n
-      count: 10
-    keep_receiver:
-    - type: last_n
-      count: 10
-`
-	manual := `
-  snapshotting:
-    type: manual
-`
-	periodic := `
-  snapshotting:
-    type: periodic
-    prefix: zrepl_
-    timestamp_format: dense
-    interval: 10m
-`
-	cron := `
-  snapshotting:
-    type: cron
-    prefix: zrepl_
-    timestamp_format: human
-    cron: "10 * * * *"
-`
-
-	periodicDaily := `
-  snapshotting:
-    type: periodic
-    prefix: zrepl_
-    interval: 1d
-`
-
-	hooks := `
-  snapshotting:
-    type: periodic
-    prefix: zrepl_
-    interval: 10m
-    hooks:
-    - type: command
-      path: /tmp/path/to/command
-    - type: command
-      path: /tmp/path/to/command
-      filesystems: { "zroot<": true, "<": false }
-    - type: postgres-checkpoint
-      dsn: "host=localhost port=5432 user=postgres sslmode=disable"
-      filesystems: {
-          "tank/postgres/data11": true
-      }
-    - type: mysql-lock-tables
-      dsn: "root@tcp(localhost)/"
-      filesystems: {
-        "tank/mysql": true
-      }
-`
-
-	fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
-	var c *Config
-
-	t.Run("manual", func(t *testing.T) {
-		c = testValidConfig(t, fillSnapshotting(manual))
-		snm := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingManual)
-		assert.Equal(t, "manual", snm.Type)
-	})
-
-	t.Run("periodic", func(t *testing.T) {
-		c = testValidConfig(t, fillSnapshotting(periodic))
-		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
-		assert.Equal(t, "periodic", snp.Type)
-		assert.Equal(t, 10*time.Minute, snp.Interval.Duration())
-		assert.Equal(t, "zrepl_", snp.Prefix)
-	})
-
-	t.Run("periodicDaily", func(t *testing.T) {
-		c = testValidConfig(t, fillSnapshotting(periodicDaily))
-		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
-		assert.Equal(t, "periodic", snp.Type)
-		assert.Equal(t, 24*time.Hour, snp.Interval.Duration())
-		assert.Equal(t, "zrepl_", snp.Prefix)
-		assert.Equal(t, "dense", snp.TimestampFormat)
-	})
-
-	t.Run("cron", func(t *testing.T) {
-		c = testValidConfig(t, fillSnapshotting(cron))
-		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingCron)
-		assert.Equal(t, "cron", snp.Type)
-		assert.Equal(t, "zrepl_", snp.Prefix)
-		assert.Equal(t, "human", snp.TimestampFormat)
-	})
-
-	t.Run("hooks", func(t *testing.T) {
-		c = testValidConfig(t, fillSnapshotting(hooks))
-		hs := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic).Hooks
-		assert.Equal(t, hs[0].Ret.(*HookCommand).Filesystems["<"], true)
-		assert.Equal(t, hs[1].Ret.(*HookCommand).Filesystems["zroot<"], true)
-		assert.Equal(t, hs[2].Ret.(*HookPostgresCheckpoint).Filesystems["tank/postgres/data11"], true)
-		assert.Equal(t, hs[3].Ret.(*HookMySQLLockTables).Filesystems["tank/mysql"], true)
-	})
-
-}
-
-func TestSnapshottingTimestampDefaults(t *testing.T) {
-	tmpl := `
-jobs:
-- name: foo
-  type: push
-  connect:
-    type: local
-    listener_name: foo
-    client_identity: bar
-  filesystems: {"<": true}
-  %s
-  pruning:
-    keep_sender:
-    - type: last_n
-      count: 10
-    keep_receiver:
-    - type: last_n
-      count: 10
-`
-
-	periodic := `
-  snapshotting:
-    type: periodic
-    prefix: zrepl_
-    interval: 10m
-`
-	cron := `
-  snapshotting:
-    type: cron
-    prefix: zrepl_
-    cron: "10 * * * *"
-`
-
-	fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
-	var c *Config
-
-	t.Run("periodic", func(t *testing.T) {
-		c = testValidConfig(t, fillSnapshotting(periodic))
-		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
-		assert.Equal(t, "periodic", snp.Type)
-		assert.Equal(t, 10*time.Minute, snp.Interval.Duration())
-		assert.Equal(t, "zrepl_", snp.Prefix)
-		assert.Equal(t, "dense", snp.TimestampFormat) // default was set correctly
-	})
-
-	t.Run("cron", func(t *testing.T) {
-		c = testValidConfig(t, fillSnapshotting(cron))
-		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingCron)
-		assert.Equal(t, "cron", snp.Type)
-		assert.Equal(t, "zrepl_", snp.Prefix)
-		assert.Equal(t, "dense", snp.TimestampFormat) // default was set correctly
-	})
-}

config/config_test.go

@@ -1,140 +0,0 @@
-package config
-
-import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"path"
-	"path/filepath"
-	"strings"
-	"testing"
-	"text/template"
-
-	"github.com/kr/pretty"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"github.com/zrepl/yaml-config"
-)
-
-func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
-	paths, err := filepath.Glob("./samples/*")
-	if err != nil {
-		t.Errorf("glob failed: %+v", err)
-	}
-
-	for _, p := range paths {
-
-		if path.Ext(p) != ".yml" {
-			t.Logf("skipping file %s", p)
-			continue
-		}
-
-		t.Run(p, func(t *testing.T) {
-			c, err := ParseConfig(p)
-			if err != nil {
-				t.Errorf("error parsing %s:\n%+v", p, err)
-			}
-
-			t.Logf("file: %s", p)
-			t.Log(pretty.Sprint(c))
-		})
-
-	}
-
-}
-
-// template must be a template/text template with a single '{{ . }}' as placeholder for val
-//
-//nolint:deadcode,unused
-func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
-	tmp, err := template.New("master").Parse(tmpl)
-	if err != nil {
-		panic(err)
-	}
-	var buf bytes.Buffer
-	err = tmp.Execute(&buf, val)
-	if err != nil {
-		panic(err)
-	}
-	return testValidConfig(t, buf.String())
-}
-
-func testValidConfig(t *testing.T, input string) *Config {
-	t.Helper()
-	conf, err := testConfig(t, input)
-	require.NoError(t, err)
-	require.NotNil(t, conf)
-	return conf
-}
-
-func testConfig(t *testing.T, input string) (*Config, error) {
-	t.Helper()
-	return ParseConfigBytes([]byte(input))
-}
-
-func trimSpaceEachLineAndPad(s, pad string) string {
-	var out strings.Builder
-	scan := bufio.NewScanner(strings.NewReader(s))
-	for scan.Scan() {
-		fmt.Fprintf(&out, "%s%s\n", pad, bytes.TrimSpace(scan.Bytes()))
-	}
-	return out.String()
-}
-
-func TestTrimSpaceEachLineAndPad(t *testing.T) {
-	foo := `
-	foo
-	bar baz 
-	`
-	assert.Equal(t, "  \n  foo\n  bar baz\n  \n", trimSpaceEachLineAndPad(foo, "  "))
-}
-
-func TestCronSpec(t *testing.T) {
-
-	expectAccept := []string{
-		`"* * * * *"`,
-		`"0-10 * * * *"`,
-		`"* 0-5,8,12 * * *"`,
-	}
-
-	expectFail := []string{
-		`* * * *`,
-		``,
-		`23`,
-		`"@reboot"`,
-		`"@every 1h30m"`,
-		`"@daily"`,
-		`* * * * * *`,
-	}
-
-	for _, input := range expectAccept {
-		t.Run(input, func(t *testing.T) {
-			s := fmt.Sprintf("spec: %s\n", input)
-			var v struct {
-				Spec CronSpec
-			}
-			v.Spec.Schedule = nil
-			t.Logf("input:\n%s", s)
-			err := yaml.UnmarshalStrict([]byte(s), &v)
-			t.Logf("error: %T %s", err, err)
-			require.NoError(t, err)
-			require.NotNil(t, v.Spec.Schedule)
-		})
-	}
-
-	for _, input := range expectFail {
-		t.Run(input, func(t *testing.T) {
-			s := fmt.Sprintf("spec: %s\n", input)
-			var v struct {
-				Spec CronSpec
-			}
-			v.Spec.Schedule = nil
-			t.Logf("input: %q", s)
-			err := yaml.UnmarshalStrict([]byte(s), &v)
-			t.Logf("error: %T %s", err, err)
-			require.Error(t, err)
-			require.Nil(t, v.Spec.Schedule)
-		})
-	}
-
-}

config/config_transport_test.go

@@ -1,99 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestTransportConnect(t *testing.T) {
-	tmpl := `
-jobs:
-- name: foo
-  type: push
-  connect:
-%s
-  filesystems: {"<": true}
-  snapshotting:
-    type: manual
-  pruning:
-    keep_sender:
-    - type: last_n
-      count: 10
-    keep_receiver:
-    - type: last_n
-      count: 10
-`
-
-	mconf := func(s string) string { return fmt.Sprintf(tmpl, s) }
-
-	type test struct {
-		Name        string
-		ExpectError bool
-		Connect     string
-	}
-
-	testTable := []test{
-		{
-			Name:        "tcp_with_address_and_port",
-			ExpectError: false,
-			Connect: `
-			type: tcp
-			address: 10.0.0.23:42
-			`,
-		},
-		{
-			Name:        "tls_with_host_and_port",
-			ExpectError: false,
-			Connect: `
-			type: tls
-			address: "server1.foo.bar:8888"
-			ca:   /etc/zrepl/ca.crt
-			cert: /etc/zrepl/backupserver.fullchain
-			key:  /etc/zrepl/backupserver.key
-			server_cn: "server1"
-			`,
-		},
-		{
-			Name:        "tcp_without_port",
-			ExpectError: true,
-			Connect: `
-			type: tcp
-			address: 10.0.0.23
-			`,
-		},
-		{
-			Name:        "tls_without_port",
-			ExpectError: true,
-			Connect: `
-			type: tls
-			address: 10.0.0.23
-			ca:   /etc/zrepl/ca.crt
-			cert: /etc/zrepl/backupserver.fullchain
-			key:  /etc/zrepl/backupserver.key
-			server_cn: "server1"
-			`,
-		},
-	}
-
-	for _, tc := range testTable {
-		t.Run(tc.Name, func(t *testing.T) {
-			require.NotEmpty(t, tc.Connect)
-			connect := trimSpaceEachLineAndPad(tc.Connect, "    ")
-			conf := mconf(connect)
-			config, err := testConfig(t, conf)
-			if tc.ExpectError && err == nil {
-				t.Errorf("expected test failure, but got valid config %v", config)
-				return
-			}
-			if !tc.ExpectError && err != nil {
-				t.Errorf("not expecint test failure but got error: %s", err)
-				return
-			}
-			t.Logf("error=%v", err)
-			t.Logf("config=%v", config)
-		})
-	}
-
-}

config/retentiongrid.go

@@ -1,119 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"regexp"
-	"strconv"
-	"strings"
-	"time"
-)
-
-type RetentionIntervalList []RetentionInterval
-
-type PruneGrid struct {
-	Type  string                `yaml:"type"`
-	Grid  RetentionIntervalList `yaml:"grid"`
-	Regex string                `yaml:"regex"`
-}
-
-type RetentionInterval struct {
-	length    time.Duration
-	keepCount int
-}
-
-func (i *RetentionInterval) Length() time.Duration {
-	return i.length
-}
-
-func (i *RetentionInterval) KeepCount() int {
-	return i.keepCount
-}
-
-const RetentionGridKeepCountAll int = -1
-
-func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
-	var in string
-	if err := u(&in, true); err != nil {
-		return err
-	}
-
-	intervals, err := ParseRetentionIntervalSpec(in)
-	if err != nil {
-		return err
-	}
-
-	*t = intervals
-
-	return nil
-}
-
-var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
-
-func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {
-
-	comps := retentionStringIntervalRegex.FindStringSubmatch(e)
-	if comps == nil {
-		err = fmt.Errorf("retention string does not match expected format")
-		return
-	}
-
-	times, err := strconv.Atoi(comps[1])
-	if err != nil {
-		return nil, err
-	} else if times <= 0 {
-		return nil, fmt.Errorf("contains factor <= 0")
-	}
-
-	duration, err := parsePositiveDuration(comps[2])
-	if err != nil {
-		return nil, err
-	}
-
-	keepCount := 1
-	if comps[3] != "" {
-		// Decompose key=value, comma separated
-		// For now, only keep_count is supported
-		re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
-		res := re.FindStringSubmatch(comps[4])
-		if res == nil || len(res) != 2 {
-			err = fmt.Errorf("interval parameter contains unknown parameters")
-			return
-		}
-		if res[1] == "all" {
-			keepCount = RetentionGridKeepCountAll
-		} else {
-			keepCount, err = strconv.Atoi(res[1])
-			if err != nil {
-				err = fmt.Errorf("cannot parse keep_count value")
-				return
-			}
-		}
-	}
-
-	intervals = make([]RetentionInterval, times)
-	for i := range intervals {
-		intervals[i] = RetentionInterval{
-			length:    duration,
-			keepCount: keepCount,
-		}
-	}
-
-	return
-
-}
-
-func ParseRetentionIntervalSpec(s string) (intervals []RetentionInterval, err error) {
-
-	ges := strings.Split(s, "|")
-	intervals = make([]RetentionInterval, 0, 7*len(ges))
-
-	for intervalIdx, e := range ges {
-		parsed, err := parseRetentionGridIntervalString(e)
-		if err != nil {
-			return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
-		}
-		intervals = append(intervals, parsed...)
-	}
-
-	return
-}

config/samples/bandwidth_limit.yml

@@ -1,41 +0,0 @@
-
-jobs:
-  - type: sink
-    name: "limited_sink"
-    root_fs: "fs0"
-    recv:
-      bandwidth_limit:
-        max: 12345 B
-    serve:
-      type: local
-      listener_name: localsink
-
-  - type: push
-    name: "limited_push"
-    connect:
-      type: local
-      listener_name: localsink
-      client_identity: local_backup
-    filesystems: {
-      "root<": true,
-    }
-    send:
-      bandwidth_limit:
-        max: 54321 B
-        bucket_capacity: 1024 B
-    snapshotting:
-      type: manual
-    pruning:
-      keep_sender:
-        - type: last_n
-          count: 1
-      keep_receiver:
-        - type: last_n
-          count: 1
-
-  - type: sink
-    name: "nolimit_sink"
-    root_fs: "fs1"
-    serve:
-      type: local
-      listener_name: localsink

config/samples/hooks/template.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-[ "$ZREPL_DRYRUN" = "true" ] && DRYRUN="echo DRYRUN: "
-
-pre_snapshot() {
-    $DRYRUN date
-}
-
-post_snapshot() {
-    $DRYRUN date
-}
-
-case "$ZREPL_HOOKTYPE" in
-    pre_snapshot|post_snapshot)
-        "$ZREPL_HOOKTYPE"
-        ;;
-    *)
-        printf 'Unrecognized hook type: %s\n' "$ZREPL_HOOKTYPE"
-        exit 255
-        ;;
-esac

config/samples/local.yml

@@ -1,31 +0,0 @@
-
-jobs:
-  - type: sink
-    name: "local_sink"
-    root_fs: "storage/zrepl/sink"
-    serve:
-      type: local
-      listener_name: localsink
-
-  - type: push
-    name: "backup_system"
-    connect:
-      type: local
-      listener_name: localsink
-      client_identity: local_backup
-    filesystems: {
-      "system<": true,
-    }
-    snapshotting:
-      type: periodic
-      interval: 10m
-      prefix: zrepl_
-    pruning:
-      keep_sender:
-      - type: not_replicated
-      - type: last_n
-        count: 10
-      keep_receiver:
-      - type: grid
-        grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-        regex: "zrepl_.*"

config/samples/pull.yml

@@ -1,24 +0,0 @@
-jobs:
-- name: pull_servers
-  type: pull
-  connect:
-    type: tls
-    address: "server1.foo.bar:8888"
-    ca: "/certs/ca.crt"
-    cert: "/certs/cert.crt"
-    key: "/certs/key.pem"
-    server_cn: "server1"
-  root_fs: "pool2/backup_servers"
-  interval: 10m
-  pruning:
-    keep_sender:
-    - type: not_replicated
-    - type: last_n
-      count: 10
-    - type: grid
-      grid: 1x1h(keep=all) | 24x1h | 14x1d
-      regex: "zrepl_.*"
-    keep_receiver:
-    - type: grid
-      grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-      regex: "zrepl_.*"

config/samples/pull_ssh.yml

@@ -1,28 +0,0 @@
-jobs:
-
-- name: pull_servers
-  type: pull
-  connect:
-    type: ssh+stdinserver
-    host: app-srv.example.com
-    user: root
-    port: 22
-    identity_file: /etc/zrepl/ssh/identity
-    options: # optional, default [], `-o` arguments passed to ssh
-    - "Compression=yes"
-  root_fs: "pool2/backup_servers"
-  interval: 10m
-  pruning:
-    keep_sender:
-    - type: not_replicated
-    - type: last_n
-      count: 10
-    - type: grid
-      grid: 1x1h(keep=all) | 24x1h | 14x1d
-      regex: "^zrepl_.*"
-    keep_receiver:
-    - type: regex
-      regex: keep_
-    - type: grid
-      grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-      regex: "^zrepl_.*"

config/samples/push.yml

@@ -1,26 +0,0 @@
-jobs:
-  - type: push
-    name: "push"
-    filesystems: {
-      "<": true,
-      "tmp": false
-    }
-    connect:
-      type: tcp
-      address: "backup-server.foo.bar:8888"
-    snapshotting:
-      type: manual
-    send:
-      encrypted: false
-    pruning:
-      keep_sender:
-        - type: not_replicated
-        - type: last_n
-          count: 10
-        - type: grid
-          grid: 1x1h(keep=all) | 24x1h | 14x1d
-          regex: "^zrepl_.*"
-      keep_receiver:
-        - type: grid
-          grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-          regex: "^zrepl_.*"

config/samples/quickstart_backup_to_external_disk.yml

@@ -1,88 +0,0 @@
-# This config serves as an example for a local zrepl installation that
-# backups the entire zpool `system` to `backuppool/zrepl/sink`
-#
-# The requirements covered by this setup are described in the zrepl documentation's
-# quick start section which inlines this example.
-#
-# CUSTOMIZATIONS YOU WILL LIKELY WANT TO APPLY:
-# - adjust the name of the production pool `system` in the `filesystems` filter of jobs `snapjob` and `push_to_drive`
-# - adjust the name of the backup pool `backuppool` in the `backuppool_sink` job
-# - adjust the occurences of `myhostname` to the name of the system you are backing up (cannot be easily changed once you start replicating)
-# - make sure the `zrepl_` prefix is not being used by any other zfs tools you might have installed (it likely isn't)
-
-jobs:
-
-# this job takes care of snapshot creation + pruning
-- name: snapjob
-  type: snap
-  filesystems: {
-      "system<": true,
-  }
-  # create snapshots with prefix `zrepl_` every 15 minutes
-  snapshotting:
-    type: periodic
-    interval: 15m
-    prefix: zrepl_
-  pruning:
-    keep:
-    # fade-out scheme for snapshots starting with `zrepl_`
-    # - keep all created in the last hour
-    # - then destroy snapshots such that we keep 24 each 1 hour apart
-    # - then destroy snapshots such that we keep 14 each 1 day apart
-    # - then destroy all older snapshots
-    - type: grid
-      grid: 1x1h(keep=all) | 24x1h | 14x1d
-      regex: "^zrepl_.*"
-    # keep all snapshots that don't have the `zrepl_` prefix
-    - type: regex
-      negate: true
-      regex: "^zrepl_.*"
-
-# This job pushes to the local sink defined in job `backuppool_sink`.
-# We trigger replication manually from the command line / udev rules using
-#  `zrepl signal wakeup push_to_drive`
-- type: push
-  name: push_to_drive
-  connect:
-    type: local
-    listener_name: backuppool_sink
-    client_identity: myhostname
-  filesystems: {
-      "system<": true
-  }
-  send:
-    encrypted: true
-  replication:
-    protection:
-      initial: guarantee_resumability
-      # Downgrade protection to guarantee_incremental which uses zfs bookmarks instead of zfs holds.
-      # Thus, when we yank out the backup drive during replication
-      # - we might not be able to resume the interrupted replication step because the partially received `to` snapshot of a `from`->`to` step may be pruned any time
-      # - but in exchange we get back the disk space allocated by `to` when we prune it
-      # - and because we still have the bookmarks created by `guarantee_incremental`, we can still do incremental replication of `from`->`to2` in the future
-      incremental: guarantee_incremental
-  snapshotting:
-    type: manual
-  pruning:
-    # no-op prune rule on sender (keep all snapshots), job `snapshot` takes care of this
-    keep_sender:
-    - type: regex
-      regex: ".*"
-    # retain 
-    keep_receiver:
-    # longer retention on the backup drive, we have more space there
-    - type: grid
-      grid: 1x1h(keep=all) | 24x1h | 360x1d
-      regex: "^zrepl_.*"
-    # retain all non-zrepl snapshots on the backup drive
-    - type: regex
-      negate: true
-      regex: "^zrepl_.*"
-
-# This job receives from job `push_to_drive` into `backuppool/zrepl/sink/myhostname`
-- type: sink
-  name: backuppool_sink
-  root_fs: "backuppool/zrepl/sink"
-  serve:
-    type: local
-    listener_name: backuppool_sink

config/samples/quickstart_continuous_server_backup_receiver.yml

@@ -1,12 +0,0 @@
-jobs:
-- name: sink
-  type: sink
-  serve:
-      type: tls
-      listen: ":8888"
-      ca: "/etc/zrepl/prod.crt"
-      cert: "/etc/zrepl/backups.crt"
-      key: "/etc/zrepl/backups.key"
-      client_cns:
-        - "prod"
-  root_fs: "storage/zrepl/sink"

config/samples/quickstart_continuous_server_backup_sender.yml

@@ -1,28 +0,0 @@
-jobs:
-- name: prod_to_backups
-  type: push
-  connect:
-    type: tls
-    address: "backups.example.com:8888"
-    ca: /etc/zrepl/backups.crt
-    cert: /etc/zrepl/prod.crt
-    key:  /etc/zrepl/prod.key
-    server_cn: "backups"
-  filesystems: {
-    "zroot<": true,
-    "zroot/var/tmp<": false,
-    "zroot/usr/home/paranoid": false
-  }
-  snapshotting:
-    type: periodic
-    prefix: zrepl_
-    interval: 10m
-  pruning:
-    keep_sender:
-    - type: not_replicated
-    - type: last_n
-      count: 10
-    keep_receiver:
-    - type: grid
-      grid: 1x1h(keep=all) | 24x1h | 30x1d | 6x30d
-      regex: "^zrepl_"

config/samples/quickstart_fan_out_replication_source.yml

@@ -1,59 +0,0 @@
-jobs:
-# Separate job for snapshots and pruning
-- name: snapshots
-  type: snap
-  filesystems:
-    'tank<': true # all filesystems
-  snapshotting:
-    type: periodic
-    prefix: zrepl_
-    interval: 10m
-  pruning:
-    keep:
-      # Keep non-zrepl snapshots
-      - type: regex
-        negate: true
-        regex: '^zrepl_'
-      # Time-based snapshot retention
-      - type: grid
-        grid: 1x1h(keep=all) | 24x1h | 30x1d | 12x30d
-        regex: '^zrepl_'
-
-# Source job for target B
-- name: target_b
-  type: source
-  serve:
-    type: tls
-    listen: :8888
-    ca: /etc/zrepl/b.example.com.crt
-    cert: /etc/zrepl/a.example.com.crt
-    key: /etc/zrepl/a.example.com.key
-    client_cns:
-      - b.example.com
-  filesystems:
-    'tank<': true # all filesystems
-  # Snapshots are handled by the separate snap job
-  snapshotting:
-    type: manual
-
-# Source job for target C
-- name: target_c
-  type: source
-  serve:
-    type: tls
-    listen: :8889
-    ca: /etc/zrepl/c.example.com.crt
-    cert: /etc/zrepl/a.example.com.crt
-    key: /etc/zrepl/a.example.com.key
-    client_cns:
-      - c.example.com
-  filesystems:
-    'tank<': true # all filesystems
-  # Snapshots are handled by the separate snap job
-  snapshotting:
-    type: manual
-
-# Source jobs for remaining targets. Each one should listen on a different port
-# and reference the correct certificate and client CN.
-# - name: target_c
-#   ...

config/samples/quickstart_fan_out_replication_target.yml

@@ -1,30 +0,0 @@
-jobs:
-# Pull from source server A
-- name: source_a
-  type: pull
-  connect:
-    type: tls
-    # Use the correct port for this specific client (eg. B is 8888, C is 8889, etc.)
-    address: a.example.com:8888
-    ca: /etc/zrepl/a.example.com.crt
-    # Use the correct key pair for this specific client
-    cert: /etc/zrepl/b.example.com.crt
-    key: /etc/zrepl/b.example.com.key
-    server_cn: a.example.com
-  root_fs: pool0/backup
-  interval: 10m
-  pruning:
-    keep_sender:
-      # Source does the pruning in its snap job
-      - type: regex
-        regex: '.*'
-    # Receiver-side pruning can be configured as desired on each target server
-    keep_receiver:
-      # Keep non-zrepl snapshots
-      - type: regex
-        negate: true
-        regex: '^zrepl_'
-      # Time-based snapshot retention
-      - type: grid
-        grid: 1x1h(keep=all) | 24x1h | 30x1d | 12x30d
-        regex: '^zrepl_'

config/samples/sink.yml

@@ -1,13 +0,0 @@
-jobs:
-  - type: sink
-    name: "laptop_sink"
-    root_fs: "pool2/backup_laptops"
-    serve:
-      type: tls
-      listen: "192.168.122.189:8888"
-      ca: "ca.pem"
-      cert: "cert.pem"
-      key: "key.pem"
-      client_cns:
-        - "laptop1"
-        - "homeserver"

config/samples/snap.yml

@@ -1,14 +0,0 @@
-jobs:
-- name: snapjob
-  type: snap
-  filesystems: {
-    "tank/frequently_changed<": true,
-  }
-  snapshotting:
-    type: periodic
-    interval: 2m
-    prefix: zrepl_snapjob_
-  pruning:
-    keep:
-      - type: last_n
-        count: 60

config/samples/snap_cron.yml

@@ -1,14 +0,0 @@
-jobs:
-- name: snapjob
-  type: snap
-  filesystems: {
-    "tank<": true,
-  }
-  snapshotting:
-    type: cron
-    prefix: zrepl_snapjob_
-    cron: "*/5 * * * *"
-  pruning:
-    keep:
-      - type: last_n
-        count: 60

config/samples/source.yml

@@ -1,21 +0,0 @@
-jobs:
-- name: pull_source
-  type: source
-  serve:
-    type: tcp
-    listen: "0.0.0.0:8888"
-    clients: {
-      "192.168.122.123" :               "mysql01",
-      "192.168.122.42" :                "mx01",
-      "2001:0db8:85a3::8a2e:0370:7334": "gateway",
-      "10.23.42.0/24":       "cluster-*",
-      "fde4:8dba:82e1::/64": "san-*",
-    }
-  filesystems: {
-    "<": true,
-    "secret": false
-  }
-  snapshotting:
-    type: periodic
-    interval: 10m
-    prefix: zrepl_

config/samples/source_ssh.yml

@@ -1,17 +0,0 @@
-jobs:
-- name: pull_source
-  type: source
-  serve:
-    type: stdinserver
-    client_identities:
-      - "client1"
-      - "client2"
-  filesystems: {
-    "<": true,
-    "secret": false
-  }
-  snapshotting:
-    type: periodic
-    interval: 10m
-    prefix: zrepl_
-

daemon/control.go

@@ -1,293 +0,0 @@
-package daemon
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net"
-	"net/http"
-	"os"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/daemon/nethelpers"
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/util/envconst"
-	"github.com/zrepl/zrepl/version"
-	"github.com/zrepl/zrepl/zfs"
-	"github.com/zrepl/zrepl/zfs/zfscmd"
-)
-
-type controlJob struct {
-	sockaddr *net.UnixAddr
-	jobs     *jobs
-}
-
-func newControlJob(sockpath string, jobs *jobs) (j *controlJob, err error) {
-	j = &controlJob{jobs: jobs}
-
-	j.sockaddr, err = net.ResolveUnixAddr("unix", sockpath)
-	if err != nil {
-		err = errors.Wrap(err, "cannot resolve unix address")
-		return
-	}
-
-	return
-}
-
-func (j *controlJob) Name() string { return jobNameControl }
-
-func (j *controlJob) Status() *job.Status { return &job.Status{Type: job.TypeInternal} }
-
-func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
-
-func (j *controlJob) SenderConfig() *endpoint.SenderConfig { return nil }
-
-var promControl struct {
-	requestBegin    *prometheus.CounterVec
-	requestFinished *prometheus.HistogramVec
-}
-
-func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) {
-	promControl.requestBegin = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "zrepl",
-		Subsystem: "control",
-		Name:      "request_begin",
-		Help:      "number of request we started to handle",
-	}, []string{"endpoint"})
-
-	promControl.requestFinished = prometheus.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace: "zrepl",
-		Subsystem: "control",
-		Name:      "request_finished",
-		Help:      "time it took a request to finish",
-		Buckets:   []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3, 10e-3, 100e-3, 200e-3, 400e-3, 800e-3, 1, 10, 20},
-	}, []string{"endpoint"})
-	registerer.MustRegister(promControl.requestBegin)
-	registerer.MustRegister(promControl.requestFinished)
-}
-
-const (
-	ControlJobEndpointPProf   string = "/debug/pprof"
-	ControlJobEndpointVersion string = "/version"
-	ControlJobEndpointStatus  string = "/status"
-	ControlJobEndpointSignal  string = "/signal"
-)
-
-func (j *controlJob) Run(ctx context.Context) {
-
-	log := job.GetLogger(ctx)
-	defer log.Info("control job finished")
-
-	l, err := nethelpers.ListenUnixPrivate(j.sockaddr)
-	if err != nil {
-		log.WithError(err).Error("error listening")
-		return
-	}
-
-	pprofServer := NewPProfServer(ctx)
-	if listen := envconst.String("ZREPL_DAEMON_AUTOSTART_PPROF_SERVER", ""); listen != "" {
-		pprofServer.Control(PprofServerControlMsg{
-			Run:               true,
-			HttpListenAddress: listen,
-		})
-	}
-
-	mux := http.NewServeMux()
-	mux.Handle(ControlJobEndpointPProf,
-		requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) {
-			var msg PprofServerControlMsg
-			err := decoder(&msg)
-			if err != nil {
-				return nil, errors.Errorf("decode failed")
-			}
-			pprofServer.Control(msg)
-			return struct{}{}, nil
-		}}})
-
-	mux.Handle(ControlJobEndpointVersion,
-		requestLogger{log: log, handler: jsonResponder{log, func() (interface{}, error) {
-			return version.NewZreplVersionInformation(), nil
-		}}})
-
-	mux.Handle(ControlJobEndpointStatus,
-		// don't log requests to status endpoint, too spammy
-		jsonResponder{log, func() (interface{}, error) {
-			jobs := j.jobs.status()
-			globalZFS := zfscmd.GetReport()
-			envconstReport := envconst.GetReport()
-			s := Status{
-				Jobs: jobs,
-				Global: GlobalStatus{
-					ZFSCmds:   globalZFS,
-					Envconst:  envconstReport,
-					OsEnviron: os.Environ(),
-				}}
-			return s, nil
-		}})
-
-	mux.Handle(ControlJobEndpointSignal,
-		requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) {
-			type reqT struct {
-				Name string
-				Op   string
-			}
-			var req reqT
-			if decoder(&req) != nil {
-				return nil, errors.Errorf("decode failed")
-			}
-
-			var err error
-			switch req.Op {
-			case "wakeup":
-				err = j.jobs.wakeup(req.Name)
-			case "reset":
-				err = j.jobs.reset(req.Name)
-			default:
-				err = fmt.Errorf("operation %q is invalid", req.Op)
-			}
-
-			return struct{}{}, err
-		}}})
-	server := http.Server{
-		Handler: mux,
-		// control socket is local, 1s timeout should be more than sufficient, even on a loaded system
-		WriteTimeout: envconst.Duration("ZREPL_DAEMON_CONTROL_SERVER_WRITE_TIMEOUT", 1*time.Second),
-		ReadTimeout:  envconst.Duration("ZREPL_DAEMON_CONTROL_SERVER_READ_TIMEOUT", 1*time.Second),
-	}
-
-outer:
-	for {
-
-		served := make(chan error)
-		go func() {
-			served <- server.Serve(l)
-			close(served)
-		}()
-
-		select {
-		case <-ctx.Done():
-			log.WithError(ctx.Err()).Info("context done")
-			err := server.Shutdown(context.Background())
-			if err != nil {
-				log.WithError(err).Error("cannot shutdown server")
-			}
-			break outer
-		case err = <-served:
-			if err != nil {
-				log.WithError(err).Error("error serving")
-				break outer
-			}
-		}
-
-	}
-
-}
-
-type jsonResponder struct {
-	log      Logger
-	producer func() (interface{}, error)
-}
-
-func (j jsonResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	logIoErr := func(err error) {
-		if err != nil {
-			j.log.WithError(err).Error("control handler io error")
-		}
-	}
-	res, err := j.producer()
-	if err != nil {
-		j.log.WithError(err).Error("control handler error")
-		w.WriteHeader(http.StatusInternalServerError)
-		_, err = io.WriteString(w, err.Error())
-		logIoErr(err)
-		return
-	}
-	var buf bytes.Buffer
-	err = json.NewEncoder(&buf).Encode(res)
-	if err != nil {
-		j.log.WithError(err).Error("control handler json marshal error")
-		w.WriteHeader(http.StatusInternalServerError)
-		_, err = io.WriteString(w, err.Error())
-	} else {
-		_, err = io.Copy(w, &buf)
-	}
-	logIoErr(err)
-}
-
-type jsonDecoder = func(interface{}) error
-
-type jsonRequestResponder struct {
-	log      Logger
-	producer func(decoder jsonDecoder) (interface{}, error)
-}
-
-func (j jsonRequestResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	logIoErr := func(err error) {
-		if err != nil {
-			j.log.WithError(err).Error("control handler io error")
-		}
-	}
-
-	var decodeError error
-	decoder := func(i interface{}) error {
-		err := json.NewDecoder(r.Body).Decode(&i)
-		decodeError = err
-		return err
-	}
-	res, producerErr := j.producer(decoder)
-
-	//If we had a decode error ignore output of producer and return error
-	if decodeError != nil {
-		w.WriteHeader(http.StatusBadRequest)
-		_, err := io.WriteString(w, decodeError.Error())
-		logIoErr(err)
-		return
-	}
-	if producerErr != nil {
-		j.log.WithError(producerErr).Error("control handler error")
-		w.WriteHeader(http.StatusInternalServerError)
-		_, err := io.WriteString(w, producerErr.Error())
-		logIoErr(err)
-		return
-	}
-
-	var buf bytes.Buffer
-	encodeErr := json.NewEncoder(&buf).Encode(res)
-	if encodeErr != nil {
-		j.log.WithError(producerErr).Error("control handler json marshal error")
-		w.WriteHeader(http.StatusInternalServerError)
-		_, err := io.WriteString(w, encodeErr.Error())
-		logIoErr(err)
-	} else {
-		_, err := io.Copy(w, &buf)
-		logIoErr(err)
-	}
-}
-
-type requestLogger struct {
-	log         logger.Logger
-	handler     http.Handler
-	handlerFunc http.HandlerFunc
-}
-
-func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	log := l.log.WithField("method", r.Method).WithField("url", r.URL)
-	log.Debug("start")
-	promControl.requestBegin.WithLabelValues(r.URL.Path).Inc()
-	defer prometheus.NewTimer(promControl.requestFinished.WithLabelValues(r.URL.Path)).ObserveDuration()
-	if l.handlerFunc != nil {
-		l.handlerFunc(w, r)
-	} else if l.handler != nil {
-		l.handler.ServeHTTP(w, r)
-	} else {
-		log.Error("no handler or handlerFunc configured")
-	}
-	log.Debug("finish")
-}

daemon/daemon.go

@@ -1,258 +0,0 @@
-package daemon
-
-import (
-	"context"
-	"fmt"
-	"math/rand"
-	"os"
-	"os/signal"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/zrepl/zrepl/daemon/logging/trace"
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/util/envconst"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/daemon/job/reset"
-	"github.com/zrepl/zrepl/daemon/job/wakeup"
-	"github.com/zrepl/zrepl/daemon/logging"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/version"
-	"github.com/zrepl/zrepl/zfs/zfscmd"
-)
-
-func Run(ctx context.Context, conf *config.Config) error {
-	ctx, cancel := context.WithCancel(ctx)
-
-	defer cancel()
-	sigChan := make(chan os.Signal, 1)
-	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
-	go func() {
-		<-sigChan
-		cancel()
-	}()
-
-	// The math/rand package is used presently for generating trace IDs, we
-	// seed it with the current time and pid so that the IDs are mostly
-	// unique.
-	rand.Seed(time.Now().UnixNano())
-	rand.Seed(int64(os.Getpid()))
-
-	outlets, err := logging.OutletsFromConfig(*conf.Global.Logging)
-	if err != nil {
-		return errors.Wrap(err, "cannot build logging from config")
-	}
-	outlets.Add(newPrometheusLogOutlet(), logger.Debug)
-
-	confJobs, err := job.JobsFromConfig(conf, config.ParseFlagsNone)
-	if err != nil {
-		return errors.Wrap(err, "cannot build jobs from config")
-	}
-
-	log := logger.NewLogger(outlets, 1*time.Second)
-	log.Info(version.NewZreplVersionInformation().String())
-
-	ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
-	trace.RegisterCallback(trace.Callback{
-		OnBegin: func(ctx context.Context) { logging.GetLogger(ctx, logging.SubsysTraceData).Debug("begin span") },
-		OnEnd: func(ctx context.Context, spanInfo trace.SpanInfo) {
-			logging.
-				GetLogger(ctx, logging.SubsysTraceData).
-				WithField("duration_s", spanInfo.EndedAt().Sub(spanInfo.StartedAt()).Seconds()).
-				Debug("finished span " + spanInfo.TaskAndSpanStack(trace.SpanStackKindAnnotation))
-		},
-	})
-
-	for _, job := range confJobs {
-		if IsInternalJobName(job.Name()) {
-			panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME
-		}
-	}
-
-	jobs := newJobs()
-
-	// start control socket
-	controlJob, err := newControlJob(conf.Global.Control.SockPath, jobs)
-	if err != nil {
-		panic(err) // FIXME
-	}
-	jobs.start(ctx, controlJob, true)
-
-	for i, jc := range conf.Global.Monitoring {
-		var (
-			job job.Job
-			err error
-		)
-		switch v := jc.Ret.(type) {
-		case *config.PrometheusMonitoring:
-			job, err = newPrometheusJobFromConfig(v)
-		default:
-			return errors.Errorf("unknown monitoring job #%d (type %T)", i, v)
-		}
-		if err != nil {
-			return errors.Wrapf(err, "cannot build monitoring job #%d", i)
-		}
-		jobs.start(ctx, job, true)
-	}
-
-	// register global (=non job-local) metrics
-	version.PrometheusRegister(prometheus.DefaultRegisterer)
-	zfscmd.RegisterMetrics(prometheus.DefaultRegisterer)
-	trace.RegisterMetrics(prometheus.DefaultRegisterer)
-	endpoint.RegisterMetrics(prometheus.DefaultRegisterer)
-
-	log.Info("starting daemon")
-
-	// start regular jobs
-	for _, j := range confJobs {
-		jobs.start(ctx, j, false)
-	}
-
-	select {
-	case <-jobs.wait():
-		log.Info("all jobs finished")
-	case <-ctx.Done():
-		log.WithError(ctx.Err()).Info("context finished")
-	}
-	log.Info("waiting for jobs to finish")
-	<-jobs.wait()
-	log.Info("daemon exiting")
-	return nil
-}
-
-type jobs struct {
-	wg sync.WaitGroup
-
-	// m protects all fields below it
-	m       sync.RWMutex
-	wakeups map[string]wakeup.Func // by Job.Name
-	resets  map[string]reset.Func  // by Job.Name
-	jobs    map[string]job.Job
-}
-
-func newJobs() *jobs {
-	return &jobs{
-		wakeups: make(map[string]wakeup.Func),
-		resets:  make(map[string]reset.Func),
-		jobs:    make(map[string]job.Job),
-	}
-}
-
-func (s *jobs) wait() <-chan struct{} {
-	ch := make(chan struct{})
-	go func() {
-		s.wg.Wait()
-		close(ch)
-	}()
-	return ch
-}
-
-type Status struct {
-	Jobs   map[string]*job.Status
-	Global GlobalStatus
-}
-
-type GlobalStatus struct {
-	ZFSCmds   *zfscmd.Report
-	Envconst  *envconst.Report
-	OsEnviron []string
-}
-
-func (s *jobs) status() map[string]*job.Status {
-	s.m.RLock()
-	defer s.m.RUnlock()
-
-	type res struct {
-		name   string
-		status *job.Status
-	}
-	var wg sync.WaitGroup
-	c := make(chan res, len(s.jobs))
-	for name, j := range s.jobs {
-		wg.Add(1)
-		go func(name string, j job.Job) {
-			defer wg.Done()
-			c <- res{name: name, status: j.Status()}
-		}(name, j)
-	}
-	wg.Wait()
-	close(c)
-	ret := make(map[string]*job.Status, len(s.jobs))
-	for res := range c {
-		ret[res.name] = res.status
-	}
-	return ret
-}
-
-func (s *jobs) wakeup(job string) error {
-	s.m.RLock()
-	defer s.m.RUnlock()
-
-	wu, ok := s.wakeups[job]
-	if !ok {
-		return errors.Errorf("Job %s does not exist", job)
-	}
-	return wu()
-}
-
-func (s *jobs) reset(job string) error {
-	s.m.RLock()
-	defer s.m.RUnlock()
-
-	wu, ok := s.resets[job]
-	if !ok {
-		return errors.Errorf("Job %s does not exist", job)
-	}
-	return wu()
-}
-
-const (
-	jobNamePrometheus = "_prometheus"
-	jobNameControl    = "_control"
-)
-
-func IsInternalJobName(s string) bool {
-	return strings.HasPrefix(s, "_")
-}
-
-func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
-	s.m.Lock()
-	defer s.m.Unlock()
-
-	ctx = logging.WithInjectedField(ctx, logging.JobField, j.Name())
-
-	jobName := j.Name()
-	if !internal && IsInternalJobName(jobName) {
-		panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName))
-	}
-	if internal && !IsInternalJobName(jobName) {
-		panic(fmt.Sprintf("internal job does not use internal job name %s", jobName))
-	}
-	if _, ok := s.jobs[jobName]; ok {
-		panic(fmt.Sprintf("duplicate job name %s", jobName))
-	}
-
-	j.RegisterMetrics(prometheus.DefaultRegisterer)
-
-	s.jobs[jobName] = j
-	ctx = zfscmd.WithJobID(ctx, j.Name())
-	ctx, wakeup := wakeup.Context(ctx)
-	ctx, resetFunc := reset.Context(ctx)
-	s.wakeups[jobName] = wakeup
-	s.resets[jobName] = resetFunc
-
-	s.wg.Add(1)
-	go func() {
-		defer s.wg.Done()
-		job.GetLogger(ctx).Info("starting job")
-		defer job.GetLogger(ctx).Info("job exited")
-		j.Run(ctx)
-	}()
-}

daemon/filters/fsmapfilter.go

@@ -1,275 +0,0 @@
-package filters
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/endpoint"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type DatasetMapFilter struct {
-	entries []datasetMapFilterEntry
-
-	// if set, only valid filter entries can be added using Add()
-	// and Map() will always return an error
-	filterMode bool
-}
-
-type datasetMapFilterEntry struct {
-	path *zfs.DatasetPath
-	// the mapping. since this datastructure acts as both mapping and filter
-	// we have to convert it to the desired rep dynamically
-	mapping      string
-	subtreeMatch bool
-}
-
-func NewDatasetMapFilter(capacity int, filterMode bool) *DatasetMapFilter {
-	return &DatasetMapFilter{
-		entries:    make([]datasetMapFilterEntry, 0, capacity),
-		filterMode: filterMode,
-	}
-}
-
-func (m *DatasetMapFilter) Add(pathPattern, mapping string) (err error) {
-
-	if m.filterMode {
-		if _, err = m.parseDatasetFilterResult(mapping); err != nil {
-			return
-		}
-	}
-
-	// assert path glob adheres to spec
-	const SUBTREE_PATTERN string = "<"
-	patternCount := strings.Count(pathPattern, SUBTREE_PATTERN)
-	switch {
-	case patternCount > 1:
-	case patternCount == 1 && !strings.HasSuffix(pathPattern, SUBTREE_PATTERN):
-		err = fmt.Errorf("pattern invalid: only one '<' at end of string allowed")
-		return
-	}
-
-	pathStr := strings.TrimSuffix(pathPattern, SUBTREE_PATTERN)
-	path, err := zfs.NewDatasetPath(pathStr)
-	if err != nil {
-		return fmt.Errorf("pattern is not a dataset path: %s", err)
-	}
-
-	entry := datasetMapFilterEntry{
-		path:         path,
-		mapping:      mapping,
-		subtreeMatch: patternCount > 0,
-	}
-	m.entries = append(m.entries, entry)
-	return
-
-}
-
-// find the most specific prefix mapping we have
-//
-// longer prefix wins over shorter prefix, direct wins over glob
-func (m DatasetMapFilter) mostSpecificPrefixMapping(path *zfs.DatasetPath) (idx int, found bool) {
-	lcp, lcp_entry_idx := -1, -1
-	direct_idx := -1
-	for e := range m.entries {
-		entry := m.entries[e]
-		ep := m.entries[e].path
-		lep := ep.Length()
-
-		switch {
-		case !entry.subtreeMatch && ep.Equal(path):
-			direct_idx = e
-			continue
-		case entry.subtreeMatch && path.HasPrefix(ep) && lep > lcp:
-			lcp = lep
-			lcp_entry_idx = e
-		default:
-			continue
-		}
-	}
-
-	if lcp_entry_idx >= 0 || direct_idx >= 0 {
-		found = true
-		switch {
-		case direct_idx >= 0:
-			idx = direct_idx
-		case lcp_entry_idx >= 0:
-			idx = lcp_entry_idx
-		}
-	}
-	return
-}
-
-// Returns target == nil if there is no mapping
-func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error) {
-
-	if m.filterMode {
-		err = fmt.Errorf("using a filter for mapping simply does not work")
-		return
-	}
-
-	mi, hasMapping := m.mostSpecificPrefixMapping(source)
-	if !hasMapping {
-		return nil, nil
-	}
-	me := m.entries[mi]
-
-	if me.mapping == "" {
-		// Special case treatment: 'foo/bar<' => ''
-		if !me.subtreeMatch {
-			return nil, fmt.Errorf("mapping to '' must be a subtree match")
-		}
-		// ok...
-	} else {
-		if strings.HasPrefix("!", me.mapping) {
-			// reject mapping
-			return nil, nil
-		}
-	}
-
-	target, err = zfs.NewDatasetPath(me.mapping)
-	if err != nil {
-		err = fmt.Errorf("mapping target is not a dataset path: %s", err)
-		return
-	}
-	if me.subtreeMatch {
-		// strip common prefix ('<' wildcards are no special case here)
-		extendComps := source.Copy()
-		extendComps.TrimPrefix(me.path)
-		target.Extend(extendComps)
-	}
-	return
-}
-
-func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
-
-	if !m.filterMode {
-		err = fmt.Errorf("using a mapping as a filter does not work")
-		return
-	}
-
-	mi, hasMapping := m.mostSpecificPrefixMapping(p)
-	if !hasMapping {
-		pass = false
-		return
-	}
-	me := m.entries[mi]
-	pass, err = m.parseDatasetFilterResult(me.mapping)
-	return
-}
-
-// Construct a new filter-only DatasetMapFilter from a mapping
-// The new filter allows exactly those paths that were not forbidden by the mapping.
-func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
-
-	if m.filterMode {
-		err = errors.Errorf("can only invert mappings")
-		return
-	}
-
-	inv = &DatasetMapFilter{
-		make([]datasetMapFilterEntry, len(m.entries)),
-		true,
-	}
-
-	for i, e := range m.entries {
-		inv.entries[i].path, err = zfs.NewDatasetPath(e.mapping)
-		if err != nil {
-			err = errors.Wrapf(err, "mapping cannot be inverted: '%s' is not a dataset path", e.mapping)
-			return
-		}
-		inv.entries[i].mapping = MapFilterResultOk
-		inv.entries[i].subtreeMatch = e.subtreeMatch
-	}
-
-	return inv, nil
-}
-
-// FIXME investigate whether we can support more...
-func (m DatasetMapFilter) Invert() (endpoint.FSMap, error) {
-
-	if m.filterMode {
-		return nil, errors.Errorf("can only invert mappings")
-	}
-
-	if len(m.entries) != 1 {
-		return nil, errors.Errorf("inversion of complicated mappings is not implemented") // FIXME
-	}
-
-	e := m.entries[0]
-
-	inv := &DatasetMapFilter{
-		make([]datasetMapFilterEntry, len(m.entries)),
-		false,
-	}
-	mp, err := zfs.NewDatasetPath(e.mapping)
-	if err != nil {
-		return nil, err
-	}
-
-	inv.entries[0] = datasetMapFilterEntry{
-		path:         mp,
-		mapping:      e.path.ToString(),
-		subtreeMatch: e.subtreeMatch,
-	}
-
-	return inv, nil
-}
-
-// Creates a new DatasetMapFilter in filter mode from a mapping
-// All accepting mapping results are mapped to accepting filter results
-// All rejecting mapping results are mapped to rejecting filter results
-func (m DatasetMapFilter) AsFilter() endpoint.FSFilter {
-
-	f := &DatasetMapFilter{
-		make([]datasetMapFilterEntry, len(m.entries)),
-		true,
-	}
-
-	for i, e := range m.entries {
-		var newe datasetMapFilterEntry = e
-		if strings.HasPrefix(newe.mapping, "!") {
-			newe.mapping = MapFilterResultOmit
-		} else {
-			newe.mapping = MapFilterResultOk
-		}
-		f.entries[i] = newe
-	}
-
-	return f
-}
-
-const (
-	MapFilterResultOk   string = "ok"
-	MapFilterResultOmit string = "!"
-)
-
-// Parse a dataset filter result
-func (m DatasetMapFilter) parseDatasetFilterResult(result string) (pass bool, err error) {
-	l := strings.ToLower(result)
-	if l == MapFilterResultOk {
-		return true, nil
-	}
-	if l == MapFilterResultOmit {
-		return false, nil
-	}
-	return false, fmt.Errorf("'%s' is not a valid filter result", result)
-}
-
-func DatasetMapFilterFromConfig(in map[string]bool) (f *DatasetMapFilter, err error) {
-
-	f = NewDatasetMapFilter(len(in), true)
-	for pathPattern, accept := range in {
-		mapping := MapFilterResultOmit
-		if accept {
-			mapping = MapFilterResultOk
-		}
-		if err = f.Add(pathPattern, mapping); err != nil {
-			err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err)
-			return
-		}
-	}
-	return
-}

daemon/filters/fsmapfilter_test.go

@@ -1,89 +0,0 @@
-package filters
-
-import (
-	"testing"
-
-	"github.com/zrepl/zrepl/zfs"
-)
-
-func TestDatasetMapFilter(t *testing.T) {
-
-	type testCase struct {
-		name      string
-		filter    map[string]string
-		checkPass map[string]bool // each entry is checked to match the filter's `pass` return value
-	}
-
-	tcs := []testCase{
-		{
-			"default_no_match",
-			map[string]string{},
-			map[string]bool{
-				"":      false,
-				"foo":   false,
-				"zroot": false,
-			},
-		},
-		{
-			"more_specific_path_has_precedence",
-			map[string]string{
-				"tank<":         "ok",
-				"tank/tmp<":     "!",
-				"tank/home/x<":  "!",
-				"tank/home/x/1": "ok",
-			},
-			map[string]bool{
-				"zroot":         false,
-				"tank":          true,
-				"tank/tmp":      false,
-				"tank/tmp/foo":  false,
-				"tank/home/x":   false,
-				"tank/home/y":   true,
-				"tank/home/x/1": true,
-				"tank/home/x/2": false,
-			},
-		},
-		{
-			"precedence_of_specific_over_subtree_wildcard_on_same_path",
-			map[string]string{
-				"tank/home/bob":  "ok",
-				"tank/home/bob<": "!",
-			},
-			map[string]bool{
-				"tank/home/bob":           true,
-				"tank/home/bob/downloads": false,
-			},
-		},
-	}
-
-	for tc := range tcs {
-		t.Run(tcs[tc].name, func(t *testing.T) {
-			c := tcs[tc]
-			f := NewDatasetMapFilter(len(c.filter), true)
-			for p, a := range c.filter {
-				err := f.Add(p, a)
-				if err != nil {
-					t.Fatalf("incorrect filter spec: %s", err)
-				}
-			}
-			for p, checkPass := range c.checkPass {
-				zp, err := zfs.NewDatasetPath(p)
-				if err != nil {
-					t.Fatalf("incorrect path spec: %s", err)
-				}
-				pass, err := f.Filter(zp)
-				if err != nil {
-					t.Fatalf("unexpected filter error: %s", err)
-				}
-				ok := pass == checkPass
-				failstr := "OK"
-				if !ok {
-					failstr = "FAIL"
-					t.Fail()
-				}
-				t.Logf("%-40q  %5v  (exp=%v act=%v)", p, failstr, checkPass, pass)
-			}
-		})
-	}
-
-}

daemon/hooks/edge_string.go

@@ -1,35 +0,0 @@
-// Code generated by "stringer -type=Edge"; DO NOT EDIT.
-
-package hooks
-
-import "strconv"
-
-func _() {
-	// An "invalid array index" compiler error signifies that the constant values have changed.
-	// Re-run the stringer command to generate them again.
-	var x [1]struct{}
-	_ = x[Pre-1]
-	_ = x[Callback-2]
-	_ = x[Post-4]
-}
-
-const (
-	_Edge_name_0 = "PreCallback"
-	_Edge_name_1 = "Post"
-)
-
-var (
-	_Edge_index_0 = [...]uint8{0, 3, 11}
-)
-
-func (i Edge) String() string {
-	switch {
-	case 1 <= i && i <= 2:
-		i -= 1
-		return _Edge_name_0[_Edge_index_0[i]:_Edge_index_0[i+1]]
-	case i == 4:
-		return _Edge_name_1
-	default:
-		return "Edge(" + strconv.FormatInt(int64(i), 10) + ")"
-	}
-}

daemon/hooks/hook_config.go

@@ -1,52 +0,0 @@
-package hooks
-
-import (
-	"fmt"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type List []Hook
-
-func HookFromConfig(in config.HookEnum) (Hook, error) {
-	switch v := in.Ret.(type) {
-	case *config.HookCommand:
-		return NewCommandHook(v)
-	case *config.HookPostgresCheckpoint:
-		return PgChkptHookFromConfig(v)
-	case *config.HookMySQLLockTables:
-		return MyLockTablesFromConfig(v)
-	default:
-		return nil, fmt.Errorf("unknown hook type %T", v)
-	}
-}
-
-func ListFromConfig(in *config.HookList) (r *List, err error) {
-	hl := make(List, len(*in))
-
-	for i, h := range *in {
-		hl[i], err = HookFromConfig(h)
-		if err != nil {
-			return nil, fmt.Errorf("create hook #%d: %s", i+1, err)
-		}
-	}
-
-	return &hl, nil
-}
-
-func (l List) CopyFilteredForFilesystem(fs *zfs.DatasetPath) (ret List, err error) {
-	ret = make(List, 0, len(l))
-
-	for _, h := range l {
-		var passFilesystem bool
-		if passFilesystem, err = h.Filesystems().Filter(fs); err != nil {
-			return nil, err
-		}
-		if passFilesystem {
-			ret = append(ret, h)
-		}
-	}
-
-	return ret, nil
-}

daemon/hooks/hook_docs.go

@@ -1,33 +0,0 @@
-// Package hooks implements pre- and post snapshot hooks.
-//
-// Plan is a generic executor for ExpectStepReports before and after an activity specified in a callback.
-// It provides a reporting facility that can be polled while the plan is executing to gather progress information.
-//
-// This package also provides all supported hook type implementations and abstractions around them.
-//
-// # Use For Other Kinds Of ExpectStepReports
-//
-// This package REQUIRES REFACTORING before it can be used for other activities than snapshots, e.g. pre- and post-replication:
-//
-// The Hook interface requires a hook to provide a Filesystems() filter, which doesn't make sense for
-// all kinds of activities.
-//
-// The hook implementations should move out of this package.
-// However, there is a lot of tight coupling which to untangle isn't worth it ATM.
-//
-// # How This Package Is Used By Package Snapper
-//
-// Deserialize a config.List using ListFromConfig().
-// Then it MUST filter the list to only contain hooks for a particular filesystem using
-// hooksList.CopyFilteredForFilesystem(fs).
-//
-// Then create a CallbackHook using NewCallbackHookForFilesystem().
-//
-// Pass all of the above to NewPlan() which provides a Report() and Run() method:
-//
-// Plan.Run(ctx context.Context,dryRun bool) executes the plan and take a context as argument that should contain a logger added using hooks.WithLogger()).
-// The value of dryRun is passed through to the hooks' Run() method.
-// Command hooks make it available in the environment variable ZREPL_DRYRUN.
-//
-// Plan.Report() can be called while Plan.Run() is executing to give an overview of plan execution progress (future use in "zrepl status").
-package hooks

daemon/hooks/hook_exec.go

@@ -1,292 +0,0 @@
-package hooks
-
-import (
-	"context"
-	"fmt"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/zrepl/zrepl/zfs"
-)
-
-// Re-export type here so that
-// every file in package hooks doesn't
-// have to import github.com/zrepl/zrepl/zfs
-type Filter zfs.DatasetFilter
-
-type Hook interface {
-	Filesystems() Filter
-
-	// If true and the Pre edge invocation of Run fails, Post edge will not run and other Pre edges will not run.
-	ErrIsFatal() bool
-
-	// Run is invoked by HookPlan for a Pre edge.
-	// If HookReport.HadError() == false, the Post edge will be invoked, too.
-	Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport
-
-	String() string
-}
-
-type Phase string
-
-const (
-	PhaseSnapshot = Phase("snapshot")
-	PhaseTesting  = Phase("testing")
-)
-
-func (p Phase) String() string {
-	return string(p)
-}
-
-//go:generate stringer -type=Edge
-type Edge uint
-
-const (
-	Pre = Edge(1 << iota)
-	Callback
-	Post
-)
-
-func (e Edge) StringForPhase(phase Phase) string {
-	return fmt.Sprintf("%s_%s", e.String(), phase.String())
-}
-
-//go:generate enumer -type=StepStatus -trimprefix=Step
-type StepStatus int
-
-const (
-	StepPending StepStatus = 1 << iota
-	StepExec
-	StepOk
-	StepErr
-	StepSkippedDueToFatalErr
-	StepSkippedDueToPreErr
-)
-
-type HookReport interface {
-	String() string
-	HadError() bool
-	Error() string
-}
-
-type Step struct {
-	Hook       Hook
-	Edge       Edge
-	Status     StepStatus
-	Begin, End time.Time
-	// Report may be nil
-	// FIXME cannot serialize this for client status, but contains interesting info (like what error happened)
-	Report HookReport
-	state  map[interface{}]interface{}
-}
-
-func (s Step) String() (out string) {
-	fatal := "~"
-	if s.Hook.ErrIsFatal() && s.Edge == Pre {
-		fatal = "!"
-	}
-	runTime := "..."
-	if s.Status != StepPending {
-		t := s.End.Sub(s.Begin)
-		runTime = t.Round(time.Millisecond).String()
-	}
-	return fmt.Sprintf("[%s] [%5s] %s [%s]  %s", s.Status, runTime, fatal, s.Edge, s.Hook)
-}
-
-type Plan struct {
-	mtx sync.RWMutex
-
-	steps []*Step
-	pre   []*Step // protected by mtx
-	cb    *Step
-	post  []*Step // not reversed, i.e. entry at index i corresponds to pre-edge in pre[i]
-
-	phase Phase
-	env   Env
-}
-
-func NewPlan(hooks *List, phase Phase, cb *CallbackHook, extra Env) (*Plan, error) {
-
-	var pre, post []*Step
-	// TODO sanity check unique name of hook?
-	for _, hook := range *hooks {
-		state := make(map[interface{}]interface{})
-		preE := &Step{
-			Hook:   hook,
-			Edge:   Pre,
-			Status: StepPending,
-			state:  state,
-		}
-		pre = append(pre, preE)
-		postE := &Step{
-			Hook:   hook,
-			Edge:   Post,
-			Status: StepPending,
-			state:  state,
-		}
-		post = append(post, postE)
-	}
-
-	cbE := &Step{
-		Hook:   cb,
-		Edge:   Callback,
-		Status: StepPending,
-	}
-
-	steps := make([]*Step, 0, len(pre)+len(post)+1)
-	steps = append(steps, pre...)
-	steps = append(steps, cbE)
-	for i := len(post) - 1; i >= 0; i-- {
-		steps = append(steps, post[i])
-	}
-
-	plan := &Plan{
-		phase: phase,
-		env:   extra,
-		steps: steps,
-		pre:   pre,
-		post:  post,
-		cb:    cbE,
-	}
-
-	return plan, nil
-}
-
-type PlanReport []Step
-
-func (p *Plan) Report() PlanReport {
-	p.mtx.RLock()
-	defer p.mtx.RUnlock()
-	rep := make([]Step, len(p.steps))
-	for i := range rep {
-		rep[i] = *p.steps[i]
-	}
-	return rep
-}
-
-func (r PlanReport) HadError() bool {
-	for _, e := range r {
-		if e.Status == StepErr {
-			return true
-		}
-	}
-	return false
-}
-
-func (r PlanReport) HadFatalError() bool {
-	for _, e := range r {
-		if e.Status == StepSkippedDueToFatalErr {
-			return true
-		}
-	}
-	return false
-}
-
-func (r PlanReport) String() string {
-	stepStrings := make([]string, len(r))
-	for i, e := range r {
-		stepStrings[i] = fmt.Sprintf("%02d %s", i+1, e)
-	}
-	return strings.Join(stepStrings, "\n")
-}
-
-func (p *Plan) Run(ctx context.Context, dryRun bool) {
-	p.mtx.RLock()
-	defer p.mtx.RUnlock()
-	w := func(f func()) {
-		p.mtx.RUnlock()
-		defer p.mtx.RLock()
-		p.mtx.Lock()
-		defer p.mtx.Unlock()
-		f()
-	}
-	runHook := func(s *Step, ctx context.Context, edge Edge) HookReport {
-		w(func() { s.Status = StepExec })
-		begin := time.Now()
-		r := s.Hook.Run(ctx, edge, p.phase, dryRun, p.env, s.state)
-		end := time.Now()
-		w(func() {
-			s.Report = r
-			s.Status = StepOk
-			if r.HadError() {
-				s.Status = StepErr
-			}
-			s.Begin, s.End = begin, end
-		})
-		return r
-	}
-
-	l := getLogger(ctx)
-
-	// it's a stack, execute until we reach the end of the list (last item in)
-	// or fail inbetween
-	l.Info("run pre-edges in configuration order")
-	next := 0
-	for ; next < len(p.pre); next++ {
-		e := p.pre[next]
-		l := l.WithField("hook", e.Hook)
-		r := runHook(e, ctx, Pre)
-		if r.HadError() {
-			l.WithError(r).Error("hook invocation failed for pre-edge")
-			if e.Hook.ErrIsFatal() {
-				l.Error("the hook run was aborted due to a fatal error in this hook")
-				break
-			}
-		}
-	}
-
-	hadFatalErr := next != len(p.pre)
-	if hadFatalErr {
-		l.Error("fatal error in a pre-snapshot hook invocation")
-		l.Error("no snapshot will be taken")
-		l.Error("only running post-edges for successful pre-edges")
-		w(func() {
-			p.post[next].Status = StepSkippedDueToFatalErr
-			for i := next + 1; i < len(p.pre); i++ {
-				p.pre[i].Status = StepSkippedDueToFatalErr
-				p.post[i].Status = StepSkippedDueToFatalErr
-			}
-			p.cb.Status = StepSkippedDueToFatalErr
-		})
-		return
-	}
-
-	l.Info("running callback")
-	cbR := runHook(p.cb, ctx, Callback)
-	if cbR.HadError() {
-		l.WithError(cbR).Error("callback failed")
-	}
-
-	l.Info("run post-edges for successful pre-edges in reverse configuration order")
-
-	// the constructor produces pre and post entries
-	// post is NOT reversed
-	next-- // now at index of last executed pre-edge
-	for ; next >= 0; next-- {
-		e := p.post[next]
-		l := l.WithField("hook", e.Hook)
-
-		if p.pre[next].Status != StepOk {
-			if p.pre[next].Status != StepErr {
-				panic(fmt.Sprintf("expecting a pre-edge hook report to be either Ok or Err, got %s", p.pre[next].Status))
-			}
-			l.Info("skip post-edge because pre-edge failed")
-			w(func() {
-				e.Status = StepSkippedDueToPreErr
-			})
-			continue
-		}
-
-		report := runHook(e, ctx, Post)
-
-		if report.HadError() {
-			l.WithError(report).Error("hook invocation failed for post-edge")
-			l.Error("subsequent post-edges run regardless of this post-edge failure")
-		}
-
-		// ErrIsFatal is only relevant for Pre
-
-	}
-
-}

daemon/hooks/hook_logging.go

@@ -1,91 +0,0 @@
-package hooks
-
-import (
-	"bufio"
-	"bytes"
-	"context"
-	"sync"
-
-	"github.com/zrepl/zrepl/daemon/logging"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/util/envconst"
-)
-
-type Logger = logger.Logger
-
-func GetLogger(ctx context.Context) Logger { return getLogger(ctx) }
-
-func getLogger(ctx context.Context) Logger {
-	return logging.GetLogger(ctx, logging.SubsysHooks)
-}
-
-const MAX_HOOK_LOG_SIZE_DEFAULT int = 1 << 20
-
-type logWriter struct {
-	/*
-		Mutex prevents:
-			concurrent writes to buf, scanner in Write([]byte)
-			data race on scanner vs Write([]byte)
-				and concurrent write to buf (call to buf.Reset())
-				in Close()
-
-		(Also, Close() should generally block until any Write() call completes.)
-	*/
-	mtx     *sync.Mutex
-	buf     bytes.Buffer
-	scanner *bufio.Scanner
-	logger  Logger
-	level   logger.Level
-	field   string
-}
-
-func NewLogWriter(mtx *sync.Mutex, logger Logger, level logger.Level, field string) *logWriter {
-	w := new(logWriter)
-	w.mtx = mtx
-	w.scanner = bufio.NewScanner(&w.buf)
-	w.logger = logger
-	w.level = level
-	w.field = field
-	return w
-}
-
-func (w *logWriter) log(line string) {
-	w.logger.WithField(w.field, line).Log(w.level, "hook output")
-}
-
-func (w *logWriter) logUnreadBytes() error {
-	for w.scanner.Scan() {
-		w.log(w.scanner.Text())
-	}
-	if w.buf.Cap() > envconst.Int("ZREPL_MAX_HOOK_LOG_SIZE", MAX_HOOK_LOG_SIZE_DEFAULT) {
-		w.buf.Reset()
-	}
-
-	return nil
-}
-
-func (w *logWriter) Write(in []byte) (int, error) {
-	w.mtx.Lock()
-	defer w.mtx.Unlock()
-
-	n, bufErr := w.buf.Write(in)
-	if bufErr != nil {
-		return n, bufErr
-	}
-
-	err := w.logUnreadBytes()
-	if err != nil {
-		return n, err
-	}
-	// Always reset the scanner for the next Write
-	w.scanner = bufio.NewScanner(&w.buf)
-
-	return n, nil
-}
-
-func (w *logWriter) Close() (err error) {
-	w.mtx.Lock()
-	defer w.mtx.Unlock()
-
-	return w.logUnreadBytes()
-}

daemon/hooks/hook_type_callback.go

@@ -1,65 +0,0 @@
-package hooks
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type HookJobCallback func(ctx context.Context) error
-
-type CallbackHook struct {
-	cb            HookJobCallback
-	filter        Filter
-	displayString string
-}
-
-func NewCallbackHookForFilesystem(displayString string, fs *zfs.DatasetPath, cb HookJobCallback) *CallbackHook {
-	filter, _ := filters.DatasetMapFilterFromConfig(map[string]bool{fs.ToString(): true})
-	return NewCallbackHook(displayString, cb, filter)
-}
-
-func NewCallbackHook(displayString string, cb HookJobCallback, filter Filter) *CallbackHook {
-	return &CallbackHook{
-		cb:            cb,
-		filter:        filter,
-		displayString: displayString,
-	}
-}
-
-func (h *CallbackHook) Filesystems() Filter {
-	return h.filter
-}
-
-func (h *CallbackHook) ErrIsFatal() bool {
-	return false // callback is by definition
-}
-
-func (h *CallbackHook) String() string {
-	return h.displayString
-}
-
-type CallbackHookReport struct {
-	Name string
-	Err  error
-}
-
-func (r *CallbackHookReport) String() string {
-	if r.HadError() {
-		return r.Error()
-	}
-	return r.Name
-}
-
-func (r *CallbackHookReport) HadError() bool { return r.Err != nil }
-
-func (r *CallbackHookReport) Error() string {
-	return fmt.Sprintf("%s error: %s", r.Name, r.Err)
-}
-
-func (h *CallbackHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
-	err := h.cb(ctx)
-	return &CallbackHookReport{h.displayString, err}
-}

daemon/hooks/hook_type_command.go

@@ -1,199 +0,0 @@
-package hooks
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"math"
-	"os"
-	"os/exec"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/util/circlog"
-	"github.com/zrepl/zrepl/util/envconst"
-)
-
-type HookEnvVar string
-
-const (
-	EnvType     HookEnvVar = "ZREPL_HOOKTYPE"
-	EnvDryRun   HookEnvVar = "ZREPL_DRYRUN"
-	EnvFS       HookEnvVar = "ZREPL_FS"
-	EnvSnapshot HookEnvVar = "ZREPL_SNAPNAME"
-	EnvTimeout  HookEnvVar = "ZREPL_TIMEOUT"
-)
-
-type Env map[HookEnvVar]string
-
-func NewHookEnv(edge Edge, phase Phase, dryRun bool, timeout time.Duration, extra Env) Env {
-	r := Env{
-		EnvTimeout: fmt.Sprintf("%.f", math.Floor(timeout.Seconds())),
-	}
-
-	edgeString := edge.StringForPhase(phase)
-	r[EnvType] = strings.ToLower(edgeString)
-
-	var dryRunString string
-	if dryRun {
-		dryRunString = "true"
-	} else {
-		dryRunString = ""
-	}
-	r[EnvDryRun] = dryRunString
-
-	for k, v := range extra {
-		r[k] = v
-	}
-
-	return r
-}
-
-type CommandHook struct {
-	edge       Edge
-	filter     Filter
-	errIsFatal bool
-	command    string
-	timeout    time.Duration
-}
-
-type CommandHookReport struct {
-	Command                      string
-	Args                         []string // currently always empty
-	Env                          Env
-	Err                          error
-	CapturedStdoutStderrCombined []byte
-}
-
-func (r *CommandHookReport) String() string {
-	// Reproduces a POSIX shell-compatible command line
-	var cmdLine strings.Builder
-	sep := ""
-
-	// Make sure environment variables are always
-	// printed in the same order
-	var hookEnvKeys []HookEnvVar
-	for k := range r.Env {
-		hookEnvKeys = append(hookEnvKeys, k)
-	}
-	sort.Slice(hookEnvKeys, func(i, j int) bool { return string(hookEnvKeys[i]) < string(hookEnvKeys[j]) })
-
-	for _, k := range hookEnvKeys {
-		cmdLine.WriteString(fmt.Sprintf("%s%s='%s'", sep, k, r.Env[k]))
-		sep = " "
-	}
-
-	cmdLine.WriteString(fmt.Sprintf("%s%s", sep, r.Command))
-	for _, a := range r.Args {
-		cmdLine.WriteString(fmt.Sprintf("%s'%s'", sep, a))
-	}
-
-	var msg string
-	if r.Err == nil {
-		msg = "command hook"
-	} else {
-		msg = fmt.Sprintf("command hook failed with %q", r.Err)
-	}
-
-	return fmt.Sprintf("%s: \"%s\"", msg, cmdLine.String()) // no %q to make copy-pastable
-}
-func (r *CommandHookReport) Error() string {
-	if r.Err == nil {
-		return ""
-	}
-	return fmt.Sprintf("%s FAILED with error: %s", r.String(), r.Err)
-}
-
-func (r *CommandHookReport) HadError() bool {
-	return r.Err != nil
-}
-
-func NewCommandHook(in *config.HookCommand) (r *CommandHook, err error) {
-	r = &CommandHook{
-		errIsFatal: in.ErrIsFatal,
-		command:    in.Path,
-		timeout:    in.Timeout,
-	}
-
-	r.filter, err = filters.DatasetMapFilterFromConfig(in.Filesystems)
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse filesystem filter: %s", err)
-	}
-
-	r.edge = Pre | Post
-
-	return r, nil
-}
-
-func (h *CommandHook) Filesystems() Filter {
-	return h.filter
-}
-
-func (h *CommandHook) ErrIsFatal() bool {
-	return h.errIsFatal
-}
-
-func (h *CommandHook) String() string {
-	return h.command
-}
-
-func (h *CommandHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
-	l := getLogger(ctx).WithField("command", h.command)
-
-	cmdCtx, cancel := context.WithTimeout(ctx, h.timeout)
-	defer cancel()
-
-	cmdExec := exec.CommandContext(cmdCtx, h.command)
-
-	hookEnv := NewHookEnv(edge, phase, dryRun, h.timeout, extra)
-	cmdEnv := os.Environ()
-	for k, v := range hookEnv {
-		cmdEnv = append(cmdEnv, fmt.Sprintf("%s=%s", k, v))
-	}
-	cmdExec.Env = cmdEnv
-
-	var scanMutex sync.Mutex
-	combinedOutput, err := circlog.NewCircularLog(envconst.Int("ZREPL_MAX_HOOK_LOG_SIZE", MAX_HOOK_LOG_SIZE_DEFAULT))
-	if err != nil {
-		return &CommandHookReport{Err: err}
-	}
-	logErrWriter := NewLogWriter(&scanMutex, l, logger.Warn, "stderr")
-	logOutWriter := NewLogWriter(&scanMutex, l, logger.Info, "stdout")
-	defer logErrWriter.Close()
-	defer logOutWriter.Close()
-
-	cmdExec.Stderr = io.MultiWriter(logErrWriter, combinedOutput)
-	cmdExec.Stdout = io.MultiWriter(logOutWriter, combinedOutput)
-
-	report := &CommandHookReport{
-		Command: h.command,
-		Env:     hookEnv,
-		// no report.Args
-	}
-
-	err = cmdExec.Start()
-	if err != nil {
-		report.Err = err
-		return report
-	}
-
-	err = cmdExec.Wait()
-	combinedOutputBytes := combinedOutput.Bytes()
-	report.CapturedStdoutStderrCombined = make([]byte, len(combinedOutputBytes))
-	copy(report.CapturedStdoutStderrCombined, combinedOutputBytes)
-	if err != nil {
-		if cmdCtx.Err() == context.DeadlineExceeded {
-			report.Err = fmt.Errorf("timed out after %s: %s", h.timeout, err)
-			return report
-		}
-		report.Err = err
-		return report
-	}
-
-	return report
-}

daemon/hooks/hook_type_mysql_lock_tables.go

@@ -1,148 +0,0 @@
-package hooks
-
-import (
-	"context"
-	"database/sql"
-	"fmt"
-	"strings"
-
-	sqldriver "database/sql/driver"
-
-	"github.com/go-sql-driver/mysql"
-
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-// Hook to implement the following recommmendation from MySQL docs
-// https://dev.mysql.com/doc/mysql-backup-excerpt/5.7/en/backup-methods.html
-//
-//	Making Backups Using a File System Snapshot:
-//
-//	If you are using a Veritas file system, you can make a backup like this:
-//
-//	   	From a client program, execute FLUSH TABLES WITH READ LOCK.
-//	   	From another shell, execute mount vxfs snapshot.
-//	   	From the first client, execute UNLOCK TABLES.
-//	   	Copy files from the snapshot.
-//	   	Unmount the snapshot.
-//
-//		Similar snapshot capabilities may be available in other file systems, such as LVM or ZFS.
-//
-
-type MySQLLockTables struct {
-	errIsFatal  bool
-	connector   sqldriver.Connector
-	filesystems Filter
-}
-
-type myLockTablesStateKey int
-
-const (
-	myLockTablesConnection myLockTablesStateKey = 1 + iota
-)
-
-func MyLockTablesFromConfig(in *config.HookMySQLLockTables) (*MySQLLockTables, error) {
-	conf, err := mysql.ParseDSN(in.DSN)
-	if err != nil {
-		return nil, errors.Wrap(err, "`dsn` invalid")
-	}
-	cn, err := mysql.NewConnector(conf)
-	if err != nil {
-		return nil, errors.Wrap(err, "`connect` invalid")
-	}
-
-	filesystems, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
-	if err != nil {
-		return nil, errors.Wrap(err, "`filesystems` invalid")
-	}
-
-	return &MySQLLockTables{
-		in.ErrIsFatal,
-		cn,
-		filesystems,
-	}, nil
-}
-
-func (h *MySQLLockTables) ErrIsFatal() bool    { return h.errIsFatal }
-func (h *MySQLLockTables) Filesystems() Filter { return h.filesystems }
-func (h *MySQLLockTables) String() string      { return "MySQL FLUSH TABLES WITH READ LOCK" }
-
-type MyLockTablesReport struct {
-	What string
-	Err  error
-}
-
-func (r *MyLockTablesReport) HadError() bool { return r.Err != nil }
-func (r *MyLockTablesReport) Error() string  { return r.String() }
-func (r *MyLockTablesReport) String() string {
-	var s strings.Builder
-	s.WriteString(r.What)
-	if r.Err != nil {
-		fmt.Fprintf(&s, ": %s", r.Err)
-	}
-	return s.String()
-}
-
-func (h *MySQLLockTables) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
-	fs, ok := extra[EnvFS]
-	if !ok {
-		panic(extra)
-	}
-	dp, err := zfs.NewDatasetPath(fs)
-	if err != nil {
-		panic(err)
-	}
-	if pass, err := h.filesystems.Filter(dp); err != nil {
-		return &MyLockTablesReport{What: "filesystem filter", Err: err}
-	} else if !pass {
-		getLogger(ctx).Debug("filesystem does not match filter, skipping")
-		return &MyLockTablesReport{What: "filesystem filter skipped this filesystem", Err: nil}
-	}
-
-	switch edge {
-	case Pre:
-		err := h.doRunPre(ctx, dp, dryRun, state)
-		return &MyLockTablesReport{"FLUSH TABLES WITH READ LOCK", err}
-	case Post:
-		err := h.doRunPost(ctx, dp, dryRun, state)
-		return &MyLockTablesReport{"UNLOCK TABLES", err}
-	}
-	return &MyLockTablesReport{What: "skipped this edge", Err: nil}
-}
-
-func (h *MySQLLockTables) doRunPre(ctx context.Context, fs *zfs.DatasetPath, dry bool, state map[interface{}]interface{}) (err error) {
-	db := sql.OpenDB(h.connector)
-	defer func(err *error) {
-		if *err != nil {
-			db.Close()
-		}
-	}(&err)
-
-	getLogger(ctx).Debug("do FLUSH TABLES WITH READ LOCK")
-	_, err = db.ExecContext(ctx, "FLUSH TABLES WITH READ LOCK")
-	if err != nil {
-		return
-	}
-
-	state[myLockTablesConnection] = db
-
-	return nil
-}
-
-func (h *MySQLLockTables) doRunPost(ctx context.Context, fs *zfs.DatasetPath, dry bool, state map[interface{}]interface{}) error {
-
-	db := state[myLockTablesConnection].(*sql.DB)
-	defer db.Close()
-
-	getLogger(ctx).Debug("do UNLOCK TABLES")
-	_, err := db.ExecContext(ctx, "UNLOCK TABLES")
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

daemon/hooks/hook_type_postgres_checkpoint.go

@@ -1,98 +0,0 @@
-package hooks
-
-import (
-	"context"
-	"database/sql"
-	"fmt"
-	"math"
-	"time"
-
-	"github.com/lib/pq"
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type PgChkptHook struct {
-	errIsFatal  bool
-	connector   *pq.Connector
-	filesystems Filter
-}
-
-func PgChkptHookFromConfig(in *config.HookPostgresCheckpoint) (*PgChkptHook, error) {
-	filesystems, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
-	if err != nil {
-		return nil, errors.Wrap(err, "`filesystems` invalid")
-	}
-	cn, err := pq.NewConnector(in.DSN)
-	if err != nil {
-		return nil, errors.Wrap(err, "`dsn` invalid")
-	}
-
-	return &PgChkptHook{
-		in.ErrIsFatal,
-		cn,
-		filesystems,
-	}, nil
-}
-
-func (h *PgChkptHook) ErrIsFatal() bool    { return h.errIsFatal }
-func (h *PgChkptHook) Filesystems() Filter { return h.filesystems }
-func (h *PgChkptHook) String() string      { return "postgres checkpoint" }
-
-type PgChkptHookReport struct{ Err error }
-
-func (r *PgChkptHookReport) HadError() bool { return r.Err != nil }
-func (r *PgChkptHookReport) Error() string  { return r.Err.Error() }
-func (r *PgChkptHookReport) String() string {
-	if r.Err != nil {
-		return fmt.Sprintf("postgres CHECKPOINT failed: %s", r.Err)
-	} else {
-		return "postgres CHECKPOINT completed"
-	}
-}
-
-func (h *PgChkptHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
-	if edge != Pre {
-		return &PgChkptHookReport{nil}
-	}
-	fs, ok := extra[EnvFS]
-	if !ok {
-		panic(extra)
-	}
-	dp, err := zfs.NewDatasetPath(fs)
-	if err != nil {
-		panic(err)
-	}
-	err = h.doRunPre(ctx, dp, dryRun)
-	return &PgChkptHookReport{err}
-}
-
-func (h *PgChkptHook) doRunPre(ctx context.Context, fs *zfs.DatasetPath, dry bool) error {
-
-	if pass, err := h.filesystems.Filter(fs); err != nil || !pass {
-		getLogger(ctx).Debug("filesystem does not match filter, skipping")
-		return err
-	}
-
-	db := sql.OpenDB(h.connector)
-	defer db.Close()
-	dl, ok := ctx.Deadline()
-	if ok {
-		timeout := uint64(math.Floor(time.Until(dl).Seconds() * 1000)) // TODO go1.13 milliseconds
-		getLogger(ctx).WithField("statement_timeout", timeout).Debug("setting statement timeout for CHECKPOINT")
-		_, err := db.ExecContext(ctx, "SET statement_timeout TO ?", timeout)
-		if err != nil {
-			return err
-		}
-	}
-	if dry {
-		getLogger(ctx).Debug("dry-run - use ping instead of CHECKPOINT")
-		return db.PingContext(ctx)
-	}
-	getLogger(ctx).Info("execute CHECKPOINT command")
-	_, err := db.ExecContext(ctx, "CHECKPOINT")
-	return err
-}

daemon/hooks/hooks_test.go

@@ -1,489 +0,0 @@
-package hooks_test
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"os"
-	"regexp"
-	"testing"
-	"text/template"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/zrepl/zrepl/daemon/logging/trace"
-
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/hooks"
-	"github.com/zrepl/zrepl/daemon/logging"
-	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type comparisonAssertionFunc func(require.TestingT, interface{}, interface{}, ...interface{})
-type valueAssertionFunc func(require.TestingT, interface{}, ...interface{})
-
-type expectStep struct {
-	ExpectedEdge hooks.Edge
-	ExpectStatus hooks.StepStatus
-	OutputTest   valueAssertionFunc
-	ErrorTest    valueAssertionFunc
-}
-
-type testCase struct {
-	Name           string
-	Config         []string
-	IsSlow         bool
-	SuppressOutput bool
-
-	ExpectCallbackSkipped bool
-	ExpectHadFatalErr     bool
-	ExpectHadError        bool
-	ExpectStepReports     []expectStep
-}
-
-func curryLeft(f comparisonAssertionFunc, expected interface{}) valueAssertionFunc {
-	return curry(f, expected, false)
-}
-
-func curryRight(f comparisonAssertionFunc, expected interface{}) valueAssertionFunc {
-	return curry(f, expected, true)
-}
-
-func curry(f comparisonAssertionFunc, expected interface{}, right bool) (ret valueAssertionFunc) {
-	ret = func(t require.TestingT, s interface{}, v ...interface{}) {
-		var x interface{}
-		var y interface{}
-		if right {
-			x = s
-			y = expected
-		} else {
-			x = expected
-			y = s
-		}
-
-		if len(v) > 0 {
-			f(t, x, y, v)
-		} else {
-			f(t, x, y)
-		}
-	}
-	return
-}
-
-func TestHooks(t *testing.T) {
-	ctx, end := trace.WithTaskFromStack(context.Background())
-	defer end()
-
-	testFSName := "testpool/testdataset"
-	testSnapshotName := "testsnap"
-
-	tmpl, err := template.New("TestHooks").Parse(`
-jobs:
-- name: TestHooks
-  type: snap
-  filesystems: {"<": true}
-  snapshotting:
-    type: periodic
-    interval: 1m
-    prefix: zrepl_snapjob_
-    hooks:
-    {{- template "List" . }}
-  pruning:
-    keep:
-      - type: last_n
-        count: 10
-`)
-	if err != nil {
-		panic(err)
-	}
-
-	regexpTest := func(s string) valueAssertionFunc {
-		return curryLeft(require.Regexp, regexp.MustCompile(s))
-	}
-
-	containsTest := func(s string) valueAssertionFunc {
-		return curryRight(require.Contains, s)
-	}
-
-	testTable := []testCase{
-		testCase{
-			Name: "no_hooks",
-			ExpectStepReports: []expectStep{
-				{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
-			},
-		},
-		testCase{
-			Name:           "timeout",
-			IsSlow:         true,
-			ExpectHadError: true,
-			Config:         []string{`{type: command, path: {{.WorkDir}}/test/test-timeout.sh, timeout: 2s}`},
-			ExpectStepReports: []expectStep{
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepErr,
-					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s ZREPL_TIMEOUT=2", testFSName, testSnapshotName)),
-					ErrorTest:    regexpTest(`timed out after 2(.\d+)?s`),
-				},
-				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepSkippedDueToPreErr,
-				},
-			},
-		},
-		testCase{
-			Name:           "check_env",
-			Config:         []string{`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`},
-			ExpectHadError: false,
-			ExpectStepReports: []expectStep{
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
-				},
-				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
-				},
-			},
-		},
-
-		testCase{
-			Name:                  "nonfatal_pre_error_continues",
-			ExpectCallbackSkipped: false,
-			ExpectHadError:        true,
-			Config:                []string{`{type: command, path: {{.WorkDir}}/test/test-error.sh}`},
-			ExpectStepReports: []expectStep{
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepErr,
-					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
-					ErrorTest:    regexpTest("^command hook failed.*exit status 1$"),
-				},
-				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepSkippedDueToPreErr, // post-edge is not executed for failing pre-edge
-				},
-			},
-		},
-
-		testCase{
-			Name:                  "pre_error_fatal_skips_subsequent_pre_edges_and_callback_and_its_post_edge_and_post_edges",
-			ExpectCallbackSkipped: true,
-			ExpectHadFatalErr:     true,
-			ExpectHadError:        true,
-			Config: []string{
-				`{type: command, path: {{.WorkDir}}/test/test-error.sh, err_is_fatal: true}`,
-				`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`,
-			},
-			ExpectStepReports: []expectStep{
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepErr,
-					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
-					ErrorTest:    regexpTest("^command hook failed.*exit status 1$"),
-				},
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepSkippedDueToFatalErr,
-				},
-				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepSkippedDueToFatalErr},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepSkippedDueToFatalErr,
-				},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepSkippedDueToFatalErr,
-				},
-			},
-		},
-
-		testCase{
-			Name:              "post_error_fails_are_ignored_even_if_fatal",
-			ExpectHadFatalErr: false, // only occurs during Post, so it's not a fatal error
-			ExpectHadError:    true,
-			Config: []string{
-				`{type: command, path: {{.WorkDir}}/test/test-post-error.sh, err_is_fatal: true}`,
-				`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`,
-			},
-			ExpectStepReports: []expectStep{
-				expectStep{
-					// No-action run of test-post-error.sh
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   require.Empty,
-				},
-				expectStep{
-					// Pre run of test-report-env.sh
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
-				},
-				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
-				},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepErr,
-					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR post_testing %s@%s", testFSName, testSnapshotName)),
-					ErrorTest:    regexpTest("^command hook failed.*exit status 1$"),
-				},
-			},
-		},
-
-		testCase{
-			Name:   "cleanup_check_env",
-			Config: []string{`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`},
-			ExpectStepReports: []expectStep{
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
-				},
-				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
-				},
-			},
-		},
-
-		testCase{
-			Name:              "pre_error_cancels_post",
-			Config:            []string{`{type: command, path: {{.WorkDir}}/test/test-pre-error-post-ok.sh}`},
-			ExpectHadError:    true,
-			ExpectHadFatalErr: false,
-			ExpectStepReports: []expectStep{
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepErr,
-					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
-					ErrorTest:    regexpTest("^command hook failed.*exit status 1$"),
-				},
-				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepSkippedDueToPreErr,
-				},
-			},
-		},
-
-		testCase{
-			Name: "pre_error_does_not_cancel_other_posts_but_itself",
-			Config: []string{
-				`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`,
-				`{type: command, path: {{.WorkDir}}/test/test-pre-error-post-ok.sh}`,
-			},
-			ExpectHadError:    true,
-			ExpectHadFatalErr: false,
-			ExpectStepReports: []expectStep{
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
-				},
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepErr,
-					OutputTest:   containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
-					ErrorTest:    regexpTest("^command hook failed.*exit status 1$"),
-				},
-				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepSkippedDueToPreErr,
-				},
-				expectStep{
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
-				},
-			},
-		},
-
-		testCase{
-			Name:              "exceed_buffer_limit",
-			SuppressOutput:    true,
-			Config:            []string{`{type: command, path: {{.WorkDir}}/test/test-large-stdout.sh}`},
-			ExpectHadError:    false,
-			ExpectHadFatalErr: false,
-			ExpectStepReports: []expectStep{
-				expectStep{
-					ExpectedEdge: hooks.Pre,
-					ExpectStatus: hooks.StepOk,
-					OutputTest: func(t require.TestingT, s interface{}, v ...interface{}) {
-						require.Len(t, s, 1<<20)
-					},
-				},
-				expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
-				expectStep{
-					// No-action run of above hook
-					ExpectedEdge: hooks.Post,
-					ExpectStatus: hooks.StepOk,
-					OutputTest:   require.Empty,
-				},
-			},
-		},
-
-		/*
-			Following not intended to test functionality of
-			filter package. Only to demonstrate that hook
-			filters are being applied. The following should
-			result in NO hooks running. If it does run, a
-			fatal hooks.RunReport will be returned.
-		*/
-		testCase{
-			Name: "exclude_all_filesystems",
-			Config: []string{
-				`{type: command, path: {{.WorkDir}}/test/test-error.sh, err_is_fatal: true, filesystems: {"<": false}}`,
-			},
-			ExpectStepReports: []expectStep{
-				expectStep{
-					ExpectedEdge: hooks.Callback,
-					ExpectStatus: hooks.StepOk,
-				},
-			},
-		},
-	}
-
-	parseHookConfig := func(t *testing.T, in string) *config.Config {
-		t.Helper()
-		conf, err := config.ParseConfigBytes([]byte(in))
-		require.NoError(t, err)
-		require.NotNil(t, conf)
-		return conf
-	}
-
-	fillHooks := func(tt *testCase) string {
-		// make hook path absolute
-		cwd, err := os.Getwd()
-		if err != nil {
-			panic("os.Getwd() failed")
-		}
-		var hooksTmpl string = "\n"
-		for _, l := range tt.Config {
-			hooksTmpl += fmt.Sprintf("    - %s\n", l)
-		}
-		tmpl.New("List").Parse(hooksTmpl)
-
-		var outBytes bytes.Buffer
-		data := struct {
-			WorkDir string
-		}{
-			WorkDir: cwd,
-		}
-		if err := tmpl.Execute(&outBytes, data); err != nil {
-			panic(err)
-		}
-
-		return outBytes.String()
-	}
-
-	var c *config.Config
-	fs, err := zfs.NewDatasetPath(testFSName)
-	require.NoError(t, err)
-
-	log := logger.NewTestLogger(t)
-
-	var cbReached bool
-	cb := hooks.NewCallbackHookForFilesystem("testcallback", fs, func(_ context.Context) error {
-		cbReached = true
-		return nil
-	})
-
-	hookEnvExtra := hooks.Env{
-		hooks.EnvFS:       fs.ToString(),
-		hooks.EnvSnapshot: testSnapshotName,
-	}
-
-	for _, tt := range testTable {
-		if testing.Short() && tt.IsSlow {
-			continue
-		}
-
-		t.Run(tt.Name, func(t *testing.T) {
-			c = parseHookConfig(t, fillHooks(&tt))
-			snp := c.Jobs[0].Ret.(*config.SnapJob).Snapshotting.Ret.(*config.SnapshottingPeriodic)
-			hookList, err := hooks.ListFromConfig(&snp.Hooks)
-			require.NoError(t, err)
-
-			filteredHooks, err := hookList.CopyFilteredForFilesystem(fs)
-			require.NoError(t, err)
-			plan, err := hooks.NewPlan(&filteredHooks, hooks.PhaseTesting, cb, hookEnvExtra)
-			require.NoError(t, err)
-			t.Logf("REPORT PRE EXECUTION:\n%s", plan.Report())
-
-			cbReached = false
-
-			if testing.Verbose() && !tt.SuppressOutput {
-				ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
-			}
-			plan.Run(ctx, false)
-			report := plan.Report()
-
-			t.Logf("REPORT POST EXECUTION:\n%s", report)
-
-			/*
-			 * TEST ASSERTIONS
-			 */
-
-			t.Logf("len(runReports)=%v", len(report))
-			t.Logf("len(tt.ExpectStepReports)=%v", len(tt.ExpectStepReports))
-			require.Equal(t, len(tt.ExpectStepReports), len(report), "ExpectStepReports must be same length as expected number of hook runs, excluding possible Callback")
-
-			// Check if callback ran, when required
-			if tt.ExpectCallbackSkipped {
-				require.False(t, cbReached, "callback ran but should not have run")
-			} else {
-				require.True(t, cbReached, "callback should have run but did not")
-			}
-
-			// Check if a fatal run error occurred and was expected
-			require.Equal(t, tt.ExpectHadFatalErr, report.HadFatalError(), "non-matching HadFatalError")
-			require.Equal(t, tt.ExpectHadError, report.HadError(), "non-matching HadError")
-
-			if tt.ExpectHadFatalErr {
-				require.True(t, tt.ExpectHadError, "ExpectHadFatalErr implies ExpectHadError")
-			}
-			if !tt.ExpectHadError {
-				require.False(t, tt.ExpectHadFatalErr, "!ExpectHadError implies !ExpectHadFatalErr")
-			}
-
-			// Iterate through each expected hook run
-			for i, hook := range tt.ExpectStepReports {
-				t.Logf("expecting report conforming to %v", hook)
-
-				exp, act := hook.ExpectStatus, report[i].Status
-				require.Equal(t, exp, act, "%s != %s", exp, act)
-
-				// Check for required ExpectedEdge
-				require.NotZero(t, hook.ExpectedEdge, "each hook must have an ExpectedEdge")
-				require.Equal(t, hook.ExpectedEdge, report[i].Edge,
-					"incorrect edge: expected %q, actual %q", hook.ExpectedEdge.String(), report[i].Edge.String(),
-				)
-
-				// Check for expected output
-				if hook.OutputTest != nil {
-					require.IsType(t, (*hooks.CommandHookReport)(nil), report[i].Report)
-					chr := report[i].Report.(*hooks.CommandHookReport)
-					hook.OutputTest(t, string(chr.CapturedStdoutStderrCombined))
-				}
-
-				// Check for expected errors
-				if hook.ErrorTest != nil {
-					hook.ErrorTest(t, string(report[i].Report.Error()))
-				}
-			}
-
-		})
-	}
-}

daemon/hooks/stepstatus_enumer.go

@@ -1,77 +0,0 @@
-// Code generated by "enumer -type=StepStatus -trimprefix=Step"; DO NOT EDIT.
-
-//
-package hooks
-
-import (
-	"fmt"
-)
-
-const (
-	_StepStatusName_0 = "PendingExec"
-	_StepStatusName_1 = "Ok"
-	_StepStatusName_2 = "Err"
-	_StepStatusName_3 = "SkippedDueToFatalErr"
-	_StepStatusName_4 = "SkippedDueToPreErr"
-)
-
-var (
-	_StepStatusIndex_0 = [...]uint8{0, 7, 11}
-	_StepStatusIndex_1 = [...]uint8{0, 2}
-	_StepStatusIndex_2 = [...]uint8{0, 3}
-	_StepStatusIndex_3 = [...]uint8{0, 20}
-	_StepStatusIndex_4 = [...]uint8{0, 18}
-)
-
-func (i StepStatus) String() string {
-	switch {
-	case 1 <= i && i <= 2:
-		i -= 1
-		return _StepStatusName_0[_StepStatusIndex_0[i]:_StepStatusIndex_0[i+1]]
-	case i == 4:
-		return _StepStatusName_1
-	case i == 8:
-		return _StepStatusName_2
-	case i == 16:
-		return _StepStatusName_3
-	case i == 32:
-		return _StepStatusName_4
-	default:
-		return fmt.Sprintf("StepStatus(%d)", i)
-	}
-}
-
-var _StepStatusValues = []StepStatus{1, 2, 4, 8, 16, 32}
-
-var _StepStatusNameToValueMap = map[string]StepStatus{
-	_StepStatusName_0[0:7]:  1,
-	_StepStatusName_0[7:11]: 2,
-	_StepStatusName_1[0:2]:  4,
-	_StepStatusName_2[0:3]:  8,
-	_StepStatusName_3[0:20]: 16,
-	_StepStatusName_4[0:18]: 32,
-}
-
-// StepStatusString retrieves an enum value from the enum constants string name.
-// Throws an error if the param is not part of the enum.
-func StepStatusString(s string) (StepStatus, error) {
-	if val, ok := _StepStatusNameToValueMap[s]; ok {
-		return val, nil
-	}
-	return 0, fmt.Errorf("%s does not belong to StepStatus values", s)
-}
-
-// StepStatusValues returns all values of the enum
-func StepStatusValues() []StepStatus {
-	return _StepStatusValues
-}
-
-// IsAStepStatus returns "true" if the value is listed in the enum definition. "false" otherwise
-func (i StepStatus) IsAStepStatus() bool {
-	for _, v := range _StepStatusValues {
-		if i == v {
-			return true
-		}
-	}
-	return false
-}

daemon/hooks/test/test-error.sh

@@ -1,5 +0,0 @@
-#!/bin/sh -eu
-
->&2 echo "TEST ERROR $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
-
-exit 1

daemon/hooks/test/test-large-stdout.sh

@@ -1,18 +0,0 @@
-#!/bin/sh -eu
-
-# Exceed default hook log size of 1<<20 bytes by double
-# The Go test should fail if buffer size exceeds 1MB
-
-pre_testing() {
-	ln=0
-	while :; do printf '%06d: 012345678901234567890123456789012345678901234567890123456789012345678901\n' "$ln"; ln="$(($ln + 1))"; done \
-	| head -c$(( 2 << 20 ))
-}
-
-case "$ZREPL_HOOKTYPE" in
-	pre_testing)
-		"$ZREPL_HOOKTYPE";;
-	*)
-		# Not handled by this script
-		exit 0;;
-esac

daemon/hooks/test/test-post-error.sh

@@ -1,15 +0,0 @@
-#!/bin/sh -eu
-
-post_testing() {
-	>&2 echo "TEST ERROR $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
-
-	exit 1
-}
-
-case "$ZREPL_HOOKTYPE" in
-	post_testing)
-		"$ZREPL_HOOKTYPE";;
-	*)
-		# Not handled by this script
-		exit 0;;
-esac

daemon/hooks/test/test-pre-error-post-ok.sh

@@ -1,11 +0,0 @@
-#!/bin/sh -eu
-
-if [ "$ZREPL_HOOKTYPE" = "pre_testing" ]; then
-	>&2 echo "TEST ERROR $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
-	exit 1
-elif [ "$ZREPL_HOOKTYPE" = "post_testing" ]; then
-	echo "TEST $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
-else
-	printf "Unknown hook type: %s" "$ZREPL_HOOKTYPE" 
-	exit 255
-fi

daemon/hooks/test/test-report-env.sh

@@ -1,3 +0,0 @@
-#!/bin/sh -eu
-
-echo "TEST $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME ${ZREPL_DRYRUN:+(dry run)}"