Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Christian Schwarz
|
aa7a13a1c1 | ||
|
Christian Schwarz
|
f76a0dec6d | ||
|
Christian Schwarz
|
b34d0e1041 |
@@ -1,145 +0,0 @@
|
||||
version: 2.0
|
||||
workflows:
|
||||
version: 2
|
||||
build:
|
||||
jobs:
|
||||
- build-1.11
|
||||
- build-1.12
|
||||
- build-1.13
|
||||
- build-1.14
|
||||
- build-latest
|
||||
- test-build-in-docker
|
||||
jobs:
|
||||
|
||||
# build-latest serves as the template
|
||||
# we use YAML anchors & aliases to exchange the docker image (and hence Go version used for the build)
|
||||
build-latest: &build-latest
|
||||
description: Builds zrepl
|
||||
parameters:
|
||||
image:
|
||||
description: "the docker image that the job should use"
|
||||
type: string
|
||||
docker:
|
||||
- image: circleci/golang:latest
|
||||
environment:
|
||||
# required by lazy.sh
|
||||
TERM: xterm
|
||||
working_directory: /go/src/github.com/zrepl/zrepl
|
||||
steps:
|
||||
- run:
|
||||
name: Setup environment variables
|
||||
command: |
|
||||
# used by pip (for docs)
|
||||
echo 'export PATH="$HOME/.local/bin:$PATH"' >> $BASH_ENV
|
||||
# we use modules
|
||||
echo 'export GO111MODULE=on' >> $BASH_ENV
|
||||
|
||||
- restore_cache:
|
||||
keys:
|
||||
- source
|
||||
- protobuf
|
||||
|
||||
- checkout
|
||||
|
||||
- save_cache:
|
||||
key: source
|
||||
paths:
|
||||
- ".git"
|
||||
|
||||
# install deps
|
||||
- run: wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
|
||||
- run: echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807 protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
|
||||
- run: sudo unzip -d /usr protoc-3.6.1-linux-x86_64.zip
|
||||
- save_cache:
|
||||
key: protobuf
|
||||
paths:
|
||||
- "/usr/include/google/protobuf"
|
||||
|
||||
- run: sudo apt update && sudo apt install python3 python3-pip libgirepository1.0-dev gawk
|
||||
- run: ./lazy.sh devsetup
|
||||
|
||||
- run: make zrepl-bin
|
||||
- run: make vet
|
||||
- run: make test
|
||||
- run: make lint
|
||||
- run: make release
|
||||
|
||||
|
||||
- store_artifacts:
|
||||
path: ./artifacts/release
|
||||
when: always
|
||||
|
||||
- run:
|
||||
shell: /bin/bash -eo pipefail
|
||||
when: always
|
||||
command: |
|
||||
if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
|
||||
echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
|
||||
exit 0
|
||||
fi
|
||||
set -u # from now on
|
||||
|
||||
# Download and install minio
|
||||
curl -sSL https://dl.minio.io/client/mc/release/linux-amd64/mc -o ${GOPATH}/bin/mc
|
||||
chmod +x ${GOPATH}/bin/mc
|
||||
mc config host add --api s3v4 zrepl-minio https://minio.cschwarz.com ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
|
||||
|
||||
# Upload artifacts
|
||||
echo "$CIRCLE_BUILD_URL" > ./artifacts/release/cirlceci_build_url
|
||||
mc cp -r artifacts/release "zrepl-minio/zrepl-ci-artifacts/${CIRCLE_SHA1}/${CIRCLE_JOB}/"
|
||||
|
||||
# Push Artifact Link to GitHub
|
||||
REPO="zrepl/zrepl"
|
||||
COMMIT="${CIRCLE_SHA1}"
|
||||
JOB_NAME="${CIRCLE_JOB}"
|
||||
curl "https://api.github.com/repos/$REPO/statuses/$COMMIT?access_token=$GITHUB_COMMIT_STATUS_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST \
|
||||
-d '{"context":"zrepl/publish-ci-artifacts", "state": "success", "description":"CI Build Artifacts for '"$JOB_NAME"'", "target_url":"https://minio.cschwarz.com/minio/zrepl-ci-artifacts/'"$COMMIT"'/"}'
|
||||
|
||||
build-1.11:
|
||||
<<: *build-latest
|
||||
docker:
|
||||
- image: circleci/golang:1.11
|
||||
|
||||
build-1.12:
|
||||
<<: *build-latest
|
||||
docker:
|
||||
- image: circleci/golang:1.12
|
||||
|
||||
build-1.13:
|
||||
<<: *build-latest
|
||||
docker:
|
||||
- image: circleci/golang:1.13
|
||||
|
||||
build-1.14:
|
||||
<<: *build-latest
|
||||
docker:
|
||||
- image: circleci/golang:1.14
|
||||
|
||||
# this job tries to mimic the build-in-docker instructions
|
||||
# given in docs/installation.rst
|
||||
#
|
||||
# However, CircleCI doesn't support volume mounts, so we have to copy
|
||||
# the source into the build-container by modifying the Dockerfile
|
||||
test-build-in-docker:
|
||||
description: Check that build-in-docker works
|
||||
docker:
|
||||
- image: circleci/golang:latest
|
||||
environment:
|
||||
working_directory: /go/src/github.com/zrepl/zrepl
|
||||
steps:
|
||||
- checkout
|
||||
- setup_remote_docker
|
||||
- run:
|
||||
name: (hacky) circleci doesn't allow volume mounts, so copy src to container
|
||||
command: echo "ADD . /src" >> build.Dockerfile
|
||||
- run:
|
||||
name: (hacky) commit modified Dockerfile to avoid failing git clean check in Makefile
|
||||
command: git -c user.name='circleci' -c user.email='circleci@localhost' commit -m 'CIRCLECI modified Dockerfile with zrepl src' --author 'autoauthor <circleci@localhost>' -- build.Dockerfile
|
||||
- run:
|
||||
name: build the build image (build deps)
|
||||
command: docker build -t zrepl_build -f build.Dockerfile .
|
||||
- run:
|
||||
name: try compiling
|
||||
command: docker run -it zrepl_build make release
|
||||
@@ -1,4 +0,0 @@
|
||||
patreon: zrepl
|
||||
liberapay: zrepl
|
||||
custom: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96
|
||||
ko_fi: problame
|
||||
@@ -1,10 +0,0 @@
|
||||
linters:
|
||||
enable:
|
||||
- goimports
|
||||
|
||||
issues:
|
||||
exclude-rules:
|
||||
- path: _test\.go
|
||||
linters:
|
||||
- errcheck
|
||||
|
||||
-70
@@ -1,70 +0,0 @@
|
||||
dist: xenial
|
||||
services:
|
||||
- docker
|
||||
|
||||
env: # for allow_failures: https://docs.travis-ci.com/user/customizing-the-build/
|
||||
matrix:
|
||||
include:
|
||||
|
||||
- language: go
|
||||
name: "Build in Docker (docs/installation.rst)"
|
||||
script:
|
||||
- sudo docker build -t zrepl_build -f build.Dockerfile .
|
||||
- |
|
||||
sudo docker run -it --rm \
|
||||
-v "${PWD}:/go/src/github.com/zrepl/zrepl" \
|
||||
--user "$(id -u):$(id -g)" \
|
||||
zrepl_build make vendordeps release
|
||||
|
||||
- &zrepl_build_template
|
||||
language: go
|
||||
go_import_path: github.com/zrepl/zrepl
|
||||
before_install:
|
||||
- wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
|
||||
- echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807 protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
|
||||
- sudo unzip -d /usr protoc-3.6.1-linux-x86_64.zip
|
||||
- ./lazy.sh godep
|
||||
- make vendordeps
|
||||
script:
|
||||
- make
|
||||
- make vet
|
||||
- make test
|
||||
- make lint
|
||||
- make artifacts/zrepl-freebsd-amd64
|
||||
- make artifacts/zrepl-linux-amd64
|
||||
- make artifacts/zrepl-darwin-amd64
|
||||
go:
|
||||
- "1.11"
|
||||
|
||||
- <<: *zrepl_build_template
|
||||
go:
|
||||
- "1.12"
|
||||
|
||||
- <<: *zrepl_build_template
|
||||
go:
|
||||
- "master"
|
||||
|
||||
- &zrepl_docs_template
|
||||
language: python
|
||||
python:
|
||||
- "3.4"
|
||||
install:
|
||||
- sudo apt-get install libgirepository1.0-dev
|
||||
- pip install -r docs/requirements.txt
|
||||
script:
|
||||
- make docs
|
||||
- <<: *zrepl_docs_template
|
||||
python:
|
||||
- "3.5"
|
||||
- <<: *zrepl_docs_template
|
||||
python:
|
||||
- "3.6"
|
||||
- <<: *zrepl_docs_template
|
||||
python:
|
||||
- "3.7"
|
||||
|
||||
|
||||
allow_failures:
|
||||
- <<: *zrepl_build_template
|
||||
go:
|
||||
- "master"
|
||||
Generated
+123
@@ -0,0 +1,123 @@
|
||||
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
|
||||
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/davecgh/go-spew"
|
||||
packages = ["spew"]
|
||||
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
|
||||
version = "v1.1.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/dustin/go-humanize"
|
||||
packages = ["."]
|
||||
revision = "bb3d318650d48840a39aa21a027c6630e198e626"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/ftrvxmtrx/fd"
|
||||
packages = ["."]
|
||||
revision = "c6d800382fff6dc1412f34269f71b7f83bd059ad"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/go-logfmt/logfmt"
|
||||
packages = ["."]
|
||||
revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5"
|
||||
version = "v0.3.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "v2"
|
||||
name = "github.com/go-yaml/yaml"
|
||||
packages = ["."]
|
||||
revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/jinzhu/copier"
|
||||
packages = ["."]
|
||||
revision = "db4671f3a9b8df855e993f7c94ec5ef1ffb0a23b"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/kr/logfmt"
|
||||
packages = ["."]
|
||||
revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/kr/pretty"
|
||||
packages = ["."]
|
||||
revision = "cfb55aafdaf3ec08f0db22699ab822c50091b1c4"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/kr/text"
|
||||
packages = ["."]
|
||||
revision = "7cafcd837844e784b526369c9bce262804aebc60"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/mattn/go-isatty"
|
||||
packages = ["."]
|
||||
revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
|
||||
version = "v0.0.3"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/mitchellh/mapstructure"
|
||||
packages = ["."]
|
||||
revision = "d0303fe809921458f417bcf828397a65db30a7e4"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/pkg/errors"
|
||||
packages = ["."]
|
||||
revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
|
||||
version = "v0.8.0"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/pmezard/go-difflib"
|
||||
packages = ["difflib"]
|
||||
revision = "792786c7400a136282c1664665ae0a8db921c6c2"
|
||||
version = "v1.0.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/problame/go-netssh"
|
||||
packages = ["."]
|
||||
revision = "ffa145d2506e222977205e7666a9722d6b9959ac"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/problame/go-rwccmd"
|
||||
packages = ["."]
|
||||
revision = "391d2c78c8404a9683d79f75dd24ab53040f89f7"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/spf13/cobra"
|
||||
packages = ["."]
|
||||
revision = "b78744579491c1ceeaaa3b40205e56b0591b93a3"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/spf13/pflag"
|
||||
packages = ["."]
|
||||
revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66"
|
||||
version = "v1.0.0"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/stretchr/testify"
|
||||
packages = ["assert"]
|
||||
revision = "69483b4bd14f5845b5a1e55bca19e954e827f1d0"
|
||||
version = "v1.1.4"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "golang.org/x/sys"
|
||||
packages = ["unix"]
|
||||
revision = "bf42f188b9bc6f2cf5b8ee5a912ef1aedd0eba4c"
|
||||
|
||||
[solve-meta]
|
||||
analyzer-name = "dep"
|
||||
analyzer-version = 1
|
||||
inputs-digest = "fcfc66322b5f9dbe329411b7688a4e9dcebe3c529c1dadf4f78bcdb2bcfb8fbf"
|
||||
solver-name = "gps-cdcl"
|
||||
solver-version = 1
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
ignored = [ "github.com/inconshreveable/mousetrap" ]
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/ftrvxmtrx/fd"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/jinzhu/copier"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/kr/pretty"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/mitchellh/go-homedir"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/mitchellh/mapstructure"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/pkg/errors"
|
||||
version = "0.8.0"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/spf13/cobra"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/spf13/viper"
|
||||
version = "1.0.0"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/stretchr/testify"
|
||||
version = "1.1.4"
|
||||
|
||||
[[constraint]]
|
||||
branch = "v2"
|
||||
name = "github.com/go-yaml/yaml"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/go-logfmt/logfmt"
|
||||
version = "*"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/problame/go-rwccmd"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/problame/go-netssh"
|
||||
branch = "master"
|
||||
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2017-2018 Christian Schwarz
|
||||
Copyright (c) 2017 Christian Schwarz
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@@ -1,185 +1,71 @@
|
||||
.PHONY: generate build test vet cover release docs docs-clean clean format lint platformtest
|
||||
.PHONY: release bins-all release-noarch
|
||||
.DEFAULT_GOAL := zrepl-bin
|
||||
.PHONY: generate build test vet cover release docs docs-clean clean vendordeps
|
||||
.DEFAULT_GOAL := build
|
||||
|
||||
ROOT := github.com/zrepl/zrepl
|
||||
SUBPKGS := cmd logger rpc util zfs
|
||||
|
||||
_TESTPKGS := $(ROOT) $(foreach p,$(SUBPKGS),$(ROOT)/$(p))
|
||||
|
||||
ARTIFACTDIR := artifacts
|
||||
|
||||
ifdef ZREPL_VERSION
|
||||
_ZREPL_VERSION := $(ZREPL_VERSION)
|
||||
endif
|
||||
ifndef _ZREPL_VERSION
|
||||
_ZREPL_VERSION := $(shell git describe --always --dirty 2>/dev/null || echo "ZREPL_BUILD_INVALID_VERSION" )
|
||||
ifeq ($(_ZREPL_VERSION),ZREPL_BUILD_INVALID_VERSION) # can't use .SHELLSTATUS because Debian Stretch is still on gmake 4.1
|
||||
ifndef ZREPL_VERSION
|
||||
ZREPL_VERSION := $(shell git describe --dirty 2>/dev/null || echo "ZREPL_BUILD_INVALID_VERSION" )
|
||||
ifeq ($(ZREPL_VERSION),ZREPL_BUILD_INVALID_VERSION) # can't use .SHELLSTATUS because Debian Stretch is still on gmake 4.1
|
||||
$(error cannot infer variable ZREPL_VERSION using git and variable is not overriden by make invocation)
|
||||
endif
|
||||
endif
|
||||
GO_LDFLAGS := "-X github.com/zrepl/zrepl/cmd.zreplVersion=$(ZREPL_VERSION)"
|
||||
|
||||
GO := go
|
||||
GOOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOOS"')
|
||||
GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"')
|
||||
GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"')
|
||||
GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"')
|
||||
GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
|
||||
GO_ENV_VARS := GO111MODULE=on
|
||||
GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
|
||||
GO_MOD_READONLY := -mod=readonly
|
||||
GO_BUILDFLAGS := $(GO_MOD_READONLY)
|
||||
GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
|
||||
GOLANGCI_LINT := golangci-lint
|
||||
ifneq ($(GOARM),)
|
||||
ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM)
|
||||
else
|
||||
ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)
|
||||
endif
|
||||
GO_BUILD := go build -ldflags $(GO_LDFLAGS)
|
||||
|
||||
.PHONY: printvars
|
||||
printvars:
|
||||
@echo GOOS=$(GOOS)
|
||||
@echo GOARCH=$(GOARCH)
|
||||
@echo GOARM=$(GOARM)
|
||||
RELEASE_BINS := $(ARTIFACTDIR)/zrepl-freebsd-amd64 $(ARTIFACTDIR)/zrepl-linux-amd64
|
||||
RELEASE_NOARCH := $(ARTIFACTDIR)/zrepl-noarch.tar
|
||||
THIS_PLATFORM_RELEASE_BIN := $(shell bash -c 'source <(go env) && echo "zrepl-$${GOOS}-$${GOARCH}"' )
|
||||
|
||||
vendordeps:
|
||||
dep ensure -v -vendor-only
|
||||
|
||||
##################### PRODUCING A RELEASE #############
|
||||
.PHONY: release wrapup-and-checksum check-git-clean sign clean
|
||||
generate: #not part of the build, must do that manually
|
||||
@for pkg in $(_TESTPKGS); do\
|
||||
go generate "$$pkg" || exit 1; \
|
||||
done;
|
||||
|
||||
release: clean
|
||||
# no cross-platform support for target test
|
||||
$(MAKE) test
|
||||
$(MAKE) bins-all
|
||||
$(MAKE) noarch
|
||||
$(MAKE) wrapup-and-checksum
|
||||
$(MAKE) check-git-clean
|
||||
ifeq (SIGN, 1)
|
||||
$(make) sign
|
||||
endif
|
||||
@echo "ZREPL RELEASE ARTIFACTS AVAILABLE IN artifacts/release"
|
||||
build:
|
||||
@echo "INFO: In case of missing dependencies, run 'make vendordeps'"
|
||||
$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl"
|
||||
|
||||
# expects `release` target to have run before
|
||||
NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
|
||||
wrapup-and-checksum:
|
||||
rm -f $(NOARCH_TARBALL)
|
||||
tar --mtime='1970-01-01' --sort=name \
|
||||
--transform 's/$(ARTIFACTDIR)/zrepl-$(_ZREPL_VERSION)-noarch/' \
|
||||
--transform 's#dist#zrepl-$(_ZREPL_VERSION)-noarch/dist#' \
|
||||
--transform 's#config/samples#zrepl-$(_ZREPL_VERSION)-noarch/config#' \
|
||||
-acf $(NOARCH_TARBALL) \
|
||||
$(ARTIFACTDIR)/docs/html \
|
||||
$(ARTIFACTDIR)/bash_completion \
|
||||
$(ARTIFACTDIR)/go_env.txt \
|
||||
dist \
|
||||
config/samples
|
||||
rm -rf "$(ARTIFACTDIR)/release"
|
||||
mkdir -p "$(ARTIFACTDIR)/release"
|
||||
cp -l $(ARTIFACTDIR)/zrepl-* \
|
||||
$(ARTIFACTDIR)/platformtest-* \
|
||||
"$(ARTIFACTDIR)/release"
|
||||
cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
|
||||
|
||||
check-git-clean:
|
||||
@# note that we use ZREPL_VERSION and not _ZREPL_VERSION because we want to detect the override
|
||||
@if git describe --always --dirty 2>/dev/null | grep dirty >/dev/null; then \
|
||||
echo '[INFO] either git reports checkout is dirty or git is not installed or this is not a git checkout'; \
|
||||
if [ "$(ZREPL_VERSION)" = "" ]; then \
|
||||
echo '[WARN] git checkout is dirty and make variable ZREPL_VERSION was not used to override'; \
|
||||
git status; \
|
||||
echo "git diff:"; \
|
||||
git diff | cat; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
fi;
|
||||
|
||||
sign:
|
||||
gpg -u "89BC 5D89 C845 568B F578 B306 CDBD 8EC8 E27C A5FC" \
|
||||
--armor \
|
||||
--detach-sign $(ARTIFACTDIR)/release/sha512sum.txt
|
||||
|
||||
clean: docs-clean
|
||||
rm -rf "$(ARTIFACTDIR)"
|
||||
|
||||
##################### BINARIES #####################
|
||||
.PHONY: bins-all lint test vet zrepl-bin platformtest-bin
|
||||
|
||||
BINS_ALL_TARGETS := zrepl-bin platformtest-bin vet lint
|
||||
GO_SUPPORTS_ILLUMOS := $(shell $(GO) version | gawk -F '.' '/^go version /{split($$0, comps, " "); split(comps[3], v, "."); if (v[1] == "go1" && v[2] >= 13) { print "illumos"; } else { print "noillumos"; }}')
|
||||
bins-all:
|
||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd GOARCH=amd64
|
||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd GOARCH=386
|
||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux GOARCH=amd64
|
||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux GOARCH=arm64
|
||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux GOARCH=arm GOARM=7
|
||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux GOARCH=386
|
||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=darwin GOARCH=amd64
|
||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=solaris GOARCH=amd64
|
||||
ifeq ($(GO_SUPPORTS_ILLUMOS), illumos)
|
||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=illumos GOARCH=amd64
|
||||
else ifeq ($(GO_SUPPORTS_ILLUMOS), noillumos)
|
||||
@echo "SKIPPING ILLUMOS BUILD BECAUSE GO VERSION DOESN'T SUPPORT IT"
|
||||
else
|
||||
@echo "CANNOT DETERMINE WHETHER GO VERSION SUPPORTS GOOS=illumos"; exit 1
|
||||
endif
|
||||
|
||||
lint:
|
||||
$(GO_ENV_VARS) $(GOLANGCI_LINT) run ./...
|
||||
test:
|
||||
$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) ./...
|
||||
@for pkg in $(_TESTPKGS); do \
|
||||
echo "Testing $$pkg"; \
|
||||
go test "$$pkg" || exit 1; \
|
||||
done;
|
||||
|
||||
vet:
|
||||
$(GO_ENV_VARS) $(GO) vet $(GO_BUILDFLAGS) ./...
|
||||
|
||||
zrepl-bin:
|
||||
$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(ZREPL_TARGET_TUPLE)"
|
||||
|
||||
platformtest-bin:
|
||||
$(GO_BUILD) -o "$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" ./platformtest/harness
|
||||
|
||||
##################### DEV TARGETS #####################
|
||||
# not part of the build, must do that manually
|
||||
.PHONY: generate format platformtest
|
||||
|
||||
generate:
|
||||
protoc -I=replication/logic/pdu --go_out=plugins=grpc:replication/logic/pdu replication/logic/pdu/pdu.proto
|
||||
$(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./...
|
||||
|
||||
format:
|
||||
goimports -srcdir . -local 'github.com/zrepl/zrepl' -w $(shell find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go')
|
||||
|
||||
ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
|
||||
ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
|
||||
ZREPL_PLATFORMTEST_MOUNTPOINT := /tmp/zreplplatformtest.pool
|
||||
ZREPL_PLATFORMTEST_ZFS_LOG := /tmp/zreplplatformtest.zfs.log
|
||||
# ZREPL_PLATFORMTEST_STOP_AND_KEEP := -failure.stop-and-keep-pool
|
||||
ZREPL_PLATFORMTEST_ARGS :=
|
||||
platformtest: # do not track dependency on platformtest-bin to allow build of platformtest outside of test VM
|
||||
rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
|
||||
platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
|
||||
"$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" \
|
||||
-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
|
||||
-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
|
||||
-mountpoint "$(ZREPL_PLATFORMTEST_MOUNTPOINT)" \
|
||||
$(ZREPL_PLATFORMTEST_STOP_AND_KEEP) \
|
||||
$(ZREPL_PLATFORMTEST_ARGS)
|
||||
|
||||
##################### NOARCH #####################
|
||||
.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
|
||||
@for pkg in $(_TESTPKGS); do \
|
||||
echo "Vetting $$pkg"; \
|
||||
go vet "$$pkg" || exit 1; \
|
||||
done;
|
||||
|
||||
cover: artifacts
|
||||
@for pkg in $(_TESTPKGS); do \
|
||||
profile="$(ARTIFACTDIR)/cover-$$(basename $$pkg).out"; \
|
||||
go test -coverprofile "$$profile" $$pkg || exit 1; \
|
||||
if [ -f "$$profile" ]; then \
|
||||
go tool cover -html="$$profile" -o "$${profile}.html" || exit 2; \
|
||||
fi; \
|
||||
done;
|
||||
|
||||
$(ARTIFACTDIR):
|
||||
mkdir -p "$@"
|
||||
|
||||
$(ARTIFACTDIR)/docs: $(ARTIFACTDIR)
|
||||
mkdir -p "$@"
|
||||
|
||||
noarch: $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs
|
||||
# pass
|
||||
$(ARTIFACTDIR)/bash_completion: $(RELEASE_BINS)
|
||||
artifacts/$(THIS_PLATFORM_RELEASE_BIN) bashcomp "$@"
|
||||
|
||||
$(ARTIFACTDIR)/bash_completion:
|
||||
$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
|
||||
artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion bash "$@"
|
||||
|
||||
$(ARTIFACTDIR)/_zrepl.zsh_completion:
|
||||
$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
|
||||
artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion zsh "$@"
|
||||
|
||||
$(ARTIFACTDIR)/go_env.txt:
|
||||
$(GO_ENV_VARS) $(GO) env > $@
|
||||
$(ARTIFACTDIR)/go_version.txt:
|
||||
go version > $@
|
||||
|
||||
docs: $(ARTIFACTDIR)/docs
|
||||
make -C docs \
|
||||
@@ -190,3 +76,31 @@ docs-clean:
|
||||
make -C docs \
|
||||
clean \
|
||||
BUILDDIR=../artifacts/docs
|
||||
|
||||
|
||||
.PHONY: $(RELEASE_BINS)
|
||||
# TODO: two wildcards possible
|
||||
$(RELEASE_BINS): $(ARTIFACTDIR)/zrepl-%-amd64: generate $(ARTIFACTDIR) vet test
|
||||
@echo "INFO: In case of missing dependencies, run 'make vendordeps'"
|
||||
GOOS=$* GOARCH=amd64 $(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$*-amd64"
|
||||
|
||||
$(RELEASE_NOARCH): docs $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_version.txt
|
||||
tar --mtime='1970-01-01' --sort=name \
|
||||
--transform 's/$(ARTIFACTDIR)/zrepl-$(ZREPL_VERSION)-noarch/' \
|
||||
-acf $@ \
|
||||
$(ARTIFACTDIR)/docs/html \
|
||||
$(ARTIFACTDIR)/bash_completion \
|
||||
$(ARTIFACTDIR)/go_version.txt
|
||||
|
||||
release: $(RELEASE_BINS) $(RELEASE_NOARCH)
|
||||
rm -rf "$(ARTIFACTDIR)/release"
|
||||
mkdir -p "$(ARTIFACTDIR)/release"
|
||||
cp $^ "$(ARTIFACTDIR)/release"
|
||||
cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
|
||||
@if echo "$(ZREPL_VERSION)" | grep dirty > /dev/null; then\
|
||||
echo '[WARN] Do not publish the artifacts, make variable ZREPL_VERSION=$(ZREPL_VERSION) indicates they are dirty!'; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
clean: docs-clean
|
||||
rm -rf "$(ARTIFACTDIR)"
|
||||
|
||||
@@ -1,13 +1,5 @@
|
||||
[](https://github.com/zrepl/zrepl/blob/master/LICENSE)
|
||||
[](https://golang.org/)
|
||||
[](https://zrepl.github.io)
|
||||
[](https://www.patreon.com/zrepl)
|
||||
[](https://liberapay.com/zrepl/donate)
|
||||
[](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
|
||||
[](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl)
|
||||
|
||||
# zrepl
|
||||
zrepl is a one-stop ZFS backup & replication solution.
|
||||
zrepl is a ZFS filesystem backup & replication solution written in Go.
|
||||
|
||||
## User Documentation
|
||||
|
||||
@@ -24,7 +16,6 @@ zrepl is a one-stop ZFS backup & replication solution.
|
||||
If so, think of an expressive configuration example.
|
||||
2. Think of at least one use case that generalizes from your concrete application.
|
||||
3. Open an issue on GitHub with example conf & use case attached.
|
||||
4. **Optional**: [Post a bounty](https://www.bountysource.com/teams/zrepl) on the issue, or [contact Christian Schwarz](https://cschwarz.com) for contract work.
|
||||
|
||||
The above does not apply if you already implemented everything.
|
||||
Check out the *Coding Workflow* section below for details.
|
||||
@@ -35,73 +26,35 @@ Check out the *Coding Workflow* section below for details.
|
||||
* Make sure your distro is compatible with the paths in `docs/installation.rst`.
|
||||
* Ship a default config that adheres to your distro's `hier` and logging system.
|
||||
* Ship a service manager file and _please_ try to upstream it to this repository.
|
||||
* `dist/systemd` contains a Systemd unit template.
|
||||
* Ship other material provided in `./dist`, e.g. in `/usr/share/zrepl/`.
|
||||
* Use `make release ZREPL_VERSION='mydistro-1.2.3_1'`
|
||||
* Your distro's name and any versioning supplemental to zrepl's (e.g. package revision) should be in this string
|
||||
* Use `make platformtest` **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
|
||||
* Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
|
||||
|
||||
## Developer Documentation
|
||||
|
||||
zrepl is written in [Go](https://golang.org) and uses [Go modules](https://github.com/golang/go/wiki/Modules) to manage dependencies.
|
||||
The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
|
||||
First, use `./lazy.sh devsetup` to install build dependencies and read `docs/installation.rst -> Compiling from Source`.
|
||||
|
||||
To get started, run `./lazy.sh devsetup` to easily install build dependencies and read `docs/installation.rst -> Compiling from Source`.
|
||||
`lazy.sh` uses `python3-pip` to fetch the build dependencies for the docs - you might want to use a [venv](https://docs.python.org/3/library/venv.html).
|
||||
If you just want to install the Go dependencies, run `./lazy.sh godep`.
|
||||
### Overall Architecture
|
||||
|
||||
The application architecture is documented as part of the user docs in the *Implementation* section (`docs/content/impl`).
|
||||
Make sure to develop an understanding how zrepl is typically used by studying the user docs first.
|
||||
|
||||
### Project Structure
|
||||
|
||||
```
|
||||
├── artifacts # build artifcats generate by make
|
||||
├── cli # wrapper around CLI package cobra
|
||||
├── client # all subcommands that are not `daemon`
|
||||
├── config # config data types (=> package yaml-config)
|
||||
│ └── samples
|
||||
├── daemon # the implementation of `zrepl daemon` subcommand
|
||||
│ ├── filters
|
||||
│ ├── hooks # snapshot hooks
|
||||
│ ├── job # job implementations
|
||||
│ ├── logging # logging outlets + formatters
|
||||
│ ├── nethelpers
|
||||
│ ├── prometheus
|
||||
│ ├── pruner # pruner implementation
|
||||
│ ├── snapper # snapshotter implementation
|
||||
├── dist # supplemental material for users & package maintainers
|
||||
├── cmd
|
||||
│ ├── sampleconf # example configuration
|
||||
├── docs # sphinx-based documentation
|
||||
│ ├── **/*.rst # documentation in reStructuredText
|
||||
│ ├── sphinxconf
|
||||
│ │ └── conf.py # sphinx config (see commit 445a280 why its not in docs/)
|
||||
│ ├── requirements.txt # pip3 requirements to build documentation
|
||||
│ ├── publish.sh # shell script for automated rendering & deploy to zrepl.github.io repo
|
||||
│ └── public_git # checkout of zrepl.github.io managed by above shell script
|
||||
├── endpoint # implementation of replication endpoints (=> package replication)
|
||||
├── logger # our own logger package
|
||||
├── platformtest # test suite for our zfs abstractions (error classification, etc)
|
||||
├── pruning # pruning rules (the logic, not the actual execution)
|
||||
│ └── retentiongrid
|
||||
├── replication
|
||||
│ ├── driver # the driver of the replication logic (status reporting, error handling)
|
||||
│ ├── logic # planning & executing replication steps via rpc
|
||||
| | └── pdu # the generated gRPC & protobuf code used in replication (and endpoints)
|
||||
│ └── report # the JSON-serializable report datastructures exposed to the client
|
||||
├── rpc # the hybrid gRPC + ./dataconn RPC client: connects to a remote replication.Endpoint
|
||||
│ ├── dataconn # Bulk data-transfer RPC protocol
|
||||
│ ├── grpcclientidentity # adaptor to inject package transport's 'client identity' concept into gRPC contexts
|
||||
│ ├── netadaptor # adaptor to convert a package transport's Connecter and Listener into net.* primitives
|
||||
│ ├── transportmux # TCP connecter and listener used to split control & data traffic
|
||||
│ └── versionhandshake # replication protocol version handshake perfomed on newly established connections
|
||||
├── tlsconf # abstraction for Go TLS server + client config
|
||||
├── transport # transport implementations
|
||||
│ ├── fromconfig
|
||||
│ ├── local
|
||||
│ ├── ssh
|
||||
│ ├── tcp
|
||||
│ └── tls
|
||||
│ ├── public_git # checkout of zrepl.github.io managed by above shell script
|
||||
├── logger # logger package used by zrepl
|
||||
├── rpc # rpc protocol implementation
|
||||
├── util
|
||||
├── version # abstraction for versions (filled during build by Makefile)
|
||||
└── zfs # zfs(8) wrappers
|
||||
└── zfs # ZFS wrappers, filesystemm diffing
|
||||
```
|
||||
|
||||
### Coding Workflow
|
||||
@@ -134,15 +87,3 @@ There will not be a big refactoring (an attempt was made, but it's destroying to
|
||||
|
||||
However, new contributions & patches should fix naming without further notice in the commit message.
|
||||
|
||||
### RPC debugging
|
||||
|
||||
Optionally, there are various RPC-related environment variables, that if set to something != `""` will produce additional debug output on stderr:
|
||||
|
||||
https://github.com/zrepl/zrepl/blob/master/rpc/rpc_debug.go#L11
|
||||
|
||||
https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/dataconn_debug.go#L11
|
||||
|
||||
https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/stream/stream_debug.go#L11
|
||||
|
||||
https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/heartbeatconn/heartbeatconn_debug.go#L11
|
||||
|
||||
|
||||
+5
-19
@@ -1,30 +1,16 @@
|
||||
FROM golang:latest
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
python3-pip \
|
||||
unzip \
|
||||
gawk
|
||||
|
||||
ADD build.installprotoc.bash ./
|
||||
RUN bash build.installprotoc.bash
|
||||
python3-pip
|
||||
|
||||
ADD lazy.sh /tmp/lazy.sh
|
||||
ADD docs/requirements.txt /tmp/requirements.txt
|
||||
ENV ZREPL_LAZY_DOCS_REQPATH=/tmp/requirements.txt
|
||||
RUN /tmp/lazy.sh docdep
|
||||
RUN /tmp/lazy.sh devsetup
|
||||
|
||||
# prepare volume mount of git checkout to /zrepl
|
||||
RUN mkdir -p /src/github.com/zrepl/zrepl
|
||||
RUN mkdir -p /.cache && chmod -R 0777 /.cache
|
||||
|
||||
# $GOPATH is /go
|
||||
# Go 1.12 doesn't use modules within GOPATH, but 1.13 and later do
|
||||
# => store source outside of GOPATH
|
||||
WORKDIR /src
|
||||
|
||||
# Install build tools (e.g. protobuf generator, stringer) into $GOPATH/bin
|
||||
ADD build/ /tmp/build
|
||||
RUN /tmp/lazy.sh godep
|
||||
|
||||
RUN mkdir -p /go/src/github.com/zrepl/zrepl
|
||||
RUN chmod -R 0777 /go
|
||||
WORKDIR /go/src/github.com/zrepl/zrepl
|
||||
|
||||
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
set -x
|
||||
|
||||
MACH=$(uname -m)
|
||||
MACH="${MACH/aarch64/aarch_64}"
|
||||
|
||||
VERSION=3.6.1
|
||||
FILENAME=protoc-"$VERSION"-linux-"$MACH".zip
|
||||
|
||||
if [ -e "$FILENAME" ]; then
|
||||
echo "$FILENAME" already exists 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
wget https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
|
||||
|
||||
stat "$FILENAME"
|
||||
|
||||
sha256sum -c --ignore-missing <<EOF
|
||||
6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807 protoc-3.6.1-linux-x86_64.zip
|
||||
af8e5aaaf39ddec62ec8dd2be1b8d9602c6da66564883a16393ade5f71170922 protoc-3.6.1-linux-aarch_64.zip
|
||||
EOF
|
||||
|
||||
unzip -d /usr "$FILENAME"
|
||||
@@ -1,51 +0,0 @@
|
||||
module github.com/zrepl/zrepl/build
|
||||
|
||||
go 1.12
|
||||
|
||||
require (
|
||||
github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 // indirect
|
||||
github.com/alvaroloes/enumer v1.1.1
|
||||
github.com/fatih/color v1.7.0 // indirect
|
||||
github.com/go-critic/go-critic v0.3.4 // indirect
|
||||
github.com/gogo/protobuf v1.2.1 // indirect
|
||||
github.com/golang/mock v1.2.0 // indirect
|
||||
github.com/golang/protobuf v1.2.0
|
||||
github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 // indirect
|
||||
github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 // indirect
|
||||
github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d // indirect
|
||||
github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 // indirect
|
||||
github.com/golangci/golangci-lint v1.17.1
|
||||
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb // indirect
|
||||
github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 // indirect
|
||||
github.com/golangci/misspell v0.3.4 // indirect
|
||||
github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 // indirect
|
||||
github.com/google/go-cmp v0.3.0 // indirect
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/pkg/errors v0.8.1 // indirect
|
||||
github.com/sirupsen/logrus v1.4.2 // indirect
|
||||
github.com/spf13/afero v1.2.2 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
github.com/spf13/viper v1.3.2 // indirect
|
||||
github.com/stretchr/testify v1.4.0 // indirect
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 // indirect
|
||||
golang.org/x/sys v0.0.0-20191010194322-b09406accb47 // indirect
|
||||
golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b
|
||||
mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe // indirect
|
||||
sourcegraph.com/sqs/pbtypes v1.0.0 // indirect
|
||||
)
|
||||
|
||||
// invalid dates in transitive dependencies (first validated in Go 1.13, didn't fail in earlier Go versions)
|
||||
replace (
|
||||
github.com/go-critic/go-critic v0.0.0-20181204210945-1df300866540 => github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540
|
||||
github.com/go-critic/go-critic v0.0.0-20181204210945-ee9bf5809ead => github.com/go-critic/go-critic v0.3.5-0.20190210220443-ee9bf5809ead
|
||||
github.com/golangci/errcheck v0.0.0-20181003203344-ef45e06d44b6 => github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6
|
||||
github.com/golangci/go-tools v0.0.0-20180109140146-35a9f45a5db0 => github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0
|
||||
github.com/golangci/go-tools v0.0.0-20180109140146-af6baa5dc196 => github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196
|
||||
github.com/golangci/gofmt v0.0.0-20181105071733-0b8337e80d98 => github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98
|
||||
github.com/golangci/gosec v0.0.0-20180901114220-66fb7fc33547 => github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547
|
||||
github.com/golangci/ineffassign v0.0.0-20180808204949-42439a7714cc => github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc
|
||||
github.com/golangci/lint-1 v0.0.0-20180610141402-ee948d087217 => github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217
|
||||
golang.org/x/tools v0.0.0-20190125232054-379209517ffe => golang.org/x/tools v0.0.0-20190205201329-379209517ffe
|
||||
mvdan.cc/unparam v0.0.0-20190124213536-fbb59629db34 => mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34
|
||||
)
|
||||
-280
@@ -1,280 +0,0 @@
|
||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/OpenPeeDeeP/depguard v0.0.0-20180806142446-a69c782687b2/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
|
||||
github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 h1:pFks+oaqVWDFq0KsLQZFB2pQGB5Us0HfMSBENtieXOs=
|
||||
github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
|
||||
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
github.com/alvaroloes/enumer v1.1.1 h1:v+1scNqEhSFAPb9tLwblQegeTXJhnw8hf9O0amTbOvQ=
|
||||
github.com/alvaroloes/enumer v1.1.1/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
|
||||
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/go-critic/go-critic v0.3.4 h1:FYaiaLjX0Nqei80KPhm4CyFQUBbmJwSrHxQ73taaGBc=
|
||||
github.com/go-critic/go-critic v0.3.4/go.mod h1:AHR42Lk/E/aOznsrYdMYeIQS5RH10HZHSqP+rD6AJrc=
|
||||
github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
|
||||
github.com/go-lintpack/lintpack v0.5.1/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
|
||||
github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0=
|
||||
github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
|
||||
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
|
||||
github.com/go-toolsmith/astcast v0.0.0-20181028201508-b7a89ed70af1/go.mod h1:TEo3Ghaj7PsZawQHxT/oBvo4HK/sl1RcuUHDKTTju+o=
|
||||
github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=
|
||||
github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
|
||||
github.com/go-toolsmith/astcopy v0.0.0-20180903214859-79b422d080c4/go.mod h1:c9CPdq2AzM8oPomdlPniEfPAC6g1s7NqZzODt8y6ib8=
|
||||
github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8=
|
||||
github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
|
||||
github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
|
||||
github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ=
|
||||
github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
|
||||
github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
|
||||
github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k=
|
||||
github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
|
||||
github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
|
||||
github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
|
||||
github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg=
|
||||
github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
|
||||
github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8 h1:vVouagbdmqTVlCIAxpyYsNNTbkKZ3V66VpKOLU/s6W4=
|
||||
github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
|
||||
github.com/go-toolsmith/pkgload v1.0.0 h1:4DFWWMXVfbcN5So1sBNW9+yeiMqLFGl1wFLTL5R0Tgg=
|
||||
github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
|
||||
github.com/go-toolsmith/strparse v0.0.0-20180903215201-830b6daa1241/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
|
||||
github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4=
|
||||
github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
|
||||
github.com/go-toolsmith/typep v0.0.0-20181030061450-d63dc7650676/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
|
||||
github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA=
|
||||
github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
|
||||
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
|
||||
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
|
||||
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
||||
github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0=
|
||||
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
|
||||
github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
|
||||
github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
|
||||
github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w=
|
||||
github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
|
||||
github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw=
|
||||
github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
|
||||
github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 h1:MRhC9XbUjE6XDOInSJ8pwHuPagqsyO89QDU9IdVhe3o=
|
||||
github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
|
||||
github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
|
||||
github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8=
|
||||
github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
|
||||
github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
|
||||
github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d h1:pXTK/gkVNs7Zyy7WKgLXmpQ5bHTrq5GDsp8R9Qs67g0=
|
||||
github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
|
||||
github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU=
|
||||
github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
|
||||
github.com/golangci/golangci-lint v1.17.1 h1:lc8Hf9GPCjIr0hg3S/xhvFT1+Hydass8F1xchr8jkME=
|
||||
github.com/golangci/golangci-lint v1.17.1/go.mod h1:+5sJSl2h3aly+fpmL2meSP8CaSKua2E4Twi9LPy7b1g=
|
||||
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb h1:Bi7BYmZVg4C+mKGi8LeohcP2GGUl2XJD4xCkJoZSaYc=
|
||||
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb/go.mod h1:ON/c2UR0VAAv6ZEAFKhjCLplESSmRFfZcDLASbI1GWo=
|
||||
github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
|
||||
github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
|
||||
github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
|
||||
github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 h1:utua3L2IbQJmauC5IXdEA547bcoU5dozgQAfc8Onsg4=
|
||||
github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
|
||||
github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
|
||||
github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
|
||||
github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
|
||||
github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
|
||||
github.com/golangci/misspell v0.3.4 h1:kAD5J0611Zo2VirUn9KFP15RjEqkmfEfnFxyIVxZCYg=
|
||||
github.com/golangci/misspell v0.3.4/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
|
||||
github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us=
|
||||
github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
|
||||
github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
|
||||
github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 h1:XQKc8IYQOeRwVs36tDrEmTgDgP88d5iEURwpmtiAlOM=
|
||||
github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
|
||||
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
|
||||
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
|
||||
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw=
|
||||
github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
|
||||
github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
|
||||
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM=
|
||||
github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
|
||||
github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
|
||||
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
|
||||
github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
|
||||
github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
|
||||
github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663 h1:Ri1EhipkbhWsffPJ3IPlrb4SkTOPa2PfRXp3jchBczw=
|
||||
github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
|
||||
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
|
||||
github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
|
||||
github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 h1:/I3lTljEEDNYLho3/FUB7iD/oc2cEFgVmbHzV+O0PtU=
|
||||
github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ=
|
||||
github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
|
||||
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.2.1/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
|
||||
github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
|
||||
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM=
|
||||
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
|
||||
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc=
|
||||
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
|
||||
github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
|
||||
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs=
|
||||
github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
|
||||
github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
|
||||
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
|
||||
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
|
||||
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v0.0.2 h1:NfkwRbgViGoyjBKsLI0QMDcuMnhM+SBg3T0cGfpvKDE=
|
||||
github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
|
||||
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
|
||||
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
|
||||
github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
|
||||
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb h1:lI9ufgFfvuqRctP9Ny8lDDLbSWCMxBPletcSqrnyFYM=
|
||||
github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
|
||||
github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
|
||||
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190313220215-9f648a60d977 h1:actzWV6iWn3GLqN8dZjzsB+CLt+gaV2+wsxroxiQI8I=
|
||||
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
|
||||
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181205014116-22934f0fdb62/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8 h1:b0PLhFjEMqrIqsD4rS5sp0YxBYgdqKzX0KkkGGKLV8I=
|
||||
golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b h1:iEAPfYPbYbxG/2lNN4cMOHkmgKNsCuUwkxlDCK46UlU=
|
||||
golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
|
||||
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
|
||||
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=
|
||||
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
|
||||
mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
|
||||
mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe h1:Ekmnp+NcP2joadI9pbK4Bva87QKZSeY7le//oiMrc9g=
|
||||
mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe/go.mod h1:BnhuWBAqxH3+J5bDybdxgw5ZfS+DsVd4iylsKQePN8o=
|
||||
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
|
||||
sourcegraph.com/sqs/pbtypes v1.0.0 h1:f7lAwqviDEGvON4kRv0o5V7FT/IQK+tbkF664XMbP3o=
|
||||
sourcegraph.com/sqs/pbtypes v1.0.0/go.mod h1:3AciMUv4qUuRHRHhOG4TZOB+72GdPVz5k+c648qsFS4=
|
||||
@@ -1,11 +0,0 @@
|
||||
// +build tools
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
_ "github.com/alvaroloes/enumer"
|
||||
_ "github.com/golang/protobuf/protoc-gen-go"
|
||||
_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
|
||||
_ "golang.org/x/tools/cmd/goimports"
|
||||
_ "golang.org/x/tools/cmd/stringer"
|
||||
)
|
||||
-148
@@ -1,148 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
)
|
||||
|
||||
var rootArgs struct {
|
||||
configPath string
|
||||
}
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "zrepl",
|
||||
Short: "One-stop ZFS replication solution",
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
|
||||
}
|
||||
|
||||
var genCompletionCmd = &cobra.Command{
|
||||
Use: "gencompletion",
|
||||
Short: "generate shell auto-completions",
|
||||
}
|
||||
|
||||
type completionCmdInfo struct {
|
||||
genFunc func(outpath string) error
|
||||
help string
|
||||
}
|
||||
|
||||
var completionCmdMap = map[string]completionCmdInfo{
|
||||
"zsh": {
|
||||
rootCmd.GenZshCompletionFile,
|
||||
" save to file `_zrepl` in your zsh's $fpath",
|
||||
},
|
||||
"bash": {
|
||||
rootCmd.GenBashCompletionFile,
|
||||
" save to a path and source that path in your .bashrc",
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
for sh, info := range completionCmdMap {
|
||||
sh, info := sh, info
|
||||
genCompletionCmd.AddCommand(&cobra.Command{
|
||||
Use: fmt.Sprintf("%s path/to/out/file", sh),
|
||||
Short: fmt.Sprintf("generate %s completions", sh),
|
||||
Example: info.help,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
if len(args) != 1 {
|
||||
fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
|
||||
err := cmd.Usage()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := info.genFunc(args[0]); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error generating %s completion: %s", sh, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
rootCmd.AddCommand(genCompletionCmd)
|
||||
}
|
||||
|
||||
type Subcommand struct {
|
||||
Use string
|
||||
Short string
|
||||
Example string
|
||||
NoRequireConfig bool
|
||||
Run func(subcommand *Subcommand, args []string) error
|
||||
SetupFlags func(f *pflag.FlagSet)
|
||||
SetupSubcommands func() []*Subcommand
|
||||
|
||||
config *config.Config
|
||||
configErr error
|
||||
}
|
||||
|
||||
func (s *Subcommand) ConfigParsingError() error {
|
||||
return s.configErr
|
||||
}
|
||||
|
||||
func (s *Subcommand) Config() *config.Config {
|
||||
if !s.NoRequireConfig && s.config == nil {
|
||||
panic("command that requires config is running and has no config set")
|
||||
}
|
||||
return s.config
|
||||
}
|
||||
|
||||
func (s *Subcommand) run(cmd *cobra.Command, args []string) {
|
||||
s.tryParseConfig()
|
||||
err := s.Run(s, args)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Subcommand) tryParseConfig() {
|
||||
config, err := config.ParseConfig(rootArgs.configPath)
|
||||
s.configErr = err
|
||||
if err != nil {
|
||||
if s.NoRequireConfig {
|
||||
// doesn't matter
|
||||
return
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "could not parse config: %s\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
s.config = config
|
||||
}
|
||||
|
||||
func AddSubcommand(s *Subcommand) {
|
||||
addSubcommandToCobraCmd(rootCmd, s)
|
||||
}
|
||||
|
||||
func addSubcommandToCobraCmd(c *cobra.Command, s *Subcommand) {
|
||||
cmd := cobra.Command{
|
||||
Use: s.Use,
|
||||
Short: s.Short,
|
||||
Example: s.Example,
|
||||
}
|
||||
if s.SetupSubcommands == nil {
|
||||
cmd.Run = s.run
|
||||
} else {
|
||||
for _, sub := range s.SetupSubcommands() {
|
||||
addSubcommandToCobraCmd(&cmd, sub)
|
||||
}
|
||||
}
|
||||
if s.SetupFlags != nil {
|
||||
s.SetupFlags(cmd.Flags())
|
||||
}
|
||||
c.AddCommand(&cmd)
|
||||
}
|
||||
|
||||
func Run() {
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -1,120 +0,0 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/kr/pretty"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
"github.com/zrepl/yaml-config"
|
||||
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/job"
|
||||
"github.com/zrepl/zrepl/daemon/logging"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
)
|
||||
|
||||
var configcheckArgs struct {
|
||||
format string
|
||||
what string
|
||||
}
|
||||
|
||||
var ConfigcheckCmd = &cli.Subcommand{
|
||||
Use: "configcheck",
|
||||
Short: "check if config can be parsed without errors",
|
||||
SetupFlags: func(f *pflag.FlagSet) {
|
||||
f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]")
|
||||
f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]")
|
||||
},
|
||||
Run: func(subcommand *cli.Subcommand, args []string) error {
|
||||
formatMap := map[string]func(interface{}){
|
||||
"": func(i interface{}) {},
|
||||
"pretty": func(i interface{}) {
|
||||
if _, err := pretty.Println(i); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
},
|
||||
"json": func(i interface{}) {
|
||||
if err := json.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
},
|
||||
"yaml": func(i interface{}) {
|
||||
if err := yaml.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
formatter, ok := formatMap[configcheckArgs.format]
|
||||
if !ok {
|
||||
return fmt.Errorf("unsupported --format %q", configcheckArgs.format)
|
||||
}
|
||||
|
||||
var hadErr bool
|
||||
// further: try to build jobs
|
||||
confJobs, err := job.JobsFromConfig(subcommand.Config())
|
||||
if err != nil {
|
||||
err := errors.Wrap(err, "cannot build jobs from config")
|
||||
if configcheckArgs.what == "jobs" {
|
||||
return err
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "%s\n", err)
|
||||
confJobs = nil
|
||||
hadErr = true
|
||||
}
|
||||
}
|
||||
|
||||
// further: try to build logging outlets
|
||||
outlets, err := logging.OutletsFromConfig(*subcommand.Config().Global.Logging)
|
||||
if err != nil {
|
||||
err := errors.Wrap(err, "cannot build logging from config")
|
||||
if configcheckArgs.what == "logging" {
|
||||
return err
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "%s\n", err)
|
||||
outlets = nil
|
||||
hadErr = true
|
||||
}
|
||||
}
|
||||
|
||||
whatMap := map[string]func(){
|
||||
"all": func() {
|
||||
o := struct {
|
||||
config *config.Config
|
||||
jobs []job.Job
|
||||
logging *logger.Outlets
|
||||
}{
|
||||
subcommand.Config(),
|
||||
confJobs,
|
||||
outlets,
|
||||
}
|
||||
formatter(o)
|
||||
},
|
||||
"config": func() {
|
||||
formatter(subcommand.Config())
|
||||
},
|
||||
"jobs": func() {
|
||||
formatter(confJobs)
|
||||
},
|
||||
"logging": func() {
|
||||
formatter(outlets)
|
||||
},
|
||||
}
|
||||
|
||||
wf, ok := whatMap[configcheckArgs.what]
|
||||
if !ok {
|
||||
return fmt.Errorf("unsupported --format %q", configcheckArgs.what)
|
||||
}
|
||||
wf()
|
||||
|
||||
if hadErr {
|
||||
return fmt.Errorf("config parsing failed")
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
func controlHttpClient(sockpath string) (client http.Client, err error) {
|
||||
return http.Client{
|
||||
Transport: &http.Transport{
|
||||
DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
|
||||
return net.Dial("unix", sockpath)
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func jsonRequestResponse(c http.Client, endpoint string, req interface{}, res interface{}) error {
|
||||
var buf bytes.Buffer
|
||||
encodeErr := json.NewEncoder(&buf).Encode(req)
|
||||
if encodeErr != nil {
|
||||
return encodeErr
|
||||
}
|
||||
|
||||
resp, err := c.Post("http://unix"+endpoint, "application/json", &buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
var msg bytes.Buffer
|
||||
_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
|
||||
return errors.Errorf("%s", msg.String())
|
||||
}
|
||||
|
||||
decodeError := json.NewDecoder(resp.Body).Decode(&res)
|
||||
if decodeError != nil {
|
||||
return decodeError
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,271 +0,0 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/kr/pretty"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/job"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
)
|
||||
|
||||
var (
|
||||
MigrateCmd = &cli.Subcommand{
|
||||
Use: "migrate",
|
||||
Short: "perform migration of the on-disk / zfs properties",
|
||||
SetupSubcommands: func() []*cli.Subcommand {
|
||||
return migrations
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
var migrations = []*cli.Subcommand{
|
||||
&cli.Subcommand{
|
||||
Use: "0.0.X:0.1:placeholder",
|
||||
Run: doMigratePlaceholder0_1,
|
||||
SetupFlags: func(f *pflag.FlagSet) {
|
||||
f.BoolVar(&migratePlaceholder0_1Args.dryRun, "dry-run", false, "dry run")
|
||||
},
|
||||
},
|
||||
&cli.Subcommand{
|
||||
Use: "replication-cursor:v1-v2",
|
||||
Run: doMigrateReplicationCursor,
|
||||
SetupFlags: func(f *pflag.FlagSet) {
|
||||
f.BoolVar(&migrateReplicationCursorArgs.dryRun, "dry-run", false, "dry run")
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
var migratePlaceholder0_1Args struct {
|
||||
dryRun bool
|
||||
}
|
||||
|
||||
func doMigratePlaceholder0_1(sc *cli.Subcommand, args []string) error {
|
||||
if len(args) != 0 {
|
||||
return fmt.Errorf("migration does not take arguments, got %v", args)
|
||||
}
|
||||
|
||||
cfg := sc.Config()
|
||||
|
||||
ctx := context.Background()
|
||||
allFSS, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "cannot list filesystems")
|
||||
}
|
||||
|
||||
type workItem struct {
|
||||
jobName string
|
||||
rootFS *zfs.DatasetPath
|
||||
fss []*zfs.DatasetPath
|
||||
}
|
||||
var wis []workItem
|
||||
for i, j := range cfg.Jobs {
|
||||
var rfsS string
|
||||
switch job := j.Ret.(type) {
|
||||
case *config.SinkJob:
|
||||
rfsS = job.RootFS
|
||||
case *config.PullJob:
|
||||
rfsS = job.RootFS
|
||||
default:
|
||||
fmt.Printf("ignoring job %q (%d/%d, type %T)\n", j.Name(), i, len(cfg.Jobs), j.Ret)
|
||||
continue
|
||||
}
|
||||
rfs, err := zfs.NewDatasetPath(rfsS)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "root fs for job %q is not a valid dataset path", j.Name())
|
||||
}
|
||||
var fss []*zfs.DatasetPath
|
||||
for _, fs := range allFSS {
|
||||
if fs.HasPrefix(rfs) {
|
||||
fss = append(fss, fs)
|
||||
}
|
||||
}
|
||||
wis = append(wis, workItem{j.Name(), rfs, fss})
|
||||
}
|
||||
|
||||
for _, wi := range wis {
|
||||
fmt.Printf("job %q => migrate filesystems below root_fs %q\n", wi.jobName, wi.rootFS.ToString())
|
||||
if len(wi.fss) == 0 {
|
||||
fmt.Printf("\tno filesystems\n")
|
||||
continue
|
||||
}
|
||||
for _, fs := range wi.fss {
|
||||
fmt.Printf("\t%q ... ", fs.ToString())
|
||||
r, err := zfs.ZFSMigrateHashBasedPlaceholderToCurrent(ctx, fs, migratePlaceholder0_1Args.dryRun)
|
||||
if err != nil {
|
||||
fmt.Printf("error: %s\n", err)
|
||||
} else if !r.NeedsModification {
|
||||
fmt.Printf("unchanged (placeholder=%v)\n", r.OriginalState.IsPlaceholder)
|
||||
} else {
|
||||
fmt.Printf("migrate (placeholder=%v) (old value = %q)\n",
|
||||
r.OriginalState.IsPlaceholder, r.OriginalState.RawLocalPropertyValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var migrateReplicationCursorArgs struct {
|
||||
dryRun bool
|
||||
}
|
||||
|
||||
var bold = color.New(color.Bold)
|
||||
var succ = color.New(color.FgGreen)
|
||||
var fail = color.New(color.FgRed)
|
||||
|
||||
var migrateReplicationCursorSkipSentinel = fmt.Errorf("skipping this filesystem")
|
||||
|
||||
func doMigrateReplicationCursor(sc *cli.Subcommand, args []string) error {
|
||||
if len(args) != 0 {
|
||||
return fmt.Errorf("migration does not take arguments, got %v", args)
|
||||
}
|
||||
|
||||
cfg := sc.Config()
|
||||
jobs, err := job.JobsFromConfig(cfg)
|
||||
if err != nil {
|
||||
fmt.Printf("cannot parse config:\n%s\n\n", err)
|
||||
fmt.Printf("NOTE: this migration was released together with a change in job name requirements.\n")
|
||||
return fmt.Errorf("exiting migration after error")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
v1cursorJobs := make([]job.Job, 0, len(cfg.Jobs))
|
||||
for i, j := range cfg.Jobs {
|
||||
if jobs[i].Name() != j.Name() {
|
||||
panic("implementation error")
|
||||
}
|
||||
switch j.Ret.(type) {
|
||||
case *config.PushJob:
|
||||
v1cursorJobs = append(v1cursorJobs, jobs[i])
|
||||
case *config.SourceJob:
|
||||
v1cursorJobs = append(v1cursorJobs, jobs[i])
|
||||
default:
|
||||
fmt.Printf("ignoring job %q (%d/%d, type %T), not supposed to create v1 replication cursors\n", j.Name(), i, len(cfg.Jobs), j.Ret)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// scan all filesystems for v1 replication cursors
|
||||
|
||||
fss, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "list filesystems")
|
||||
}
|
||||
|
||||
var hadError bool
|
||||
for _, fs := range fss {
|
||||
|
||||
bold.Printf("INSPECT FILESYSTEM %q\n", fs.ToString())
|
||||
|
||||
err := doMigrateReplicationCursorFS(ctx, v1cursorJobs, fs)
|
||||
if err == migrateReplicationCursorSkipSentinel {
|
||||
bold.Printf("FILESYSTEM SKIPPED\n")
|
||||
} else if err != nil {
|
||||
hadError = true
|
||||
fail.Printf("MIGRATION FAILED: %s\n", err)
|
||||
} else {
|
||||
succ.Printf("FILESYSTEM %q COMPLETE\n", fs.ToString())
|
||||
}
|
||||
}
|
||||
|
||||
if hadError {
|
||||
fail.Printf("\n\none or more filesystems could not be migrated, please inspect output and or re-run migration")
|
||||
return errors.Errorf("")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func doMigrateReplicationCursorFS(ctx context.Context, v1CursorJobs []job.Job, fs *zfs.DatasetPath) error {
|
||||
|
||||
var owningJob job.Job = nil
|
||||
for _, job := range v1CursorJobs {
|
||||
conf := job.SenderConfig()
|
||||
if conf == nil {
|
||||
continue
|
||||
}
|
||||
pass, err := conf.FSF.Filter(fs)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "filesystem filter error in job %q for fs %q", job.Name(), fs.ToString())
|
||||
}
|
||||
if !pass {
|
||||
continue
|
||||
}
|
||||
if owningJob != nil {
|
||||
return errors.Errorf("jobs %q and %q both match %q\ncannot attribute replication cursor to either one", owningJob.Name(), job.Name(), fs)
|
||||
}
|
||||
owningJob = job
|
||||
}
|
||||
if owningJob == nil {
|
||||
fmt.Printf("no job's Filesystems filter matches\n")
|
||||
return migrateReplicationCursorSkipSentinel
|
||||
}
|
||||
fmt.Printf("identified owning job %q\n", owningJob.Name())
|
||||
|
||||
bookmarks, err := zfs.ZFSListFilesystemVersions(fs, zfs.ListFilesystemVersionsOptions{
|
||||
Types: zfs.Bookmarks,
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "list filesystem versions of %q", fs.ToString())
|
||||
}
|
||||
|
||||
var oldCursor *zfs.FilesystemVersion
|
||||
for i, fsv := range bookmarks {
|
||||
_, _, err := endpoint.ParseReplicationCursorBookmarkName(fsv.ToAbsPath(fs))
|
||||
if err != endpoint.ErrV1ReplicationCursor {
|
||||
continue
|
||||
}
|
||||
|
||||
if oldCursor != nil {
|
||||
fmt.Printf("unexpected v1 replication cursor candidate: %q", fsv.ToAbsPath(fs))
|
||||
return errors.Wrap(err, "multiple filesystem versions identified as v1 replication cursors")
|
||||
}
|
||||
|
||||
oldCursor = &bookmarks[i]
|
||||
|
||||
}
|
||||
|
||||
if oldCursor == nil {
|
||||
bold.Printf("no v1 replication cursor found for filesystem %q\n", fs.ToString())
|
||||
return migrateReplicationCursorSkipSentinel
|
||||
}
|
||||
|
||||
fmt.Printf("found v1 replication cursor:\n%s\n", pretty.Sprint(oldCursor))
|
||||
|
||||
mostRecentNew, err := endpoint.GetMostRecentReplicationCursorOfJob(ctx, fs.ToString(), owningJob.SenderConfig().JobID)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "get most recent v2 replication cursor")
|
||||
}
|
||||
|
||||
if mostRecentNew == nil {
|
||||
return errors.Errorf("no v2 replication cursor found for job %q on filesystem %q", owningJob.SenderConfig().JobID, fs.ToString())
|
||||
}
|
||||
|
||||
fmt.Printf("most recent v2 replication cursor:\n%#v", oldCursor)
|
||||
|
||||
if !(mostRecentNew.CreateTXG >= oldCursor.CreateTXG) {
|
||||
return errors.Errorf("v1 replication cursor createtxg is higher than v2 cursor's, skipping this filesystem")
|
||||
}
|
||||
|
||||
fmt.Printf("determined that v2 cursor is bookmark of same or newer version than v1 cursor\n")
|
||||
fmt.Printf("destroying v1 cursor %q\n", oldCursor.ToAbsPath(fs))
|
||||
|
||||
if migrateReplicationCursorArgs.dryRun {
|
||||
succ.Printf("DRY RUN\n")
|
||||
} else {
|
||||
if err := zfs.ZFSDestroyFilesystemVersion(ctx, fs, oldCursor); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
package client
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestMigrationsUnambiguousNames(t *testing.T) {
|
||||
names := make(map[string]bool)
|
||||
for _, mig := range migrations {
|
||||
if _, ok := names[mig.Use]; ok {
|
||||
t.Errorf("duplicate migration name %q", mig.Use)
|
||||
t.FailNow()
|
||||
return
|
||||
} else {
|
||||
names[mig.Use] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon"
|
||||
)
|
||||
|
||||
var pprofArgs struct {
|
||||
daemon.PprofServerControlMsg
|
||||
}
|
||||
|
||||
var PprofCmd = &cli.Subcommand{
|
||||
Use: "pprof off | [on TCP_LISTEN_ADDRESS]",
|
||||
Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
|
||||
Run: func(subcommand *cli.Subcommand, args []string) error {
|
||||
if len(args) < 1 {
|
||||
goto enargs
|
||||
}
|
||||
switch args[0] {
|
||||
case "on":
|
||||
pprofArgs.Run = true
|
||||
if len(args) != 2 {
|
||||
return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
|
||||
}
|
||||
pprofArgs.HttpListenAddress = args[1]
|
||||
case "off":
|
||||
if len(args) != 1 {
|
||||
goto enargs
|
||||
}
|
||||
pprofArgs.Run = false
|
||||
}
|
||||
|
||||
RunPProf(subcommand.Config())
|
||||
return nil
|
||||
enargs:
|
||||
return errors.New("invalid number of positional arguments")
|
||||
|
||||
},
|
||||
}
|
||||
|
||||
func RunPProf(conf *config.Config) {
|
||||
log := log.New(os.Stderr, "", 0)
|
||||
|
||||
die := func() {
|
||||
log.Printf("exiting after error")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
log.Printf("connecting to zrepl daemon")
|
||||
|
||||
httpc, err := controlHttpClient(conf.Global.Control.SockPath)
|
||||
if err != nil {
|
||||
log.Printf("error creating http client: %s", err)
|
||||
die()
|
||||
}
|
||||
err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofArgs.PprofServerControlMsg, struct{}{})
|
||||
if err != nil {
|
||||
log.Printf("error sending control message: %s", err)
|
||||
die()
|
||||
}
|
||||
log.Printf("finished")
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon"
|
||||
)
|
||||
|
||||
var SignalCmd = &cli.Subcommand{
|
||||
Use: "signal [wakeup|reset] JOB",
|
||||
Short: "wake up a job from wait state or abort its current invocation",
|
||||
Run: func(subcommand *cli.Subcommand, args []string) error {
|
||||
return runSignalCmd(subcommand.Config(), args)
|
||||
},
|
||||
}
|
||||
|
||||
func runSignalCmd(config *config.Config, args []string) error {
|
||||
if len(args) != 2 {
|
||||
return errors.Errorf("Expected 2 arguments: [wakeup|reset] JOB")
|
||||
}
|
||||
|
||||
httpc, err := controlHttpClient(config.Global.Control.SockPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = jsonRequestResponse(httpc, daemon.ControlJobEndpointSignal,
|
||||
struct {
|
||||
Name string
|
||||
Op string
|
||||
}{
|
||||
Name: args[1],
|
||||
Op: args[0],
|
||||
},
|
||||
struct{}{},
|
||||
)
|
||||
return err
|
||||
}
|
||||
@@ -1,810 +0,0 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
// tcell is the termbox-compatible library for abstracting away escape sequences, etc.
|
||||
// as of tcell#252, the number of default distributed terminals is relatively limited
|
||||
// additional terminal definitions can be included via side-effect import
|
||||
// See https://github.com/gdamore/tcell/blob/master/terminfo/base/base.go
|
||||
// See https://github.com/gdamore/tcell/issues/252#issuecomment-533836078
|
||||
"github.com/gdamore/tcell/termbox"
|
||||
_ "github.com/gdamore/tcell/terminfo/s/screen" // tmux on FreeBSD 11 & 12 without ncurses
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
"github.com/zrepl/yaml-config"
|
||||
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/daemon"
|
||||
"github.com/zrepl/zrepl/daemon/job"
|
||||
"github.com/zrepl/zrepl/daemon/pruner"
|
||||
"github.com/zrepl/zrepl/daemon/snapper"
|
||||
"github.com/zrepl/zrepl/replication/report"
|
||||
)
|
||||
|
||||
type byteProgressMeasurement struct {
|
||||
time time.Time
|
||||
val int64
|
||||
}
|
||||
|
||||
type bytesProgressHistory struct {
|
||||
last *byteProgressMeasurement // pointer as poor man's optional
|
||||
changeCount int
|
||||
lastChange time.Time
|
||||
bpsAvg float64
|
||||
}
|
||||
|
||||
func (p *bytesProgressHistory) Update(currentVal int64) (bytesPerSecondAvg int64, changeCount int) {
|
||||
|
||||
if p.last == nil {
|
||||
p.last = &byteProgressMeasurement{
|
||||
time: time.Now(),
|
||||
val: currentVal,
|
||||
}
|
||||
return 0, 0
|
||||
}
|
||||
|
||||
if p.last.val != currentVal {
|
||||
p.changeCount++
|
||||
p.lastChange = time.Now()
|
||||
}
|
||||
|
||||
if time.Since(p.lastChange) > 3*time.Second {
|
||||
p.last = nil
|
||||
return 0, 0
|
||||
}
|
||||
|
||||
deltaV := currentVal - p.last.val
|
||||
deltaT := time.Since(p.last.time)
|
||||
rate := float64(deltaV) / deltaT.Seconds()
|
||||
|
||||
factor := 0.3
|
||||
p.bpsAvg = (1-factor)*p.bpsAvg + factor*rate
|
||||
|
||||
p.last.time = time.Now()
|
||||
p.last.val = currentVal
|
||||
|
||||
return int64(p.bpsAvg), p.changeCount
|
||||
}
|
||||
|
||||
type tui struct {
|
||||
x, y int
|
||||
indent int
|
||||
|
||||
lock sync.Mutex //For report and error
|
||||
report map[string]*job.Status
|
||||
err error
|
||||
|
||||
jobFilter string
|
||||
|
||||
replicationProgress map[string]*bytesProgressHistory // by job name
|
||||
}
|
||||
|
||||
func newTui() tui {
|
||||
return tui{
|
||||
replicationProgress: make(map[string]*bytesProgressHistory),
|
||||
}
|
||||
}
|
||||
|
||||
const INDENT_MULTIPLIER = 4
|
||||
|
||||
func (t *tui) moveLine(dl int, col int) {
|
||||
t.y += dl
|
||||
t.x = t.indent*INDENT_MULTIPLIER + col
|
||||
}
|
||||
|
||||
func (t *tui) write(text string) {
|
||||
for _, c := range text {
|
||||
if c == '\n' {
|
||||
t.newline()
|
||||
continue
|
||||
}
|
||||
termbox.SetCell(t.x, t.y, c, termbox.ColorDefault, termbox.ColorDefault)
|
||||
t.x += 1
|
||||
}
|
||||
}
|
||||
|
||||
func (t *tui) printf(text string, a ...interface{}) {
|
||||
t.write(fmt.Sprintf(text, a...))
|
||||
}
|
||||
|
||||
func wrap(s string, width int) string {
|
||||
var b strings.Builder
|
||||
for len(s) > 0 {
|
||||
rem := width
|
||||
if rem > len(s) {
|
||||
rem = len(s)
|
||||
}
|
||||
if idx := strings.IndexAny(s, "\n\r"); idx != -1 && idx < rem {
|
||||
rem = idx + 1
|
||||
}
|
||||
untilNewline := strings.TrimRight(s[:rem], "\n\r")
|
||||
s = s[rem:]
|
||||
if len(untilNewline) == 0 {
|
||||
continue
|
||||
}
|
||||
b.WriteString(untilNewline)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return strings.TrimRight(b.String(), "\n\r")
|
||||
}
|
||||
|
||||
func (t *tui) printfDrawIndentedAndWrappedIfMultiline(format string, a ...interface{}) {
|
||||
whole := fmt.Sprintf(format, a...)
|
||||
width, _ := termbox.Size()
|
||||
if !strings.ContainsAny(whole, "\n\r") && t.x+len(whole) <= width {
|
||||
t.printf(format, a...)
|
||||
} else {
|
||||
t.addIndent(1)
|
||||
t.newline()
|
||||
t.write(wrap(whole, width-INDENT_MULTIPLIER*t.indent))
|
||||
t.addIndent(-1)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *tui) newline() {
|
||||
t.moveLine(1, 0)
|
||||
}
|
||||
|
||||
func (t *tui) setIndent(indent int) {
|
||||
t.indent = indent
|
||||
t.moveLine(0, 0)
|
||||
}
|
||||
|
||||
func (t *tui) addIndent(indent int) {
|
||||
t.indent += indent
|
||||
t.moveLine(0, 0)
|
||||
}
|
||||
|
||||
var statusFlags struct {
|
||||
Raw bool
|
||||
Job string
|
||||
}
|
||||
|
||||
var StatusCmd = &cli.Subcommand{
|
||||
Use: "status",
|
||||
Short: "show job activity or dump as JSON for monitoring",
|
||||
SetupFlags: func(f *pflag.FlagSet) {
|
||||
f.BoolVar(&statusFlags.Raw, "raw", false, "dump raw status description from zrepl daemon")
|
||||
f.StringVar(&statusFlags.Job, "job", "", "only dump specified job")
|
||||
},
|
||||
Run: runStatus,
|
||||
}
|
||||
|
||||
func runStatus(s *cli.Subcommand, args []string) error {
|
||||
httpc, err := controlHttpClient(s.Config().Global.Control.SockPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if statusFlags.Raw {
|
||||
resp, err := httpc.Get("http://unix" + daemon.ControlJobEndpointStatus)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
fmt.Fprintf(os.Stderr, "Received error response:\n")
|
||||
_, err := io.CopyN(os.Stderr, resp.Body, 4096)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return errors.Errorf("exit")
|
||||
}
|
||||
if _, err := io.Copy(os.Stdout, resp.Body); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
t := newTui()
|
||||
t.lock.Lock()
|
||||
t.err = errors.New("Got no report yet")
|
||||
t.lock.Unlock()
|
||||
t.jobFilter = statusFlags.Job
|
||||
|
||||
err = termbox.Init()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer termbox.Close()
|
||||
|
||||
update := func() {
|
||||
var m daemon.Status
|
||||
|
||||
err2 := jsonRequestResponse(httpc, daemon.ControlJobEndpointStatus,
|
||||
struct{}{},
|
||||
&m,
|
||||
)
|
||||
|
||||
t.lock.Lock()
|
||||
t.err = err2
|
||||
t.report = m.Jobs
|
||||
t.lock.Unlock()
|
||||
t.draw()
|
||||
}
|
||||
update()
|
||||
|
||||
ticker := time.NewTicker(500 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
go func() {
|
||||
for range ticker.C {
|
||||
update()
|
||||
}
|
||||
}()
|
||||
|
||||
termbox.HideCursor()
|
||||
termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
|
||||
|
||||
loop:
|
||||
for {
|
||||
switch ev := termbox.PollEvent(); ev.Type {
|
||||
case termbox.EventKey:
|
||||
switch ev.Key {
|
||||
case termbox.KeyEsc:
|
||||
break loop
|
||||
case termbox.KeyCtrlC:
|
||||
break loop
|
||||
}
|
||||
case termbox.EventResize:
|
||||
t.draw()
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func (t *tui) getReplicationProgressHistory(jobName string) *bytesProgressHistory {
|
||||
p, ok := t.replicationProgress[jobName]
|
||||
if !ok {
|
||||
p = &bytesProgressHistory{}
|
||||
t.replicationProgress[jobName] = p
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func (t *tui) draw() {
|
||||
t.lock.Lock()
|
||||
defer t.lock.Unlock()
|
||||
|
||||
termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
|
||||
t.x = 0
|
||||
t.y = 0
|
||||
t.indent = 0
|
||||
|
||||
if t.err != nil {
|
||||
t.write(t.err.Error())
|
||||
} else {
|
||||
//Iterate over map in alphabetical order
|
||||
keys := make([]string, 0, len(t.report))
|
||||
for k := range t.report {
|
||||
if len(k) == 0 || daemon.IsInternalJobName(k) { //Internal job
|
||||
continue
|
||||
}
|
||||
if t.jobFilter != "" && k != t.jobFilter {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
if len(keys) == 0 {
|
||||
t.setIndent(0)
|
||||
t.printf("no jobs to display")
|
||||
t.newline()
|
||||
termbox.Flush()
|
||||
return
|
||||
}
|
||||
|
||||
for _, k := range keys {
|
||||
v := t.report[k]
|
||||
|
||||
t.setIndent(0)
|
||||
|
||||
t.printf("Job: %s", k)
|
||||
t.setIndent(1)
|
||||
t.newline()
|
||||
t.printf("Type: %s", v.Type)
|
||||
t.setIndent(1)
|
||||
t.newline()
|
||||
|
||||
if v.Type == job.TypePush || v.Type == job.TypePull {
|
||||
activeStatus, ok := v.JobSpecific.(*job.ActiveSideStatus)
|
||||
if !ok || activeStatus == nil {
|
||||
t.printf("ActiveSideStatus is null")
|
||||
t.newline()
|
||||
continue
|
||||
}
|
||||
|
||||
t.printf("Replication:")
|
||||
t.newline()
|
||||
t.addIndent(1)
|
||||
t.renderReplicationReport(activeStatus.Replication, t.getReplicationProgressHistory(k))
|
||||
t.addIndent(-1)
|
||||
|
||||
t.printf("Pruning Sender:")
|
||||
t.newline()
|
||||
t.addIndent(1)
|
||||
t.renderPrunerReport(activeStatus.PruningSender)
|
||||
t.addIndent(-1)
|
||||
|
||||
t.printf("Pruning Receiver:")
|
||||
t.newline()
|
||||
t.addIndent(1)
|
||||
t.renderPrunerReport(activeStatus.PruningReceiver)
|
||||
t.addIndent(-1)
|
||||
|
||||
if v.Type == job.TypePush {
|
||||
t.printf("Snapshotting:")
|
||||
t.newline()
|
||||
t.addIndent(1)
|
||||
t.renderSnapperReport(activeStatus.Snapshotting)
|
||||
t.addIndent(-1)
|
||||
}
|
||||
|
||||
} else if v.Type == job.TypeSnap {
|
||||
snapStatus, ok := v.JobSpecific.(*job.SnapJobStatus)
|
||||
if !ok || snapStatus == nil {
|
||||
t.printf("SnapJobStatus is null")
|
||||
t.newline()
|
||||
continue
|
||||
}
|
||||
t.printf("Pruning snapshots:")
|
||||
t.newline()
|
||||
t.addIndent(1)
|
||||
t.renderPrunerReport(snapStatus.Pruning)
|
||||
t.addIndent(-1)
|
||||
t.printf("Snapshotting:")
|
||||
t.newline()
|
||||
t.addIndent(1)
|
||||
t.renderSnapperReport(snapStatus.Snapshotting)
|
||||
t.addIndent(-1)
|
||||
} else if v.Type == job.TypeSource {
|
||||
|
||||
st := v.JobSpecific.(*job.PassiveStatus)
|
||||
t.printf("Snapshotting:\n")
|
||||
t.addIndent(1)
|
||||
t.renderSnapperReport(st.Snapper)
|
||||
t.addIndent(-1)
|
||||
|
||||
} else {
|
||||
t.printf("No status representation for job type '%s', dumping as YAML", v.Type)
|
||||
t.newline()
|
||||
asYaml, err := yaml.Marshal(v.JobSpecific)
|
||||
if err != nil {
|
||||
t.printf("Error marshaling status to YAML: %s", err)
|
||||
t.newline()
|
||||
continue
|
||||
}
|
||||
t.write(string(asYaml))
|
||||
t.newline()
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
termbox.Flush()
|
||||
}
|
||||
|
||||
func (t *tui) renderReplicationReport(rep *report.Report, history *bytesProgressHistory) {
|
||||
if rep == nil {
|
||||
t.printf("...\n")
|
||||
return
|
||||
}
|
||||
|
||||
if rep.WaitReconnectError != nil {
|
||||
t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: %s", rep.WaitReconnectError)
|
||||
t.newline()
|
||||
}
|
||||
if !rep.WaitReconnectSince.IsZero() {
|
||||
delta := time.Until(rep.WaitReconnectUntil).Round(time.Second)
|
||||
if rep.WaitReconnectUntil.IsZero() || delta > 0 {
|
||||
var until string
|
||||
if rep.WaitReconnectUntil.IsZero() {
|
||||
until = "waiting indefinitely"
|
||||
} else {
|
||||
until = fmt.Sprintf("hard fail in %s @ %s", delta, rep.WaitReconnectUntil)
|
||||
}
|
||||
t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnecting with exponential backoff (since %s) (%s)",
|
||||
rep.WaitReconnectSince, until)
|
||||
} else {
|
||||
t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnects reached hard-fail timeout @ %s", rep.WaitReconnectUntil)
|
||||
}
|
||||
t.newline()
|
||||
}
|
||||
|
||||
// TODO visualize more than the latest attempt by folding all attempts into one
|
||||
if len(rep.Attempts) == 0 {
|
||||
t.printf("no attempts made yet")
|
||||
return
|
||||
} else {
|
||||
t.printf("Attempt #%d", len(rep.Attempts))
|
||||
if len(rep.Attempts) > 1 {
|
||||
t.printf(". Previous attempts failed with the following statuses:")
|
||||
t.newline()
|
||||
t.addIndent(1)
|
||||
for i, a := range rep.Attempts[:len(rep.Attempts)-1] {
|
||||
t.printfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt))
|
||||
t.newline()
|
||||
}
|
||||
t.addIndent(-1)
|
||||
} else {
|
||||
t.newline()
|
||||
}
|
||||
}
|
||||
|
||||
latest := rep.Attempts[len(rep.Attempts)-1]
|
||||
sort.Slice(latest.Filesystems, func(i, j int) bool {
|
||||
return latest.Filesystems[i].Info.Name < latest.Filesystems[j].Info.Name
|
||||
})
|
||||
|
||||
t.printf("Status: %s", latest.State)
|
||||
t.newline()
|
||||
if latest.State == report.AttemptPlanningError {
|
||||
t.printf("Problem: ")
|
||||
t.printfDrawIndentedAndWrappedIfMultiline("%s", latest.PlanError)
|
||||
t.newline()
|
||||
} else if latest.State == report.AttemptFanOutError {
|
||||
t.printf("Problem: one or more of the filesystems encountered errors")
|
||||
t.newline()
|
||||
}
|
||||
|
||||
if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
|
||||
// Draw global progress bar
|
||||
// Progress: [---------------]
|
||||
expected, replicated, containsInvalidSizeEstimates := latest.BytesSum()
|
||||
rate, changeCount := history.Update(replicated)
|
||||
eta := time.Duration(0)
|
||||
if rate > 0 {
|
||||
eta = time.Duration((expected-replicated)/rate) * time.Second
|
||||
}
|
||||
t.write("Progress: ")
|
||||
t.drawBar(50, replicated, expected, changeCount)
|
||||
t.write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinary(replicated), ByteCountBinary(expected), ByteCountBinary(rate)))
|
||||
if eta != 0 {
|
||||
t.write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
|
||||
}
|
||||
t.newline()
|
||||
if containsInvalidSizeEstimates {
|
||||
t.write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
|
||||
t.newline()
|
||||
}
|
||||
|
||||
var maxFSLen int
|
||||
for _, fs := range latest.Filesystems {
|
||||
if len(fs.Info.Name) > maxFSLen {
|
||||
maxFSLen = len(fs.Info.Name)
|
||||
}
|
||||
}
|
||||
for _, fs := range latest.Filesystems {
|
||||
t.printFilesystemStatus(fs, false, maxFSLen) // FIXME bring 'active' flag back
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (t *tui) renderPrunerReport(r *pruner.Report) {
|
||||
if r == nil {
|
||||
t.printf("...\n")
|
||||
return
|
||||
}
|
||||
|
||||
state, err := pruner.StateString(r.State)
|
||||
if err != nil {
|
||||
t.printf("Status: %q (parse error: %q)\n", r.State, err)
|
||||
return
|
||||
}
|
||||
|
||||
t.printf("Status: %s", state)
|
||||
t.newline()
|
||||
|
||||
if r.Error != "" {
|
||||
t.printf("Error: %s\n", r.Error)
|
||||
}
|
||||
|
||||
type commonFS struct {
|
||||
*pruner.FSReport
|
||||
completed bool
|
||||
}
|
||||
all := make([]commonFS, 0, len(r.Pending)+len(r.Completed))
|
||||
for i := range r.Pending {
|
||||
all = append(all, commonFS{&r.Pending[i], false})
|
||||
}
|
||||
for i := range r.Completed {
|
||||
all = append(all, commonFS{&r.Completed[i], true})
|
||||
}
|
||||
|
||||
switch state {
|
||||
case pruner.Plan:
|
||||
fallthrough
|
||||
case pruner.PlanErr:
|
||||
return
|
||||
}
|
||||
|
||||
if len(all) == 0 {
|
||||
t.printf("nothing to do\n")
|
||||
return
|
||||
}
|
||||
|
||||
var totalDestroyCount, completedDestroyCount int
|
||||
var maxFSname int
|
||||
for _, fs := range all {
|
||||
totalDestroyCount += len(fs.DestroyList)
|
||||
if fs.completed {
|
||||
completedDestroyCount += len(fs.DestroyList)
|
||||
}
|
||||
if maxFSname < len(fs.Filesystem) {
|
||||
maxFSname = len(fs.Filesystem)
|
||||
}
|
||||
}
|
||||
|
||||
// global progress bar
|
||||
progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
|
||||
t.write("Progress: ")
|
||||
t.write("[")
|
||||
t.write(times("=", progress))
|
||||
t.write(">")
|
||||
t.write(times("-", 80-progress))
|
||||
t.write("]")
|
||||
t.printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
|
||||
t.newline()
|
||||
|
||||
sort.SliceStable(all, func(i, j int) bool {
|
||||
return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
|
||||
})
|
||||
|
||||
// Draw a table-like representation of 'all'
|
||||
for _, fs := range all {
|
||||
t.write(rightPad(fs.Filesystem, maxFSname, " "))
|
||||
t.write(" ")
|
||||
if !fs.SkipReason.NotSkipped() {
|
||||
t.printf("skipped: %s\n", fs.SkipReason)
|
||||
continue
|
||||
}
|
||||
if fs.LastError != "" {
|
||||
if strings.ContainsAny(fs.LastError, "\r\n") {
|
||||
t.printf("ERROR:")
|
||||
t.printfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError)
|
||||
} else {
|
||||
t.printfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError)
|
||||
}
|
||||
t.newline()
|
||||
continue
|
||||
}
|
||||
|
||||
pruneRuleActionStr := fmt.Sprintf("(destroy %d of %d snapshots)",
|
||||
len(fs.DestroyList), len(fs.SnapshotList))
|
||||
|
||||
if fs.completed {
|
||||
t.printf("Completed %s\n", pruneRuleActionStr)
|
||||
continue
|
||||
}
|
||||
|
||||
t.write("Pending ") // whitespace is padding 10
|
||||
if len(fs.DestroyList) == 1 {
|
||||
t.write(fs.DestroyList[0].Name)
|
||||
} else {
|
||||
t.write(pruneRuleActionStr)
|
||||
}
|
||||
t.newline()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (t *tui) renderSnapperReport(r *snapper.Report) {
|
||||
if r == nil {
|
||||
t.printf("<snapshot type does not have a report>\n")
|
||||
return
|
||||
}
|
||||
|
||||
t.printf("Status: %s", r.State)
|
||||
t.newline()
|
||||
|
||||
if r.Error != "" {
|
||||
t.printf("Error: %s\n", r.Error)
|
||||
}
|
||||
if !r.SleepUntil.IsZero() {
|
||||
t.printf("Sleep until: %s\n", r.SleepUntil)
|
||||
}
|
||||
|
||||
sort.Slice(r.Progress, func(i, j int) bool {
|
||||
return strings.Compare(r.Progress[i].Path, r.Progress[j].Path) == -1
|
||||
})
|
||||
|
||||
t.addIndent(1)
|
||||
defer t.addIndent(-1)
|
||||
dur := func(d time.Duration) string {
|
||||
return d.Round(100 * time.Millisecond).String()
|
||||
}
|
||||
|
||||
type row struct {
|
||||
path, state, duration, remainder, hookReport string
|
||||
}
|
||||
var widths struct {
|
||||
path, state, duration int
|
||||
}
|
||||
rows := make([]*row, len(r.Progress))
|
||||
for i, fs := range r.Progress {
|
||||
r := &row{
|
||||
path: fs.Path,
|
||||
state: fs.State.String(),
|
||||
}
|
||||
if fs.HooksHadError {
|
||||
r.hookReport = fs.Hooks // FIXME render here, not in daemon
|
||||
}
|
||||
switch fs.State {
|
||||
case snapper.SnapPending:
|
||||
r.duration = "..."
|
||||
r.remainder = ""
|
||||
case snapper.SnapStarted:
|
||||
r.duration = dur(time.Since(fs.StartAt))
|
||||
r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
|
||||
case snapper.SnapDone:
|
||||
fallthrough
|
||||
case snapper.SnapError:
|
||||
r.duration = dur(fs.DoneAt.Sub(fs.StartAt))
|
||||
r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
|
||||
}
|
||||
rows[i] = r
|
||||
if len(r.path) > widths.path {
|
||||
widths.path = len(r.path)
|
||||
}
|
||||
if len(r.state) > widths.state {
|
||||
widths.state = len(r.state)
|
||||
}
|
||||
if len(r.duration) > widths.duration {
|
||||
widths.duration = len(r.duration)
|
||||
}
|
||||
}
|
||||
|
||||
for _, r := range rows {
|
||||
path := rightPad(r.path, widths.path, " ")
|
||||
state := rightPad(r.state, widths.state, " ")
|
||||
duration := rightPad(r.duration, widths.duration, " ")
|
||||
t.printf("%s %s %s", path, state, duration)
|
||||
t.printfDrawIndentedAndWrappedIfMultiline(" %s", r.remainder)
|
||||
if r.hookReport != "" {
|
||||
t.printfDrawIndentedAndWrappedIfMultiline("%s", r.hookReport)
|
||||
}
|
||||
t.newline()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func times(str string, n int) (out string) {
|
||||
for i := 0; i < n; i++ {
|
||||
out += str
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func rightPad(str string, length int, pad string) string {
|
||||
if len(str) > length {
|
||||
return str[:length]
|
||||
}
|
||||
return str + strings.Repeat(pad, length-len(str))
|
||||
}
|
||||
|
||||
var arrowPositions = `>\|/`
|
||||
|
||||
// changeCount = 0 indicates stall / no progress
|
||||
func (t *tui) drawBar(length int, bytes, totalBytes int64, changeCount int) {
|
||||
var completedLength int
|
||||
if totalBytes > 0 {
|
||||
completedLength = int(int64(length) * bytes / totalBytes)
|
||||
if completedLength > length {
|
||||
completedLength = length
|
||||
}
|
||||
} else if totalBytes == bytes {
|
||||
completedLength = length
|
||||
}
|
||||
|
||||
t.write("[")
|
||||
t.write(times("=", completedLength))
|
||||
t.write(string(arrowPositions[changeCount%len(arrowPositions)]))
|
||||
t.write(times("-", length-completedLength))
|
||||
t.write("]")
|
||||
}
|
||||
|
||||
func (t *tui) printFilesystemStatus(rep *report.FilesystemReport, active bool, maxFS int) {
|
||||
|
||||
expected, replicated, containsInvalidSizeEstimates := rep.BytesSum()
|
||||
sizeEstimationImpreciseNotice := ""
|
||||
if containsInvalidSizeEstimates {
|
||||
sizeEstimationImpreciseNotice = " (some steps lack size estimation)"
|
||||
}
|
||||
if rep.CurrentStep < len(rep.Steps) && rep.Steps[rep.CurrentStep].Info.BytesExpected == 0 {
|
||||
sizeEstimationImpreciseNotice = " (step lacks size estimation)"
|
||||
}
|
||||
|
||||
status := fmt.Sprintf("%s (step %d/%d, %s/%s)%s",
|
||||
strings.ToUpper(string(rep.State)),
|
||||
rep.CurrentStep, len(rep.Steps),
|
||||
ByteCountBinary(replicated), ByteCountBinary(expected),
|
||||
sizeEstimationImpreciseNotice,
|
||||
)
|
||||
|
||||
activeIndicator := " "
|
||||
if active {
|
||||
activeIndicator = "*"
|
||||
}
|
||||
t.printf("%s %s %s ",
|
||||
activeIndicator,
|
||||
rightPad(rep.Info.Name, maxFS, " "),
|
||||
status)
|
||||
|
||||
next := ""
|
||||
if err := rep.Error(); err != nil {
|
||||
next = err.Err
|
||||
} else if rep.State != report.FilesystemDone {
|
||||
if nextStep := rep.NextStep(); nextStep != nil {
|
||||
if nextStep.IsIncremental() {
|
||||
next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
|
||||
} else {
|
||||
next = fmt.Sprintf("next: full send %s", nextStep.Info.To)
|
||||
}
|
||||
attribs := []string{}
|
||||
|
||||
if nextStep.Info.Resumed {
|
||||
attribs = append(attribs, "resumed")
|
||||
}
|
||||
|
||||
attribs = append(attribs, fmt.Sprintf("encrypted=%s", nextStep.Info.Encrypted))
|
||||
|
||||
next += fmt.Sprintf(" (%s)", strings.Join(attribs, ", "))
|
||||
} else {
|
||||
next = "" // individual FSes may still be in planning state
|
||||
}
|
||||
|
||||
}
|
||||
t.printfDrawIndentedAndWrappedIfMultiline("%s", next)
|
||||
|
||||
t.newline()
|
||||
}
|
||||
|
||||
func ByteCountBinary(b int64) string {
|
||||
const unit = 1024
|
||||
if b < unit {
|
||||
return fmt.Sprintf("%d B", b)
|
||||
}
|
||||
div, exp := int64(unit), 0
|
||||
for n := b / unit; n >= unit; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
|
||||
func humanizeDuration(duration time.Duration) string {
|
||||
days := int64(duration.Hours() / 24)
|
||||
hours := int64(math.Mod(duration.Hours(), 24))
|
||||
minutes := int64(math.Mod(duration.Minutes(), 60))
|
||||
seconds := int64(math.Mod(duration.Seconds(), 60))
|
||||
|
||||
var parts []string
|
||||
|
||||
force := false
|
||||
chunks := []int64{days, hours, minutes, seconds}
|
||||
for i, chunk := range chunks {
|
||||
if force || chunk > 0 {
|
||||
padding := 0
|
||||
if force {
|
||||
padding = 2
|
||||
}
|
||||
parts = append(parts, fmt.Sprintf("%*d%c", padding, chunk, "dhms"[i]))
|
||||
force = true
|
||||
}
|
||||
}
|
||||
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
@@ -1,214 +0,0 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
var TestCmd = &cli.Subcommand{
|
||||
Use: "test",
|
||||
SetupSubcommands: func() []*cli.Subcommand {
|
||||
return []*cli.Subcommand{testFilter, testPlaceholder, testDecodeResumeToken}
|
||||
},
|
||||
}
|
||||
|
||||
var testFilterArgs struct {
|
||||
job string
|
||||
all bool
|
||||
input string
|
||||
}
|
||||
|
||||
var testFilter = &cli.Subcommand{
|
||||
Use: "filesystems --job JOB [--all | --input INPUT]",
|
||||
Short: "test filesystems filter specified in push or source job",
|
||||
SetupFlags: func(f *pflag.FlagSet) {
|
||||
f.StringVar(&testFilterArgs.job, "job", "", "the name of the push or source job")
|
||||
f.StringVar(&testFilterArgs.input, "input", "", "a filesystem name to test against the job's filters")
|
||||
f.BoolVar(&testFilterArgs.all, "all", false, "test all local filesystems")
|
||||
},
|
||||
Run: runTestFilterCmd,
|
||||
}
|
||||
|
||||
func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
|
||||
|
||||
if testFilterArgs.job == "" {
|
||||
return fmt.Errorf("must specify --job flag")
|
||||
}
|
||||
if !(testFilterArgs.all != (testFilterArgs.input != "")) { // xor
|
||||
return fmt.Errorf("must set one: --all or --input")
|
||||
}
|
||||
|
||||
conf := subcommand.Config()
|
||||
ctx := context.Background()
|
||||
|
||||
var confFilter config.FilesystemsFilter
|
||||
job, err := conf.Job(testFilterArgs.job)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch j := job.Ret.(type) {
|
||||
case *config.SourceJob:
|
||||
confFilter = j.Filesystems
|
||||
case *config.PushJob:
|
||||
confFilter = j.Filesystems
|
||||
case *config.SnapJob:
|
||||
confFilter = j.Filesystems
|
||||
default:
|
||||
return fmt.Errorf("job type %T does not have filesystems filter", j)
|
||||
}
|
||||
|
||||
f, err := filters.DatasetMapFilterFromConfig(confFilter)
|
||||
if err != nil {
|
||||
return fmt.Errorf("filter invalid: %s", err)
|
||||
}
|
||||
|
||||
var fsnames []string
|
||||
if testFilterArgs.input != "" {
|
||||
fsnames = []string{testFilterArgs.input}
|
||||
} else {
|
||||
out, err := zfs.ZFSList(ctx, []string{"name"})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not list ZFS filesystems: %s", err)
|
||||
}
|
||||
for _, row := range out {
|
||||
|
||||
fsnames = append(fsnames, row[0])
|
||||
}
|
||||
}
|
||||
|
||||
fspaths := make([]*zfs.DatasetPath, len(fsnames))
|
||||
for i, fsname := range fsnames {
|
||||
path, err := zfs.NewDatasetPath(fsname)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fspaths[i] = path
|
||||
}
|
||||
|
||||
hadFilterErr := false
|
||||
for _, in := range fspaths {
|
||||
var res string
|
||||
var errStr string
|
||||
pass, err := f.Filter(in)
|
||||
if err != nil {
|
||||
res = "ERROR"
|
||||
errStr = err.Error()
|
||||
hadFilterErr = true
|
||||
} else if pass {
|
||||
res = "ACCEPT"
|
||||
} else {
|
||||
res = "REJECT"
|
||||
}
|
||||
fmt.Printf("%s\t%s\t%s\n", res, in.ToString(), errStr)
|
||||
}
|
||||
|
||||
if hadFilterErr {
|
||||
return fmt.Errorf("filter errors occurred")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var testPlaceholderArgs struct {
|
||||
ds string
|
||||
all bool
|
||||
}
|
||||
|
||||
var testPlaceholder = &cli.Subcommand{
|
||||
Use: "placeholder [--all | --dataset DATASET]",
|
||||
Short: fmt.Sprintf("list received placeholder filesystems (zfs property %q)", zfs.PlaceholderPropertyName),
|
||||
Example: `
|
||||
placeholder --all
|
||||
placeholder --dataset path/to/sink/clientident/fs`,
|
||||
NoRequireConfig: true,
|
||||
SetupFlags: func(f *pflag.FlagSet) {
|
||||
f.StringVar(&testPlaceholderArgs.ds, "dataset", "", "dataset path (not required to exist)")
|
||||
f.BoolVar(&testPlaceholderArgs.all, "all", false, "list tab-separated placeholder status of all filesystems")
|
||||
},
|
||||
Run: runTestPlaceholder,
|
||||
}
|
||||
|
||||
func runTestPlaceholder(subcommand *cli.Subcommand, args []string) error {
|
||||
|
||||
var checkDPs []*zfs.DatasetPath
|
||||
ctx := context.Background()
|
||||
|
||||
// all actions first
|
||||
if testPlaceholderArgs.all {
|
||||
out, err := zfs.ZFSList(ctx, []string{"name"})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not list ZFS filesystems")
|
||||
}
|
||||
for _, row := range out {
|
||||
dp, err := zfs.NewDatasetPath(row[0])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
checkDPs = append(checkDPs, dp)
|
||||
}
|
||||
} else {
|
||||
dp, err := zfs.NewDatasetPath(testPlaceholderArgs.ds)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dp.Empty() {
|
||||
return fmt.Errorf("must specify --dataset DATASET or --all")
|
||||
}
|
||||
checkDPs = append(checkDPs, dp)
|
||||
}
|
||||
|
||||
fmt.Printf("IS_PLACEHOLDER\tDATASET\tzrepl:placeholder\n")
|
||||
for _, dp := range checkDPs {
|
||||
ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, dp)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "cannot get placeholder state")
|
||||
}
|
||||
if !ph.FSExists {
|
||||
panic("placeholder state inconsistent: filesystem " + ph.FS + " must exist in this context")
|
||||
}
|
||||
is := "yes"
|
||||
if !ph.IsPlaceholder {
|
||||
is = "no"
|
||||
}
|
||||
fmt.Printf("%s\t%s\t%s\n", is, dp.ToString(), ph.RawLocalPropertyValue)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var testDecodeResumeTokenArgs struct {
|
||||
token string
|
||||
}
|
||||
|
||||
var testDecodeResumeToken = &cli.Subcommand{
|
||||
Use: "decoderesumetoken --token TOKEN",
|
||||
Short: "decode resume token",
|
||||
SetupFlags: func(f *pflag.FlagSet) {
|
||||
f.StringVar(&testDecodeResumeTokenArgs.token, "token", "", "the resume token obtained from the receive_resume_token property")
|
||||
},
|
||||
Run: runTestDecodeResumeTokenCmd,
|
||||
}
|
||||
|
||||
func runTestDecodeResumeTokenCmd(subcommand *cli.Subcommand, args []string) error {
|
||||
if testDecodeResumeTokenArgs.token == "" {
|
||||
return fmt.Errorf("token argument must be specified")
|
||||
}
|
||||
token, err := zfs.ParseResumeToken(context.Background(), testDecodeResumeTokenArgs.token)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
if err := enc.Encode(&token); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon"
|
||||
"github.com/zrepl/zrepl/version"
|
||||
)
|
||||
|
||||
var versionArgs struct {
|
||||
Show string
|
||||
Config *config.Config
|
||||
ConfigErr error
|
||||
}
|
||||
|
||||
var VersionCmd = &cli.Subcommand{
|
||||
Use: "version",
|
||||
Short: "print version of zrepl binary and running daemon",
|
||||
NoRequireConfig: true,
|
||||
SetupFlags: func(f *pflag.FlagSet) {
|
||||
f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)")
|
||||
},
|
||||
Run: func(subcommand *cli.Subcommand, args []string) error {
|
||||
versionArgs.Config = subcommand.Config()
|
||||
versionArgs.ConfigErr = subcommand.ConfigParsingError()
|
||||
return runVersionCmd()
|
||||
},
|
||||
}
|
||||
|
||||
func runVersionCmd() error {
|
||||
args := versionArgs
|
||||
|
||||
if args.Show != "daemon" && args.Show != "client" && args.Show != "" {
|
||||
return fmt.Errorf("show flag must be 'client' or 'server' or be left empty")
|
||||
}
|
||||
|
||||
var clientVersion, daemonVersion *version.ZreplVersionInformation
|
||||
if args.Show == "client" || args.Show == "" {
|
||||
clientVersion = version.NewZreplVersionInformation()
|
||||
fmt.Printf("client: %s\n", clientVersion.String())
|
||||
}
|
||||
if args.Show == "daemon" || args.Show == "" {
|
||||
|
||||
if args.ConfigErr != nil {
|
||||
return fmt.Errorf("config parsing error: %s", args.ConfigErr)
|
||||
}
|
||||
|
||||
httpc, err := controlHttpClient(args.Config.Global.Control.SockPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("server: error: %s\n", err)
|
||||
}
|
||||
|
||||
var info version.ZreplVersionInformation
|
||||
err = jsonRequestResponse(httpc, daemon.ControlJobEndpointVersion, "", &info)
|
||||
if err != nil {
|
||||
return fmt.Errorf("server: error: %s\n", err)
|
||||
}
|
||||
daemonVersion = &info
|
||||
fmt.Printf("server: %s\n", daemonVersion.String())
|
||||
}
|
||||
|
||||
if args.Show == "" {
|
||||
if clientVersion.Version != daemonVersion.Version {
|
||||
fmt.Fprintf(os.Stderr, "WARNING: client version != daemon version, restart zrepl daemon\n")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,147 +0,0 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
var (
|
||||
ZFSAbstractionsCmd = &cli.Subcommand{
|
||||
Use: "zfs-abstraction",
|
||||
Short: "manage abstractions that zrepl builds on top of ZFS",
|
||||
SetupSubcommands: func() []*cli.Subcommand {
|
||||
return []*cli.Subcommand{
|
||||
zabsCmdList,
|
||||
zabsCmdReleaseAll,
|
||||
zabsCmdReleaseStale,
|
||||
zabsCmdCreate,
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
// a common set of CLI flags that map to the fields of an
|
||||
// endpoint.ListZFSHoldsAndBookmarksQuery
|
||||
type zabsFilterFlags struct {
|
||||
Filesystems FilesystemsFilterFlag
|
||||
Job JobIDFlag
|
||||
Types AbstractionTypesFlag
|
||||
Concurrency int64
|
||||
}
|
||||
|
||||
// produce a query from the CLI flags
|
||||
func (f zabsFilterFlags) Query() (endpoint.ListZFSHoldsAndBookmarksQuery, error) {
|
||||
q := endpoint.ListZFSHoldsAndBookmarksQuery{
|
||||
FS: f.Filesystems.FlagValue(),
|
||||
What: f.Types.FlagValue(),
|
||||
JobID: f.Job.FlagValue(),
|
||||
Concurrency: f.Concurrency,
|
||||
}
|
||||
return q, q.Validate()
|
||||
}
|
||||
|
||||
func (f *zabsFilterFlags) registerZabsFilterFlags(s *pflag.FlagSet, verb string) {
|
||||
// Note: the default value is defined in the .FlagValue methods
|
||||
s.Var(&f.Filesystems, "fs", fmt.Sprintf("only %s holds on the specified filesystem [default: all filesystems] [comma-separated list of <dataset-pattern>:<ok|!> pairs]", verb))
|
||||
s.Var(&f.Job, "job", fmt.Sprintf("only %s holds created by the specified job [default: any job]", verb))
|
||||
|
||||
variants := make([]string, 0, len(endpoint.AbstractionTypesAll))
|
||||
for v := range endpoint.AbstractionTypesAll {
|
||||
variants = append(variants, string(v))
|
||||
}
|
||||
variants = sort.StringSlice(variants)
|
||||
variantsJoined := strings.Join(variants, "|")
|
||||
s.Var(&f.Types, "type", fmt.Sprintf("only %s holds of the specified type [default: all] [comma-separated list of %s]", verb, variantsJoined))
|
||||
|
||||
s.Int64VarP(&f.Concurrency, "concurrency", "p", 1, "number of concurrently queried filesystems")
|
||||
}
|
||||
|
||||
type JobIDFlag struct{ J *endpoint.JobID }
|
||||
|
||||
func (f *JobIDFlag) Set(s string) error {
|
||||
if len(s) == 0 {
|
||||
*f = JobIDFlag{J: nil}
|
||||
return nil
|
||||
}
|
||||
|
||||
jobID, err := endpoint.MakeJobID(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*f = JobIDFlag{J: &jobID}
|
||||
return nil
|
||||
}
|
||||
func (f JobIDFlag) Type() string { return "job-ID" }
|
||||
func (f JobIDFlag) String() string { return fmt.Sprint(f.J) }
|
||||
func (f JobIDFlag) FlagValue() *endpoint.JobID { return f.J }
|
||||
|
||||
type AbstractionTypesFlag map[endpoint.AbstractionType]bool
|
||||
|
||||
func (f *AbstractionTypesFlag) Set(s string) error {
|
||||
ats, err := endpoint.AbstractionTypeSetFromStrings(strings.Split(s, ","))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*f = AbstractionTypesFlag(ats)
|
||||
return nil
|
||||
}
|
||||
func (f AbstractionTypesFlag) Type() string { return "abstraction-type" }
|
||||
func (f AbstractionTypesFlag) String() string {
|
||||
return endpoint.AbstractionTypeSet(f).String()
|
||||
}
|
||||
func (f AbstractionTypesFlag) FlagValue() map[endpoint.AbstractionType]bool {
|
||||
if len(f) > 0 {
|
||||
return f
|
||||
}
|
||||
return endpoint.AbstractionTypesAll
|
||||
}
|
||||
|
||||
type FilesystemsFilterFlag struct {
|
||||
F endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter
|
||||
}
|
||||
|
||||
func (flag *FilesystemsFilterFlag) Set(s string) error {
|
||||
mappings := strings.Split(s, ",")
|
||||
if len(mappings) == 1 && !strings.Contains(mappings[0], ":") {
|
||||
flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
|
||||
FS: &mappings[0],
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
f := filters.NewDatasetMapFilter(len(mappings), true)
|
||||
for _, m := range mappings {
|
||||
thisMappingErr := fmt.Errorf("expecting comma-separated list of <dataset-pattern>:<ok|!> pairs, got %q", m)
|
||||
lhsrhs := strings.SplitN(m, ":", 2)
|
||||
if len(lhsrhs) != 2 {
|
||||
return thisMappingErr
|
||||
}
|
||||
err := f.Add(lhsrhs[0], lhsrhs[1])
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %s", thisMappingErr, err)
|
||||
}
|
||||
}
|
||||
flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
|
||||
Filter: f,
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (flag FilesystemsFilterFlag) Type() string { return "filesystem filter spec" }
|
||||
func (flag FilesystemsFilterFlag) String() string {
|
||||
return fmt.Sprintf("%v", flag.F)
|
||||
}
|
||||
func (flag FilesystemsFilterFlag) FlagValue() endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter {
|
||||
var z FilesystemsFilterFlag
|
||||
if flag == z {
|
||||
return endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{Filter: zfs.NoFilter()}
|
||||
}
|
||||
return flag.F
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
package client
|
||||
|
||||
import "github.com/zrepl/zrepl/cli"
|
||||
|
||||
var zabsCmdCreate = &cli.Subcommand{
|
||||
Use: "create",
|
||||
NoRequireConfig: true,
|
||||
Short: `create zrepl ZFS abstractions (mostly useful for debugging & development, users should not need to use this command)`,
|
||||
SetupSubcommands: func() []*cli.Subcommand {
|
||||
return []*cli.Subcommand{
|
||||
zabsCmdCreateStepHold,
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
var zabsCreateStepHoldFlags struct {
|
||||
target string
|
||||
jobid JobIDFlag
|
||||
}
|
||||
|
||||
var zabsCmdCreateStepHold = &cli.Subcommand{
|
||||
Use: "step",
|
||||
Run: doZabsCreateStep,
|
||||
NoRequireConfig: true,
|
||||
Short: `create a step hold or bookmark`,
|
||||
SetupFlags: func(f *pflag.FlagSet) {
|
||||
f.StringVarP(&zabsCreateStepHoldFlags.target, "target", "t", "", "snapshot to be held / bookmark to be held")
|
||||
f.VarP(&zabsCreateStepHoldFlags.jobid, "jobid", "j", "jobid for which the hold is installed")
|
||||
},
|
||||
}
|
||||
|
||||
func doZabsCreateStep(sc *cli.Subcommand, args []string) error {
|
||||
if len(args) > 0 {
|
||||
return errors.New("subcommand takes no arguments")
|
||||
}
|
||||
|
||||
f := &zabsCreateStepHoldFlags
|
||||
|
||||
fs, _, _, err := zfs.DecomposeVersionString(f.target)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "%q invalid target", f.target)
|
||||
}
|
||||
|
||||
if f.jobid.FlagValue() == nil {
|
||||
return errors.Errorf("jobid must be set")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
v, err := zfs.ZFSGetFilesystemVersion(ctx, f.target)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "get info about target %q", f.target)
|
||||
}
|
||||
|
||||
step, err := endpoint.HoldStep(ctx, fs, v, *f.jobid.FlagValue())
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "create step hold")
|
||||
}
|
||||
fmt.Println(step.String())
|
||||
return nil
|
||||
}
|
||||
@@ -1,100 +0,0 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/util/chainlock"
|
||||
)
|
||||
|
||||
var zabsListFlags struct {
|
||||
Filter zabsFilterFlags
|
||||
Json bool
|
||||
}
|
||||
|
||||
var zabsCmdList = &cli.Subcommand{
|
||||
Use: "list",
|
||||
Short: `list zrepl ZFS abstractions`,
|
||||
Run: doZabsList,
|
||||
NoRequireConfig: true,
|
||||
SetupFlags: func(f *pflag.FlagSet) {
|
||||
zabsListFlags.Filter.registerZabsFilterFlags(f, "list")
|
||||
f.BoolVar(&zabsListFlags.Json, "json", false, "emit JSON")
|
||||
},
|
||||
}
|
||||
|
||||
func doZabsList(sc *cli.Subcommand, args []string) error {
|
||||
var err error
|
||||
ctx := context.Background()
|
||||
|
||||
if len(args) > 0 {
|
||||
return errors.New("this subcommand takes no positional arguments")
|
||||
}
|
||||
|
||||
q, err := zabsListFlags.Filter.Query()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "invalid filter specification on command line")
|
||||
}
|
||||
|
||||
abstractions, errors, err := endpoint.ListAbstractionsStreamed(ctx, q)
|
||||
if err != nil {
|
||||
return err // context clear by invocation of command
|
||||
}
|
||||
|
||||
var line chainlock.L
|
||||
var wg sync.WaitGroup
|
||||
defer wg.Wait()
|
||||
wg.Add(1)
|
||||
|
||||
// print results
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
for a := range abstractions {
|
||||
func() {
|
||||
defer line.Lock().Unlock()
|
||||
if zabsListFlags.Json {
|
||||
enc.SetIndent("", " ")
|
||||
if err := enc.Encode(abstractions); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println()
|
||||
} else {
|
||||
fmt.Println(a)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}()
|
||||
|
||||
// print errors to stderr
|
||||
errorColor := color.New(color.FgRed)
|
||||
var errorsSlice []endpoint.ListAbstractionsError
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for err := range errors {
|
||||
func() {
|
||||
defer line.Lock().Unlock()
|
||||
errorsSlice = append(errorsSlice, err)
|
||||
errorColor.Fprintf(os.Stderr, "%s\n", err)
|
||||
}()
|
||||
}
|
||||
}()
|
||||
wg.Wait()
|
||||
if len(errorsSlice) > 0 {
|
||||
errorColor.Add(color.Bold).Fprintf(os.Stderr, "there were errors in listing the abstractions")
|
||||
return fmt.Errorf("")
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,145 +0,0 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
)
|
||||
|
||||
// shared between release-all and release-step
|
||||
var zabsReleaseFlags struct {
|
||||
Filter zabsFilterFlags
|
||||
Json bool
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
func registerZabsReleaseFlags(s *pflag.FlagSet) {
|
||||
zabsReleaseFlags.Filter.registerZabsFilterFlags(s, "release")
|
||||
s.BoolVar(&zabsReleaseFlags.Json, "json", false, "emit json instead of pretty-printed")
|
||||
s.BoolVar(&zabsReleaseFlags.DryRun, "dry-run", false, "do a dry-run")
|
||||
}
|
||||
|
||||
var zabsCmdReleaseAll = &cli.Subcommand{
|
||||
Use: "release-all",
|
||||
Run: doZabsReleaseAll,
|
||||
NoRequireConfig: true,
|
||||
Short: `(DANGEROUS) release ALL zrepl ZFS abstractions (mostly useful for uninstalling zrepl completely or for "de-zrepl-ing" a filesystem)`,
|
||||
SetupFlags: registerZabsReleaseFlags,
|
||||
}
|
||||
|
||||
var zabsCmdReleaseStale = &cli.Subcommand{
|
||||
Use: "release-stale",
|
||||
Run: doZabsReleaseStale,
|
||||
NoRequireConfig: true,
|
||||
Short: `release stale zrepl ZFS abstractions (useful if zrepl has a bug and does not do it by itself)`,
|
||||
SetupFlags: registerZabsReleaseFlags,
|
||||
}
|
||||
|
||||
func doZabsReleaseAll(sc *cli.Subcommand, args []string) error {
|
||||
var err error
|
||||
ctx := context.Background()
|
||||
|
||||
if len(args) > 0 {
|
||||
return errors.New("this subcommand takes no positional arguments")
|
||||
}
|
||||
|
||||
q, err := zabsReleaseFlags.Filter.Query()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "invalid filter specification on command line")
|
||||
}
|
||||
|
||||
abstractions, listErrors, err := endpoint.ListAbstractions(ctx, q)
|
||||
if err != nil {
|
||||
return err // context clear by invocation of command
|
||||
}
|
||||
if len(listErrors) > 0 {
|
||||
color.New(color.FgRed).Fprintf(os.Stderr, "there were errors in listing the abstractions:\n%s\n", listErrors)
|
||||
// proceed anyways with rest of abstractions
|
||||
}
|
||||
|
||||
return doZabsRelease_Common(ctx, abstractions)
|
||||
}
|
||||
|
||||
func doZabsReleaseStale(sc *cli.Subcommand, args []string) error {
|
||||
|
||||
var err error
|
||||
ctx := context.Background()
|
||||
|
||||
if len(args) > 0 {
|
||||
return errors.New("this subcommand takes no positional arguments")
|
||||
}
|
||||
|
||||
q, err := zabsReleaseFlags.Filter.Query()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "invalid filter specification on command line")
|
||||
}
|
||||
|
||||
stalenessInfo, err := endpoint.ListStale(ctx, q)
|
||||
if err != nil {
|
||||
return err // context clear by invocation of command
|
||||
}
|
||||
|
||||
return doZabsRelease_Common(ctx, stalenessInfo.Stale)
|
||||
}
|
||||
|
||||
func doZabsRelease_Common(ctx context.Context, destroy []endpoint.Abstraction) error {
|
||||
|
||||
if zabsReleaseFlags.DryRun {
|
||||
if zabsReleaseFlags.Json {
|
||||
m, err := json.MarshalIndent(destroy, "", " ")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if _, err := os.Stdout.Write(m); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println()
|
||||
} else {
|
||||
for _, a := range destroy {
|
||||
fmt.Printf("would destroy %s\n", a)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
outcome := endpoint.BatchDestroy(ctx, destroy)
|
||||
hadErr := false
|
||||
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
colorErr := color.New(color.FgRed)
|
||||
printfSuccess := color.New(color.FgGreen).FprintfFunc()
|
||||
printfSection := color.New(color.Bold).FprintfFunc()
|
||||
|
||||
for res := range outcome {
|
||||
hadErr = hadErr || res.DestroyErr != nil
|
||||
if zabsReleaseFlags.Json {
|
||||
err := enc.Encode(res)
|
||||
if err != nil {
|
||||
colorErr.Fprintf(os.Stderr, "cannot marshal there were errors in destroying the abstractions")
|
||||
}
|
||||
} else {
|
||||
printfSection(os.Stdout, "destroy %s ...", res.Abstraction)
|
||||
if res.DestroyErr != nil {
|
||||
colorErr.Fprintf(os.Stdout, " failed:\n%s\n", res.DestroyErr)
|
||||
} else {
|
||||
printfSuccess(os.Stdout, " OK\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if hadErr {
|
||||
colorErr.Add(color.Bold).Fprintf(os.Stderr, "there were errors in destroying the abstractions")
|
||||
return fmt.Errorf("")
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type IntervalAutosnap struct {
|
||||
task *Task
|
||||
DatasetFilter zfs.DatasetFilter
|
||||
Prefix string
|
||||
SnapshotInterval time.Duration
|
||||
}
|
||||
|
||||
func (a *IntervalAutosnap) filterFilesystems() (fss []*zfs.DatasetPath, stop bool) {
|
||||
a.task.Enter("filter_filesystems")
|
||||
defer a.task.Finish()
|
||||
fss, err := zfs.ZFSListMapping(a.DatasetFilter)
|
||||
stop = err != nil
|
||||
if err != nil {
|
||||
a.task.Log().WithError(err).Error("cannot list datasets")
|
||||
}
|
||||
if len(fss) == 0 {
|
||||
a.task.Log().Warn("no filesystem matching filesystem filter")
|
||||
}
|
||||
return fss, stop
|
||||
}
|
||||
|
||||
func (a *IntervalAutosnap) findSyncPoint(fss []*zfs.DatasetPath) (syncPoint time.Time, err error) {
|
||||
a.task.Enter("find_sync_point")
|
||||
defer a.task.Finish()
|
||||
type snapTime struct {
|
||||
ds *zfs.DatasetPath
|
||||
time time.Time
|
||||
}
|
||||
|
||||
if len(fss) == 0 {
|
||||
return time.Now(), nil
|
||||
}
|
||||
|
||||
snaptimes := make([]snapTime, 0, len(fss))
|
||||
|
||||
now := time.Now()
|
||||
|
||||
a.task.Log().Debug("examine filesystem state")
|
||||
for _, d := range fss {
|
||||
|
||||
l := a.task.Log().WithField(logFSField, d.ToString())
|
||||
|
||||
fsvs, err := zfs.ZFSListFilesystemVersions(d, NewPrefixFilter(a.Prefix))
|
||||
if err != nil {
|
||||
l.WithError(err).Error("cannot list filesystem versions")
|
||||
continue
|
||||
}
|
||||
if len(fsvs) <= 0 {
|
||||
l.WithField("prefix", a.Prefix).Info("no filesystem versions with prefix")
|
||||
continue
|
||||
}
|
||||
|
||||
// Sort versions by creation
|
||||
sort.SliceStable(fsvs, func(i, j int) bool {
|
||||
return fsvs[i].CreateTXG < fsvs[j].CreateTXG
|
||||
})
|
||||
|
||||
latest := fsvs[len(fsvs)-1]
|
||||
l.WithField("creation", latest.Creation).
|
||||
Debug("found latest snapshot")
|
||||
|
||||
since := now.Sub(latest.Creation)
|
||||
if since < 0 {
|
||||
l.WithField("snapshot", latest.Name).
|
||||
WithField("creation", latest.Creation).
|
||||
Error("snapshot is from the future")
|
||||
continue
|
||||
}
|
||||
next := now
|
||||
if since < a.SnapshotInterval {
|
||||
next = latest.Creation.Add(a.SnapshotInterval)
|
||||
}
|
||||
snaptimes = append(snaptimes, snapTime{d, next})
|
||||
}
|
||||
|
||||
if len(snaptimes) == 0 {
|
||||
snaptimes = append(snaptimes, snapTime{nil, now})
|
||||
}
|
||||
|
||||
sort.Slice(snaptimes, func(i, j int) bool {
|
||||
return snaptimes[i].time.Before(snaptimes[j].time)
|
||||
})
|
||||
|
||||
return snaptimes[0].time, nil
|
||||
|
||||
}
|
||||
|
||||
func (a *IntervalAutosnap) waitForSyncPoint(ctx context.Context, syncPoint time.Time) {
|
||||
a.task.Enter("wait_sync_point")
|
||||
defer a.task.Finish()
|
||||
|
||||
const LOG_TIME_FMT string = time.ANSIC
|
||||
|
||||
a.task.Log().WithField("sync_point", syncPoint.Format(LOG_TIME_FMT)).
|
||||
Info("wait for sync point")
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
a.task.Log().WithError(ctx.Err()).Info("context done")
|
||||
return
|
||||
case <-time.After(syncPoint.Sub(time.Now())):
|
||||
}
|
||||
}
|
||||
|
||||
func (a *IntervalAutosnap) syncUpRun(ctx context.Context, didSnaps chan struct{}) (stop bool) {
|
||||
a.task.Enter("sync_up")
|
||||
defer a.task.Finish()
|
||||
|
||||
fss, stop := a.filterFilesystems()
|
||||
if stop {
|
||||
return true
|
||||
}
|
||||
|
||||
syncPoint, err := a.findSyncPoint(fss)
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
|
||||
a.waitForSyncPoint(ctx, syncPoint)
|
||||
|
||||
a.task.Log().Debug("snapshot all filesystems to enable further snaps in lockstep")
|
||||
a.doSnapshots(didSnaps)
|
||||
return false
|
||||
}
|
||||
|
||||
func (a *IntervalAutosnap) Run(ctx context.Context, didSnaps chan struct{}) {
|
||||
|
||||
if a.syncUpRun(ctx, didSnaps) {
|
||||
a.task.Log().Error("stoppping autosnap after error in sync up")
|
||||
return
|
||||
}
|
||||
|
||||
// task drops back to idle here
|
||||
|
||||
a.task.Log().Debug("setting up ticker in SnapshotInterval")
|
||||
ticker := time.NewTicker(a.SnapshotInterval)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
ticker.Stop()
|
||||
a.task.Log().WithError(ctx.Err()).Info("context done")
|
||||
return
|
||||
|
||||
case <-ticker.C:
|
||||
a.doSnapshots(didSnaps)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (a *IntervalAutosnap) doSnapshots(didSnaps chan struct{}) {
|
||||
|
||||
a.task.Enter("do_snapshots")
|
||||
defer a.task.Finish()
|
||||
|
||||
// don't cache the result from previous run in case the user added
|
||||
// a new dataset in the meantime
|
||||
fss, stop := a.filterFilesystems()
|
||||
if stop {
|
||||
return
|
||||
}
|
||||
|
||||
a.task.Log().Info("beginning parallel snapshots")
|
||||
|
||||
// TODO channel programs -> allow a little jitter?
|
||||
var wg sync.WaitGroup
|
||||
for fsi := range fss {
|
||||
wg.Add(1)
|
||||
go func(fs *zfs.DatasetPath) {
|
||||
defer wg.Done()
|
||||
|
||||
suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
|
||||
snapname := fmt.Sprintf("%s%s", a.Prefix, suffix)
|
||||
|
||||
l := a.task.Log().WithField(logFSField, fs.ToString()).
|
||||
WithField("snapname", snapname)
|
||||
|
||||
l.Info("create snapshot")
|
||||
err := zfs.ZFSSnapshot(fs, snapname, false)
|
||||
if err != nil {
|
||||
l.WithError(err).Error("cannot create snapshot")
|
||||
return
|
||||
}
|
||||
|
||||
l.Info("create corresponding bookmark")
|
||||
err = zfs.ZFSBookmark(fs, snapname, snapname)
|
||||
if err != nil {
|
||||
l.WithError(err).Error("cannot create bookmark")
|
||||
}
|
||||
|
||||
}(fss[fsi])
|
||||
}
|
||||
|
||||
a.task.Log().Info("waiting for parallel snapshots to finish")
|
||||
wg.Wait()
|
||||
a.task.Log().Info("snapshots finished")
|
||||
|
||||
select {
|
||||
case didSnaps <- struct{}{}:
|
||||
default:
|
||||
a.task.Log().Error("warning: callback channel is full, discarding")
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"os"
|
||||
)
|
||||
|
||||
var bashcompCmd = &cobra.Command{
|
||||
Use: "bashcomp path/to/out/file",
|
||||
Short: "generate bash completions",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
if len(args) != 1 {
|
||||
fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
|
||||
cmd.Usage()
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := RootCmd.GenBashCompletionFile(args[0]); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error generating bash completion: %s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
},
|
||||
Hidden: true,
|
||||
}
|
||||
|
||||
func init() {
|
||||
RootCmd.AddCommand(bashcompCmd)
|
||||
}
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"fmt"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Global Global
|
||||
Jobs map[string]Job
|
||||
}
|
||||
|
||||
func (c *Config) LookupJob(name string) (j Job, err error) {
|
||||
j, ok := c.Jobs[name]
|
||||
if !ok {
|
||||
return nil, errors.Errorf("job '%s' is not defined", name)
|
||||
}
|
||||
return j, nil
|
||||
}
|
||||
|
||||
type Global struct {
|
||||
Serve struct {
|
||||
Stdinserver struct {
|
||||
SockDir string
|
||||
}
|
||||
}
|
||||
Control struct {
|
||||
Sockpath string
|
||||
}
|
||||
logging *LoggingConfig
|
||||
}
|
||||
|
||||
type JobDebugSettings struct {
|
||||
Conn struct {
|
||||
ReadDump string `mapstructure:"read_dump"`
|
||||
WriteDump string `mapstructure:"write_dump"`
|
||||
}
|
||||
RPC struct {
|
||||
Log bool
|
||||
}
|
||||
}
|
||||
|
||||
type RWCConnecter interface {
|
||||
Connect() (io.ReadWriteCloser, error)
|
||||
}
|
||||
type AuthenticatedChannelListenerFactory interface {
|
||||
Listen() (AuthenticatedChannelListener, error)
|
||||
}
|
||||
|
||||
type AuthenticatedChannelListener interface {
|
||||
Accept() (ch io.ReadWriteCloser, err error)
|
||||
Close() (err error)
|
||||
}
|
||||
|
||||
type SSHStdinServerConnectDescr struct {
|
||||
}
|
||||
|
||||
type PrunePolicy interface {
|
||||
// Prune filters versions and decide which to keep and which to remove.
|
||||
// Prune **does not** implement the actual removal of the versions.
|
||||
Prune(fs *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error)
|
||||
}
|
||||
|
||||
type PruningJob interface {
|
||||
Pruner(task *Task, side PrunePolicySide, dryRun bool) (Pruner, error)
|
||||
}
|
||||
|
||||
// A type for constants describing different prune policies of a PruningJob
|
||||
// This is mostly a special-case for LocalJob, which is the only job that has two prune policies
|
||||
// instead of one.
|
||||
// It implements github.com/spf13/pflag.Value to be used as CLI flag for the test subcommand
|
||||
type PrunePolicySide string
|
||||
|
||||
const (
|
||||
PrunePolicySideDefault PrunePolicySide = ""
|
||||
PrunePolicySideLeft PrunePolicySide = "left"
|
||||
PrunePolicySideRight PrunePolicySide = "right"
|
||||
)
|
||||
|
||||
func (s *PrunePolicySide) String() string {
|
||||
return string(*s)
|
||||
}
|
||||
|
||||
func (s *PrunePolicySide) Set(news string) error {
|
||||
p := PrunePolicySide(news)
|
||||
switch p {
|
||||
case PrunePolicySideRight:
|
||||
fallthrough
|
||||
case PrunePolicySideLeft:
|
||||
*s = p
|
||||
default:
|
||||
return errors.Errorf("must be either %s or %s", PrunePolicySideLeft, PrunePolicySideRight)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *PrunePolicySide) Type() string {
|
||||
return fmt.Sprintf("%s | %s", PrunePolicySideLeft, PrunePolicySideRight)
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"context"
|
||||
"github.com/jinzhu/copier"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/problame/go-netssh"
|
||||
)
|
||||
|
||||
type SSHStdinserverConnecter struct {
|
||||
Host string
|
||||
User string
|
||||
Port uint16
|
||||
IdentityFile string `mapstructure:"identity_file"`
|
||||
TransportOpenCommand []string `mapstructure:"transport_open_command"`
|
||||
SSHCommand string `mapstructure:"ssh_command"`
|
||||
Options []string
|
||||
}
|
||||
|
||||
func parseSSHStdinserverConnecter(i map[string]interface{}) (c *SSHStdinserverConnecter, err error) {
|
||||
|
||||
c = &SSHStdinserverConnecter{}
|
||||
if err = mapstructure.Decode(i, c); err != nil {
|
||||
err = errors.New(fmt.Sprintf("could not parse ssh transport: %s", err))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// TODO assert fields are filled
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
func (c *SSHStdinserverConnecter) Connect() (rwc io.ReadWriteCloser, err error) {
|
||||
|
||||
var endpoint netssh.Endpoint
|
||||
if err = copier.Copy(&endpoint, c); err != nil {
|
||||
return
|
||||
}
|
||||
if rwc, err = netssh.Dial(context.TODO(), endpoint); err != nil {
|
||||
err = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type PrefixFilter struct {
|
||||
prefix string
|
||||
fstype zfs.VersionType
|
||||
fstypeSet bool // optionals anyone?
|
||||
}
|
||||
|
||||
func NewPrefixFilter(prefix string) *PrefixFilter {
|
||||
return &PrefixFilter{prefix: prefix}
|
||||
}
|
||||
|
||||
func NewTypedPrefixFilter(prefix string, versionType zfs.VersionType) *PrefixFilter {
|
||||
return &PrefixFilter{prefix, versionType, true}
|
||||
}
|
||||
|
||||
func parseSnapshotPrefix(i string) (p string, err error) {
|
||||
if len(i) <= 0 {
|
||||
err = errors.Errorf("snapshot prefix must not be empty string")
|
||||
return
|
||||
}
|
||||
p = i
|
||||
return
|
||||
}
|
||||
|
||||
func (f *PrefixFilter) Filter(fsv zfs.FilesystemVersion) (accept bool, err error) {
|
||||
fstypeMatches := (!f.fstypeSet || fsv.Type == f.fstype)
|
||||
prefixMatches := strings.HasPrefix(fsv.Name, f.prefix)
|
||||
return fstypeMatches && prefixMatches, nil
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"github.com/pkg/errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type ControlJob struct {
|
||||
Name string
|
||||
sockaddr *net.UnixAddr
|
||||
}
|
||||
|
||||
func NewControlJob(name, sockpath string) (j *ControlJob, err error) {
|
||||
j = &ControlJob{Name: name}
|
||||
|
||||
j.sockaddr, err = net.ResolveUnixAddr("unix", sockpath)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "cannot resolve unix address")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (j *ControlJob) JobName() string {
|
||||
return j.Name
|
||||
}
|
||||
|
||||
func (j *ControlJob) JobStatus(ctx context.Context) (*JobStatus, error) {
|
||||
return &JobStatus{Tasks: nil}, nil
|
||||
}
|
||||
|
||||
const (
|
||||
ControlJobEndpointPProf string = "/debug/pprof"
|
||||
ControlJobEndpointVersion string = "/version"
|
||||
ControlJobEndpointStatus string = "/status"
|
||||
)
|
||||
|
||||
func (j *ControlJob) JobStart(ctx context.Context) {
|
||||
|
||||
log := ctx.Value(contextKeyLog).(Logger)
|
||||
defer log.Info("control job finished")
|
||||
|
||||
daemon := ctx.Value(contextKeyDaemon).(*Daemon)
|
||||
|
||||
l, err := ListenUnixPrivate(j.sockaddr)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("error listening")
|
||||
return
|
||||
}
|
||||
|
||||
pprofServer := NewPProfServer(ctx)
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle(ControlJobEndpointPProf, requestLogger{log: log, handlerFunc: func(w http.ResponseWriter, r *http.Request) {
|
||||
var msg PprofServerControlMsg
|
||||
err := json.NewDecoder(r.Body).Decode(&msg)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("bad pprof request from client")
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
}
|
||||
pprofServer.Control(msg)
|
||||
w.WriteHeader(200)
|
||||
}})
|
||||
mux.Handle(ControlJobEndpointVersion,
|
||||
requestLogger{log: log, handler: jsonResponder{func() (interface{}, error) {
|
||||
return NewZreplVersionInformation(), nil
|
||||
}}})
|
||||
mux.Handle(ControlJobEndpointStatus,
|
||||
requestLogger{log: log, handler: jsonResponder{func() (interface{}, error) {
|
||||
return daemon.Status(), nil
|
||||
}}})
|
||||
server := http.Server{Handler: mux}
|
||||
|
||||
outer:
|
||||
for {
|
||||
|
||||
served := make(chan error)
|
||||
go func() {
|
||||
served <- server.Serve(l)
|
||||
close(served)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.WithError(ctx.Err()).Info("context done")
|
||||
server.Shutdown(context.Background())
|
||||
break outer
|
||||
case err = <-served:
|
||||
if err != nil {
|
||||
log.WithError(err).Error("error serving")
|
||||
break outer
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
type jsonResponder struct {
|
||||
producer func() (interface{}, error)
|
||||
}
|
||||
|
||||
func (j jsonResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
res, err := j.producer()
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
io.WriteString(w, err.Error())
|
||||
return
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
err = json.NewEncoder(&buf).Encode(res)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
io.WriteString(w, err.Error())
|
||||
} else {
|
||||
io.Copy(w, &buf)
|
||||
}
|
||||
}
|
||||
|
||||
type requestLogger struct {
|
||||
log Logger
|
||||
handler http.Handler
|
||||
handlerFunc http.HandlerFunc
|
||||
}
|
||||
|
||||
func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
log := l.log.WithField("method", r.Method).WithField("url", r.URL)
|
||||
log.Info("start")
|
||||
if l.handlerFunc != nil {
|
||||
l.handlerFunc(w, r)
|
||||
} else if l.handler != nil {
|
||||
l.handler.ServeHTTP(w, r)
|
||||
} else {
|
||||
log.Error("no handler or handlerFunc configured")
|
||||
}
|
||||
log.Info("finish")
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"context"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/zrepl/zrepl/rpc"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type LocalJob struct {
|
||||
Name string
|
||||
Mapping *DatasetMapFilter
|
||||
SnapshotPrefix string
|
||||
Interval time.Duration
|
||||
InitialReplPolicy InitialReplPolicy
|
||||
PruneLHS PrunePolicy
|
||||
PruneRHS PrunePolicy
|
||||
Debug JobDebugSettings
|
||||
snapperTask *Task
|
||||
mainTask *Task
|
||||
handlerTask *Task
|
||||
pruneRHSTask *Task
|
||||
pruneLHSTask *Task
|
||||
}
|
||||
|
||||
func parseLocalJob(c JobParsingContext, name string, i map[string]interface{}) (j *LocalJob, err error) {
|
||||
|
||||
var asMap struct {
|
||||
Mapping map[string]string
|
||||
SnapshotPrefix string `mapstructure:"snapshot_prefix"`
|
||||
Interval string
|
||||
InitialReplPolicy string `mapstructure:"initial_repl_policy"`
|
||||
PruneLHS map[string]interface{} `mapstructure:"prune_lhs"`
|
||||
PruneRHS map[string]interface{} `mapstructure:"prune_rhs"`
|
||||
Debug map[string]interface{}
|
||||
}
|
||||
|
||||
if err = mapstructure.Decode(i, &asMap); err != nil {
|
||||
err = errors.Wrap(err, "mapstructure error")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
j = &LocalJob{Name: name}
|
||||
|
||||
if j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
|
||||
err = errors.Wrap(err, "cannot parse interval")
|
||||
return
|
||||
}
|
||||
|
||||
if j.InitialReplPolicy, err = parseInitialReplPolicy(asMap.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if j.PruneLHS, err = parsePrunePolicy(asMap.PruneLHS, true); err != nil {
|
||||
err = errors.Wrap(err, "cannot parse 'prune_lhs'")
|
||||
return
|
||||
}
|
||||
if j.PruneRHS, err = parsePrunePolicy(asMap.PruneRHS, false); err != nil {
|
||||
err = errors.Wrap(err, "cannot parse 'prune_rhs'")
|
||||
return
|
||||
}
|
||||
|
||||
if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
|
||||
err = errors.Wrap(err, "cannot parse 'debug'")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (j *LocalJob) JobName() string {
|
||||
return j.Name
|
||||
}
|
||||
|
||||
func (j *LocalJob) JobStart(ctx context.Context) {
|
||||
|
||||
rootLog := ctx.Value(contextKeyLog).(Logger)
|
||||
|
||||
j.snapperTask = NewTask("snapshot", rootLog)
|
||||
j.mainTask = NewTask("main", rootLog)
|
||||
j.handlerTask = NewTask("handler", rootLog)
|
||||
j.pruneRHSTask = NewTask("prune_rhs", rootLog)
|
||||
j.pruneLHSTask = NewTask("prune_lhs", rootLog)
|
||||
|
||||
local := rpc.NewLocalRPC()
|
||||
// Allow access to any dataset since we control what mapping
|
||||
// is passed to the pull routine.
|
||||
// All local datasets will be passed to its Map() function,
|
||||
// but only those for which a mapping exists will actually be pulled.
|
||||
// We can pay this small performance penalty for now.
|
||||
handler := NewHandler(j.handlerTask.Log(), localPullACL{}, NewPrefixFilter(j.SnapshotPrefix))
|
||||
|
||||
registerEndpoints(local, handler)
|
||||
|
||||
snapper := IntervalAutosnap{
|
||||
task: j.snapperTask,
|
||||
DatasetFilter: j.Mapping.AsFilter(),
|
||||
Prefix: j.SnapshotPrefix,
|
||||
SnapshotInterval: j.Interval,
|
||||
}
|
||||
|
||||
plhs, err := j.Pruner(j.pruneLHSTask, PrunePolicySideLeft, false)
|
||||
if err != nil {
|
||||
rootLog.WithError(err).Error("error creating lhs pruner")
|
||||
return
|
||||
}
|
||||
prhs, err := j.Pruner(j.pruneRHSTask, PrunePolicySideRight, false)
|
||||
if err != nil {
|
||||
rootLog.WithError(err).Error("error creating rhs pruner")
|
||||
return
|
||||
}
|
||||
|
||||
didSnaps := make(chan struct{})
|
||||
go snapper.Run(ctx, didSnaps)
|
||||
|
||||
outer:
|
||||
for {
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
j.mainTask.Log().WithError(ctx.Err()).Info("context")
|
||||
break outer
|
||||
case <-didSnaps:
|
||||
j.mainTask.Log().Debug("finished taking snapshots")
|
||||
j.mainTask.Log().Info("starting replication procedure")
|
||||
}
|
||||
|
||||
j.mainTask.Log().Debug("replicating from lhs to rhs")
|
||||
j.mainTask.Enter("replicate")
|
||||
puller := Puller{j.mainTask, local, j.Mapping, j.InitialReplPolicy}
|
||||
puller.Pull()
|
||||
j.mainTask.Finish()
|
||||
|
||||
// use a ctx as soon as Pull gains ctx support
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
break outer
|
||||
default:
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
j.mainTask.Log().Info("pruning lhs")
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
plhs.Run(ctx)
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
j.mainTask.Log().Info("pruning rhs")
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
prhs.Run(ctx)
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (j *LocalJob) JobStatus(ctxt context.Context) (*JobStatus, error) {
|
||||
return &JobStatus{Tasks: []*TaskStatus{
|
||||
j.snapperTask.Status(),
|
||||
j.pruneLHSTask.Status(),
|
||||
j.pruneRHSTask.Status(),
|
||||
j.mainTask.Status(),
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func (j *LocalJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) {
|
||||
|
||||
var dsfilter zfs.DatasetFilter
|
||||
var pp PrunePolicy
|
||||
switch side {
|
||||
case PrunePolicySideLeft:
|
||||
pp = j.PruneLHS
|
||||
dsfilter = j.Mapping.AsFilter()
|
||||
case PrunePolicySideRight:
|
||||
pp = j.PruneRHS
|
||||
dsfilter, err = j.Mapping.InvertedFilter()
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "cannot invert mapping for prune_rhs")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = errors.Errorf("must be either left or right side")
|
||||
return
|
||||
}
|
||||
|
||||
p = Pruner{
|
||||
task,
|
||||
time.Now(),
|
||||
dryRun,
|
||||
dsfilter,
|
||||
j.SnapshotPrefix,
|
||||
pp,
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/zrepl/zrepl/rpc"
|
||||
"github.com/zrepl/zrepl/util"
|
||||
)
|
||||
|
||||
type PullJob struct {
|
||||
Name string
|
||||
Connect RWCConnecter
|
||||
Interval time.Duration
|
||||
Mapping *DatasetMapFilter
|
||||
// constructed from mapping during parsing
|
||||
pruneFilter *DatasetMapFilter
|
||||
SnapshotPrefix string
|
||||
InitialReplPolicy InitialReplPolicy
|
||||
Prune PrunePolicy
|
||||
Debug JobDebugSettings
|
||||
|
||||
task *Task
|
||||
}
|
||||
|
||||
func parsePullJob(c JobParsingContext, name string, i map[string]interface{}) (j *PullJob, err error) {
|
||||
|
||||
var asMap struct {
|
||||
Connect map[string]interface{}
|
||||
Interval string
|
||||
Mapping map[string]string
|
||||
InitialReplPolicy string `mapstructure:"initial_repl_policy"`
|
||||
Prune map[string]interface{}
|
||||
SnapshotPrefix string `mapstructure:"snapshot_prefix"`
|
||||
Debug map[string]interface{}
|
||||
}
|
||||
|
||||
if err = mapstructure.Decode(i, &asMap); err != nil {
|
||||
err = errors.Wrap(err, "mapstructure error")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
j = &PullJob{Name: name}
|
||||
|
||||
j.Connect, err = parseSSHStdinserverConnecter(asMap.Connect)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "cannot parse 'connect'")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
|
||||
err = errors.Wrap(err, "cannot parse 'interval'")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
j.Mapping, err = parseDatasetMapFilter(asMap.Mapping, false)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "cannot parse 'mapping'")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if j.pruneFilter, err = j.Mapping.InvertedFilter(); err != nil {
|
||||
err = errors.Wrap(err, "cannot automatically invert 'mapping' for prune job")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
j.InitialReplPolicy, err = parseInitialReplPolicy(asMap.InitialReplPolicy, DEFAULT_INITIAL_REPL_POLICY)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "cannot parse 'initial_repl_policy'")
|
||||
return
|
||||
}
|
||||
|
||||
if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if j.Prune, err = parsePrunePolicy(asMap.Prune, false); err != nil {
|
||||
err = errors.Wrap(err, "cannot parse prune policy")
|
||||
return
|
||||
}
|
||||
|
||||
if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
|
||||
err = errors.Wrap(err, "cannot parse 'debug'")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (j *PullJob) JobName() string {
|
||||
return j.Name
|
||||
}
|
||||
|
||||
func (j *PullJob) JobStart(ctx context.Context) {
|
||||
|
||||
log := ctx.Value(contextKeyLog).(Logger)
|
||||
defer log.Info("exiting")
|
||||
j.task = NewTask("main", log)
|
||||
|
||||
// j.task is idle here idle here
|
||||
|
||||
ticker := time.NewTicker(j.Interval)
|
||||
for {
|
||||
j.doRun(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
j.task.Log().WithError(ctx.Err()).Info("context")
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (j *PullJob) doRun(ctx context.Context) {
|
||||
|
||||
j.task.Enter("run")
|
||||
defer j.task.Finish()
|
||||
|
||||
j.task.Log().Info("connecting")
|
||||
rwc, err := j.Connect.Connect()
|
||||
if err != nil {
|
||||
j.task.Log().WithError(err).Error("error connecting")
|
||||
return
|
||||
}
|
||||
|
||||
rwc, err = util.NewReadWriteCloserLogger(rwc, j.Debug.Conn.ReadDump, j.Debug.Conn.WriteDump)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
client := rpc.NewClient(rwc)
|
||||
if j.Debug.RPC.Log {
|
||||
client.SetLogger(j.task.Log(), true)
|
||||
}
|
||||
|
||||
j.task.Enter("pull")
|
||||
puller := Puller{j.task, client, j.Mapping, j.InitialReplPolicy}
|
||||
puller.Pull()
|
||||
closeRPCWithTimeout(j.task, client, time.Second*1, "")
|
||||
rwc.Close()
|
||||
j.task.Finish()
|
||||
|
||||
j.task.Enter("prune")
|
||||
pruner, err := j.Pruner(j.task, PrunePolicySideDefault, false)
|
||||
if err != nil {
|
||||
j.task.Log().WithError(err).Error("error creating pruner")
|
||||
} else {
|
||||
pruner.Run(ctx)
|
||||
}
|
||||
j.task.Finish()
|
||||
|
||||
}
|
||||
|
||||
func (j *PullJob) JobStatus(ctxt context.Context) (*JobStatus, error) {
|
||||
return &JobStatus{Tasks: []*TaskStatus{j.task.Status()}}, nil
|
||||
}
|
||||
|
||||
func (j *PullJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) {
|
||||
p = Pruner{
|
||||
task,
|
||||
time.Now(),
|
||||
dryRun,
|
||||
j.pruneFilter,
|
||||
j.SnapshotPrefix,
|
||||
j.Prune,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func closeRPCWithTimeout(task *Task, remote rpc.RPCClient, timeout time.Duration, goodbye string) {
|
||||
|
||||
task.Log().Info("closing rpc connection")
|
||||
|
||||
ch := make(chan error)
|
||||
go func() {
|
||||
ch <- remote.Close()
|
||||
close(ch)
|
||||
}()
|
||||
|
||||
var err error
|
||||
select {
|
||||
case <-time.After(timeout):
|
||||
err = fmt.Errorf("timeout exceeded (%s)", timeout)
|
||||
case closeRequestErr := <-ch:
|
||||
err = closeRequestErr
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
task.Log().WithError(err).Error("error closing connection")
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
mapstructure "github.com/mitchellh/mapstructure"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/zrepl/zrepl/rpc"
|
||||
"github.com/zrepl/zrepl/util"
|
||||
)
|
||||
|
||||
type SourceJob struct {
|
||||
Name string
|
||||
Serve AuthenticatedChannelListenerFactory
|
||||
Filesystems *DatasetMapFilter
|
||||
SnapshotPrefix string
|
||||
Interval time.Duration
|
||||
Prune PrunePolicy
|
||||
Debug JobDebugSettings
|
||||
serveTask *Task
|
||||
autosnapTask *Task
|
||||
pruneTask *Task
|
||||
}
|
||||
|
||||
func parseSourceJob(c JobParsingContext, name string, i map[string]interface{}) (j *SourceJob, err error) {
|
||||
|
||||
var asMap struct {
|
||||
Serve map[string]interface{}
|
||||
Filesystems map[string]string
|
||||
SnapshotPrefix string `mapstructure:"snapshot_prefix"`
|
||||
Interval string
|
||||
Prune map[string]interface{}
|
||||
Debug map[string]interface{}
|
||||
}
|
||||
|
||||
if err = mapstructure.Decode(i, &asMap); err != nil {
|
||||
err = errors.Wrap(err, "mapstructure error")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
j = &SourceJob{Name: name}
|
||||
|
||||
if j.Serve, err = parseAuthenticatedChannelListenerFactory(c, asMap.Serve); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if j.Filesystems, err = parseDatasetMapFilter(asMap.Filesystems, true); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if j.SnapshotPrefix, err = parseSnapshotPrefix(asMap.SnapshotPrefix); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if j.Interval, err = parsePostitiveDuration(asMap.Interval); err != nil {
|
||||
err = errors.Wrap(err, "cannot parse 'interval'")
|
||||
return
|
||||
}
|
||||
|
||||
if j.Prune, err = parsePrunePolicy(asMap.Prune, true); err != nil {
|
||||
err = errors.Wrap(err, "cannot parse 'prune'")
|
||||
return
|
||||
}
|
||||
|
||||
if err = mapstructure.Decode(asMap.Debug, &j.Debug); err != nil {
|
||||
err = errors.Wrap(err, "cannot parse 'debug'")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (j *SourceJob) JobName() string {
|
||||
return j.Name
|
||||
}
|
||||
|
||||
func (j *SourceJob) JobStart(ctx context.Context) {
|
||||
|
||||
log := ctx.Value(contextKeyLog).(Logger)
|
||||
defer log.Info("exiting")
|
||||
|
||||
j.autosnapTask = NewTask("autosnap", log)
|
||||
j.pruneTask = NewTask("prune", log)
|
||||
j.serveTask = NewTask("serve", log)
|
||||
|
||||
a := IntervalAutosnap{j.autosnapTask, j.Filesystems, j.SnapshotPrefix, j.Interval}
|
||||
p, err := j.Pruner(j.pruneTask, PrunePolicySideDefault, false)
|
||||
|
||||
if err != nil {
|
||||
log.WithError(err).Error("error creating pruner")
|
||||
return
|
||||
}
|
||||
|
||||
didSnaps := make(chan struct{})
|
||||
|
||||
go j.serve(ctx, j.serveTask)
|
||||
go a.Run(ctx, didSnaps)
|
||||
|
||||
outer:
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
break outer
|
||||
case <-didSnaps:
|
||||
log.Info("starting pruner")
|
||||
p.Run(ctx)
|
||||
log.Info("pruner done")
|
||||
}
|
||||
}
|
||||
log.WithError(ctx.Err()).Info("context")
|
||||
|
||||
}
|
||||
|
||||
func (j *SourceJob) JobStatus(ctxt context.Context) (*JobStatus, error) {
|
||||
return &JobStatus{
|
||||
Tasks: []*TaskStatus{
|
||||
j.autosnapTask.Status(),
|
||||
j.pruneTask.Status(),
|
||||
j.serveTask.Status(),
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func (j *SourceJob) Pruner(task *Task, side PrunePolicySide, dryRun bool) (p Pruner, err error) {
|
||||
p = Pruner{
|
||||
task,
|
||||
time.Now(),
|
||||
dryRun,
|
||||
j.Filesystems,
|
||||
j.SnapshotPrefix,
|
||||
j.Prune,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (j *SourceJob) serve(ctx context.Context, task *Task) {
|
||||
|
||||
listener, err := j.Serve.Listen()
|
||||
if err != nil {
|
||||
task.Log().WithError(err).Error("error listening")
|
||||
return
|
||||
}
|
||||
|
||||
type rwcChanMsg struct {
|
||||
rwc io.ReadWriteCloser
|
||||
err error
|
||||
}
|
||||
rwcChan := make(chan rwcChanMsg)
|
||||
|
||||
// Serve connections until interrupted or error
|
||||
outer:
|
||||
for {
|
||||
|
||||
go func() {
|
||||
rwc, err := listener.Accept()
|
||||
if err != nil {
|
||||
rwcChan <- rwcChanMsg{rwc, err}
|
||||
close(rwcChan)
|
||||
return
|
||||
}
|
||||
rwcChan <- rwcChanMsg{rwc, err}
|
||||
}()
|
||||
|
||||
select {
|
||||
|
||||
case rwcMsg := <-rwcChan:
|
||||
|
||||
if rwcMsg.err != nil {
|
||||
task.Log().WithError(err).Error("error accepting connection")
|
||||
break outer
|
||||
}
|
||||
|
||||
j.handleConnection(rwcMsg.rwc, task)
|
||||
|
||||
case <-ctx.Done():
|
||||
task.Log().WithError(ctx.Err()).Info("context")
|
||||
break outer
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
task.Enter("close_listener")
|
||||
defer task.Finish()
|
||||
err = listener.Close()
|
||||
if err != nil {
|
||||
task.Log().WithError(err).Error("error closing listener")
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
func (j *SourceJob) handleConnection(rwc io.ReadWriteCloser, task *Task) {
|
||||
|
||||
task.Enter("handle_connection")
|
||||
defer task.Finish()
|
||||
|
||||
task.Log().Info("handling client connection")
|
||||
|
||||
rwc, err := util.NewReadWriteCloserLogger(rwc, j.Debug.Conn.ReadDump, j.Debug.Conn.WriteDump)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// construct connection handler
|
||||
handler := NewHandler(task.Log(), j.Filesystems, NewPrefixFilter(j.SnapshotPrefix))
|
||||
|
||||
// handle connection
|
||||
rpcServer := rpc.NewServer(rwc)
|
||||
if j.Debug.RPC.Log {
|
||||
rpclog := task.Log().WithField("subsystem", "rpc")
|
||||
rpcServer.SetLogger(rpclog, true)
|
||||
}
|
||||
registerEndpoints(rpcServer, handler)
|
||||
if err = rpcServer.Serve(); err != nil {
|
||||
task.Log().WithError(err).Error("error serving connection")
|
||||
}
|
||||
|
||||
// wait for client to close connection
|
||||
// FIXME: we cannot just close it like we would to with a TCP socket because
|
||||
// FIXME: go-nettsh's Close() may overtake the remaining data in the pipe
|
||||
const CLIENT_HANGUP_TIMEOUT = 1 * time.Second
|
||||
task.Log().
|
||||
WithField("timeout", CLIENT_HANGUP_TIMEOUT).
|
||||
Debug("waiting for client to hang up")
|
||||
|
||||
wchan := make(chan error)
|
||||
go func() {
|
||||
var pseudo [1]byte
|
||||
_, err := io.ReadFull(rwc, pseudo[:])
|
||||
wchan <- err
|
||||
}()
|
||||
var werr error
|
||||
select {
|
||||
case werr = <-wchan:
|
||||
// all right
|
||||
case <-time.After(CLIENT_HANGUP_TIMEOUT):
|
||||
werr = errors.New("client did not close connection within timeout")
|
||||
}
|
||||
if werr != nil && werr != io.EOF {
|
||||
task.Log().WithError(werr).
|
||||
Error("error waiting for client to hang up")
|
||||
}
|
||||
task.Log().Info("closing client connection")
|
||||
if err = rwc.Close(); err != nil {
|
||||
task.Log().WithError(err).Error("error force-closing connection")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"github.com/mattn/go-isatty"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
type LoggingConfig struct {
|
||||
Outlets *logger.Outlets
|
||||
}
|
||||
|
||||
type MetadataFlags int64
|
||||
|
||||
const (
|
||||
MetadataTime MetadataFlags = 1 << iota
|
||||
MetadataLevel
|
||||
|
||||
MetadataNone MetadataFlags = 0
|
||||
MetadataAll MetadataFlags = ^0
|
||||
)
|
||||
|
||||
func parseLogging(i interface{}) (c *LoggingConfig, err error) {
|
||||
|
||||
c = &LoggingConfig{}
|
||||
c.Outlets = logger.NewOutlets()
|
||||
|
||||
var asList []interface{}
|
||||
if err = mapstructure.Decode(i, &asList); err != nil {
|
||||
return nil, errors.Wrap(err, "mapstructure error")
|
||||
}
|
||||
if len(asList) == 0 {
|
||||
// Default config
|
||||
out := WriterOutlet{&HumanFormatter{}, os.Stdout}
|
||||
c.Outlets.Add(out, logger.Warn)
|
||||
return
|
||||
}
|
||||
|
||||
var syslogOutlets, stdoutOutlets int
|
||||
for lei, le := range asList {
|
||||
|
||||
outlet, minLevel, err := parseOutlet(le)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "cannot parse outlet #%d", lei)
|
||||
}
|
||||
var _ logger.Outlet = WriterOutlet{}
|
||||
var _ logger.Outlet = &SyslogOutlet{}
|
||||
switch outlet.(type) {
|
||||
case *SyslogOutlet:
|
||||
syslogOutlets++
|
||||
case WriterOutlet:
|
||||
stdoutOutlets++
|
||||
}
|
||||
|
||||
c.Outlets.Add(outlet, minLevel)
|
||||
|
||||
}
|
||||
|
||||
if syslogOutlets > 1 {
|
||||
return nil, errors.Errorf("can only define one 'syslog' outlet")
|
||||
}
|
||||
if stdoutOutlets > 1 {
|
||||
return nil, errors.Errorf("can only define one 'stdout' outlet")
|
||||
}
|
||||
|
||||
return c, nil
|
||||
|
||||
}
|
||||
|
||||
func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
|
||||
var is string
|
||||
switch j := i.(type) {
|
||||
case string:
|
||||
is = j
|
||||
default:
|
||||
return nil, errors.Errorf("invalid log format: wrong type: %T", i)
|
||||
}
|
||||
|
||||
switch is {
|
||||
case "human":
|
||||
return &HumanFormatter{}, nil
|
||||
case "logfmt":
|
||||
return &LogfmtFormatter{}, nil
|
||||
case "json":
|
||||
return &JSONFormatter{}, nil
|
||||
default:
|
||||
return nil, errors.Errorf("invalid log format: '%s'", is)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func parseOutlet(i interface{}) (o logger.Outlet, level logger.Level, err error) {
|
||||
|
||||
var in struct {
|
||||
Outlet string
|
||||
Level string
|
||||
Format string
|
||||
}
|
||||
if err = mapstructure.Decode(i, &in); err != nil {
|
||||
err = errors.Wrap(err, "mapstructure error")
|
||||
return
|
||||
}
|
||||
if in.Outlet == "" || in.Level == "" || in.Format == "" {
|
||||
err = errors.Errorf("must specify 'outlet', 'level' and 'format' field")
|
||||
return
|
||||
}
|
||||
|
||||
minLevel, err := logger.ParseLevel(in.Level)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "cannot parse 'level' field")
|
||||
return
|
||||
}
|
||||
formatter, err := parseLogFormat(in.Format)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "cannot parse")
|
||||
return
|
||||
}
|
||||
|
||||
switch in.Outlet {
|
||||
case "stdout":
|
||||
o, err = parseStdoutOutlet(i, formatter)
|
||||
case "tcp":
|
||||
o, err = parseTCPOutlet(i, formatter)
|
||||
case "syslog":
|
||||
o, err = parseSyslogOutlet(i, formatter)
|
||||
default:
|
||||
err = errors.Errorf("unknown outlet type '%s'", in.Outlet)
|
||||
}
|
||||
return o, minLevel, err
|
||||
|
||||
}
|
||||
|
||||
func parseStdoutOutlet(i interface{}, formatter EntryFormatter) (WriterOutlet, error) {
|
||||
|
||||
var in struct {
|
||||
Time bool
|
||||
}
|
||||
if err := mapstructure.Decode(i, &in); err != nil {
|
||||
return WriterOutlet{}, errors.Wrap(err, "invalid structure for stdout outlet")
|
||||
}
|
||||
|
||||
flags := MetadataAll
|
||||
writer := os.Stdout
|
||||
if !isatty.IsTerminal(writer.Fd()) && !in.Time {
|
||||
flags &= ^MetadataTime
|
||||
}
|
||||
|
||||
formatter.SetMetadataFlags(flags)
|
||||
return WriterOutlet{
|
||||
formatter,
|
||||
os.Stdout,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseTCPOutlet(i interface{}, formatter EntryFormatter) (out *TCPOutlet, err error) {
|
||||
|
||||
var in struct {
|
||||
Net string
|
||||
Address string
|
||||
RetryInterval string `mapstructure:"retry_interval"`
|
||||
TLS *struct {
|
||||
CA string
|
||||
Cert string
|
||||
Key string
|
||||
}
|
||||
}
|
||||
if err = mapstructure.Decode(i, &in); err != nil {
|
||||
return nil, errors.Wrap(err, "mapstructure error")
|
||||
}
|
||||
|
||||
retryInterval, err := time.ParseDuration(in.RetryInterval)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
|
||||
}
|
||||
|
||||
if len(in.Net) == 0 {
|
||||
return nil, errors.New("field 'net' must not be empty")
|
||||
}
|
||||
if len(in.Address) == 0 {
|
||||
return nil, errors.New("field 'address' must not be empty")
|
||||
}
|
||||
|
||||
var tlsConfig *tls.Config
|
||||
if in.TLS != nil {
|
||||
|
||||
cert, err := tls.LoadX509KeyPair(in.TLS.Cert, in.TLS.Key)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannot load client cert")
|
||||
}
|
||||
|
||||
var rootCAs *x509.CertPool
|
||||
if in.TLS.CA == "" {
|
||||
if rootCAs, err = x509.SystemCertPool(); err != nil {
|
||||
return nil, errors.Wrap(err, "cannot open system cert pool")
|
||||
}
|
||||
} else {
|
||||
rootCAs = x509.NewCertPool()
|
||||
rootCAPEM, err := ioutil.ReadFile(in.TLS.CA)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannot load CA cert")
|
||||
}
|
||||
if !rootCAs.AppendCertsFromPEM(rootCAPEM) {
|
||||
return nil, errors.New("cannot parse CA cert")
|
||||
}
|
||||
}
|
||||
if err != nil && in.TLS.CA == "" {
|
||||
return nil, errors.Wrap(err, "cannot load root ca pool")
|
||||
}
|
||||
|
||||
tlsConfig = &tls.Config{
|
||||
Certificates: []tls.Certificate{cert},
|
||||
RootCAs: rootCAs,
|
||||
}
|
||||
|
||||
tlsConfig.BuildNameToCertificate()
|
||||
}
|
||||
|
||||
formatter.SetMetadataFlags(MetadataAll)
|
||||
return NewTCPOutlet(formatter, in.Net, in.Address, tlsConfig, retryInterval), nil
|
||||
|
||||
}
|
||||
|
||||
func parseSyslogOutlet(i interface{}, formatter EntryFormatter) (out *SyslogOutlet, err error) {
|
||||
|
||||
var in struct {
|
||||
RetryInterval string `mapstructure:"retry_interval"`
|
||||
}
|
||||
if err = mapstructure.Decode(i, &in); err != nil {
|
||||
return nil, errors.Wrap(err, "mapstructure error")
|
||||
}
|
||||
|
||||
out = &SyslogOutlet{}
|
||||
out.Formatter = formatter
|
||||
out.Formatter.SetMetadataFlags(MetadataNone)
|
||||
|
||||
out.RetryInterval = 0 // default to 0 as we assume local syslog will just work
|
||||
if in.RetryInterval != "" {
|
||||
out.RetryInterval, err = time.ParseDuration(in.RetryInterval)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannot parse 'retry_interval'")
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
@@ -1,12 +1,11 @@
|
||||
package filters
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/mitchellh/mapstructure"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
@@ -102,7 +101,6 @@ func (m DatasetMapFilter) mostSpecificPrefixMapping(path *zfs.DatasetPath) (idx
|
||||
return
|
||||
}
|
||||
|
||||
// Returns target == nil if there is no mapping
|
||||
func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error) {
|
||||
|
||||
if m.filterMode {
|
||||
@@ -116,17 +114,9 @@ func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath,
|
||||
}
|
||||
me := m.entries[mi]
|
||||
|
||||
if me.mapping == "" {
|
||||
// Special case treatment: 'foo/bar<' => ''
|
||||
if !me.subtreeMatch {
|
||||
return nil, fmt.Errorf("mapping to '' must be a subtree match")
|
||||
}
|
||||
// ok...
|
||||
} else {
|
||||
if strings.HasPrefix("!", me.mapping) {
|
||||
// reject mapping
|
||||
return nil, nil
|
||||
}
|
||||
if strings.HasPrefix("!", me.mapping) {
|
||||
// reject mapping
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
target, err = zfs.NewDatasetPath(me.mapping)
|
||||
@@ -161,7 +151,7 @@ func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
|
||||
}
|
||||
|
||||
// Construct a new filter-only DatasetMapFilter from a mapping
|
||||
// The new filter allows exactly those paths that were not forbidden by the mapping.
|
||||
// The new filter allows excactly those paths that were not forbidden by the mapping.
|
||||
func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
|
||||
|
||||
if m.filterMode {
|
||||
@@ -177,7 +167,7 @@ func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
|
||||
for i, e := range m.entries {
|
||||
inv.entries[i].path, err = zfs.NewDatasetPath(e.mapping)
|
||||
if err != nil {
|
||||
err = errors.Wrapf(err, "mapping cannot be inverted: '%s' is not a dataset path", e.mapping)
|
||||
err = errors.Wrapf(err, "mapping cannot be inverted: '%s' is not a dataset path: %s", e.mapping)
|
||||
return
|
||||
}
|
||||
inv.entries[i].mapping = MapFilterResultOk
|
||||
@@ -187,43 +177,12 @@ func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
|
||||
return inv, nil
|
||||
}
|
||||
|
||||
// FIXME investigate whether we can support more...
|
||||
func (m DatasetMapFilter) Invert() (endpoint.FSMap, error) {
|
||||
|
||||
if m.filterMode {
|
||||
return nil, errors.Errorf("can only invert mappings")
|
||||
}
|
||||
|
||||
if len(m.entries) != 1 {
|
||||
return nil, errors.Errorf("inversion of complicated mappings is not implemented") // FIXME
|
||||
}
|
||||
|
||||
e := m.entries[0]
|
||||
|
||||
inv := &DatasetMapFilter{
|
||||
make([]datasetMapFilterEntry, len(m.entries)),
|
||||
false,
|
||||
}
|
||||
mp, err := zfs.NewDatasetPath(e.mapping)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
inv.entries[0] = datasetMapFilterEntry{
|
||||
path: mp,
|
||||
mapping: e.path.ToString(),
|
||||
subtreeMatch: e.subtreeMatch,
|
||||
}
|
||||
|
||||
return inv, nil
|
||||
}
|
||||
|
||||
// Creates a new DatasetMapFilter in filter mode from a mapping
|
||||
// All accepting mapping results are mapped to accepting filter results
|
||||
// All rejecting mapping results are mapped to rejecting filter results
|
||||
func (m DatasetMapFilter) AsFilter() endpoint.FSFilter {
|
||||
func (m DatasetMapFilter) AsFilter() (f *DatasetMapFilter) {
|
||||
|
||||
f := &DatasetMapFilter{
|
||||
f = &DatasetMapFilter{
|
||||
make([]datasetMapFilterEntry, len(m.entries)),
|
||||
true,
|
||||
}
|
||||
@@ -258,14 +217,16 @@ func (m DatasetMapFilter) parseDatasetFilterResult(result string) (pass bool, er
|
||||
return false, fmt.Errorf("'%s' is not a valid filter result", result)
|
||||
}
|
||||
|
||||
func DatasetMapFilterFromConfig(in map[string]bool) (f *DatasetMapFilter, err error) {
|
||||
func parseDatasetMapFilter(mi interface{}, filterMode bool) (f *DatasetMapFilter, err error) {
|
||||
|
||||
f = NewDatasetMapFilter(len(in), true)
|
||||
for pathPattern, accept := range in {
|
||||
mapping := MapFilterResultOmit
|
||||
if accept {
|
||||
mapping = MapFilterResultOk
|
||||
}
|
||||
var m map[string]string
|
||||
if err = mapstructure.Decode(mi, &m); err != nil {
|
||||
err = fmt.Errorf("maps / filters must be specified as map[string]string: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
f = NewDatasetMapFilter(len(m), filterMode)
|
||||
for pathPattern, mapping := range m {
|
||||
if err = f.Add(pathPattern, mapping); err != nil {
|
||||
err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err)
|
||||
return
|
||||
@@ -0,0 +1,294 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
|
||||
"fmt"
|
||||
yaml "github.com/go-yaml/yaml"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
"github.com/pkg/errors"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
var ConfigFileDefaultLocations []string = []string{
|
||||
"/etc/zrepl/zrepl.yml",
|
||||
"/usr/local/etc/zrepl/zrepl.yml",
|
||||
}
|
||||
|
||||
const (
|
||||
JobNameControl string = "control"
|
||||
)
|
||||
|
||||
var ReservedJobNames []string = []string{
|
||||
JobNameControl,
|
||||
}
|
||||
|
||||
type ConfigParsingContext struct {
|
||||
Global *Global
|
||||
}
|
||||
|
||||
func ParseConfig(path string) (config *Config, err error) {
|
||||
|
||||
if path == "" {
|
||||
// Try default locations
|
||||
for _, l := range ConfigFileDefaultLocations {
|
||||
stat, err := os.Stat(l)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if !stat.Mode().IsRegular() {
|
||||
err = errors.Errorf("file at default location is not a regular file: %s", l)
|
||||
continue
|
||||
}
|
||||
path = l
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
var i interface{}
|
||||
|
||||
var bytes []byte
|
||||
|
||||
if bytes, err = ioutil.ReadFile(path); err != nil {
|
||||
err = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = yaml.Unmarshal(bytes, &i); err != nil {
|
||||
err = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
return parseConfig(i)
|
||||
}
|
||||
|
||||
func parseConfig(i interface{}) (c *Config, err error) {
|
||||
|
||||
var asMap struct {
|
||||
Global map[string]interface{}
|
||||
Jobs []map[string]interface{}
|
||||
}
|
||||
if err := mapstructure.Decode(i, &asMap); err != nil {
|
||||
return nil, errors.Wrap(err, "config root must be a dict")
|
||||
}
|
||||
|
||||
c = &Config{}
|
||||
|
||||
// Parse global with defaults
|
||||
c.Global.Serve.Stdinserver.SockDir = "/var/run/zrepl/stdinserver"
|
||||
c.Global.Control.Sockpath = "/var/run/zrepl/control"
|
||||
|
||||
err = mapstructure.Decode(asMap.Global, &c.Global)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "mapstructure error on 'global' section: %s")
|
||||
return
|
||||
}
|
||||
|
||||
if c.Global.logging, err = parseLogging(asMap.Global["logging"]); err != nil {
|
||||
return nil, errors.Wrap(err, "cannot parse logging section")
|
||||
}
|
||||
|
||||
cpc := ConfigParsingContext{&c.Global}
|
||||
jpc := JobParsingContext{cpc}
|
||||
|
||||
// Jobs
|
||||
c.Jobs = make(map[string]Job, len(asMap.Jobs))
|
||||
for i := range asMap.Jobs {
|
||||
job, err := parseJob(jpc, asMap.Jobs[i])
|
||||
if err != nil {
|
||||
// Try to find its name
|
||||
namei, ok := asMap.Jobs[i]["name"]
|
||||
if !ok {
|
||||
namei = fmt.Sprintf("<no name, entry #%d in list>", i)
|
||||
}
|
||||
err = errors.Wrapf(err, "cannot parse job '%v'", namei)
|
||||
return nil, err
|
||||
}
|
||||
jn := job.JobName()
|
||||
if _, ok := c.Jobs[jn]; ok {
|
||||
err = errors.Errorf("duplicate job name: %s", jn)
|
||||
return nil, err
|
||||
}
|
||||
c.Jobs[job.JobName()] = job
|
||||
}
|
||||
|
||||
cj, err := NewControlJob(JobNameControl, jpc.Global.Control.Sockpath)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "cannot create control job")
|
||||
return
|
||||
}
|
||||
c.Jobs[JobNameControl] = cj
|
||||
|
||||
return c, nil
|
||||
|
||||
}
|
||||
|
||||
func extractStringField(i map[string]interface{}, key string, notempty bool) (field string, err error) {
|
||||
vi, ok := i[key]
|
||||
if !ok {
|
||||
err = errors.Errorf("must have field '%s'", key)
|
||||
return "", err
|
||||
}
|
||||
field, ok = vi.(string)
|
||||
if !ok {
|
||||
err = errors.Errorf("'%s' field must have type string", key)
|
||||
return "", err
|
||||
}
|
||||
if notempty && len(field) <= 0 {
|
||||
err = errors.Errorf("'%s' field must not be empty", key)
|
||||
return "", err
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type JobParsingContext struct {
|
||||
ConfigParsingContext
|
||||
}
|
||||
|
||||
func parseJob(c JobParsingContext, i map[string]interface{}) (j Job, err error) {
|
||||
|
||||
name, err := extractStringField(i, "name", true)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, r := range ReservedJobNames {
|
||||
if name == r {
|
||||
err = errors.Errorf("job name '%s' is reserved", name)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
jobtype, err := extractStringField(i, "type", true)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
switch jobtype {
|
||||
case "pull":
|
||||
return parsePullJob(c, name, i)
|
||||
case "source":
|
||||
return parseSourceJob(c, name, i)
|
||||
case "local":
|
||||
return parseLocalJob(c, name, i)
|
||||
default:
|
||||
return nil, errors.Errorf("unknown job type '%s'", jobtype)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func parseConnect(i map[string]interface{}) (c RWCConnecter, err error) {
|
||||
|
||||
t, err := extractStringField(i, "type", true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch t {
|
||||
case "ssh+stdinserver":
|
||||
return parseSSHStdinserverConnecter(i)
|
||||
default:
|
||||
return nil, errors.Errorf("unknown connection type '%s'", t)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func parseInitialReplPolicy(v interface{}, defaultPolicy InitialReplPolicy) (p InitialReplPolicy, err error) {
|
||||
s, ok := v.(string)
|
||||
if !ok {
|
||||
goto err
|
||||
}
|
||||
|
||||
switch {
|
||||
case s == "":
|
||||
p = defaultPolicy
|
||||
case s == "most_recent":
|
||||
p = InitialReplPolicyMostRecent
|
||||
case s == "all":
|
||||
p = InitialReplPolicyAll
|
||||
default:
|
||||
goto err
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
err:
|
||||
err = errors.New(fmt.Sprintf("expected InitialReplPolicy, got %#v", v))
|
||||
return
|
||||
}
|
||||
|
||||
func parsePrunePolicy(v map[string]interface{}, willSeeBookmarks bool) (p PrunePolicy, err error) {
|
||||
|
||||
policyName, err := extractStringField(v, "policy", true)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
switch policyName {
|
||||
case "grid":
|
||||
return parseGridPrunePolicy(v, willSeeBookmarks)
|
||||
case "noprune":
|
||||
return NoPrunePolicy{}, nil
|
||||
default:
|
||||
err = errors.Errorf("unknown policy '%s'", policyName)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func parseAuthenticatedChannelListenerFactory(c JobParsingContext, v map[string]interface{}) (p AuthenticatedChannelListenerFactory, err error) {
|
||||
|
||||
t, err := extractStringField(v, "type", true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch t {
|
||||
case "stdinserver":
|
||||
return parseStdinserverListenerFactory(c, v)
|
||||
default:
|
||||
err = errors.Errorf("unknown type '%s'", t)
|
||||
return
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
|
||||
|
||||
func parsePostitiveDuration(e string) (d time.Duration, err error) {
|
||||
comps := durationStringRegex.FindStringSubmatch(e)
|
||||
if len(comps) != 3 {
|
||||
err = fmt.Errorf("does not match regex: %s %#v", e, comps)
|
||||
return
|
||||
}
|
||||
|
||||
durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if durationFactor <= 0 {
|
||||
return 0, errors.New("duration must be positive integer")
|
||||
}
|
||||
|
||||
var durationUnit time.Duration
|
||||
switch comps[2] {
|
||||
case "s":
|
||||
durationUnit = time.Second
|
||||
case "m":
|
||||
durationUnit = time.Minute
|
||||
case "h":
|
||||
durationUnit = time.Hour
|
||||
case "d":
|
||||
durationUnit = 24 * time.Hour
|
||||
case "w":
|
||||
durationUnit = 24 * 7 * time.Hour
|
||||
default:
|
||||
err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
|
||||
return
|
||||
}
|
||||
|
||||
d = time.Duration(durationFactor) * durationUnit
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/zrepl/zrepl/util"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
"math"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type GridPrunePolicy struct {
|
||||
RetentionGrid *util.RetentionGrid
|
||||
MaxBookmarks int
|
||||
}
|
||||
|
||||
const GridPrunePolicyMaxBookmarksKeepAll = -1
|
||||
|
||||
type retentionGridAdaptor struct {
|
||||
zfs.FilesystemVersion
|
||||
}
|
||||
|
||||
func (a retentionGridAdaptor) Date() time.Time {
|
||||
return a.Creation
|
||||
}
|
||||
|
||||
func (a retentionGridAdaptor) LessThan(b util.RetentionGridEntry) bool {
|
||||
return a.CreateTXG < b.(retentionGridAdaptor).CreateTXG
|
||||
}
|
||||
|
||||
// Prune filters snapshots with the retention grid.
|
||||
// Bookmarks are deleted such that KeepBookmarks are kept in the end.
|
||||
// The oldest bookmarks are removed first.
|
||||
func (p *GridPrunePolicy) Prune(_ *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error) {
|
||||
skeep, sremove := p.pruneSnapshots(versions)
|
||||
keep, remove = p.pruneBookmarks(skeep)
|
||||
remove = append(remove, sremove...)
|
||||
return keep, remove, nil
|
||||
}
|
||||
|
||||
func (p *GridPrunePolicy) pruneSnapshots(versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion) {
|
||||
|
||||
// Build adaptors for retention grid
|
||||
keep = []zfs.FilesystemVersion{}
|
||||
adaptors := make([]util.RetentionGridEntry, 0)
|
||||
for fsv := range versions {
|
||||
if versions[fsv].Type != zfs.Snapshot {
|
||||
keep = append(keep, versions[fsv])
|
||||
continue
|
||||
}
|
||||
adaptors = append(adaptors, retentionGridAdaptor{versions[fsv]})
|
||||
}
|
||||
|
||||
sort.SliceStable(adaptors, func(i, j int) bool {
|
||||
return adaptors[i].LessThan(adaptors[j])
|
||||
})
|
||||
now := adaptors[len(adaptors)-1].Date()
|
||||
|
||||
// Evaluate retention grid
|
||||
keepa, removea := p.RetentionGrid.FitEntries(now, adaptors)
|
||||
|
||||
// Revert adaptors
|
||||
for i := range keepa {
|
||||
keep = append(keep, keepa[i].(retentionGridAdaptor).FilesystemVersion)
|
||||
}
|
||||
remove = make([]zfs.FilesystemVersion, len(removea))
|
||||
for i := range removea {
|
||||
remove[i] = removea[i].(retentionGridAdaptor).FilesystemVersion
|
||||
}
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
func (p *GridPrunePolicy) pruneBookmarks(versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion) {
|
||||
|
||||
if p.MaxBookmarks == GridPrunePolicyMaxBookmarksKeepAll {
|
||||
return versions, []zfs.FilesystemVersion{}
|
||||
}
|
||||
|
||||
keep = []zfs.FilesystemVersion{}
|
||||
bookmarks := make([]zfs.FilesystemVersion, 0)
|
||||
for fsv := range versions {
|
||||
if versions[fsv].Type != zfs.Bookmark {
|
||||
keep = append(keep, versions[fsv])
|
||||
continue
|
||||
}
|
||||
bookmarks = append(bookmarks, versions[fsv])
|
||||
}
|
||||
|
||||
if len(bookmarks) == 0 {
|
||||
return keep, []zfs.FilesystemVersion{}
|
||||
}
|
||||
if len(bookmarks) < p.MaxBookmarks {
|
||||
keep = append(keep, bookmarks...)
|
||||
return keep, []zfs.FilesystemVersion{}
|
||||
}
|
||||
|
||||
// NOTE: sorting descending by descending by createtxg <=> sorting ascending wrt creation time
|
||||
sort.SliceStable(bookmarks, func(i, j int) bool {
|
||||
return (bookmarks[i].CreateTXG > bookmarks[j].CreateTXG)
|
||||
})
|
||||
|
||||
keep = append(keep, bookmarks[:p.MaxBookmarks]...)
|
||||
remove = bookmarks[p.MaxBookmarks:]
|
||||
|
||||
return keep, remove
|
||||
}
|
||||
|
||||
func parseGridPrunePolicy(e map[string]interface{}, willSeeBookmarks bool) (p *GridPrunePolicy, err error) {
|
||||
|
||||
const KeepBookmarksAllString = "all"
|
||||
var i struct {
|
||||
Grid string
|
||||
KeepBookmarks string `mapstructure:"keep_bookmarks"`
|
||||
}
|
||||
|
||||
dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{Result: &i, WeaklyTypedInput: true})
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "mapstructure error")
|
||||
return
|
||||
}
|
||||
if err = dec.Decode(e); err != nil {
|
||||
err = errors.Wrapf(err, "mapstructure error")
|
||||
return
|
||||
}
|
||||
|
||||
// Parse grid
|
||||
intervals, err := parseRetentionGridIntervalsString(i.Grid)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("cannot parse retention grid: %s", err)
|
||||
return
|
||||
}
|
||||
// Assert intervals are of increasing length (not necessarily required, but indicates config mistake)
|
||||
lastDuration := time.Duration(0)
|
||||
for i := range intervals {
|
||||
|
||||
if intervals[i].Length < lastDuration {
|
||||
// If all intervals before were keep=all, this is ok
|
||||
allPrevKeepCountAll := true
|
||||
for j := i - 1; allPrevKeepCountAll && j >= 0; j-- {
|
||||
allPrevKeepCountAll = intervals[j].KeepCount == util.RetentionGridKeepCountAll
|
||||
}
|
||||
if allPrevKeepCountAll {
|
||||
goto isMonotonicIncrease
|
||||
}
|
||||
err = errors.New("retention grid interval length must be monotonically increasing")
|
||||
return
|
||||
}
|
||||
isMonotonicIncrease:
|
||||
lastDuration = intervals[i].Length
|
||||
|
||||
}
|
||||
|
||||
// Parse KeepBookmarks
|
||||
keepBookmarks := 0
|
||||
if i.KeepBookmarks == KeepBookmarksAllString || (i.KeepBookmarks == "" && !willSeeBookmarks) {
|
||||
keepBookmarks = GridPrunePolicyMaxBookmarksKeepAll
|
||||
} else {
|
||||
i, err := strconv.ParseInt(i.KeepBookmarks, 10, 32)
|
||||
if err != nil || i <= 0 || i > math.MaxInt32 {
|
||||
return nil, errors.Errorf("keep_bookmarks must be positive integer or 'all'")
|
||||
}
|
||||
keepBookmarks = int(i)
|
||||
}
|
||||
return &GridPrunePolicy{
|
||||
util.NewRetentionGrid(intervals),
|
||||
keepBookmarks,
|
||||
}, nil
|
||||
}
|
||||
|
||||
var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
|
||||
|
||||
func parseRetentionGridIntervalString(e string) (intervals []util.RetentionInterval, err error) {
|
||||
|
||||
comps := retentionStringIntervalRegex.FindStringSubmatch(e)
|
||||
if comps == nil {
|
||||
err = fmt.Errorf("retention string does not match expected format")
|
||||
return
|
||||
}
|
||||
|
||||
times, err := strconv.Atoi(comps[1])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} else if times <= 0 {
|
||||
return nil, fmt.Errorf("contains factor <= 0")
|
||||
}
|
||||
|
||||
duration, err := parsePostitiveDuration(comps[2])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
keepCount := 1
|
||||
if comps[3] != "" {
|
||||
// Decompose key=value, comma separated
|
||||
// For now, only keep_count is supported
|
||||
re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
|
||||
res := re.FindStringSubmatch(comps[4])
|
||||
if res == nil || len(res) != 2 {
|
||||
err = fmt.Errorf("interval parameter contains unknown parameters")
|
||||
return
|
||||
}
|
||||
if res[1] == "all" {
|
||||
keepCount = util.RetentionGridKeepCountAll
|
||||
} else {
|
||||
keepCount, err = strconv.Atoi(res[1])
|
||||
if err != nil {
|
||||
err = fmt.Errorf("cannot parse keep_count value")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
intervals = make([]util.RetentionInterval, times)
|
||||
for i := range intervals {
|
||||
intervals[i] = util.RetentionInterval{
|
||||
Length: duration,
|
||||
KeepCount: keepCount,
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
func parseRetentionGridIntervalsString(s string) (intervals []util.RetentionInterval, err error) {
|
||||
|
||||
ges := strings.Split(s, "|")
|
||||
intervals = make([]util.RetentionInterval, 0, 7*len(ges))
|
||||
|
||||
for intervalIdx, e := range ges {
|
||||
parsed, err := parseRetentionGridIntervalString(e)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
|
||||
}
|
||||
intervals = append(intervals, parsed...)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package cmd
|
||||
|
||||
import "github.com/zrepl/zrepl/zfs"
|
||||
|
||||
type NoPrunePolicy struct{}
|
||||
|
||||
func (p NoPrunePolicy) Prune(fs *zfs.DatasetPath, versions []zfs.FilesystemVersion) (keep, remove []zfs.FilesystemVersion, err error) {
|
||||
keep = versions
|
||||
remove = []zfs.FilesystemVersion{}
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"github.com/mitchellh/mapstructure"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/problame/go-netssh"
|
||||
"io"
|
||||
"path"
|
||||
)
|
||||
|
||||
type StdinserverListenerFactory struct {
|
||||
ClientIdentity string `mapstructure:"client_identity"`
|
||||
sockpath string
|
||||
}
|
||||
|
||||
func parseStdinserverListenerFactory(c JobParsingContext, i map[string]interface{}) (f *StdinserverListenerFactory, err error) {
|
||||
|
||||
f = &StdinserverListenerFactory{}
|
||||
|
||||
if err = mapstructure.Decode(i, f); err != nil {
|
||||
return nil, errors.Wrap(err, "mapstructure error")
|
||||
}
|
||||
if !(len(f.ClientIdentity) > 0) {
|
||||
err = errors.Errorf("must specify 'client_identity'")
|
||||
return
|
||||
}
|
||||
|
||||
f.sockpath = path.Join(c.Global.Serve.Stdinserver.SockDir, f.ClientIdentity)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (f *StdinserverListenerFactory) Listen() (al AuthenticatedChannelListener, err error) {
|
||||
|
||||
if err = PreparePrivateSockpath(f.sockpath); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
l, err := netssh.Listen(f.sockpath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return StdinserverListener{l}, nil
|
||||
}
|
||||
|
||||
type StdinserverListener struct {
|
||||
l *netssh.Listener
|
||||
}
|
||||
|
||||
func (l StdinserverListener) Accept() (ch io.ReadWriteCloser, err error) {
|
||||
return l.l.Accept()
|
||||
}
|
||||
|
||||
func (l StdinserverListener) Close() (err error) {
|
||||
return l.l.Close()
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kr/pretty"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/zrepl/zrepl/util"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
|
||||
|
||||
paths := []string{
|
||||
"./sampleconf/localbackup/host1.yml",
|
||||
"./sampleconf/pullbackup/backuphost.yml",
|
||||
"./sampleconf/pullbackup/productionhost.yml",
|
||||
"./sampleconf/random/debugging.yml",
|
||||
"./sampleconf/random/logging.yml",
|
||||
}
|
||||
|
||||
for _, p := range paths {
|
||||
|
||||
c, err := ParseConfig(p)
|
||||
if err != nil {
|
||||
t.Errorf("error parsing %s:\n%+v", p, err)
|
||||
}
|
||||
|
||||
t.Logf("file: %s", p)
|
||||
t.Log(pretty.Sprint(c))
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestParseRetentionGridStringParsing(t *testing.T) {
|
||||
|
||||
intervals, err := parseRetentionGridIntervalsString("2x10m(keep=2) | 1x1h | 3x1w")
|
||||
|
||||
assert.Nil(t, err)
|
||||
assert.Len(t, intervals, 6)
|
||||
proto := util.RetentionInterval{
|
||||
KeepCount: 2,
|
||||
Length: 10 * time.Minute,
|
||||
}
|
||||
assert.EqualValues(t, proto, intervals[0])
|
||||
assert.EqualValues(t, proto, intervals[1])
|
||||
|
||||
proto.KeepCount = 1
|
||||
proto.Length = 1 * time.Hour
|
||||
assert.EqualValues(t, proto, intervals[2])
|
||||
|
||||
proto.Length = 7 * 24 * time.Hour
|
||||
assert.EqualValues(t, proto, intervals[3])
|
||||
assert.EqualValues(t, proto, intervals[4])
|
||||
assert.EqualValues(t, proto, intervals[5])
|
||||
|
||||
intervals, err = parseRetentionGridIntervalsString("|")
|
||||
assert.Error(t, err)
|
||||
intervals, err = parseRetentionGridIntervalsString("2x10m")
|
||||
assert.NoError(t, err)
|
||||
|
||||
intervals, err = parseRetentionGridIntervalsString("1x10m(keep=all)")
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, intervals, 1)
|
||||
assert.EqualValues(t, util.RetentionGridKeepCountAll, intervals[0].KeepCount)
|
||||
|
||||
}
|
||||
|
||||
func TestDatasetMapFilter(t *testing.T) {
|
||||
|
||||
expectMapping := func(m map[string]string, from, to string) {
|
||||
dmf, err := parseDatasetMapFilter(m, false)
|
||||
if err != nil {
|
||||
t.Logf("expect test map to be valid: %s", err)
|
||||
t.FailNow()
|
||||
}
|
||||
fromPath, err := zfs.NewDatasetPath(from)
|
||||
if err != nil {
|
||||
t.Logf("expect test from path to be valid: %s", err)
|
||||
t.FailNow()
|
||||
}
|
||||
|
||||
res, err := dmf.Map(fromPath)
|
||||
if to == "" {
|
||||
assert.Nil(t, res)
|
||||
assert.Nil(t, err)
|
||||
t.Logf("%s => NOT MAPPED", fromPath.ToString())
|
||||
return
|
||||
}
|
||||
|
||||
assert.Nil(t, err)
|
||||
toPath, err := zfs.NewDatasetPath(to)
|
||||
if err != nil {
|
||||
t.Logf("expect test to path to be valid: %s", err)
|
||||
t.FailNow()
|
||||
}
|
||||
assert.True(t, res.Equal(toPath))
|
||||
}
|
||||
|
||||
expectFilter := func(m map[string]string, path string, pass bool) {
|
||||
dmf, err := parseDatasetMapFilter(m, true)
|
||||
if err != nil {
|
||||
t.Logf("expect test filter to be valid: %s", err)
|
||||
t.FailNow()
|
||||
}
|
||||
p, err := zfs.NewDatasetPath(path)
|
||||
if err != nil {
|
||||
t.Logf("expect test path to be valid: %s", err)
|
||||
t.FailNow()
|
||||
}
|
||||
res, err := dmf.Filter(p)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, pass, res)
|
||||
}
|
||||
|
||||
map1 := map[string]string{
|
||||
"a/b/c<": "root1",
|
||||
"a/b<": "root2",
|
||||
"<": "root3/b/c",
|
||||
"b": "!",
|
||||
"a/b/c/d/e<": "!",
|
||||
"q<": "root4/1/2",
|
||||
}
|
||||
|
||||
expectMapping(map1, "a/b/c", "root1")
|
||||
expectMapping(map1, "a/b/c/d", "root1/d")
|
||||
expectMapping(map1, "a/b/c/d/e", "")
|
||||
expectMapping(map1, "a/b/e", "root2/e")
|
||||
expectMapping(map1, "a/b", "root2")
|
||||
expectMapping(map1, "x", "root3/b/c/x")
|
||||
expectMapping(map1, "x/y", "root3/b/c/x/y")
|
||||
expectMapping(map1, "q", "root4/1/2")
|
||||
expectMapping(map1, "b", "")
|
||||
expectMapping(map1, "q/r", "root4/1/2/r")
|
||||
|
||||
filter1 := map[string]string{
|
||||
"<": "!",
|
||||
"a<": "ok",
|
||||
"a/b<": "!",
|
||||
}
|
||||
|
||||
expectFilter(filter1, "b", false)
|
||||
expectFilter(filter1, "a", true)
|
||||
expectFilter(filter1, "a/d", true)
|
||||
expectFilter(filter1, "a/b", false)
|
||||
expectFilter(filter1, "a/b/c", false)
|
||||
|
||||
filter2 := map[string]string{}
|
||||
expectFilter(filter2, "foo", false) // default to omit
|
||||
|
||||
}
|
||||
|
||||
func TestDatasetMapFilter_AsFilter(t *testing.T) {
|
||||
|
||||
mapspec := map[string]string{
|
||||
"a/b/c<": "root1",
|
||||
"a/b<": "root2",
|
||||
"<": "root3/b/c",
|
||||
"b": "!",
|
||||
"a/b/c/d/e<": "!",
|
||||
"q<": "root4/1/2",
|
||||
}
|
||||
|
||||
m, err := parseDatasetMapFilter(mapspec, false)
|
||||
assert.Nil(t, err)
|
||||
|
||||
f := m.AsFilter()
|
||||
|
||||
t.Logf("Mapping:\n%s\nFilter:\n%s", pretty.Sprint(m), pretty.Sprint(f))
|
||||
|
||||
tf := func(f zfs.DatasetFilter, path string, pass bool) {
|
||||
p, err := zfs.NewDatasetPath(path)
|
||||
assert.Nil(t, err)
|
||||
r, err := f.Filter(p)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, pass, r)
|
||||
}
|
||||
|
||||
tf(f, "a/b/c", true)
|
||||
tf(f, "a/b", true)
|
||||
tf(f, "b", false)
|
||||
tf(f, "a/b/c/d/e", false)
|
||||
tf(f, "a/b/c/d/e/f", false)
|
||||
tf(f, "a", true)
|
||||
|
||||
}
|
||||
|
||||
func TestDatasetMapFilter_InvertedFilter(t *testing.T) {
|
||||
mapspec := map[string]string{
|
||||
"a/b": "1/2",
|
||||
"a/b/c<": "3",
|
||||
"a/b/c/d<": "1/2/a",
|
||||
"a/b/d": "!",
|
||||
}
|
||||
|
||||
m, err := parseDatasetMapFilter(mapspec, false)
|
||||
assert.Nil(t, err)
|
||||
|
||||
inv, err := m.InvertedFilter()
|
||||
assert.Nil(t, err)
|
||||
|
||||
t.Log(pretty.Sprint(inv))
|
||||
|
||||
expectMapping := func(m *DatasetMapFilter, ps string, expRes bool) {
|
||||
p, err := zfs.NewDatasetPath(ps)
|
||||
assert.Nil(t, err)
|
||||
r, err := m.Filter(p)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, expRes, r)
|
||||
}
|
||||
|
||||
expectMapping(inv, "4", false)
|
||||
expectMapping(inv, "3", true)
|
||||
expectMapping(inv, "3/x", true)
|
||||
expectMapping(inv, "1", false)
|
||||
expectMapping(inv, "1/2", true)
|
||||
expectMapping(inv, "1/2/3", false)
|
||||
expectMapping(inv, "1/2/a/b", true)
|
||||
|
||||
}
|
||||
+305
@@ -0,0 +1,305 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"io"
|
||||
golog "log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var controlCmd = &cobra.Command{
|
||||
Use: "control",
|
||||
Short: "control zrepl daemon",
|
||||
}
|
||||
|
||||
var pprofCmd = &cobra.Command{
|
||||
Use: "pprof off | [on TCP_LISTEN_ADDRESS]",
|
||||
Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
|
||||
Run: doControlPProf,
|
||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
if cmd.Flags().NArg() < 1 {
|
||||
goto enargs
|
||||
}
|
||||
switch cmd.Flags().Arg(0) {
|
||||
case "on":
|
||||
pprofCmdArgs.msg.Run = true
|
||||
if cmd.Flags().NArg() != 2 {
|
||||
return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
|
||||
}
|
||||
pprofCmdArgs.msg.HttpListenAddress = cmd.Flags().Arg(1)
|
||||
case "off":
|
||||
if cmd.Flags().NArg() != 1 {
|
||||
goto enargs
|
||||
}
|
||||
pprofCmdArgs.msg.Run = false
|
||||
}
|
||||
return nil
|
||||
enargs:
|
||||
return errors.New("invalid number of positional arguments")
|
||||
|
||||
},
|
||||
}
|
||||
var pprofCmdArgs struct {
|
||||
msg PprofServerControlMsg
|
||||
}
|
||||
|
||||
var controlVersionCmd = &cobra.Command{
|
||||
Use: "version",
|
||||
Short: "print version of running zrepl daemon",
|
||||
Run: doControLVersionCmd,
|
||||
}
|
||||
|
||||
var controlStatusCmdArgs struct {
|
||||
format string
|
||||
level logger.Level
|
||||
onlyShowJob string
|
||||
}
|
||||
|
||||
var controlStatusCmd = &cobra.Command{
|
||||
Use: "status [JOB_NAME]",
|
||||
Short: "get current status",
|
||||
Run: doControlStatusCmd,
|
||||
}
|
||||
|
||||
func init() {
|
||||
RootCmd.AddCommand(controlCmd)
|
||||
controlCmd.AddCommand(pprofCmd)
|
||||
controlCmd.AddCommand(controlVersionCmd)
|
||||
controlCmd.AddCommand(controlStatusCmd)
|
||||
controlStatusCmd.Flags().StringVar(&controlStatusCmdArgs.format, "format", "human", "output format (human|raw)")
|
||||
controlStatusCmdArgs.level = logger.Warn
|
||||
controlStatusCmd.Flags().Var(&controlStatusCmdArgs.level, "level", "minimum log level to show")
|
||||
}
|
||||
|
||||
func controlHttpClient() (client http.Client, err error) {
|
||||
|
||||
conf, err := ParseConfig(rootArgs.configFile)
|
||||
if err != nil {
|
||||
return http.Client{}, err
|
||||
}
|
||||
|
||||
return http.Client{
|
||||
Transport: &http.Transport{
|
||||
DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
|
||||
return net.Dial("unix", conf.Global.Control.Sockpath)
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func doControlPProf(cmd *cobra.Command, args []string) {
|
||||
|
||||
log := golog.New(os.Stderr, "", 0)
|
||||
|
||||
die := func() {
|
||||
log.Printf("exiting after error")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
log.Printf("connecting to zrepl daemon")
|
||||
httpc, err := controlHttpClient()
|
||||
if err != nil {
|
||||
log.Printf("error parsing config: %s", err)
|
||||
die()
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := json.NewEncoder(&buf).Encode(&pprofCmdArgs.msg); err != nil {
|
||||
log.Printf("error marshaling request: %s", err)
|
||||
die()
|
||||
}
|
||||
_, err = httpc.Post("http://unix"+ControlJobEndpointPProf, "application/json", &buf)
|
||||
if err != nil {
|
||||
log.Printf("error: %s", err)
|
||||
die()
|
||||
}
|
||||
|
||||
log.Printf("finished")
|
||||
}
|
||||
|
||||
func doControLVersionCmd(cmd *cobra.Command, args []string) {
|
||||
|
||||
log := golog.New(os.Stderr, "", 0)
|
||||
|
||||
die := func() {
|
||||
log.Printf("exiting after error")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
httpc, err := controlHttpClient()
|
||||
if err != nil {
|
||||
log.Printf("could not connect to daemon: %s", err)
|
||||
die()
|
||||
}
|
||||
|
||||
resp, err := httpc.Get("http://unix" + ControlJobEndpointVersion)
|
||||
if err != nil {
|
||||
log.Printf("error: %s", err)
|
||||
die()
|
||||
} else if resp.StatusCode != http.StatusOK {
|
||||
var msg bytes.Buffer
|
||||
io.CopyN(&msg, resp.Body, 4096)
|
||||
log.Printf("error: %s", msg.String())
|
||||
die()
|
||||
}
|
||||
|
||||
var info ZreplVersionInformation
|
||||
err = json.NewDecoder(resp.Body).Decode(&info)
|
||||
if err != nil {
|
||||
log.Printf("error unmarshaling response: %s", err)
|
||||
die()
|
||||
}
|
||||
|
||||
fmt.Println(info.String())
|
||||
|
||||
}
|
||||
|
||||
func doControlStatusCmd(cmd *cobra.Command, args []string) {
|
||||
|
||||
log := golog.New(os.Stderr, "", 0)
|
||||
|
||||
die := func() {
|
||||
log.Print("exiting after error")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if len(args) == 1 {
|
||||
controlStatusCmdArgs.onlyShowJob = args[0]
|
||||
} else if len(args) > 1 {
|
||||
log.Print("can only specify one job as positional argument")
|
||||
cmd.Usage()
|
||||
die()
|
||||
}
|
||||
|
||||
httpc, err := controlHttpClient()
|
||||
if err != nil {
|
||||
log.Printf("could not connect to daemon: %s", err)
|
||||
die()
|
||||
}
|
||||
|
||||
resp, err := httpc.Get("http://unix" + ControlJobEndpointStatus)
|
||||
if err != nil {
|
||||
log.Printf("error: %s", err)
|
||||
die()
|
||||
} else if resp.StatusCode != http.StatusOK {
|
||||
var msg bytes.Buffer
|
||||
io.CopyN(&msg, resp.Body, 4096)
|
||||
log.Printf("error: %s", msg.String())
|
||||
die()
|
||||
}
|
||||
|
||||
var status DaemonStatus
|
||||
err = json.NewDecoder(resp.Body).Decode(&status)
|
||||
if err != nil {
|
||||
log.Printf("error unmarshaling response: %s", err)
|
||||
die()
|
||||
}
|
||||
|
||||
switch controlStatusCmdArgs.format {
|
||||
case "raw":
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
if err := enc.Encode(status); err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
case "human":
|
||||
|
||||
formatter := HumanFormatter{}
|
||||
formatter.SetMetadataFlags(MetadataAll)
|
||||
formatter.SetIgnoreFields([]string{
|
||||
logJobField,
|
||||
})
|
||||
jobNames := make([]string, 0, len(status.Jobs))
|
||||
for name, _ := range status.Jobs {
|
||||
jobNames = append(jobNames, name)
|
||||
}
|
||||
sort.Slice(jobNames, func(i, j int) bool {
|
||||
return strings.Compare(jobNames[i], jobNames[j]) == -1
|
||||
})
|
||||
now := time.Now()
|
||||
for _, name := range jobNames {
|
||||
|
||||
if controlStatusCmdArgs.onlyShowJob != "" && name != controlStatusCmdArgs.onlyShowJob {
|
||||
continue
|
||||
}
|
||||
|
||||
job := status.Jobs[name]
|
||||
jobLogEntries := make([]logger.Entry, 0)
|
||||
informAboutError := false
|
||||
|
||||
fmt.Printf("Job '%s':\n", name)
|
||||
for _, task := range job.Tasks {
|
||||
|
||||
var header bytes.Buffer
|
||||
fmt.Fprintf(&header, " Task '%s': ", task.Name)
|
||||
if !task.Idle {
|
||||
fmt.Fprint(&header, strings.Join(task.ActivityStack, "."))
|
||||
} else {
|
||||
fmt.Fprint(&header, "<idle>")
|
||||
}
|
||||
fmt.Fprint(&header, " ")
|
||||
const TASK_STALLED_HOLDOFF_DURATION = 10 * time.Second
|
||||
sinceLastUpdate := now.Sub(task.LastUpdate)
|
||||
if !task.Idle || task.ProgressRx != 0 || task.ProgressTx != 0 {
|
||||
fmt.Fprintf(&header, "(%s / %s , Rx/Tx",
|
||||
humanize.Bytes(uint64(task.ProgressRx)),
|
||||
humanize.Bytes(uint64(task.ProgressTx)))
|
||||
if task.Idle {
|
||||
fmt.Fprint(&header, ", values from last run")
|
||||
}
|
||||
fmt.Fprint(&header, ")")
|
||||
}
|
||||
fmt.Fprint(&header, "\n")
|
||||
if !task.Idle && !task.LastUpdate.IsZero() && sinceLastUpdate >= TASK_STALLED_HOLDOFF_DURATION {
|
||||
informAboutError = true
|
||||
fmt.Fprintf(&header, " WARNING: last update %s ago at %s)",
|
||||
sinceLastUpdate.String(),
|
||||
task.LastUpdate.Format(HumanFormatterDateFormat))
|
||||
fmt.Fprint(&header, "\n")
|
||||
}
|
||||
io.Copy(os.Stdout, &header)
|
||||
|
||||
jobLogEntries = append(jobLogEntries, task.LogEntries...)
|
||||
informAboutError = informAboutError || task.MaxLogLevel >= logger.Warn
|
||||
}
|
||||
|
||||
sort.Slice(jobLogEntries, func(i, j int) bool {
|
||||
return jobLogEntries[i].Time.Before(jobLogEntries[j].Time)
|
||||
})
|
||||
if informAboutError {
|
||||
fmt.Println(" WARNING: Some tasks encountered problems since the last time they left idle state:")
|
||||
fmt.Println(" check the logs below or your log file for more information.")
|
||||
fmt.Println(" Use the --level flag if you need debug information.")
|
||||
fmt.Println()
|
||||
}
|
||||
for _, e := range jobLogEntries {
|
||||
if e.Level < controlStatusCmdArgs.level {
|
||||
continue
|
||||
}
|
||||
formatted, err := formatter.Format(&e)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Printf(" %s\n", string(formatted))
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
}
|
||||
default:
|
||||
log.Printf("invalid output format '%s'", controlStatusCmdArgs.format)
|
||||
die()
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package daemon
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
@@ -7,12 +7,11 @@ import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http/pprof"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/job"
|
||||
)
|
||||
|
||||
type pprofServer struct {
|
||||
type PProfServer struct {
|
||||
cc chan PprofServerControlMsg
|
||||
state PprofServerControlMsg
|
||||
listener net.Listener
|
||||
}
|
||||
|
||||
@@ -23,9 +22,9 @@ type PprofServerControlMsg struct {
|
||||
HttpListenAddress string
|
||||
}
|
||||
|
||||
func NewPProfServer(ctx context.Context) *pprofServer {
|
||||
func NewPProfServer(ctx context.Context) *PProfServer {
|
||||
|
||||
s := &pprofServer{
|
||||
s := &PProfServer{
|
||||
cc: make(chan PprofServerControlMsg),
|
||||
}
|
||||
|
||||
@@ -33,7 +32,7 @@ func NewPProfServer(ctx context.Context) *pprofServer {
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *pprofServer) controlLoop(ctx context.Context) {
|
||||
func (s *PProfServer) controlLoop(ctx context.Context) {
|
||||
outer:
|
||||
for {
|
||||
|
||||
@@ -64,14 +63,7 @@ outer:
|
||||
mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
|
||||
mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
|
||||
mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
|
||||
go func() {
|
||||
err := http.Serve(s.listener, mux)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
} else if err != nil {
|
||||
job.GetLogger(ctx).WithError(err).Error("pprof server serve error")
|
||||
}
|
||||
}()
|
||||
go http.Serve(s.listener, mux)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -83,6 +75,6 @@ outer:
|
||||
}
|
||||
}
|
||||
|
||||
func (s *pprofServer) Control(msg PprofServerControlMsg) {
|
||||
func (s *PProfServer) Control(msg PprofServerControlMsg) {
|
||||
s.cc <- msg
|
||||
}
|
||||
+394
@@ -0,0 +1,394 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// daemonCmd represents the daemon command
|
||||
var daemonCmd = &cobra.Command{
|
||||
Use: "daemon",
|
||||
Short: "start daemon",
|
||||
Run: doDaemon,
|
||||
}
|
||||
|
||||
func init() {
|
||||
RootCmd.AddCommand(daemonCmd)
|
||||
}
|
||||
|
||||
type Job interface {
|
||||
JobName() string
|
||||
JobStart(ctxt context.Context)
|
||||
JobStatus(ctxt context.Context) (*JobStatus, error)
|
||||
}
|
||||
|
||||
func doDaemon(cmd *cobra.Command, args []string) {
|
||||
|
||||
conf, err := ParseConfig(rootArgs.configFile)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error parsing config: %s\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
log := logger.NewLogger(conf.Global.logging.Outlets, 1*time.Second)
|
||||
|
||||
log.Info(NewZreplVersionInformation().String())
|
||||
log.Debug("starting daemon")
|
||||
ctx := context.WithValue(context.Background(), contextKeyLog, log)
|
||||
ctx = context.WithValue(ctx, contextKeyLog, log)
|
||||
|
||||
d := NewDaemon(conf)
|
||||
d.Loop(ctx)
|
||||
|
||||
}
|
||||
|
||||
type contextKey string
|
||||
|
||||
const (
|
||||
contextKeyLog contextKey = contextKey("log")
|
||||
contextKeyDaemon contextKey = contextKey("daemon")
|
||||
)
|
||||
|
||||
type Daemon struct {
|
||||
conf *Config
|
||||
startedAt time.Time
|
||||
}
|
||||
|
||||
func NewDaemon(initialConf *Config) *Daemon {
|
||||
return &Daemon{conf: initialConf}
|
||||
}
|
||||
|
||||
func (d *Daemon) Loop(ctx context.Context) {
|
||||
|
||||
d.startedAt = time.Now()
|
||||
|
||||
log := ctx.Value(contextKeyLog).(Logger)
|
||||
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
ctx = context.WithValue(ctx, contextKeyDaemon, d)
|
||||
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
finishs := make(chan Job)
|
||||
|
||||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
log.Info("starting jobs from config")
|
||||
i := 0
|
||||
for _, job := range d.conf.Jobs {
|
||||
logger := log.WithField(logJobField, job.JobName())
|
||||
logger.Info("starting")
|
||||
i++
|
||||
jobCtx := context.WithValue(ctx, contextKeyLog, logger)
|
||||
go func(j Job) {
|
||||
j.JobStart(jobCtx)
|
||||
finishs <- j
|
||||
}(job)
|
||||
}
|
||||
|
||||
finishCount := 0
|
||||
outer:
|
||||
for {
|
||||
select {
|
||||
case <-finishs:
|
||||
finishCount++
|
||||
if finishCount == len(d.conf.Jobs) {
|
||||
log.Info("all jobs finished")
|
||||
break outer
|
||||
}
|
||||
|
||||
case sig := <-sigChan:
|
||||
log.WithField("signal", sig).Info("received signal")
|
||||
log.Info("cancelling all jobs")
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
signal.Stop(sigChan)
|
||||
cancel() // make go vet happy
|
||||
|
||||
log.Info("exiting")
|
||||
|
||||
}
|
||||
|
||||
// Representation of a Job's status that is composed of Tasks
|
||||
type JobStatus struct {
|
||||
// Statuses of all tasks of this job
|
||||
Tasks []*TaskStatus
|
||||
// Error != "" if JobStatus() returned an error
|
||||
JobStatusError string
|
||||
}
|
||||
|
||||
// Representation of a Daemon's status that is composed of Jobs
|
||||
type DaemonStatus struct {
|
||||
StartedAt time.Time
|
||||
Jobs map[string]*JobStatus
|
||||
}
|
||||
|
||||
func (d *Daemon) Status() (s *DaemonStatus) {
|
||||
|
||||
s = &DaemonStatus{}
|
||||
s.StartedAt = d.startedAt
|
||||
|
||||
s.Jobs = make(map[string]*JobStatus, len(d.conf.Jobs))
|
||||
|
||||
for name, j := range d.conf.Jobs {
|
||||
status, err := j.JobStatus(context.TODO())
|
||||
if err != nil {
|
||||
s.Jobs[name] = &JobStatus{nil, err.Error()}
|
||||
continue
|
||||
}
|
||||
s.Jobs[name] = status
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Representation of a Task's status
|
||||
type TaskStatus struct {
|
||||
Name string
|
||||
// Whether the task is idle.
|
||||
Idle bool
|
||||
// The stack of activities the task is currently executing.
|
||||
// The first element is the root activity and equal to Name.
|
||||
ActivityStack []string
|
||||
// Number of bytes received by the task since it last left idle state.
|
||||
ProgressRx int64
|
||||
// Number of bytes sent by the task since it last left idle state.
|
||||
ProgressTx int64
|
||||
// Log entries emitted by the task since it last left idle state.
|
||||
// Only contains the log entries emitted through the task's logger
|
||||
// (provided by Task.Log()).
|
||||
LogEntries []logger.Entry
|
||||
// The maximum log level of LogEntries.
|
||||
// Only valid if len(LogEntries) > 0.
|
||||
MaxLogLevel logger.Level
|
||||
// Last time something about the Task changed
|
||||
LastUpdate time.Time
|
||||
}
|
||||
|
||||
// An instance of Task tracks a single thread of activity that is part of a Job.
|
||||
type Task struct {
|
||||
// Stack of activities the task is currently in
|
||||
// Members are instances of taskActivity
|
||||
activities *list.List
|
||||
// Last time activities was changed (not the activities inside, the list)
|
||||
activitiesLastUpdate time.Time
|
||||
// Protects Task members from modification
|
||||
rwl sync.RWMutex
|
||||
}
|
||||
|
||||
// Structure that describes the progress a Task has made
|
||||
type taskProgress struct {
|
||||
rx int64
|
||||
tx int64
|
||||
lastUpdate time.Time
|
||||
logEntries []logger.Entry
|
||||
mtx sync.RWMutex
|
||||
}
|
||||
|
||||
func newTaskProgress() (p *taskProgress) {
|
||||
return &taskProgress{
|
||||
logEntries: make([]logger.Entry, 0),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *taskProgress) UpdateIO(drx, dtx int64) {
|
||||
p.mtx.Lock()
|
||||
defer p.mtx.Unlock()
|
||||
p.rx += drx
|
||||
p.tx += dtx
|
||||
p.lastUpdate = time.Now()
|
||||
}
|
||||
|
||||
func (p *taskProgress) UpdateLogEntry(entry logger.Entry) {
|
||||
p.mtx.Lock()
|
||||
defer p.mtx.Unlock()
|
||||
// FIXME: ensure maximum size (issue #48)
|
||||
p.logEntries = append(p.logEntries, entry)
|
||||
p.lastUpdate = time.Now()
|
||||
}
|
||||
|
||||
func (p *taskProgress) DeepCopy() (out taskProgress) {
|
||||
p.mtx.RLock()
|
||||
defer p.mtx.RUnlock()
|
||||
out.rx, out.tx = p.rx, p.tx
|
||||
out.lastUpdate = p.lastUpdate
|
||||
out.logEntries = make([]logger.Entry, len(p.logEntries))
|
||||
for i := range p.logEntries {
|
||||
out.logEntries[i] = p.logEntries[i]
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// returns a copy of this taskProgress, the mutex carries no semantic value
|
||||
func (p *taskProgress) Read() (out taskProgress) {
|
||||
p.mtx.RLock()
|
||||
defer p.mtx.RUnlock()
|
||||
return p.DeepCopy()
|
||||
}
|
||||
|
||||
// Element of a Task's activity stack
|
||||
type taskActivity struct {
|
||||
name string
|
||||
idle bool
|
||||
logger *logger.Logger
|
||||
// The progress of the task that is updated by UpdateIO() and UpdateLogEntry()
|
||||
//
|
||||
// Progress happens on a task-level and is thus global to the task.
|
||||
// That's why progress is just a pointer to the current taskProgress:
|
||||
// we reset progress when leaving the idle root activity
|
||||
progress *taskProgress
|
||||
}
|
||||
|
||||
func NewTask(name string, lg *logger.Logger) *Task {
|
||||
t := &Task{
|
||||
activities: list.New(),
|
||||
}
|
||||
rootLogger := lg.ReplaceField(logTaskField, name).
|
||||
WithOutlet(t, logger.Debug)
|
||||
rootAct := &taskActivity{name, true, rootLogger, newTaskProgress()}
|
||||
t.activities.PushFront(rootAct)
|
||||
return t
|
||||
}
|
||||
|
||||
// callers must hold t.rwl
|
||||
func (t *Task) cur() *taskActivity {
|
||||
return t.activities.Front().Value.(*taskActivity)
|
||||
}
|
||||
|
||||
// buildActivityStack returns the stack of activity names
|
||||
// t.rwl must be held, but the slice can be returned since strings are immutable
|
||||
func (t *Task) buildActivityStack() []string {
|
||||
comps := make([]string, 0, t.activities.Len())
|
||||
for e := t.activities.Back(); e != nil; e = e.Prev() {
|
||||
act := e.Value.(*taskActivity)
|
||||
comps = append(comps, act.name)
|
||||
}
|
||||
return comps
|
||||
}
|
||||
|
||||
// Start a sub-activity.
|
||||
// Must always be matched with a call to t.Finish()
|
||||
// --- consider using defer for this purpose.
|
||||
func (t *Task) Enter(activity string) {
|
||||
t.rwl.Lock()
|
||||
defer t.rwl.Unlock()
|
||||
|
||||
prev := t.cur()
|
||||
if prev.idle {
|
||||
// reset progress when leaving idle task
|
||||
// we leave the old progress dangling to have the user not worry about
|
||||
prev.progress = newTaskProgress()
|
||||
}
|
||||
act := &taskActivity{activity, false, nil, prev.progress}
|
||||
t.activities.PushFront(act)
|
||||
stack := t.buildActivityStack()
|
||||
activityField := strings.Join(stack, ".")
|
||||
act.logger = prev.logger.ReplaceField(logTaskField, activityField)
|
||||
|
||||
t.activitiesLastUpdate = time.Now()
|
||||
}
|
||||
|
||||
func (t *Task) UpdateProgress(dtx, drx int64) {
|
||||
t.rwl.RLock()
|
||||
p := t.cur().progress // protected by own rwlock
|
||||
t.rwl.RUnlock()
|
||||
p.UpdateIO(dtx, drx)
|
||||
}
|
||||
|
||||
// Returns a wrapper io.Reader that updates this task's _current_ progress value.
|
||||
// Progress updates after this task resets its progress value are discarded.
|
||||
func (t *Task) ProgressUpdater(r io.Reader) *IOProgressUpdater {
|
||||
t.rwl.RLock()
|
||||
defer t.rwl.RUnlock()
|
||||
return &IOProgressUpdater{r, t.cur().progress}
|
||||
}
|
||||
|
||||
func (t *Task) Status() *TaskStatus {
|
||||
t.rwl.RLock()
|
||||
defer t.rwl.RUnlock()
|
||||
// NOTE
|
||||
// do not return any state in TaskStatus that is protected by t.rwl
|
||||
|
||||
cur := t.cur()
|
||||
stack := t.buildActivityStack()
|
||||
prog := cur.progress.Read()
|
||||
|
||||
var maxLevel logger.Level
|
||||
for _, entry := range prog.logEntries {
|
||||
if maxLevel < entry.Level {
|
||||
maxLevel = entry.Level
|
||||
}
|
||||
}
|
||||
|
||||
lastUpdate := prog.lastUpdate
|
||||
if lastUpdate.Before(t.activitiesLastUpdate) {
|
||||
lastUpdate = t.activitiesLastUpdate
|
||||
}
|
||||
|
||||
s := &TaskStatus{
|
||||
Name: stack[0],
|
||||
ActivityStack: stack,
|
||||
Idle: cur.idle,
|
||||
ProgressRx: prog.rx,
|
||||
ProgressTx: prog.tx,
|
||||
LogEntries: prog.logEntries,
|
||||
MaxLogLevel: maxLevel,
|
||||
LastUpdate: lastUpdate,
|
||||
}
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
// Finish a sub-activity.
|
||||
// Corresponds to a preceding call to t.Enter()
|
||||
func (t *Task) Finish() {
|
||||
t.rwl.Lock()
|
||||
defer t.rwl.Unlock()
|
||||
top := t.activities.Front()
|
||||
if top.Next() == nil {
|
||||
return // cannot remove root activity
|
||||
}
|
||||
t.activities.Remove(top)
|
||||
t.activitiesLastUpdate = time.Now()
|
||||
|
||||
}
|
||||
|
||||
// Returns a logger derived from the logger passed to the constructor function.
|
||||
// The logger's task field contains the current activity stack joined by '.'.
|
||||
func (t *Task) Log() *logger.Logger {
|
||||
t.rwl.RLock()
|
||||
defer t.rwl.RUnlock()
|
||||
// FIXME should influence TaskStatus's LastUpdate field
|
||||
return t.cur().logger
|
||||
}
|
||||
|
||||
// implement logger.Outlet interface
|
||||
func (t *Task) WriteEntry(entry logger.Entry) error {
|
||||
t.rwl.RLock()
|
||||
defer t.rwl.RUnlock()
|
||||
t.cur().progress.UpdateLogEntry(entry)
|
||||
return nil
|
||||
}
|
||||
|
||||
type IOProgressUpdater struct {
|
||||
r io.Reader
|
||||
p *taskProgress
|
||||
}
|
||||
|
||||
func (u *IOProgressUpdater) Read(p []byte) (n int, err error) {
|
||||
n, err = u.r.Read(p)
|
||||
u.p.UpdateIO(int64(n), 0)
|
||||
return
|
||||
|
||||
}
|
||||
+181
@@ -0,0 +1,181 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/zrepl/zrepl/rpc"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type DatasetMapping interface {
|
||||
Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error)
|
||||
}
|
||||
|
||||
type FilesystemRequest struct {
|
||||
Roots []string // may be nil, indicating interest in all filesystems
|
||||
}
|
||||
|
||||
type FilesystemVersionsRequest struct {
|
||||
Filesystem *zfs.DatasetPath
|
||||
}
|
||||
|
||||
type InitialTransferRequest struct {
|
||||
Filesystem *zfs.DatasetPath
|
||||
FilesystemVersion zfs.FilesystemVersion
|
||||
}
|
||||
|
||||
type IncrementalTransferRequest struct {
|
||||
Filesystem *zfs.DatasetPath
|
||||
From zfs.FilesystemVersion
|
||||
To zfs.FilesystemVersion
|
||||
}
|
||||
|
||||
type Handler struct {
|
||||
logger Logger
|
||||
dsf zfs.DatasetFilter
|
||||
fsvf zfs.FilesystemVersionFilter
|
||||
}
|
||||
|
||||
func NewHandler(logger Logger, dsfilter zfs.DatasetFilter, snapfilter zfs.FilesystemVersionFilter) (h Handler) {
|
||||
return Handler{logger, dsfilter, snapfilter}
|
||||
}
|
||||
|
||||
func registerEndpoints(server rpc.RPCServer, handler Handler) (err error) {
|
||||
err = server.RegisterEndpoint("FilesystemRequest", handler.HandleFilesystemRequest)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = server.RegisterEndpoint("FilesystemVersionsRequest", handler.HandleFilesystemVersionsRequest)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = server.RegisterEndpoint("InitialTransferRequest", handler.HandleInitialTransferRequest)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
err = server.RegisterEndpoint("IncrementalTransferRequest", handler.HandleIncrementalTransferRequest)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h Handler) HandleFilesystemRequest(r *FilesystemRequest, roots *[]*zfs.DatasetPath) (err error) {
|
||||
|
||||
log := h.logger.WithField("endpoint", "FilesystemRequest")
|
||||
|
||||
log.WithField("request", r).Debug("request")
|
||||
log.WithField("dataset_filter", h.dsf).Debug("dsf")
|
||||
|
||||
allowed, err := zfs.ZFSListMapping(h.dsf)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("error listing filesystems")
|
||||
return
|
||||
}
|
||||
|
||||
log.WithField("response", allowed).Debug("response")
|
||||
*roots = allowed
|
||||
return
|
||||
}
|
||||
|
||||
func (h Handler) HandleFilesystemVersionsRequest(r *FilesystemVersionsRequest, versions *[]zfs.FilesystemVersion) (err error) {
|
||||
|
||||
log := h.logger.WithField("endpoint", "FilesystemVersionsRequest")
|
||||
|
||||
log.WithField("request", r).Debug("request")
|
||||
|
||||
// allowed to request that?
|
||||
if h.pullACLCheck(r.Filesystem, nil); err != nil {
|
||||
log.WithError(err).Warn("pull ACL check failed")
|
||||
return
|
||||
}
|
||||
|
||||
// find our versions
|
||||
vs, err := zfs.ZFSListFilesystemVersions(r.Filesystem, h.fsvf)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("cannot list filesystem versions")
|
||||
return
|
||||
}
|
||||
|
||||
log.WithField("response", vs).Debug("response")
|
||||
|
||||
*versions = vs
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
func (h Handler) HandleInitialTransferRequest(r *InitialTransferRequest, stream *io.Reader) (err error) {
|
||||
|
||||
log := h.logger.WithField("endpoint", "InitialTransferRequest")
|
||||
|
||||
log.WithField("request", r).Debug("request")
|
||||
if err = h.pullACLCheck(r.Filesystem, &r.FilesystemVersion); err != nil {
|
||||
log.WithError(err).Warn("pull ACL check failed")
|
||||
return
|
||||
}
|
||||
|
||||
log.Debug("invoking zfs send")
|
||||
|
||||
s, err := zfs.ZFSSend(r.Filesystem, &r.FilesystemVersion, nil)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("cannot send filesystem")
|
||||
}
|
||||
*stream = s
|
||||
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
func (h Handler) HandleIncrementalTransferRequest(r *IncrementalTransferRequest, stream *io.Reader) (err error) {
|
||||
|
||||
log := h.logger.WithField("endpoint", "IncrementalTransferRequest")
|
||||
log.WithField("request", r).Debug("request")
|
||||
if err = h.pullACLCheck(r.Filesystem, &r.From); err != nil {
|
||||
log.WithError(err).Warn("pull ACL check failed")
|
||||
return
|
||||
}
|
||||
if err = h.pullACLCheck(r.Filesystem, &r.To); err != nil {
|
||||
log.WithError(err).Warn("pull ACL check failed")
|
||||
return
|
||||
}
|
||||
|
||||
log.Debug("invoking zfs send")
|
||||
|
||||
s, err := zfs.ZFSSend(r.Filesystem, &r.From, &r.To)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("cannot send filesystem")
|
||||
}
|
||||
|
||||
*stream = s
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
func (h Handler) pullACLCheck(p *zfs.DatasetPath, v *zfs.FilesystemVersion) (err error) {
|
||||
var fsAllowed, vAllowed bool
|
||||
fsAllowed, err = h.dsf.Filter(p)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("error evaluating ACL: %s", err)
|
||||
return
|
||||
}
|
||||
if !fsAllowed {
|
||||
err = fmt.Errorf("ACL prohibits access to %s", p.ToString())
|
||||
return
|
||||
}
|
||||
if v == nil {
|
||||
return
|
||||
}
|
||||
|
||||
vAllowed, err = h.fsvf.Filter(*v)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "error evaluating version filter")
|
||||
return
|
||||
}
|
||||
if !vAllowed {
|
||||
err = fmt.Errorf("ACL prohibits access to %s", v.ToAbsPath(p))
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -1,11 +1,10 @@
|
||||
package nethelpers
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"github.com/pkg/errors"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
func PreparePrivateSockpath(sockpath string) error {
|
||||
@@ -1,18 +1,20 @@
|
||||
package logging
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/go-logfmt/logfmt"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"time"
|
||||
)
|
||||
|
||||
type EntryFormatter interface {
|
||||
SetMetadataFlags(flags MetadataFlags)
|
||||
Format(e *logger.Entry) ([]byte, error)
|
||||
}
|
||||
|
||||
const (
|
||||
FieldLevel = "level"
|
||||
FieldMessage = "msg"
|
||||
@@ -20,19 +22,13 @@ const (
|
||||
)
|
||||
|
||||
const (
|
||||
JobField string = "job"
|
||||
SubsysField string = "subsystem"
|
||||
)
|
||||
|
||||
type MetadataFlags int64
|
||||
|
||||
const (
|
||||
MetadataTime MetadataFlags = 1 << iota
|
||||
MetadataLevel
|
||||
MetadataColor
|
||||
|
||||
MetadataNone MetadataFlags = 0
|
||||
MetadataAll MetadataFlags = ^0
|
||||
logJobField string = "job"
|
||||
logTaskField string = "task"
|
||||
logFSField string = "filesystem"
|
||||
logMapFromField string = "map_from"
|
||||
logMapToField string = "map_to"
|
||||
logIncFromField string = "inc_from"
|
||||
logIncToField string = "inc_to"
|
||||
)
|
||||
|
||||
type NoFormatter struct{}
|
||||
@@ -73,30 +69,40 @@ func (f *HumanFormatter) ignored(field string) bool {
|
||||
func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
|
||||
|
||||
var line bytes.Buffer
|
||||
col := color.New()
|
||||
if f.metadataFlags&MetadataColor != 0 {
|
||||
col = e.Color()
|
||||
}
|
||||
|
||||
if f.metadataFlags&MetadataTime != 0 {
|
||||
fmt.Fprintf(&line, "%s ", e.Time.Format(HumanFormatterDateFormat))
|
||||
}
|
||||
if f.metadataFlags&MetadataLevel != 0 {
|
||||
fmt.Fprintf(&line, "[%s]", col.Sprint(e.Level.Short()))
|
||||
fmt.Fprintf(&line, "[%s]", e.Level.Short())
|
||||
}
|
||||
|
||||
prefixFields := []string{JobField, SubsysField}
|
||||
prefixFields := []string{logJobField, logTaskField, logFSField}
|
||||
prefixed := make(map[string]bool, len(prefixFields)+2)
|
||||
for _, field := range prefixFields {
|
||||
val, ok := e.Fields[field]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if !f.ignored(field) {
|
||||
fmt.Fprintf(&line, "[%s]", col.Sprint(val))
|
||||
prefixed[field] = true
|
||||
val, ok := e.Fields[field].(string)
|
||||
if ok {
|
||||
if !f.ignored(field) {
|
||||
fmt.Fprintf(&line, "[%s]", val)
|
||||
prefixed[field] = true
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
// even more prefix fields
|
||||
mapFrom, mapFromOk := e.Fields[logMapFromField].(string)
|
||||
mapTo, mapToOk := e.Fields[logMapToField].(string)
|
||||
if mapFromOk && mapToOk && !f.ignored(logMapFromField) && !f.ignored(logMapToField) {
|
||||
fmt.Fprintf(&line, "[%s => %s]", mapFrom, mapTo)
|
||||
prefixed[logMapFromField], prefixed[logMapToField] = true, true
|
||||
}
|
||||
incFrom, incFromOk := e.Fields[logIncFromField].(string)
|
||||
incTo, incToOk := e.Fields[logIncToField].(string)
|
||||
if incFromOk && incToOk && !f.ignored(logIncFromField) && !f.ignored(logMapToField) {
|
||||
fmt.Fprintf(&line, "[%s => %s]", incFrom, incTo)
|
||||
prefixed[logIncFromField], prefixed[logIncToField] = true, true
|
||||
}
|
||||
|
||||
if line.Len() > 0 {
|
||||
fmt.Fprint(&line, ": ")
|
||||
@@ -104,11 +110,15 @@ func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
|
||||
fmt.Fprint(&line, e.Message)
|
||||
|
||||
if len(e.Fields)-len(prefixed) > 0 {
|
||||
fmt.Fprint(&line, " ")
|
||||
enc := logfmt.NewEncoder(&line)
|
||||
for field, value := range e.Fields {
|
||||
if prefixed[field] || f.ignored(field) {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&line, " %s=%q", col.Sprint(field), fmt.Sprint(value))
|
||||
if err := logfmtTryEncodeKeyval(enc, field, value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -161,21 +171,15 @@ func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
|
||||
enc := logfmt.NewEncoder(&buf)
|
||||
|
||||
if f.metadataFlags&MetadataTime != 0 {
|
||||
err := enc.EncodeKeyval(FieldTime, e.Time)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "logfmt: encode time")
|
||||
}
|
||||
enc.EncodeKeyval(FieldTime, e.Time)
|
||||
}
|
||||
if f.metadataFlags&MetadataLevel != 0 {
|
||||
err := enc.EncodeKeyval(FieldLevel, e.Level)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "logfmt: encode level")
|
||||
}
|
||||
enc.EncodeKeyval(FieldLevel, e.Level)
|
||||
}
|
||||
|
||||
// at least try and put job and task in front
|
||||
prefixed := make(map[string]bool, 2)
|
||||
prefix := []string{JobField, SubsysField}
|
||||
prefix := []string{logJobField, logTaskField}
|
||||
for _, pf := range prefix {
|
||||
v, ok := e.Fields[pf]
|
||||
if !ok {
|
||||
@@ -187,10 +191,8 @@ func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
|
||||
prefixed[pf] = true
|
||||
}
|
||||
|
||||
err := enc.EncodeKeyval(FieldMessage, e.Message)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "logfmt: encode message")
|
||||
}
|
||||
enc.EncodeKeyval(FieldMessage, e.Message)
|
||||
|
||||
for k, v := range e.Fields {
|
||||
if !prefixed[k] {
|
||||
if err := logfmtTryEncodeKeyval(enc, k, v); err != nil {
|
||||
@@ -209,10 +211,7 @@ func logfmtTryEncodeKeyval(enc *logfmt.Encoder, field, value interface{}) error
|
||||
case nil: // ok
|
||||
return nil
|
||||
case logfmt.ErrUnsupportedValueType:
|
||||
err := enc.EncodeKeyval(field, fmt.Sprintf("<%T>", value))
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "cannot encode unsupported value type Go type")
|
||||
}
|
||||
enc.EncodeKeyval(field, fmt.Sprintf("<%T>", value))
|
||||
return nil
|
||||
}
|
||||
return errors.Wrapf(err, "cannot encode field '%s'", field)
|
||||
@@ -1,39 +1,29 @@
|
||||
package logging
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"io"
|
||||
"log/syslog"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
)
|
||||
|
||||
type EntryFormatter interface {
|
||||
SetMetadataFlags(flags MetadataFlags)
|
||||
Format(e *logger.Entry) ([]byte, error)
|
||||
}
|
||||
|
||||
type WriterOutlet struct {
|
||||
formatter EntryFormatter
|
||||
writer io.Writer
|
||||
Formatter EntryFormatter
|
||||
Writer io.Writer
|
||||
}
|
||||
|
||||
func (h WriterOutlet) WriteEntry(entry logger.Entry) error {
|
||||
bytes, err := h.formatter.Format(&entry)
|
||||
bytes, err := h.Formatter.Format(&entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = h.writer.Write(bytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = h.writer.Write([]byte("\n"))
|
||||
_, err = h.Writer.Write(bytes)
|
||||
h.Writer.Write([]byte("\n"))
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -63,7 +53,7 @@ func NewTCPOutlet(formatter EntryFormatter, network, address string, tlsConfig *
|
||||
return
|
||||
}
|
||||
|
||||
entryChan := make(chan *bytes.Buffer, 1) // allow one message in flight while previous is in io.Copy()
|
||||
entryChan := make(chan *bytes.Buffer, 1) // allow one message in flight while previos is in io.Copy()
|
||||
|
||||
o := &TCPOutlet{
|
||||
formatter: formatter,
|
||||
@@ -97,10 +87,8 @@ func (h *TCPOutlet) outLoop(retryInterval time.Duration) {
|
||||
conn = nil
|
||||
}
|
||||
}
|
||||
err = conn.SetWriteDeadline(time.Now().Add(retryInterval))
|
||||
if err == nil {
|
||||
_, err = io.Copy(conn, msg)
|
||||
}
|
||||
conn.SetWriteDeadline(time.Now().Add(retryInterval))
|
||||
_, err = io.Copy(conn, msg)
|
||||
if err != nil {
|
||||
retry = time.Now().Add(retryInterval)
|
||||
conn.Close()
|
||||
@@ -131,7 +119,6 @@ func (h *TCPOutlet) WriteEntry(e logger.Entry) error {
|
||||
type SyslogOutlet struct {
|
||||
Formatter EntryFormatter
|
||||
RetryInterval time.Duration
|
||||
Facility syslog.Priority
|
||||
writer *syslog.Writer
|
||||
lastConnectAttempt time.Time
|
||||
}
|
||||
@@ -150,7 +137,7 @@ func (o *SyslogOutlet) WriteEntry(entry logger.Entry) error {
|
||||
if now.Sub(o.lastConnectAttempt) < o.RetryInterval {
|
||||
return nil // not an error toward logger
|
||||
}
|
||||
o.writer, err = syslog.New(o.Facility, "zrepl")
|
||||
o.writer, err = syslog.New(syslog.LOG_LOCAL0, "zrepl")
|
||||
o.lastConnectAttempt = time.Now()
|
||||
if err != nil {
|
||||
o.writer = nil
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
// zrepl replicates ZFS filesystems & volumes between pools
|
||||
//
|
||||
// Code Organization
|
||||
//
|
||||
// The cmd package uses github.com/spf13/cobra for its CLI.
|
||||
//
|
||||
// It combines the other packages in the zrepl project to implement zrepl functionality.
|
||||
//
|
||||
// Each subcommand's code is in the corresponding *.go file.
|
||||
// All other *.go files contain code shared by the subcommands.
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
)
|
||||
|
||||
//
|
||||
//type Logger interface {
|
||||
// Printf(format string, v ...interface{})
|
||||
//}
|
||||
|
||||
var (
|
||||
zreplVersion string // set by build infrastructure
|
||||
)
|
||||
|
||||
type Logger = *logger.Logger
|
||||
|
||||
var RootCmd = &cobra.Command{
|
||||
Use: "zrepl",
|
||||
Short: "ZFS dataset replication",
|
||||
Long: `Replicate ZFS filesystems & volumes between pools:
|
||||
|
||||
- push & pull mode
|
||||
- automatic snapshot creation & pruning
|
||||
- local / over the network
|
||||
- ACLs instead of blank SSH access`,
|
||||
}
|
||||
|
||||
var rootArgs struct {
|
||||
configFile string
|
||||
}
|
||||
|
||||
func init() {
|
||||
//cobra.OnInitialize(initConfig)
|
||||
RootCmd.PersistentFlags().StringVar(&rootArgs.configFile, "config", "", "config file path")
|
||||
}
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/zrepl/zrepl/util"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Pruner struct {
|
||||
task *Task
|
||||
Now time.Time
|
||||
DryRun bool
|
||||
DatasetFilter zfs.DatasetFilter
|
||||
SnapshotPrefix string
|
||||
PrunePolicy PrunePolicy
|
||||
}
|
||||
|
||||
type PruneResult struct {
|
||||
Filesystem *zfs.DatasetPath
|
||||
All []zfs.FilesystemVersion
|
||||
Keep []zfs.FilesystemVersion
|
||||
Remove []zfs.FilesystemVersion
|
||||
}
|
||||
|
||||
// FIXME must not call p.task.Enter because it runs in parallel
|
||||
func (p *Pruner) filterFilesystems() (filesystems []*zfs.DatasetPath, stop bool) {
|
||||
filesystems, err := zfs.ZFSListMapping(p.DatasetFilter)
|
||||
if err != nil {
|
||||
p.task.Log().WithError(err).Error("error applying filesystem filter")
|
||||
return nil, true
|
||||
}
|
||||
if len(filesystems) <= 0 {
|
||||
p.task.Log().Info("no filesystems matching filter")
|
||||
return nil, true
|
||||
}
|
||||
return filesystems, false
|
||||
}
|
||||
|
||||
// FIXME must not call p.task.Enter because it runs in parallel
|
||||
func (p *Pruner) filterVersions(fs *zfs.DatasetPath) (fsversions []zfs.FilesystemVersion, stop bool) {
|
||||
log := p.task.Log().WithField(logFSField, fs.ToString())
|
||||
|
||||
filter := NewPrefixFilter(p.SnapshotPrefix)
|
||||
fsversions, err := zfs.ZFSListFilesystemVersions(fs, filter)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("error listing filesytem versions")
|
||||
return nil, true
|
||||
}
|
||||
if len(fsversions) == 0 {
|
||||
log.WithField("prefix", p.SnapshotPrefix).Info("no filesystem versions matching prefix")
|
||||
return nil, true
|
||||
}
|
||||
return fsversions, false
|
||||
}
|
||||
|
||||
// FIXME must not call p.task.Enter because it runs in parallel
|
||||
func (p *Pruner) pruneFilesystem(fs *zfs.DatasetPath, destroySemaphore util.Semaphore) (r PruneResult, valid bool) {
|
||||
log := p.task.Log().WithField(logFSField, fs.ToString())
|
||||
|
||||
fsversions, stop := p.filterVersions(fs)
|
||||
if stop {
|
||||
return
|
||||
}
|
||||
|
||||
keep, remove, err := p.PrunePolicy.Prune(fs, fsversions)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("error evaluating prune policy")
|
||||
return
|
||||
}
|
||||
|
||||
log.WithField("fsversions", fsversions).
|
||||
WithField("keep", keep).
|
||||
WithField("remove", remove).
|
||||
Debug("prune policy debug dump")
|
||||
|
||||
r = PruneResult{fs, fsversions, keep, remove}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for v := range remove {
|
||||
wg.Add(1)
|
||||
go func(v zfs.FilesystemVersion) {
|
||||
defer wg.Done()
|
||||
// log fields
|
||||
fields := make(map[string]interface{})
|
||||
fields["version"] = v.ToAbsPath(fs)
|
||||
timeSince := v.Creation.Sub(p.Now)
|
||||
fields["age_ns"] = timeSince
|
||||
const day time.Duration = 24 * time.Hour
|
||||
days := timeSince / day
|
||||
remainder := timeSince % day
|
||||
fields["age_str"] = fmt.Sprintf("%dd%s", days, remainder)
|
||||
|
||||
log.WithFields(fields).Info("destroying version")
|
||||
// echo what we'll do and exec zfs destroy if not dry run
|
||||
// TODO special handling for EBUSY (zfs hold)
|
||||
// TODO error handling for clones? just echo to cli, skip over, and exit with non-zero status code (we're idempotent)
|
||||
if !p.DryRun {
|
||||
destroySemaphore.Down()
|
||||
err := zfs.ZFSDestroyFilesystemVersion(fs, v)
|
||||
destroySemaphore.Up()
|
||||
if err != nil {
|
||||
log.WithFields(fields).WithError(err).Error("error destroying version")
|
||||
}
|
||||
}
|
||||
}(remove[v])
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
return r, true
|
||||
}
|
||||
|
||||
func (p *Pruner) Run(ctx context.Context) (r []PruneResult, err error) {
|
||||
p.task.Enter("run")
|
||||
defer p.task.Finish()
|
||||
|
||||
if p.DryRun {
|
||||
p.task.Log().Info("doing dry run")
|
||||
}
|
||||
|
||||
filesystems, stop := p.filterFilesystems()
|
||||
if stop {
|
||||
return
|
||||
}
|
||||
|
||||
maxConcurrentDestroy := len(filesystems)
|
||||
p.task.Log().WithField("max_concurrent_destroy", maxConcurrentDestroy).Info("begin concurrent destroy")
|
||||
destroySem := util.NewSemaphore(maxConcurrentDestroy)
|
||||
|
||||
resChan := make(chan PruneResult, len(filesystems))
|
||||
var wg sync.WaitGroup
|
||||
for _, fs := range filesystems {
|
||||
wg.Add(1)
|
||||
go func(fs *zfs.DatasetPath) {
|
||||
defer wg.Done()
|
||||
res, ok := p.pruneFilesystem(fs, destroySem)
|
||||
if ok {
|
||||
resChan <- res
|
||||
}
|
||||
|
||||
}(fs)
|
||||
}
|
||||
wg.Wait()
|
||||
close(resChan)
|
||||
|
||||
p.task.Log().Info("destroys done")
|
||||
|
||||
r = make([]PruneResult, 0, len(filesystems))
|
||||
for res := range resChan {
|
||||
r = append(r, res)
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
}
|
||||
@@ -0,0 +1,323 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"github.com/zrepl/zrepl/rpc"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type localPullACL struct{}
|
||||
|
||||
func (a localPullACL) Filter(p *zfs.DatasetPath) (pass bool, err error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
const DEFAULT_INITIAL_REPL_POLICY = InitialReplPolicyMostRecent
|
||||
|
||||
type InitialReplPolicy string
|
||||
|
||||
const (
|
||||
InitialReplPolicyMostRecent InitialReplPolicy = "most_recent"
|
||||
InitialReplPolicyAll InitialReplPolicy = "all"
|
||||
)
|
||||
|
||||
type Puller struct {
|
||||
task *Task
|
||||
Remote rpc.RPCClient
|
||||
Mapping DatasetMapping
|
||||
InitialReplPolicy InitialReplPolicy
|
||||
}
|
||||
|
||||
type remoteLocalMapping struct {
|
||||
Remote *zfs.DatasetPath
|
||||
Local *zfs.DatasetPath
|
||||
}
|
||||
|
||||
func (p *Puller) getRemoteFilesystems() (rfs []*zfs.DatasetPath, ok bool) {
|
||||
p.task.Enter("fetch_remote_fs_list")
|
||||
defer p.task.Finish()
|
||||
|
||||
fsr := FilesystemRequest{}
|
||||
if err := p.Remote.Call("FilesystemRequest", &fsr, &rfs); err != nil {
|
||||
p.task.Log().WithError(err).Error("cannot fetch remote filesystem list")
|
||||
return nil, false
|
||||
}
|
||||
return rfs, true
|
||||
}
|
||||
|
||||
func (p *Puller) buildReplMapping(remoteFilesystems []*zfs.DatasetPath) (replMapping map[string]remoteLocalMapping, ok bool) {
|
||||
p.task.Enter("build_repl_mapping")
|
||||
defer p.task.Finish()
|
||||
|
||||
replMapping = make(map[string]remoteLocalMapping, len(remoteFilesystems))
|
||||
for fs := range remoteFilesystems {
|
||||
var err error
|
||||
var localFs *zfs.DatasetPath
|
||||
localFs, err = p.Mapping.Map(remoteFilesystems[fs])
|
||||
if err != nil {
|
||||
err := fmt.Errorf("error mapping %s: %s", remoteFilesystems[fs], err)
|
||||
p.task.Log().WithError(err).WithField(logMapFromField, remoteFilesystems[fs]).Error("cannot map")
|
||||
return nil, false
|
||||
}
|
||||
if localFs == nil {
|
||||
continue
|
||||
}
|
||||
p.task.Log().WithField(logMapFromField, remoteFilesystems[fs].ToString()).
|
||||
WithField(logMapToField, localFs.ToString()).Debug("mapping")
|
||||
m := remoteLocalMapping{remoteFilesystems[fs], localFs}
|
||||
replMapping[m.Local.ToString()] = m
|
||||
}
|
||||
return replMapping, true
|
||||
}
|
||||
|
||||
// returns true if the receiving filesystem (local side) exists and can have child filesystems
|
||||
func (p *Puller) replFilesystem(m remoteLocalMapping, localFilesystemState map[string]zfs.FilesystemState) (localExists bool) {
|
||||
|
||||
p.task.Enter("repl_fs")
|
||||
defer p.task.Finish()
|
||||
var err error
|
||||
remote := p.Remote
|
||||
|
||||
log := p.task.Log().
|
||||
WithField(logMapFromField, m.Remote.ToString()).
|
||||
WithField(logMapToField, m.Local.ToString())
|
||||
|
||||
log.Debug("examining local filesystem state")
|
||||
localState, localExists := localFilesystemState[m.Local.ToString()]
|
||||
var versions []zfs.FilesystemVersion
|
||||
switch {
|
||||
case !localExists:
|
||||
log.Info("local filesystem does not exist")
|
||||
case localState.Placeholder:
|
||||
log.Info("local filesystem is marked as placeholder")
|
||||
default:
|
||||
log.Debug("local filesystem exists")
|
||||
log.Debug("requesting local filesystem versions")
|
||||
if versions, err = zfs.ZFSListFilesystemVersions(m.Local, nil); err != nil {
|
||||
log.WithError(err).Error("cannot get local filesystem versions")
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("requesting remote filesystem versions")
|
||||
r := FilesystemVersionsRequest{
|
||||
Filesystem: m.Remote,
|
||||
}
|
||||
var theirVersions []zfs.FilesystemVersion
|
||||
if err = remote.Call("FilesystemVersionsRequest", &r, &theirVersions); err != nil {
|
||||
log.WithError(err).Error("cannot get remote filesystem versions")
|
||||
log.Warn("stopping replication for all filesystems mapped as children of receiving filesystem")
|
||||
return false
|
||||
}
|
||||
|
||||
log.Debug("computing diff between remote and local filesystem versions")
|
||||
diff := zfs.MakeFilesystemDiff(versions, theirVersions)
|
||||
log.WithField("diff", diff).Debug("diff between local and remote filesystem")
|
||||
|
||||
if localState.Placeholder && diff.Conflict != zfs.ConflictAllRight {
|
||||
panic("internal inconsistency: local placeholder implies ConflictAllRight")
|
||||
}
|
||||
|
||||
switch diff.Conflict {
|
||||
case zfs.ConflictAllRight:
|
||||
|
||||
log.WithField("replication_policy", p.InitialReplPolicy).Info("performing initial sync, following policy")
|
||||
|
||||
if p.InitialReplPolicy != InitialReplPolicyMostRecent {
|
||||
panic(fmt.Sprintf("policy '%s' not implemented", p.InitialReplPolicy))
|
||||
}
|
||||
|
||||
snapsOnly := make([]zfs.FilesystemVersion, 0, len(diff.MRCAPathRight))
|
||||
for s := range diff.MRCAPathRight {
|
||||
if diff.MRCAPathRight[s].Type == zfs.Snapshot {
|
||||
snapsOnly = append(snapsOnly, diff.MRCAPathRight[s])
|
||||
}
|
||||
}
|
||||
|
||||
if len(snapsOnly) < 1 {
|
||||
log.Warn("cannot perform initial sync: no remote snapshots")
|
||||
return false
|
||||
}
|
||||
|
||||
r := InitialTransferRequest{
|
||||
Filesystem: m.Remote,
|
||||
FilesystemVersion: snapsOnly[len(snapsOnly)-1],
|
||||
}
|
||||
|
||||
log.WithField("version", r.FilesystemVersion).Debug("requesting snapshot stream")
|
||||
|
||||
var stream io.Reader
|
||||
|
||||
if err = remote.Call("InitialTransferRequest", &r, &stream); err != nil {
|
||||
log.WithError(err).Error("cannot request initial transfer")
|
||||
return false
|
||||
}
|
||||
log.Debug("received initial transfer request response")
|
||||
|
||||
log.Debug("invoke zfs receive")
|
||||
recvArgs := []string{"-u"}
|
||||
if localState.Placeholder {
|
||||
log.Info("receive with forced rollback to replace placeholder filesystem")
|
||||
recvArgs = append(recvArgs, "-F")
|
||||
}
|
||||
progressStream := p.task.ProgressUpdater(stream)
|
||||
if err = zfs.ZFSRecv(m.Local, progressStream, recvArgs...); err != nil {
|
||||
log.WithError(err).Error("cannot receive stream")
|
||||
return false
|
||||
}
|
||||
log.Info("finished receiving stream") // TODO rx delta
|
||||
|
||||
// TODO unify with recv path of ConflictIncremental
|
||||
log.Debug("configuring properties of received filesystem")
|
||||
props := zfs.NewZFSProperties()
|
||||
props.Set("readonly", "on")
|
||||
if err = zfs.ZFSSet(m.Local, props); err != nil {
|
||||
log.WithError(err).Error("cannot set readonly property")
|
||||
}
|
||||
|
||||
log.Info("finished initial transfer")
|
||||
return true
|
||||
|
||||
case zfs.ConflictIncremental:
|
||||
|
||||
if len(diff.IncrementalPath) < 2 {
|
||||
log.Info("remote and local are in sync")
|
||||
return true
|
||||
}
|
||||
|
||||
log.Info("following incremental path from diff")
|
||||
for i := 0; i < len(diff.IncrementalPath)-1; i++ {
|
||||
|
||||
from, to := diff.IncrementalPath[i], diff.IncrementalPath[i+1]
|
||||
|
||||
log, _ := log.WithField(logIncFromField, from.Name).WithField(logIncToField, to.Name), 0
|
||||
|
||||
log.Debug("requesting incremental snapshot stream")
|
||||
r := IncrementalTransferRequest{
|
||||
Filesystem: m.Remote,
|
||||
From: from,
|
||||
To: to,
|
||||
}
|
||||
var stream io.Reader
|
||||
if err = remote.Call("IncrementalTransferRequest", &r, &stream); err != nil {
|
||||
log.WithError(err).Error("cannot request incremental snapshot stream")
|
||||
return false
|
||||
}
|
||||
|
||||
log.Debug("invoking zfs receive")
|
||||
progressStream := p.task.ProgressUpdater(stream)
|
||||
// TODO protect against malicious incremental stream
|
||||
if err = zfs.ZFSRecv(m.Local, progressStream); err != nil {
|
||||
log.WithError(err).Error("cannot receive stream")
|
||||
return false
|
||||
}
|
||||
log.Info("finished incremental transfer") // TODO increment rx
|
||||
|
||||
}
|
||||
log.Info("finished following incremental path") // TODO path rx
|
||||
return true
|
||||
|
||||
case zfs.ConflictNoCommonAncestor:
|
||||
fallthrough
|
||||
case zfs.ConflictDiverged:
|
||||
|
||||
var jsonDiff bytes.Buffer
|
||||
if err := json.NewEncoder(&jsonDiff).Encode(diff); err != nil {
|
||||
log.WithError(err).Error("cannot JSON-encode diff")
|
||||
return false
|
||||
}
|
||||
|
||||
var problem, resolution string
|
||||
|
||||
switch diff.Conflict {
|
||||
case zfs.ConflictNoCommonAncestor:
|
||||
problem = "remote and local filesystem have snapshots, but no common one"
|
||||
resolution = "perform manual establish a common snapshot history"
|
||||
case zfs.ConflictDiverged:
|
||||
problem = "remote and local filesystem share a history but have diverged"
|
||||
resolution = "perform manual replication or delete snapshots on the receiving" +
|
||||
"side to establish an incremental replication parse"
|
||||
}
|
||||
|
||||
log.WithField("diff", jsonDiff.String()).
|
||||
WithField("problem", problem).
|
||||
WithField("resolution", resolution).
|
||||
Error("manual conflict resolution required")
|
||||
|
||||
return false
|
||||
|
||||
}
|
||||
|
||||
panic("should not be reached")
|
||||
}
|
||||
|
||||
func (p *Puller) Pull() {
|
||||
p.task.Enter("run")
|
||||
defer p.task.Finish()
|
||||
|
||||
p.task.Log().Info("request remote filesystem list")
|
||||
remoteFilesystems, ok := p.getRemoteFilesystems()
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
p.task.Log().Debug("map remote filesystems to local paths and determine order for per-filesystem sync")
|
||||
replMapping, ok := p.buildReplMapping(remoteFilesystems)
|
||||
if !ok {
|
||||
|
||||
}
|
||||
|
||||
p.task.Log().Debug("build cache for already present local filesystem state")
|
||||
p.task.Enter("cache_local_fs_state")
|
||||
localFilesystemState, err := zfs.ZFSListFilesystemState()
|
||||
p.task.Finish()
|
||||
if err != nil {
|
||||
p.task.Log().WithError(err).Error("cannot request local filesystem state")
|
||||
return
|
||||
}
|
||||
|
||||
localTraversal := zfs.NewDatasetPathForest()
|
||||
for _, m := range replMapping {
|
||||
localTraversal.Add(m.Local)
|
||||
}
|
||||
|
||||
p.task.Log().Info("start per-filesystem sync")
|
||||
localTraversal.WalkTopDown(func(v zfs.DatasetPathVisit) bool {
|
||||
|
||||
p.task.Enter("tree_walk")
|
||||
defer p.task.Finish()
|
||||
|
||||
log := p.task.Log().WithField(logFSField, v.Path.ToString())
|
||||
|
||||
if v.FilledIn {
|
||||
if _, exists := localFilesystemState[v.Path.ToString()]; exists {
|
||||
// No need to verify if this is a placeholder or not. It is sufficient
|
||||
// to know we can add child filesystems to it
|
||||
return true
|
||||
}
|
||||
log.Debug("create placeholder filesystem")
|
||||
p.task.Enter("create_placeholder")
|
||||
err = zfs.ZFSCreatePlaceholderFilesystem(v.Path)
|
||||
p.task.Finish()
|
||||
if err != nil {
|
||||
log.Error("cannot create placeholder filesystem")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
m, ok := replMapping[v.Path.ToString()]
|
||||
if !ok {
|
||||
panic("internal inconsistency: replMapping should contain mapping for any path that was not filled in by WalkTopDown()")
|
||||
}
|
||||
|
||||
return p.replFilesystem(m, localFilesystemState)
|
||||
})
|
||||
|
||||
return
|
||||
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
jobs:
|
||||
- name: mirror_local
|
||||
type: local
|
||||
|
||||
# snapshot the filesystems matched by the left-hand-side of the mapping
|
||||
# every 10m with zrepl_ as prefix
|
||||
mapping: {
|
||||
"zroot/var/db<": "storage/backups/local/zroot/var/db",
|
||||
"zroot/usr/home<": "storage/backups/local/zroot/usr/home",
|
||||
"zroot/usr/home/paranoid": "!", #don't backup paranoid user
|
||||
"zroot/poudriere/ports<": "!", #don't backup the ports trees
|
||||
}
|
||||
snapshot_prefix: zrepl_
|
||||
interval: 10m
|
||||
initial_repl_policy: most_recent
|
||||
|
||||
# keep one hour of 10m interval snapshots of filesystems matched by
|
||||
# the left-hand-side of the mapping
|
||||
prune_lhs:
|
||||
policy: grid
|
||||
grid: 1x1h(keep=all)
|
||||
keep_bookmarks: all
|
||||
|
||||
# follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
|
||||
prune_rhs:
|
||||
policy: grid
|
||||
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
|
||||
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
jobs:
|
||||
- name: fullbackup_prod1
|
||||
type: pull
|
||||
# connect to remote using ssh / stdinserver command
|
||||
connect:
|
||||
type: ssh+stdinserver
|
||||
host: prod1.example.com
|
||||
user: root
|
||||
port: 22
|
||||
identity_file: /root/.ssh/id_ed25519
|
||||
|
||||
# pull (=ask for new snapshots) every 10m, prune afterwards
|
||||
# this will leave us at most 10m behind production
|
||||
interval: 10m
|
||||
|
||||
# pull all offered filesystems to storage/backups/zrepl/pull/prod1.example.com
|
||||
mapping: {
|
||||
"<":"storage/backups/zrepl/pull/prod1.example.com"
|
||||
}
|
||||
initial_repl_policy: most_recent
|
||||
|
||||
# follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
|
||||
snapshot_prefix: zrepl_
|
||||
prune:
|
||||
policy: grid
|
||||
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
global:
|
||||
serve:
|
||||
stdinserver:
|
||||
# Directory where AF_UNIX sockets for stdinserver command are placed.
|
||||
#
|
||||
# `zrepl stdinserver CLIENT_IDENTITY`
|
||||
# * connects to the socket in $sockdir/CLIENT_IDENTITY
|
||||
# * sends its stdin / stdout file descriptors to the `zrepl daemon` process (see cmsg(3))
|
||||
# * does nothing more
|
||||
#
|
||||
# This enables a setup where `zrepl daemon` is not directly exposed to the internet
|
||||
# but instead all traffic is tunnelled through SSH.
|
||||
# The server with the source job has an authorized_keys file entry for the public key
|
||||
# used by the corresponding pull job
|
||||
#
|
||||
# command="/mnt/zrepl stdinserver CLIENT_IDENTITY" ssh-ed25519 AAAAC3NzaC1E... zrepl@pullingserver
|
||||
#
|
||||
# Below is the default value.
|
||||
sockdir: /var/run/zrepl/stdinserver
|
||||
|
||||
jobs:
|
||||
|
||||
- name: fullbackup_prod1
|
||||
# expect remote to connect via ssh+stdinserver with fullbackup_prod1 as client_identity
|
||||
type: source
|
||||
serve:
|
||||
type: stdinserver # see global.serve.stdinserver for explanation
|
||||
client_identity: fullbackup_prod1
|
||||
|
||||
# snapshot these filesystems every 10m with zrepl_ as prefix
|
||||
filesystems: {
|
||||
"zroot/var/db<": "ok",
|
||||
"zroot/usr/home<": "ok",
|
||||
"zroot/var/tmp": "!", #don't backup /tmp
|
||||
}
|
||||
snapshot_prefix: zrepl_
|
||||
interval: 10m
|
||||
|
||||
|
||||
# keep 1 hour of snapshots (6 at 10m interval)
|
||||
# and one day of bookmarks in case pull doesn't work (link down, etc)
|
||||
# => keep_bookmarks = 24h / interval = 24h / 10m = 144
|
||||
prune:
|
||||
policy: grid
|
||||
grid: 1x1h(keep=all)
|
||||
keep_bookmarks: 144
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
jobs:
|
||||
- name: fullbackup_prod1
|
||||
|
||||
# expect remote to connect via ssh+stdinserver with fullbackup_prod1 as client_identity
|
||||
type: push-sink
|
||||
serve:
|
||||
type: stdinserver
|
||||
client_identity: fullbackup_prod1
|
||||
|
||||
# map all pushed datasets to storage/backups/zrepl/sink/prod1.example.com
|
||||
mapping: {
|
||||
"<":"storage/backups/zrepl/sink/prod1.example.com"
|
||||
}
|
||||
|
||||
# follow a grandfathering scheme for filesystems on the right-hand-side of the mapping
|
||||
prune:
|
||||
policy: grid
|
||||
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
|
||||
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
jobs:
|
||||
- name: fullbackup_prod1
|
||||
|
||||
# connect to remote using ssh / stdinserver command
|
||||
type: push
|
||||
connect:
|
||||
type: ssh+stdinserver
|
||||
host: prod1.example.com
|
||||
user: root
|
||||
port: 22
|
||||
identity_file: /root/.ssh/id_ed25519
|
||||
|
||||
# snapshot these datsets every 10m with zrepl_ as prefix
|
||||
filesystems: {
|
||||
"zroot/var/db<": "ok",
|
||||
"zroot/usr/home<": "!",
|
||||
}
|
||||
snapshot_prefix: zrepl_
|
||||
interval: 10m
|
||||
|
||||
# keep a one day window 10m interval snapshots in case push doesn't work (link down, etc)
|
||||
# (we cannot keep more than one day because this host will run out of disk space)
|
||||
prune:
|
||||
policy: grid
|
||||
grid: 1x1d(keep=all)
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
global:
|
||||
serve:
|
||||
stdinserver:
|
||||
sockdir: /var/run/zrepl/stdinserver
|
||||
|
||||
jobs:
|
||||
|
||||
- name: debian2_pull
|
||||
# JOB DEBUGGING OPTIONS
|
||||
# should be equal for all job types, but each job implements the debugging itself
|
||||
# => consult job documentation for supported options
|
||||
debug:
|
||||
conn: # debug the io.ReadWriteCloser connection
|
||||
read_dump: /tmp/connlog_read # dump results of Read() invocations to this file
|
||||
write_dump: /tmp/connlog_write # dump results of Write() invocations to this file
|
||||
rpc: # debug the RPC protocol implementation
|
||||
log: true # log output from rpc layer to the job log
|
||||
|
||||
# ... just to make the unit tests pass.
|
||||
# check other examples, e.g. localbackup or pullbackup for what the sutff below means
|
||||
type: source
|
||||
serve:
|
||||
type: stdinserver
|
||||
client_identity: debian2
|
||||
filesystems: {
|
||||
"pool1/db<": ok
|
||||
}
|
||||
snapshot_prefix: zrepl_
|
||||
interval: 1s
|
||||
prune:
|
||||
policy: grid
|
||||
grid: 1x10s(keep=all)
|
||||
keep_bookmarks: all
|
||||
@@ -0,0 +1,24 @@
|
||||
global:
|
||||
logging:
|
||||
|
||||
- outlet: stdout
|
||||
level: warn
|
||||
format: human
|
||||
|
||||
- outlet: tcp
|
||||
level: debug
|
||||
format: json
|
||||
net: tcp
|
||||
address: 127.0.0.1:8080
|
||||
retry_interval: 1s
|
||||
tls: # if not specified, use plain TCP
|
||||
ca: sampleconf/random/logging/logserver.crt
|
||||
cert: sampleconf/random/logging/client.crt
|
||||
key: sampleconf/random/logging/client.key
|
||||
|
||||
- outlet: syslog
|
||||
level: debug
|
||||
format: logfmt
|
||||
|
||||
|
||||
jobs: []
|
||||
@@ -0,0 +1,19 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDIzCCAgsCAQEwDQYJKoZIhvcNAQELBQAwWTELMAkGA1UEBhMCQVUxEzARBgNV
|
||||
BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0
|
||||
ZDESMBAGA1UEAwwJbG9nc2VydmVyMB4XDTE3MDkyNDEyMzAzNloXDTE3MTAyNDEy
|
||||
MzAzNlowVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNV
|
||||
BAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xpZW50MIIB
|
||||
IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/xJTUlqApeJGzRD+w2J8sZS
|
||||
Bo+s+04T987L/M6gaCo8aDSTEb/ZH3XSoU5JEmO6kPpwNNapOsaEhTCjndZQdm5F
|
||||
uqiUtAg1uW0HCkBEIDkGr9bFHDKzpewGmmMgfQ2+hfiBR/4ZCrc/vd9P0W9BiWQS
|
||||
Dtc7p22XraWPVL8HlSz5K/Ih+V6i8O+kBltZkusiJh2bWPoRp/netiTZuc6du+Wp
|
||||
kpWp1OBaTU4GXIAlLj5afF14BBphRQK983Yhaz53BkA7OQ76XxowynMjmuLQVGmK
|
||||
f1R9zEJuohTX9XIr1tp/ueRHcS4Awk6LcNZUMCV6270FNSIw2f4hbOZvep+t2wID
|
||||
AQABMA0GCSqGSIb3DQEBCwUAA4IBAQACK3OeNzScpiNwz/jpg/usQzvXbZ/wDvml
|
||||
YLjtzn/A65ox8a8BhxvH1ydyoCM2YAGYX7+y7qXJnMgRO/v8565CQIVcznHhg9ST
|
||||
3828/WqZ3bXf2DV5GxKKQf7hPmBnyVUUhn/Ny91MECED27lZucWiX/bczN8ffDeh
|
||||
M3+ngezcJxsOBd4x0gLrqIJCoaFRSeepOaFEW6GHQ8loxE9GmA7FQd2phIpJHFSd
|
||||
Z7nQl7X5C1iN2OboEApJHwtmNVC45UlOpg53vo2sDTLhSfdogstiWi8x1HmvhIGM
|
||||
j3XHs0Illvo9OwVrmgUph8zQ7pvr/AFrTOIbhgzl/9uVUk5ApwFM
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,16 @@
|
||||
-----BEGIN CERTIFICATE REQUEST-----
|
||||
MIICmzCCAYMCAQAwVjELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx
|
||||
ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xp
|
||||
ZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/xJTUlqApeJGzRD
|
||||
+w2J8sZSBo+s+04T987L/M6gaCo8aDSTEb/ZH3XSoU5JEmO6kPpwNNapOsaEhTCj
|
||||
ndZQdm5FuqiUtAg1uW0HCkBEIDkGr9bFHDKzpewGmmMgfQ2+hfiBR/4ZCrc/vd9P
|
||||
0W9BiWQSDtc7p22XraWPVL8HlSz5K/Ih+V6i8O+kBltZkusiJh2bWPoRp/netiTZ
|
||||
uc6du+WpkpWp1OBaTU4GXIAlLj5afF14BBphRQK983Yhaz53BkA7OQ76XxowynMj
|
||||
muLQVGmKf1R9zEJuohTX9XIr1tp/ueRHcS4Awk6LcNZUMCV6270FNSIw2f4hbOZv
|
||||
ep+t2wIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAKnlr0Qs5KYF85u2YA7DJ5pL
|
||||
HwAx+qNoNbox5CS1aynrDBpDTWLaErviUJ+4WxRlRyTMEscMOIOKajbYhqqFmtGZ
|
||||
mu3SshZnFihErw8TOQMyU1LGGG+l6r+6ve5TciwJRLla2Y75z7izr6cyvQNRWdLr
|
||||
PvxL1/Yqr8LKha12+7o28R4SLf6/GY0GcedqoebRmtuwA/jES0PuGauEUD5lH4cj
|
||||
Me8sqRrB+IMHQ5j8hlJX4DbA8UQRUBL64sHkQzeQfWu+qkWmS5I19CFfLNrcH+OV
|
||||
yhyjGfN0q0jHyHdpckBhgzS7IIdo6P66AIlm4qpHM7Scra3JaGM7oaZPamJ6f8U=
|
||||
-----END CERTIFICATE REQUEST-----
|
||||
@@ -0,0 +1,28 @@
|
||||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC3/ElNSWoCl4kb
|
||||
NEP7DYnyxlIGj6z7ThP3zsv8zqBoKjxoNJMRv9kfddKhTkkSY7qQ+nA01qk6xoSF
|
||||
MKOd1lB2bkW6qJS0CDW5bQcKQEQgOQav1sUcMrOl7AaaYyB9Db6F+IFH/hkKtz+9
|
||||
30/Rb0GJZBIO1zunbZetpY9UvweVLPkr8iH5XqLw76QGW1mS6yImHZtY+hGn+d62
|
||||
JNm5zp275amSlanU4FpNTgZcgCUuPlp8XXgEGmFFAr3zdiFrPncGQDs5DvpfGjDK
|
||||
cyOa4tBUaYp/VH3MQm6iFNf1civW2n+55EdxLgDCTotw1lQwJXrbvQU1IjDZ/iFs
|
||||
5m96n63bAgMBAAECggEAF4om0sWe06ARwbJJNFjCGpa3LfG5/xk5Qs5pmPnS2iD1
|
||||
Q5veaTnzjKvlfA/pF3o9B4mTS59fXY7Cq8vSU0J1XwGy2DPzeqlGPmgtq2kXjkvd
|
||||
iCfhZj8ybvsoyR3/rSBSDRADcnOXPqC9fgyRSMmESBDOoql1D3HdIzF4ii46ySIU
|
||||
/XQvExS6NWifbP+Ue6DETV8NhreO5PqjeXLITQhhndtc8MDL/8eCNOyN8XjYIWKX
|
||||
smlBYtRQYOOY9BHOQgUn6yvPHrtKJNKci+qcQNvWir66mBhY1o40MH5wTIV+8yP2
|
||||
Vbm/VzoNKIYgeROsilBW7QTwGvkDn3R11zeTqfUNSQKBgQD0eFzhJAEZi4uBw6Tg
|
||||
NKmBC5Y1IHPOsb5gKPNz9Z9j4qYRDySgYl6ISk+2EdhgUCo1NmTk8EIPQjIerUVf
|
||||
S+EogFnpsj8U9LR3OM79DaGkNULxrHqhd209/g8DtVgk7yjkxL4vmVOv8qpHMp/7
|
||||
eWsylN7AOxj2RB/eXYQBPrw+jQKBgQDAqae9HasLmvpJ9ktTv30yZSKXC+LP4A0D
|
||||
RBBmx410VpPd4CvcpCJxXmjer6B7+9L1xHYP2pvsnMBid5i0knuvyK28dYy7fldl
|
||||
CzWvb+lqNA5YYPFXQED4oEdihlQczoI1Bm06SFizeAKD1Q9e2c+lgbR/51j8xuXi
|
||||
twvhMj/YBwKBgQCZw97/iQrcC2Zq7yiUEOuQjD4lGk1c83U/vGIsTJC9XcCAOFsc
|
||||
OeMlrD/oz96d7a4unBDn4qpaOJOXsfpRT0PGmrxy/jcpMiUUW/ntNpa11v5NTeQw
|
||||
DRL8DAFbnsNbL8Yz5f+Nps35fBNYBuKTZLJlNTfKByHTO9QjpAQ0WEZEvQKBgQCi
|
||||
Ovm83EuYVSKmvxcE6Tyx/8lVqTOO2Vn7wweQlD4/lVujvE0S2L8L+XSS9w5K+GzW
|
||||
eFz10p3zarbw80YJ30L5bSEmjVE43BUZR4woMzM4M6dUsiTm1HshIE2b4ALZ0uZ/
|
||||
Ye794ceXL9nmSrVLqFsaQZLNFPCwwYb4FiyRry9lZwKBgAO9VbWcN8SEeBDKo3z8
|
||||
yRbRTc6sI+AdKY44Dfx0tqOPmTjO3mE4X1GU4sbfD2Bvg3DdjwTuxxC/jHaKu0GG
|
||||
dTM0CbrZGbDAj7E87SOcN/PWEeBckSvuQq5H3DQfwIpTmlS1l5oZn9CxRGbLqC2G
|
||||
ifnel8XWUG0ROybsr1tk4mzW
|
||||
-----END PRIVATE KEY-----
|
||||
@@ -0,0 +1,21 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDiDCCAnCgAwIBAgIJALhp/WvTQeg/MA0GCSqGSIb3DQEBCwUAMFkxCzAJBgNV
|
||||
BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
|
||||
aWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMMCWxvZ3NlcnZlcjAeFw0xNzA5MjQxMjI3
|
||||
MDRaFw0yNzA5MjIxMjI3MDRaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21l
|
||||
LVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNV
|
||||
BAMMCWxvZ3NlcnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKs3
|
||||
TLYfXhV3hap71tOkhPQlM+m0EKRAo8Nua50Cci5UhDo4JkVpyYok1h+NFkqmjU2b
|
||||
IiIuGvsZZPOWYjbWWnSJE4+n5pBFBzcfNQ4d8xVxjANImFn6Tcehhj0WkbDIv/Ge
|
||||
364XUgywS7u3EGQj/FO7vZ8KHlUxBHNuPIOPHftwIVRyleh5K32UyBaSpSmnqGos
|
||||
rvI1byMuznavcZpOs4vlebZ+Jy6a20iKf9fj/0f0t0O+F5x3JIk07D3zSywhJ4RM
|
||||
M0mGIUmYXbh2SMh+f61KDZLDANpz/pMAPbUJe0mxEtBf0tnwK1gEqc3SLwA0EwiM
|
||||
8Hnn2iaH5Ln20UE3LOkCAwEAAaNTMFEwHQYDVR0OBBYEFDXoDcwx9SngzZcRYCeP
|
||||
BplBecfiMB8GA1UdIwQYMBaAFDXoDcwx9SngzZcRYCePBplBecfiMA8GA1UdEwEB
|
||||
/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBADyNvs4AA91x3gurQb1pcPVhK6nR
|
||||
mkYSTN1AsDKSRi/X2iCUmR7G7FlF7XW8mntTpHvVzcs+gr94WckH5wqEOA5iZnaw
|
||||
PXUWexmdXUge4hmC2q6kBQ5e2ykhSJMRVZXvOLZOZV9qitceamHESV1cKZSNMvZM
|
||||
aCSVA1RK61/nUzs04pVp5PFPv9gFxJp9ki39FYFdsgZmM5RZ5I/FqxxvTJzu4RnH
|
||||
VPjsMopzARYwJw6dV2bKdFSYOE8B/Vs3Yv0GxjrABw2ko4PkBPTjLIz22x6+Hd9r
|
||||
K9BQi4pVmQfvppF5+SORSftlHSS+N47b0DD1rW1f5R6QGi71dFuJGikOwvY=
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,28 @@
|
||||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCrN0y2H14Vd4Wq
|
||||
e9bTpIT0JTPptBCkQKPDbmudAnIuVIQ6OCZFacmKJNYfjRZKpo1NmyIiLhr7GWTz
|
||||
lmI21lp0iROPp+aQRQc3HzUOHfMVcYwDSJhZ+k3HoYY9FpGwyL/xnt+uF1IMsEu7
|
||||
txBkI/xTu72fCh5VMQRzbjyDjx37cCFUcpXoeSt9lMgWkqUpp6hqLK7yNW8jLs52
|
||||
r3GaTrOL5Xm2ficumttIin/X4/9H9LdDvhecdySJNOw980ssISeETDNJhiFJmF24
|
||||
dkjIfn+tSg2SwwDac/6TAD21CXtJsRLQX9LZ8CtYBKnN0i8ANBMIjPB559omh+S5
|
||||
9tFBNyzpAgMBAAECggEBAIY8ZwJq+WKvQLb3POjWFf8so9TY/ispGrwAeJKy9j5o
|
||||
uPrERw0o8YBDfTVjclS43BQ6Srqtly3DLSjlgL8ps+WmCxYYN2ZpGE0ZRIl65bis
|
||||
O2/fnML+wbiAZTTD2xnVatfPDeP6GLQmDFpyHoHEzPIBQZvNXRbBxZGSnhMvQ/x7
|
||||
FhqSBQG4kf3b1XDCENIbFEVOBOCg7WtMiIgjEGS7QnW3I65/Zt+Ts1LXRZbz+6na
|
||||
Gmi0PGHA/oLUh1NRzsF4zuZn6fFzja5zw4mkt+JvCWEoxg1QhRAxRp6QQwmZ6MIc
|
||||
1rw1D4Z+c5UEKyqHeIwZj4M6UNPhCfTXVm47c9eSiGECgYEA4U8pB+7eRo2fqX0C
|
||||
nWsWMcmsULJvwplQnUSFenUayPn3E8ammS/ZBHksoKhj82vwIdDbtS1hQZn8Bzsi
|
||||
atc8au0wz0YRDcVDzHX4HknXVQayHtP/FTPeSr5hwpoY8vhEbySuxBTBkXCrp4dx
|
||||
u5ErfOiYEP3Q1ZvPRywelrATu20CgYEAwonV5dgOcen/4oAirlnvufc2NfqhAQwJ
|
||||
FJ/JSVMAcXxPYu3sZMv0dGWrX8mLc+P1+XMCuV/7eBM/vU2LbDzmpeUV8sJfB2jw
|
||||
wyKqKXZwBgeq60btriA4f+0ElwRGgU2KSiniUuuTX2JmyftFQx4cVAQRCFk27NY0
|
||||
09psSsYyre0CgYBo6unabdtH029EB5iOIW3GZXk+Yrk0TxyA/4WAjsOYTv5FUT4H
|
||||
G4bdVGf5sDBLDDpYJOAKsEUXvVLlMx5FzlCuIiGWg7QxS2jU7yJJSG1jhKixPlsM
|
||||
Toj3GUyAyC1SB1Ymw1g2qsuwpFzquGG3zFQJ6G3Xi7oRnmqZY+wik3+8yQKBgB11
|
||||
SdKYOPe++2SNCrNkIw0CBk9+OEs0S1u4Jn7X9sU4kbzlUlqhF89YZe8HUfqmlmTD
|
||||
qbHwet/f6lL8HxSw1Cxi2EP+cu1oUqz53tKQgL4pAxTFlNA9SND2Ty+fEh4aY8p/
|
||||
NSphSduzxuTnC8HyGVAPnZSqDcsnVLCP7r4T7TCxAoGAbJygkkk/gZ9pT4fZoIaq
|
||||
8CMR8FTfxtkwCuZsWccSMUOWtx9nqet3gbCpKHfyoYZiKB4ke+lnUz4uFS16Y3hG
|
||||
kN0hFfvfoNa8eB2Ox7vs60cMMfWJac0H7KSaDDy+EvbhE2KtQADT0eWxMyhzGR8p
|
||||
5CbIivB0QCjeQIA8dOQpE8E=
|
||||
-----END PRIVATE KEY-----
|
||||
@@ -1,51 +1,55 @@
|
||||
package client
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/problame/go-netssh"
|
||||
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/problame/go-netssh"
|
||||
"github.com/spf13/cobra"
|
||||
"log"
|
||||
"path"
|
||||
)
|
||||
|
||||
var StdinserverCmd = &cli.Subcommand{
|
||||
var StdinserverCmd = &cobra.Command{
|
||||
Use: "stdinserver CLIENT_IDENTITY",
|
||||
Short: "stdinserver transport mode (started from authorized_keys file as forced command)",
|
||||
Run: func(subcommand *cli.Subcommand, args []string) error {
|
||||
return runStdinserver(subcommand.Config(), args)
|
||||
},
|
||||
Short: "start in stdinserver mode (from authorized_keys file)",
|
||||
Run: cmdStdinServer,
|
||||
}
|
||||
|
||||
func runStdinserver(config *config.Config, args []string) error {
|
||||
func init() {
|
||||
RootCmd.AddCommand(StdinserverCmd)
|
||||
}
|
||||
|
||||
func cmdStdinServer(cmd *cobra.Command, args []string) {
|
||||
|
||||
// NOTE: the netssh proxying protocol requires exiting with non-zero status if anything goes wrong
|
||||
defer os.Exit(1)
|
||||
|
||||
log := log.New(os.Stderr, "", log.LUTC|log.Ldate|log.Ltime)
|
||||
|
||||
conf, err := ParseConfig(rootArgs.configFile)
|
||||
if err != nil {
|
||||
log.Printf("error parsing config: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(args) != 1 || args[0] == "" {
|
||||
err := errors.New("must specify client_identity as positional argument")
|
||||
return err
|
||||
log.Print("must specify client_identity as positional argument")
|
||||
return
|
||||
}
|
||||
|
||||
identity := args[0]
|
||||
unixaddr := path.Join(config.Global.Serve.StdinServer.SockDir, identity)
|
||||
unixaddr := path.Join(conf.Global.Serve.Stdinserver.SockDir, identity)
|
||||
|
||||
log.Printf("proxying client identity '%s' to zrepl daemon '%s'", identity, unixaddr)
|
||||
|
||||
ctx := netssh.ContextWithLog(context.TODO(), log)
|
||||
|
||||
err := netssh.Proxy(ctx, unixaddr)
|
||||
err = netssh.Proxy(ctx, unixaddr)
|
||||
if err == nil {
|
||||
log.Print("proxying finished successfully, exiting with status 0")
|
||||
os.Exit(0)
|
||||
}
|
||||
log.Printf("error proxying: %s", err)
|
||||
return nil
|
||||
|
||||
}
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/kr/pretty"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
"time"
|
||||
)
|
||||
|
||||
var testCmd = &cobra.Command{
|
||||
Use: "test",
|
||||
Short: "test configuration",
|
||||
PersistentPreRun: testCmdGlobalInit,
|
||||
}
|
||||
|
||||
var testCmdGlobal struct {
|
||||
log Logger
|
||||
conf *Config
|
||||
}
|
||||
|
||||
var testConfigSyntaxCmd = &cobra.Command{
|
||||
Use: "config",
|
||||
Short: "parse config file and dump parsed datastructure",
|
||||
Run: doTestConfig,
|
||||
}
|
||||
|
||||
var testDatasetMapFilter = &cobra.Command{
|
||||
Use: "pattern jobname test/zfs/dataset/path",
|
||||
Short: "test dataset mapping / filter specified in config",
|
||||
Example: ` zrepl test pattern my_pull_job tank/tmp`,
|
||||
Run: doTestDatasetMapFilter,
|
||||
}
|
||||
|
||||
var testPrunePolicyArgs struct {
|
||||
side PrunePolicySide
|
||||
showKept bool
|
||||
showRemoved bool
|
||||
}
|
||||
|
||||
var testPrunePolicyCmd = &cobra.Command{
|
||||
Use: "prune jobname",
|
||||
Short: "do a dry-run of the pruning part of a job",
|
||||
Run: doTestPrunePolicy,
|
||||
}
|
||||
|
||||
func init() {
|
||||
RootCmd.AddCommand(testCmd)
|
||||
testCmd.AddCommand(testConfigSyntaxCmd)
|
||||
testCmd.AddCommand(testDatasetMapFilter)
|
||||
|
||||
testPrunePolicyCmd.Flags().VarP(&testPrunePolicyArgs.side, "side", "s", "prune_lhs (left) or prune_rhs (right)")
|
||||
testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showKept, "kept", false, "show kept snapshots")
|
||||
testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showRemoved, "removed", true, "show removed snapshots")
|
||||
testCmd.AddCommand(testPrunePolicyCmd)
|
||||
}
|
||||
|
||||
func testCmdGlobalInit(cmd *cobra.Command, args []string) {
|
||||
|
||||
out := logger.NewOutlets()
|
||||
out.Add(WriterOutlet{&NoFormatter{}, os.Stdout}, logger.Info)
|
||||
log := logger.NewLogger(out, 1*time.Second)
|
||||
testCmdGlobal.log = log
|
||||
|
||||
var err error
|
||||
if testCmdGlobal.conf, err = ParseConfig(rootArgs.configFile); err != nil {
|
||||
testCmdGlobal.log.Printf("error parsing config file: %s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func doTestConfig(cmd *cobra.Command, args []string) {
|
||||
|
||||
log, conf := testCmdGlobal.log, testCmdGlobal.conf
|
||||
|
||||
log.Printf("config ok")
|
||||
log.Printf("%# v", pretty.Formatter(conf))
|
||||
return
|
||||
}
|
||||
|
||||
func doTestDatasetMapFilter(cmd *cobra.Command, args []string) {
|
||||
|
||||
log, conf := testCmdGlobal.log, testCmdGlobal.conf
|
||||
|
||||
if len(args) != 2 {
|
||||
log.Printf("specify job name as first postitional argument, test input as second")
|
||||
log.Printf(cmd.UsageString())
|
||||
os.Exit(1)
|
||||
}
|
||||
n, i := args[0], args[1]
|
||||
|
||||
jobi, err := conf.LookupJob(n)
|
||||
if err != nil {
|
||||
log.Printf("%s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
var mf *DatasetMapFilter
|
||||
switch j := jobi.(type) {
|
||||
case *PullJob:
|
||||
mf = j.Mapping
|
||||
case *SourceJob:
|
||||
mf = j.Filesystems
|
||||
case *LocalJob:
|
||||
mf = j.Mapping
|
||||
default:
|
||||
panic("incomplete implementation")
|
||||
}
|
||||
|
||||
ip, err := zfs.NewDatasetPath(i)
|
||||
if err != nil {
|
||||
log.Printf("cannot parse test input as ZFS dataset path: %s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if mf.filterMode {
|
||||
pass, err := mf.Filter(ip)
|
||||
if err != nil {
|
||||
log.Printf("error evaluating filter: %s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
log.Printf("filter result: %v", pass)
|
||||
} else {
|
||||
res, err := mf.Map(ip)
|
||||
if err != nil {
|
||||
log.Printf("error evaluating mapping: %s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
toStr := "NO MAPPING"
|
||||
if res != nil {
|
||||
toStr = res.ToString()
|
||||
}
|
||||
log.Printf("%s => %s", ip.ToString(), toStr)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func doTestPrunePolicy(cmd *cobra.Command, args []string) {
|
||||
|
||||
log, conf := testCmdGlobal.log, testCmdGlobal.conf
|
||||
|
||||
if cmd.Flags().NArg() != 1 {
|
||||
log.Printf("specify job name as first positional argument")
|
||||
log.Printf(cmd.UsageString())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
jobname := cmd.Flags().Arg(0)
|
||||
jobi, err := conf.LookupJob(jobname)
|
||||
if err != nil {
|
||||
log.Printf("%s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
jobp, ok := jobi.(PruningJob)
|
||||
if !ok {
|
||||
log.Printf("job doesn't do any prunes")
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
log.Printf("job dump:\n%s", pretty.Sprint(jobp))
|
||||
|
||||
task := NewTask("", log)
|
||||
pruner, err := jobp.Pruner(task, testPrunePolicyArgs.side, true)
|
||||
if err != nil {
|
||||
log.Printf("cannot create test pruner: %s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
log.Printf("start pruning")
|
||||
|
||||
ctx := context.WithValue(context.Background(), contextKeyLog, log)
|
||||
result, err := pruner.Run(ctx)
|
||||
if err != nil {
|
||||
log.Printf("error running pruner: %s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
return strings.Compare(result[i].Filesystem.ToString(), result[j].Filesystem.ToString()) == -1
|
||||
})
|
||||
|
||||
var b bytes.Buffer
|
||||
for _, r := range result {
|
||||
fmt.Fprintf(&b, "%s\n", r.Filesystem.ToString())
|
||||
|
||||
if testPrunePolicyArgs.showKept {
|
||||
fmt.Fprintf(&b, "\tkept:\n")
|
||||
for _, v := range r.Keep {
|
||||
fmt.Fprintf(&b, "\t- %s\n", v.Name)
|
||||
}
|
||||
}
|
||||
|
||||
if testPrunePolicyArgs.showRemoved {
|
||||
fmt.Fprintf(&b, "\tremoved:\n")
|
||||
for _, v := range r.Remove {
|
||||
fmt.Fprintf(&b, "\t- %s\n", v.Name)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
log.Printf("pruning result:\n%s", b.String())
|
||||
|
||||
}
|
||||
@@ -1,13 +1,24 @@
|
||||
package version
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/spf13/cobra"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
var (
|
||||
zreplVersion string // set by build infrastructure
|
||||
)
|
||||
var versionCmd = &cobra.Command{
|
||||
Use: "version",
|
||||
Short: "print version of zrepl binary (for running daemon 'zrepl control version' command)",
|
||||
Run: doVersion,
|
||||
}
|
||||
|
||||
func init() {
|
||||
RootCmd.AddCommand(versionCmd)
|
||||
}
|
||||
|
||||
func doVersion(cmd *cobra.Command, args []string) {
|
||||
fmt.Println(NewZreplVersionInformation().String())
|
||||
}
|
||||
|
||||
type ZreplVersionInformation struct {
|
||||
Version string
|
||||
@@ -1,657 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log/syslog"
|
||||
"os"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/zrepl/yaml-config"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Jobs []JobEnum `yaml:"jobs"`
|
||||
Global *Global `yaml:"global,optional,fromdefaults"`
|
||||
}
|
||||
|
||||
func (c *Config) Job(name string) (*JobEnum, error) {
|
||||
for _, j := range c.Jobs {
|
||||
if j.Name() == name {
|
||||
return &j, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("job %q not defined in config", name)
|
||||
}
|
||||
|
||||
type JobEnum struct {
|
||||
Ret interface{}
|
||||
}
|
||||
|
||||
func (j JobEnum) Name() string {
|
||||
var name string
|
||||
switch v := j.Ret.(type) {
|
||||
case *SnapJob:
|
||||
name = v.Name
|
||||
case *PushJob:
|
||||
name = v.Name
|
||||
case *SinkJob:
|
||||
name = v.Name
|
||||
case *PullJob:
|
||||
name = v.Name
|
||||
case *SourceJob:
|
||||
name = v.Name
|
||||
default:
|
||||
panic(fmt.Sprintf("unknown job type %T", v))
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
type ActiveJob struct {
|
||||
Type string `yaml:"type"`
|
||||
Name string `yaml:"name"`
|
||||
Connect ConnectEnum `yaml:"connect"`
|
||||
Pruning PruningSenderReceiver `yaml:"pruning"`
|
||||
Debug JobDebugSettings `yaml:"debug,optional"`
|
||||
}
|
||||
|
||||
type PassiveJob struct {
|
||||
Type string `yaml:"type"`
|
||||
Name string `yaml:"name"`
|
||||
Serve ServeEnum `yaml:"serve"`
|
||||
Debug JobDebugSettings `yaml:"debug,optional"`
|
||||
}
|
||||
|
||||
type SnapJob struct {
|
||||
Type string `yaml:"type"`
|
||||
Name string `yaml:"name"`
|
||||
Pruning PruningLocal `yaml:"pruning"`
|
||||
Debug JobDebugSettings `yaml:"debug,optional"`
|
||||
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
|
||||
Filesystems FilesystemsFilter `yaml:"filesystems"`
|
||||
}
|
||||
|
||||
type SendOptions struct {
|
||||
Encrypted bool `yaml:"encrypted"`
|
||||
}
|
||||
|
||||
var _ yaml.Defaulter = (*SendOptions)(nil)
|
||||
|
||||
func (l *SendOptions) SetDefault() {
|
||||
*l = SendOptions{Encrypted: false}
|
||||
}
|
||||
|
||||
type RecvOptions struct {
|
||||
// Note: we cannot enforce encrypted recv as the ZFS cli doesn't provide a mechanism for it
|
||||
// Encrypted bool `yaml:"may_encrypted"`
|
||||
|
||||
// Future:
|
||||
// Reencrypt bool `yaml:"reencrypt"`
|
||||
}
|
||||
|
||||
var _ yaml.Defaulter = (*RecvOptions)(nil)
|
||||
|
||||
func (l *RecvOptions) SetDefault() {
|
||||
*l = RecvOptions{}
|
||||
}
|
||||
|
||||
type PushJob struct {
|
||||
ActiveJob `yaml:",inline"`
|
||||
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
|
||||
Filesystems FilesystemsFilter `yaml:"filesystems"`
|
||||
Send *SendOptions `yaml:"send,fromdefaults,optional"`
|
||||
}
|
||||
|
||||
type PullJob struct {
|
||||
ActiveJob `yaml:",inline"`
|
||||
RootFS string `yaml:"root_fs"`
|
||||
Interval PositiveDurationOrManual `yaml:"interval"`
|
||||
Recv *RecvOptions `yaml:"recv,fromdefaults,optional"`
|
||||
}
|
||||
|
||||
type PositiveDurationOrManual struct {
|
||||
Interval time.Duration
|
||||
Manual bool
|
||||
}
|
||||
|
||||
var _ yaml.Unmarshaler = (*PositiveDurationOrManual)(nil)
|
||||
|
||||
func (i *PositiveDurationOrManual) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
||||
var s string
|
||||
if err := u(&s, true); err != nil {
|
||||
return err
|
||||
}
|
||||
switch s {
|
||||
case "manual":
|
||||
i.Manual = true
|
||||
i.Interval = 0
|
||||
case "":
|
||||
return fmt.Errorf("value must not be empty")
|
||||
default:
|
||||
i.Manual = false
|
||||
i.Interval, err = time.ParseDuration(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if i.Interval <= 0 {
|
||||
return fmt.Errorf("value must be a positive duration, got %q", s)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type SinkJob struct {
|
||||
PassiveJob `yaml:",inline"`
|
||||
RootFS string `yaml:"root_fs"`
|
||||
Recv *RecvOptions `yaml:"recv,optional,fromdefaults"`
|
||||
}
|
||||
|
||||
type SourceJob struct {
|
||||
PassiveJob `yaml:",inline"`
|
||||
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
|
||||
Filesystems FilesystemsFilter `yaml:"filesystems"`
|
||||
Send *SendOptions `yaml:"send,optional,fromdefaults"`
|
||||
}
|
||||
|
||||
type FilesystemsFilter map[string]bool
|
||||
|
||||
type SnapshottingEnum struct {
|
||||
Ret interface{}
|
||||
}
|
||||
|
||||
type SnapshottingPeriodic struct {
|
||||
Type string `yaml:"type"`
|
||||
Prefix string `yaml:"prefix"`
|
||||
Interval time.Duration `yaml:"interval,positive"`
|
||||
Hooks HookList `yaml:"hooks,optional"`
|
||||
}
|
||||
|
||||
type SnapshottingManual struct {
|
||||
Type string `yaml:"type"`
|
||||
}
|
||||
|
||||
type PruningSenderReceiver struct {
|
||||
KeepSender []PruningEnum `yaml:"keep_sender"`
|
||||
KeepReceiver []PruningEnum `yaml:"keep_receiver"`
|
||||
}
|
||||
|
||||
type PruningLocal struct {
|
||||
Keep []PruningEnum `yaml:"keep"`
|
||||
}
|
||||
|
||||
type LoggingOutletEnumList []LoggingOutletEnum
|
||||
|
||||
func (l *LoggingOutletEnumList) SetDefault() {
|
||||
def := `
|
||||
type: "stdout"
|
||||
time: true
|
||||
level: "warn"
|
||||
format: "human"
|
||||
`
|
||||
s := &StdoutLoggingOutlet{}
|
||||
err := yaml.UnmarshalStrict([]byte(def), &s)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
*l = []LoggingOutletEnum{LoggingOutletEnum{Ret: s}}
|
||||
}
|
||||
|
||||
var _ yaml.Defaulter = &LoggingOutletEnumList{}
|
||||
|
||||
type Global struct {
|
||||
Logging *LoggingOutletEnumList `yaml:"logging,optional,fromdefaults"`
|
||||
Monitoring []MonitoringEnum `yaml:"monitoring,optional"`
|
||||
Control *GlobalControl `yaml:"control,optional,fromdefaults"`
|
||||
Serve *GlobalServe `yaml:"serve,optional,fromdefaults"`
|
||||
}
|
||||
|
||||
func Default(i interface{}) {
|
||||
v := reflect.ValueOf(i)
|
||||
if v.Kind() != reflect.Ptr {
|
||||
panic(v)
|
||||
}
|
||||
y := `{}`
|
||||
err := yaml.Unmarshal([]byte(y), v.Interface())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
type ConnectEnum struct {
|
||||
Ret interface{}
|
||||
}
|
||||
|
||||
type ConnectCommon struct {
|
||||
Type string `yaml:"type"`
|
||||
}
|
||||
|
||||
type TCPConnect struct {
|
||||
ConnectCommon `yaml:",inline"`
|
||||
Address string `yaml:"address,hostport"`
|
||||
DialTimeout time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
|
||||
}
|
||||
|
||||
type TLSConnect struct {
|
||||
ConnectCommon `yaml:",inline"`
|
||||
Address string `yaml:"address,hostport"`
|
||||
Ca string `yaml:"ca"`
|
||||
Cert string `yaml:"cert"`
|
||||
Key string `yaml:"key"`
|
||||
ServerCN string `yaml:"server_cn"`
|
||||
DialTimeout time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
|
||||
}
|
||||
|
||||
type SSHStdinserverConnect struct {
|
||||
ConnectCommon `yaml:",inline"`
|
||||
Host string `yaml:"host"`
|
||||
User string `yaml:"user"`
|
||||
Port uint16 `yaml:"port"`
|
||||
IdentityFile string `yaml:"identity_file"`
|
||||
TransportOpenCommand []string `yaml:"transport_open_command,optional"` //TODO unused
|
||||
SSHCommand string `yaml:"ssh_command,optional"` //TODO unused
|
||||
Options []string `yaml:"options,optional"`
|
||||
DialTimeout time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
|
||||
}
|
||||
|
||||
type LocalConnect struct {
|
||||
ConnectCommon `yaml:",inline"`
|
||||
ListenerName string `yaml:"listener_name"`
|
||||
ClientIdentity string `yaml:"client_identity"`
|
||||
DialTimeout time.Duration `yaml:"dial_timeout,zeropositive,default=2s"`
|
||||
}
|
||||
|
||||
type ServeEnum struct {
|
||||
Ret interface{}
|
||||
}
|
||||
|
||||
type ServeCommon struct {
|
||||
Type string `yaml:"type"`
|
||||
}
|
||||
|
||||
type TCPServe struct {
|
||||
ServeCommon `yaml:",inline"`
|
||||
Listen string `yaml:"listen,hostport"`
|
||||
ListenFreeBind bool `yaml:"listen_freebind,default=false"`
|
||||
Clients map[string]string `yaml:"clients"`
|
||||
}
|
||||
|
||||
type TLSServe struct {
|
||||
ServeCommon `yaml:",inline"`
|
||||
Listen string `yaml:"listen,hostport"`
|
||||
ListenFreeBind bool `yaml:"listen_freebind,default=false"`
|
||||
Ca string `yaml:"ca"`
|
||||
Cert string `yaml:"cert"`
|
||||
Key string `yaml:"key"`
|
||||
ClientCNs []string `yaml:"client_cns"`
|
||||
HandshakeTimeout time.Duration `yaml:"handshake_timeout,zeropositive,default=10s"`
|
||||
}
|
||||
|
||||
type StdinserverServer struct {
|
||||
ServeCommon `yaml:",inline"`
|
||||
ClientIdentities []string `yaml:"client_identities"`
|
||||
}
|
||||
|
||||
type LocalServe struct {
|
||||
ServeCommon `yaml:",inline"`
|
||||
ListenerName string `yaml:"listener_name"`
|
||||
}
|
||||
|
||||
type PruningEnum struct {
|
||||
Ret interface{}
|
||||
}
|
||||
|
||||
type PruneKeepNotReplicated struct {
|
||||
Type string `yaml:"type"`
|
||||
KeepSnapshotAtCursor bool `yaml:"keep_snapshot_at_cursor,optional,default=true"`
|
||||
}
|
||||
|
||||
type PruneKeepLastN struct {
|
||||
Type string `yaml:"type"`
|
||||
Count int `yaml:"count"`
|
||||
}
|
||||
|
||||
type PruneKeepRegex struct { // FIXME rename to KeepRegex
|
||||
Type string `yaml:"type"`
|
||||
Regex string `yaml:"regex"`
|
||||
Negate bool `yaml:"negate,optional,default=false"`
|
||||
}
|
||||
|
||||
type LoggingOutletEnum struct {
|
||||
Ret interface{}
|
||||
}
|
||||
|
||||
type LoggingOutletCommon struct {
|
||||
Type string `yaml:"type"`
|
||||
Level string `yaml:"level"`
|
||||
Format string `yaml:"format"`
|
||||
}
|
||||
|
||||
type StdoutLoggingOutlet struct {
|
||||
LoggingOutletCommon `yaml:",inline"`
|
||||
Time bool `yaml:"time,default=true"`
|
||||
Color bool `yaml:"color,default=true"`
|
||||
}
|
||||
|
||||
type SyslogLoggingOutlet struct {
|
||||
LoggingOutletCommon `yaml:",inline"`
|
||||
Facility *SyslogFacility `yaml:"facility,optional,fromdefaults"`
|
||||
RetryInterval time.Duration `yaml:"retry_interval,positive,default=10s"`
|
||||
}
|
||||
|
||||
type TCPLoggingOutlet struct {
|
||||
LoggingOutletCommon `yaml:",inline"`
|
||||
Address string `yaml:"address,hostport"`
|
||||
Net string `yaml:"net,default=tcp"`
|
||||
RetryInterval time.Duration `yaml:"retry_interval,positive,default=10s"`
|
||||
TLS *TCPLoggingOutletTLS `yaml:"tls,optional"`
|
||||
}
|
||||
|
||||
type TCPLoggingOutletTLS struct {
|
||||
CA string `yaml:"ca"`
|
||||
Cert string `yaml:"cert"`
|
||||
Key string `yaml:"key"`
|
||||
}
|
||||
|
||||
type MonitoringEnum struct {
|
||||
Ret interface{}
|
||||
}
|
||||
|
||||
type PrometheusMonitoring struct {
|
||||
Type string `yaml:"type"`
|
||||
Listen string `yaml:"listen,hostport"`
|
||||
ListenFreeBind bool `yaml:"listen_freebind,default=false"`
|
||||
}
|
||||
|
||||
type SyslogFacility syslog.Priority
|
||||
|
||||
func (f *SyslogFacility) SetDefault() {
|
||||
*f = SyslogFacility(syslog.LOG_LOCAL0)
|
||||
}
|
||||
|
||||
var _ yaml.Defaulter = (*SyslogFacility)(nil)
|
||||
|
||||
type GlobalControl struct {
|
||||
SockPath string `yaml:"sockpath,default=/var/run/zrepl/control"`
|
||||
}
|
||||
|
||||
type GlobalServe struct {
|
||||
StdinServer *GlobalStdinServer `yaml:"stdinserver,optional,fromdefaults"`
|
||||
}
|
||||
|
||||
type GlobalStdinServer struct {
|
||||
SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
|
||||
}
|
||||
|
||||
type JobDebugSettings struct {
|
||||
Conn *struct {
|
||||
ReadDump string `yaml:"read_dump"`
|
||||
WriteDump string `yaml:"write_dump"`
|
||||
} `yaml:"conn,optional"`
|
||||
RPCLog bool `yaml:"rpc_log,optional,default=false"`
|
||||
}
|
||||
|
||||
type HookList []HookEnum
|
||||
|
||||
type HookEnum struct {
|
||||
Ret interface{}
|
||||
}
|
||||
|
||||
type HookCommand struct {
|
||||
Path string `yaml:"path"`
|
||||
Timeout time.Duration `yaml:"timeout,optional,positive,default=30s"`
|
||||
Filesystems FilesystemsFilter `yaml:"filesystems,optional,default={'<': true}"`
|
||||
HookSettingsCommon `yaml:",inline"`
|
||||
}
|
||||
|
||||
type HookPostgresCheckpoint struct {
|
||||
HookSettingsCommon `yaml:",inline"`
|
||||
DSN string `yaml:"dsn"`
|
||||
Timeout time.Duration `yaml:"timeout,optional,positive,default=30s"`
|
||||
Filesystems FilesystemsFilter `yaml:"filesystems"` // required, user should not CHECKPOINT for every FS
|
||||
}
|
||||
|
||||
type HookMySQLLockTables struct {
|
||||
HookSettingsCommon `yaml:",inline"`
|
||||
DSN string `yaml:"dsn"`
|
||||
Timeout time.Duration `yaml:"timeout,optional,positive,default=30s"`
|
||||
Filesystems FilesystemsFilter `yaml:"filesystems"`
|
||||
}
|
||||
|
||||
type HookSettingsCommon struct {
|
||||
Type string `yaml:"type"`
|
||||
ErrIsFatal bool `yaml:"err_is_fatal,optional,default=false"`
|
||||
}
|
||||
|
||||
func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) {
|
||||
var in struct {
|
||||
Type string
|
||||
}
|
||||
if err := u(&in, true); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if in.Type == "" {
|
||||
return nil, &yaml.TypeError{Errors: []string{"must specify type"}}
|
||||
}
|
||||
|
||||
v, ok := types[in.Type]
|
||||
if !ok {
|
||||
return nil, &yaml.TypeError{Errors: []string{fmt.Sprintf("invalid type name %q", in.Type)}}
|
||||
}
|
||||
if err := u(v, false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
||||
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
||||
"snap": &SnapJob{},
|
||||
"push": &PushJob{},
|
||||
"sink": &SinkJob{},
|
||||
"pull": &PullJob{},
|
||||
"source": &SourceJob{},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
func (t *ConnectEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
||||
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
||||
"tcp": &TCPConnect{},
|
||||
"tls": &TLSConnect{},
|
||||
"ssh+stdinserver": &SSHStdinserverConnect{},
|
||||
"local": &LocalConnect{},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
||||
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
||||
"tcp": &TCPServe{},
|
||||
"tls": &TLSServe{},
|
||||
"stdinserver": &StdinserverServer{},
|
||||
"local": &LocalServe{},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
||||
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
||||
"not_replicated": &PruneKeepNotReplicated{},
|
||||
"last_n": &PruneKeepLastN{},
|
||||
"grid": &PruneGrid{},
|
||||
"regex": &PruneKeepRegex{},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
func (t *SnapshottingEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
||||
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
||||
"periodic": &SnapshottingPeriodic{},
|
||||
"manual": &SnapshottingManual{},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
func (t *LoggingOutletEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
||||
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
||||
"stdout": &StdoutLoggingOutlet{},
|
||||
"syslog": &SyslogLoggingOutlet{},
|
||||
"tcp": &TCPLoggingOutlet{},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
func (t *MonitoringEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
||||
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
||||
"prometheus": &PrometheusMonitoring{},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
func (t *SyslogFacility) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
||||
var s string
|
||||
if err := u(&s, true); err != nil {
|
||||
return err
|
||||
}
|
||||
var level syslog.Priority
|
||||
switch s {
|
||||
case "kern":
|
||||
level = syslog.LOG_KERN
|
||||
case "user":
|
||||
level = syslog.LOG_USER
|
||||
case "mail":
|
||||
level = syslog.LOG_MAIL
|
||||
case "daemon":
|
||||
level = syslog.LOG_DAEMON
|
||||
case "auth":
|
||||
level = syslog.LOG_AUTH
|
||||
case "syslog":
|
||||
level = syslog.LOG_SYSLOG
|
||||
case "lpr":
|
||||
level = syslog.LOG_LPR
|
||||
case "news":
|
||||
level = syslog.LOG_NEWS
|
||||
case "uucp":
|
||||
level = syslog.LOG_UUCP
|
||||
case "cron":
|
||||
level = syslog.LOG_CRON
|
||||
case "authpriv":
|
||||
level = syslog.LOG_AUTHPRIV
|
||||
case "ftp":
|
||||
level = syslog.LOG_FTP
|
||||
case "local0":
|
||||
level = syslog.LOG_LOCAL0
|
||||
case "local1":
|
||||
level = syslog.LOG_LOCAL1
|
||||
case "local2":
|
||||
level = syslog.LOG_LOCAL2
|
||||
case "local3":
|
||||
level = syslog.LOG_LOCAL3
|
||||
case "local4":
|
||||
level = syslog.LOG_LOCAL4
|
||||
case "local5":
|
||||
level = syslog.LOG_LOCAL5
|
||||
case "local6":
|
||||
level = syslog.LOG_LOCAL6
|
||||
case "local7":
|
||||
level = syslog.LOG_LOCAL7
|
||||
default:
|
||||
return fmt.Errorf("invalid syslog level: %q", s)
|
||||
}
|
||||
*t = SyslogFacility(level)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *HookEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
||||
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
||||
"command": &HookCommand{},
|
||||
"postgres-checkpoint": &HookPostgresCheckpoint{},
|
||||
"mysql-lock-tables": &HookMySQLLockTables{},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
var ConfigFileDefaultLocations = []string{
|
||||
"/etc/zrepl/zrepl.yml",
|
||||
"/usr/local/etc/zrepl/zrepl.yml",
|
||||
}
|
||||
|
||||
func ParseConfig(path string) (i *Config, err error) {
|
||||
|
||||
if path == "" {
|
||||
// Try default locations
|
||||
for _, l := range ConfigFileDefaultLocations {
|
||||
stat, statErr := os.Stat(l)
|
||||
if statErr != nil {
|
||||
continue
|
||||
}
|
||||
if !stat.Mode().IsRegular() {
|
||||
err = errors.Errorf("file at default location is not a regular file: %s", l)
|
||||
return
|
||||
}
|
||||
path = l
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
var bytes []byte
|
||||
|
||||
if bytes, err = ioutil.ReadFile(path); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return ParseConfigBytes(bytes)
|
||||
}
|
||||
|
||||
func ParseConfigBytes(bytes []byte) (*Config, error) {
|
||||
var c *Config
|
||||
if err := yaml.UnmarshalStrict(bytes, &c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c == nil {
|
||||
return nil, fmt.Errorf("config is empty or only consists of comments")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
|
||||
|
||||
func parsePositiveDuration(e string) (d time.Duration, err error) {
|
||||
comps := durationStringRegex.FindStringSubmatch(e)
|
||||
if len(comps) != 3 {
|
||||
err = fmt.Errorf("does not match regex: %s %#v", e, comps)
|
||||
return
|
||||
}
|
||||
|
||||
durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if durationFactor <= 0 {
|
||||
return 0, errors.New("duration must be positive integer")
|
||||
}
|
||||
|
||||
var durationUnit time.Duration
|
||||
switch comps[2] {
|
||||
case "s":
|
||||
durationUnit = time.Second
|
||||
case "m":
|
||||
durationUnit = time.Minute
|
||||
case "h":
|
||||
durationUnit = time.Hour
|
||||
case "d":
|
||||
durationUnit = 24 * time.Hour
|
||||
case "w":
|
||||
durationUnit = 24 * 7 * time.Hour
|
||||
default:
|
||||
err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
|
||||
return
|
||||
}
|
||||
|
||||
d = time.Duration(durationFactor) * durationUnit
|
||||
return
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/syslog"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/zrepl/yaml-config"
|
||||
)
|
||||
|
||||
func testValidGlobalSection(t *testing.T, s string) *Config {
|
||||
jobdef := `
|
||||
jobs:
|
||||
- name: dummyjob
|
||||
type: sink
|
||||
serve:
|
||||
type: tcp
|
||||
listen: ":2342"
|
||||
clients: {
|
||||
"10.0.0.1":"foo"
|
||||
}
|
||||
root_fs: zroot/foo
|
||||
`
|
||||
_, err := ParseConfigBytes([]byte(jobdef))
|
||||
require.NoError(t, err)
|
||||
return testValidConfig(t, s+jobdef)
|
||||
}
|
||||
|
||||
func TestOutletTypes(t *testing.T) {
|
||||
conf := testValidGlobalSection(t, `
|
||||
global:
|
||||
logging:
|
||||
- type: stdout
|
||||
level: debug
|
||||
format: human
|
||||
- type: syslog
|
||||
level: info
|
||||
retry_interval: 20s
|
||||
format: human
|
||||
- type: tcp
|
||||
level: debug
|
||||
format: json
|
||||
address: logserver.example.com:1234
|
||||
- type: tcp
|
||||
level: debug
|
||||
format: json
|
||||
address: encryptedlogserver.example.com:1234
|
||||
retry_interval: 20s
|
||||
tls:
|
||||
ca: /etc/zrepl/log/ca.crt
|
||||
cert: /etc/zrepl/log/key.pem
|
||||
key: /etc/zrepl/log/cert.pem
|
||||
`)
|
||||
assert.Equal(t, 4, len(*conf.Global.Logging))
|
||||
assert.NotNil(t, (*conf.Global.Logging)[3].Ret.(*TCPLoggingOutlet).TLS)
|
||||
}
|
||||
|
||||
func TestDefaultLoggingOutlet(t *testing.T) {
|
||||
conf := testValidGlobalSection(t, "")
|
||||
assert.Equal(t, 1, len(*conf.Global.Logging))
|
||||
o := (*conf.Global.Logging)[0].Ret.(*StdoutLoggingOutlet)
|
||||
assert.Equal(t, "warn", o.Level)
|
||||
assert.Equal(t, "human", o.Format)
|
||||
}
|
||||
|
||||
func TestPrometheusMonitoring(t *testing.T) {
|
||||
conf := testValidGlobalSection(t, `
|
||||
global:
|
||||
monitoring:
|
||||
- type: prometheus
|
||||
listen: ':9091'
|
||||
`)
|
||||
assert.Equal(t, ":9091", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
|
||||
}
|
||||
|
||||
func TestSyslogLoggingOutletFacility(t *testing.T) {
|
||||
type SyslogFacilityPriority struct {
|
||||
Facility string
|
||||
Priority syslog.Priority
|
||||
}
|
||||
syslogFacilitiesPriorities := []SyslogFacilityPriority{
|
||||
{"", syslog.LOG_LOCAL0}, // default
|
||||
{"kern", syslog.LOG_KERN}, {"daemon", syslog.LOG_DAEMON}, {"auth", syslog.LOG_AUTH},
|
||||
{"syslog", syslog.LOG_SYSLOG}, {"lpr", syslog.LOG_LPR}, {"news", syslog.LOG_NEWS},
|
||||
{"uucp", syslog.LOG_UUCP}, {"cron", syslog.LOG_CRON}, {"authpriv", syslog.LOG_AUTHPRIV},
|
||||
{"ftp", syslog.LOG_FTP}, {"local0", syslog.LOG_LOCAL0}, {"local1", syslog.LOG_LOCAL1},
|
||||
{"local2", syslog.LOG_LOCAL2}, {"local3", syslog.LOG_LOCAL3}, {"local4", syslog.LOG_LOCAL4},
|
||||
{"local5", syslog.LOG_LOCAL5}, {"local6", syslog.LOG_LOCAL6}, {"local7", syslog.LOG_LOCAL7},
|
||||
}
|
||||
|
||||
for _, sFP := range syslogFacilitiesPriorities {
|
||||
logcfg := fmt.Sprintf(`
|
||||
global:
|
||||
logging:
|
||||
- type: syslog
|
||||
level: info
|
||||
format: human
|
||||
facility: %s
|
||||
`, sFP.Facility)
|
||||
conf := testValidGlobalSection(t, logcfg)
|
||||
assert.Equal(t, 1, len(*conf.Global.Logging))
|
||||
assert.True(t, SyslogFacility(sFP.Priority) == *(*conf.Global.Logging)[0].Ret.(*SyslogLoggingOutlet).Facility)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoggingOutletEnumList_SetDefaults(t *testing.T) {
|
||||
e := &LoggingOutletEnumList{}
|
||||
var i yaml.Defaulter = e
|
||||
require.NotPanics(t, func() {
|
||||
i.SetDefault()
|
||||
assert.Equal(t, "warn", (*e)[0].Ret.(*StdoutLoggingOutlet).Level)
|
||||
})
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
package config
|
||||
@@ -1,40 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestConfigEmptyFails(t *testing.T) {
|
||||
conf, err := testConfig(t, "\n")
|
||||
assert.Nil(t, conf)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestJobsOnlyWorks(t *testing.T) {
|
||||
testValidConfig(t, `
|
||||
jobs:
|
||||
- name: push
|
||||
type: push
|
||||
# snapshot the filesystems matched by the left-hand-side of the mapping
|
||||
# every 10m with zrepl_ as prefix
|
||||
connect:
|
||||
type: tcp
|
||||
address: localhost:2342
|
||||
filesystems: {
|
||||
"pool1/var/db<": true,
|
||||
"pool1/usr/home<": true,
|
||||
"pool1/usr/home/paranoid": false, #don't backup paranoid user
|
||||
"pool1/poudriere/ports<": false #don't backup the ports trees
|
||||
}
|
||||
snapshotting:
|
||||
type: manual
|
||||
pruning:
|
||||
keep_sender:
|
||||
- type: not_replicated
|
||||
keep_receiver:
|
||||
- type: last_n
|
||||
count: 1
|
||||
`)
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/zrepl/yaml-config"
|
||||
)
|
||||
|
||||
func TestPositiveDurationOrManual(t *testing.T) {
|
||||
cases := []struct {
|
||||
Comment, Input string
|
||||
Result *PositiveDurationOrManual
|
||||
}{
|
||||
{"empty is error", "", nil},
|
||||
{"negative is error", "-1s", nil},
|
||||
{"zero seconds is error", "0s", nil},
|
||||
{"zero is error", "0", nil},
|
||||
{"non-manual is error", "something", nil},
|
||||
{"positive seconds works", "1s", &PositiveDurationOrManual{Manual: false, Interval: 1 * time.Second}},
|
||||
{"manual works", "manual", &PositiveDurationOrManual{Manual: true, Interval: 0}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.Comment, func(t *testing.T) {
|
||||
var out struct {
|
||||
FieldName PositiveDurationOrManual `yaml:"fieldname"`
|
||||
}
|
||||
input := fmt.Sprintf("\nfieldname: %s\n", tc.Input)
|
||||
err := yaml.UnmarshalStrict([]byte(input), &out)
|
||||
if tc.Result == nil {
|
||||
assert.Error(t, err)
|
||||
t.Logf("%#v", out)
|
||||
} else {
|
||||
assert.Equal(t, *tc.Result, out.FieldName)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestSendOptions(t *testing.T) {
|
||||
tmpl := `
|
||||
jobs:
|
||||
- name: foo
|
||||
type: push
|
||||
connect:
|
||||
type: local
|
||||
listener_name: foo
|
||||
client_identity: bar
|
||||
filesystems: {"<": true}
|
||||
%s
|
||||
snapshotting:
|
||||
type: manual
|
||||
pruning:
|
||||
keep_sender:
|
||||
- type: last_n
|
||||
count: 10
|
||||
keep_receiver:
|
||||
- type: last_n
|
||||
count: 10
|
||||
`
|
||||
encrypted_false := `
|
||||
send:
|
||||
encrypted: false
|
||||
`
|
||||
|
||||
encrypted_true := `
|
||||
send:
|
||||
encrypted: true
|
||||
`
|
||||
|
||||
encrypted_unspecified := `
|
||||
send: {}
|
||||
`
|
||||
|
||||
send_not_specified := `
|
||||
`
|
||||
|
||||
fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
|
||||
var c *Config
|
||||
|
||||
t.Run("encrypted_false", func(t *testing.T) {
|
||||
c = testValidConfig(t, fill(encrypted_false))
|
||||
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
|
||||
assert.Equal(t, false, encrypted)
|
||||
})
|
||||
t.Run("encrypted_true", func(t *testing.T) {
|
||||
c = testValidConfig(t, fill(encrypted_true))
|
||||
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
|
||||
assert.Equal(t, true, encrypted)
|
||||
})
|
||||
|
||||
t.Run("encrypted_unspecified", func(t *testing.T) {
|
||||
c, err := testConfig(t, fill(encrypted_unspecified))
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, c)
|
||||
})
|
||||
|
||||
t.Run("send_not_specified", func(t *testing.T) {
|
||||
c, err := testConfig(t, fill(send_not_specified))
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, c)
|
||||
})
|
||||
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestSnapshotting(t *testing.T) {
|
||||
tmpl := `
|
||||
jobs:
|
||||
- name: foo
|
||||
type: push
|
||||
connect:
|
||||
type: local
|
||||
listener_name: foo
|
||||
client_identity: bar
|
||||
filesystems: {"<": true}
|
||||
%s
|
||||
pruning:
|
||||
keep_sender:
|
||||
- type: last_n
|
||||
count: 10
|
||||
keep_receiver:
|
||||
- type: last_n
|
||||
count: 10
|
||||
`
|
||||
manual := `
|
||||
snapshotting:
|
||||
type: manual
|
||||
`
|
||||
periodic := `
|
||||
snapshotting:
|
||||
type: periodic
|
||||
prefix: zrepl_
|
||||
interval: 10m
|
||||
`
|
||||
|
||||
hooks := `
|
||||
snapshotting:
|
||||
type: periodic
|
||||
prefix: zrepl_
|
||||
interval: 10m
|
||||
hooks:
|
||||
- type: command
|
||||
path: /tmp/path/to/command
|
||||
- type: command
|
||||
path: /tmp/path/to/command
|
||||
filesystems: { "zroot<": true, "<": false }
|
||||
- type: postgres-checkpoint
|
||||
dsn: "host=localhost port=5432 user=postgres sslmode=disable"
|
||||
filesystems: {
|
||||
"tank/postgres/data11": true
|
||||
}
|
||||
- type: mysql-lock-tables
|
||||
dsn: "root@tcp(localhost)/"
|
||||
filesystems: {
|
||||
"tank/mysql": true
|
||||
}
|
||||
`
|
||||
|
||||
fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
|
||||
var c *Config
|
||||
|
||||
t.Run("manual", func(t *testing.T) {
|
||||
c = testValidConfig(t, fillSnapshotting(manual))
|
||||
snm := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingManual)
|
||||
assert.Equal(t, "manual", snm.Type)
|
||||
})
|
||||
|
||||
t.Run("periodic", func(t *testing.T) {
|
||||
c = testValidConfig(t, fillSnapshotting(periodic))
|
||||
snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
|
||||
assert.Equal(t, "periodic", snp.Type)
|
||||
assert.Equal(t, 10*time.Minute, snp.Interval)
|
||||
assert.Equal(t, "zrepl_", snp.Prefix)
|
||||
})
|
||||
|
||||
t.Run("hooks", func(t *testing.T) {
|
||||
c = testValidConfig(t, fillSnapshotting(hooks))
|
||||
hs := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic).Hooks
|
||||
assert.Equal(t, hs[0].Ret.(*HookCommand).Filesystems["<"], true)
|
||||
assert.Equal(t, hs[1].Ret.(*HookCommand).Filesystems["zroot<"], true)
|
||||
assert.Equal(t, hs[2].Ret.(*HookPostgresCheckpoint).Filesystems["tank/postgres/data11"], true)
|
||||
assert.Equal(t, hs[3].Ret.(*HookMySQLLockTables).Filesystems["tank/mysql"], true)
|
||||
})
|
||||
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"text/template"
|
||||
|
||||
"github.com/kr/pretty"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
|
||||
paths, err := filepath.Glob("./samples/*")
|
||||
if err != nil {
|
||||
t.Errorf("glob failed: %+v", err)
|
||||
}
|
||||
|
||||
for _, p := range paths {
|
||||
|
||||
if path.Ext(p) != ".yml" {
|
||||
t.Logf("skipping file %s", p)
|
||||
continue
|
||||
}
|
||||
|
||||
t.Run(p, func(t *testing.T) {
|
||||
c, err := ParseConfig(p)
|
||||
if err != nil {
|
||||
t.Errorf("error parsing %s:\n%+v", p, err)
|
||||
}
|
||||
|
||||
t.Logf("file: %s", p)
|
||||
t.Log(pretty.Sprint(c))
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// template must be a template/text template with a single '{{ . }}' as placeholder for val
|
||||
//nolint[:deadcode,unused]
|
||||
func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
|
||||
tmp, err := template.New("master").Parse(tmpl)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
err = tmp.Execute(&buf, val)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return testValidConfig(t, buf.String())
|
||||
}
|
||||
|
||||
func testValidConfig(t *testing.T, input string) *Config {
|
||||
t.Helper()
|
||||
conf, err := testConfig(t, input)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, conf)
|
||||
return conf
|
||||
}
|
||||
|
||||
func testConfig(t *testing.T, input string) (*Config, error) {
|
||||
t.Helper()
|
||||
return ParseConfigBytes([]byte(input))
|
||||
}
|
||||
|
||||
func trimSpaceEachLineAndPad(s, pad string) string {
|
||||
var out strings.Builder
|
||||
scan := bufio.NewScanner(strings.NewReader(s))
|
||||
for scan.Scan() {
|
||||
fmt.Fprintf(&out, "%s%s\n", pad, bytes.TrimSpace(scan.Bytes()))
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func TestTrimSpaceEachLineAndPad(t *testing.T) {
|
||||
foo := `
|
||||
foo
|
||||
bar baz
|
||||
`
|
||||
assert.Equal(t, " \n foo\n bar baz\n \n", trimSpaceEachLineAndPad(foo, " "))
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestTransportConnect(t *testing.T) {
|
||||
tmpl := `
|
||||
jobs:
|
||||
- name: foo
|
||||
type: push
|
||||
connect:
|
||||
%s
|
||||
filesystems: {"<": true}
|
||||
snapshotting:
|
||||
type: manual
|
||||
pruning:
|
||||
keep_sender:
|
||||
- type: last_n
|
||||
count: 10
|
||||
keep_receiver:
|
||||
- type: last_n
|
||||
count: 10
|
||||
`
|
||||
|
||||
mconf := func(s string) string { return fmt.Sprintf(tmpl, s) }
|
||||
|
||||
type test struct {
|
||||
Name string
|
||||
ExpectError bool
|
||||
Connect string
|
||||
}
|
||||
|
||||
testTable := []test{
|
||||
{
|
||||
Name: "tcp_with_address_and_port",
|
||||
ExpectError: false,
|
||||
Connect: `
|
||||
type: tcp
|
||||
address: 10.0.0.23:42
|
||||
`,
|
||||
},
|
||||
{
|
||||
Name: "tls_with_host_and_port",
|
||||
ExpectError: false,
|
||||
Connect: `
|
||||
type: tls
|
||||
address: "server1.foo.bar:8888"
|
||||
ca: /etc/zrepl/ca.crt
|
||||
cert: /etc/zrepl/backupserver.fullchain
|
||||
key: /etc/zrepl/backupserver.key
|
||||
server_cn: "server1"
|
||||
`,
|
||||
},
|
||||
{
|
||||
Name: "tcp_without_port",
|
||||
ExpectError: true,
|
||||
Connect: `
|
||||
type: tcp
|
||||
address: 10.0.0.23
|
||||
`,
|
||||
},
|
||||
{
|
||||
Name: "tls_without_port",
|
||||
ExpectError: true,
|
||||
Connect: `
|
||||
type: tls
|
||||
address: 10.0.0.23
|
||||
ca: /etc/zrepl/ca.crt
|
||||
cert: /etc/zrepl/backupserver.fullchain
|
||||
key: /etc/zrepl/backupserver.key
|
||||
server_cn: "server1"
|
||||
`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testTable {
|
||||
t.Run(tc.Name, func(t *testing.T) {
|
||||
require.NotEmpty(t, tc.Connect)
|
||||
connect := trimSpaceEachLineAndPad(tc.Connect, " ")
|
||||
conf := mconf(connect)
|
||||
config, err := testConfig(t, conf)
|
||||
if tc.ExpectError && err == nil {
|
||||
t.Errorf("expected test failure, but got valid config %v", config)
|
||||
return
|
||||
}
|
||||
if !tc.ExpectError && err != nil {
|
||||
t.Errorf("not expecint test failure but got error: %s", err)
|
||||
return
|
||||
}
|
||||
t.Logf("error=%v", err)
|
||||
t.Logf("config=%v", config)
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,119 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type RetentionIntervalList []RetentionInterval
|
||||
|
||||
type PruneGrid struct {
|
||||
Type string `yaml:"type"`
|
||||
Grid RetentionIntervalList `yaml:"grid"`
|
||||
Regex string `yaml:"regex"`
|
||||
}
|
||||
|
||||
type RetentionInterval struct {
|
||||
length time.Duration
|
||||
keepCount int
|
||||
}
|
||||
|
||||
func (i *RetentionInterval) Length() time.Duration {
|
||||
return i.length
|
||||
}
|
||||
|
||||
func (i *RetentionInterval) KeepCount() int {
|
||||
return i.keepCount
|
||||
}
|
||||
|
||||
const RetentionGridKeepCountAll int = -1
|
||||
|
||||
func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
||||
var in string
|
||||
if err := u(&in, true); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
intervals, err := parseRetentionGridIntervalsString(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*t = intervals
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
|
||||
|
||||
func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {
|
||||
|
||||
comps := retentionStringIntervalRegex.FindStringSubmatch(e)
|
||||
if comps == nil {
|
||||
err = fmt.Errorf("retention string does not match expected format")
|
||||
return
|
||||
}
|
||||
|
||||
times, err := strconv.Atoi(comps[1])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} else if times <= 0 {
|
||||
return nil, fmt.Errorf("contains factor <= 0")
|
||||
}
|
||||
|
||||
duration, err := parsePositiveDuration(comps[2])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
keepCount := 1
|
||||
if comps[3] != "" {
|
||||
// Decompose key=value, comma separated
|
||||
// For now, only keep_count is supported
|
||||
re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
|
||||
res := re.FindStringSubmatch(comps[4])
|
||||
if res == nil || len(res) != 2 {
|
||||
err = fmt.Errorf("interval parameter contains unknown parameters")
|
||||
return
|
||||
}
|
||||
if res[1] == "all" {
|
||||
keepCount = RetentionGridKeepCountAll
|
||||
} else {
|
||||
keepCount, err = strconv.Atoi(res[1])
|
||||
if err != nil {
|
||||
err = fmt.Errorf("cannot parse keep_count value")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
intervals = make([]RetentionInterval, times)
|
||||
for i := range intervals {
|
||||
intervals[i] = RetentionInterval{
|
||||
length: duration,
|
||||
keepCount: keepCount,
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
func parseRetentionGridIntervalsString(s string) (intervals []RetentionInterval, err error) {
|
||||
|
||||
ges := strings.Split(s, "|")
|
||||
intervals = make([]RetentionInterval, 0, 7*len(ges))
|
||||
|
||||
for intervalIdx, e := range ges {
|
||||
parsed, err := parseRetentionGridIntervalString(e)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
|
||||
}
|
||||
intervals = append(intervals, parsed...)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
[ "$ZREPL_DRYRUN" = "true" ] && DRYRUN="echo DRYRUN: "
|
||||
|
||||
pre_snapshot() {
|
||||
$DRYRUN date
|
||||
}
|
||||
|
||||
post_snapshot() {
|
||||
$DRYRUN date
|
||||
}
|
||||
|
||||
case "$ZREPL_HOOKTYPE" in
|
||||
pre_snapshot|post_snapshot)
|
||||
"$ZREPL_HOOKTYPE"
|
||||
;;
|
||||
*)
|
||||
printf 'Unrecognized hook type: %s\n' "$ZREPL_HOOKTYPE"
|
||||
exit 255
|
||||
;;
|
||||
esac
|
||||
@@ -1,31 +0,0 @@
|
||||
|
||||
jobs:
|
||||
- type: sink
|
||||
name: "local_sink"
|
||||
root_fs: "storage/zrepl/sink"
|
||||
serve:
|
||||
type: local
|
||||
listener_name: localsink
|
||||
|
||||
- type: push
|
||||
name: "backup_system"
|
||||
connect:
|
||||
type: local
|
||||
listener_name: localsink
|
||||
client_identity: local_backup
|
||||
filesystems: {
|
||||
"system<": true,
|
||||
}
|
||||
snapshotting:
|
||||
type: periodic
|
||||
interval: 10m
|
||||
prefix: zrepl_
|
||||
pruning:
|
||||
keep_sender:
|
||||
- type: not_replicated
|
||||
- type: last_n
|
||||
count: 10
|
||||
keep_receiver:
|
||||
- type: grid
|
||||
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
|
||||
regex: "zrepl_.*"
|
||||
@@ -1,24 +0,0 @@
|
||||
jobs:
|
||||
- name: pull_servers
|
||||
type: pull
|
||||
connect:
|
||||
type: tls
|
||||
address: "server1.foo.bar:8888"
|
||||
ca: "/certs/ca.crt"
|
||||
cert: "/certs/cert.crt"
|
||||
key: "/certs/key.pem"
|
||||
server_cn: "server1"
|
||||
root_fs: "pool2/backup_servers"
|
||||
interval: 10m
|
||||
pruning:
|
||||
keep_sender:
|
||||
- type: not_replicated
|
||||
- type: last_n
|
||||
count: 10
|
||||
- type: grid
|
||||
grid: 1x1h(keep=all) | 24x1h | 14x1d
|
||||
regex: "zrepl_.*"
|
||||
keep_receiver:
|
||||
- type: grid
|
||||
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
|
||||
regex: "zrepl_.*"
|
||||
@@ -1,28 +0,0 @@
|
||||
jobs:
|
||||
|
||||
- name: pull_servers
|
||||
type: pull
|
||||
connect:
|
||||
type: ssh+stdinserver
|
||||
host: app-srv.example.com
|
||||
user: root
|
||||
port: 22
|
||||
identity_file: /etc/zrepl/ssh/identity
|
||||
options: # optional, default [], `-o` arguments passed to ssh
|
||||
- "Compression=yes"
|
||||
root_fs: "pool2/backup_servers"
|
||||
interval: 10m
|
||||
pruning:
|
||||
keep_sender:
|
||||
- type: not_replicated
|
||||
- type: last_n
|
||||
count: 10
|
||||
- type: grid
|
||||
grid: 1x1h(keep=all) | 24x1h | 14x1d
|
||||
regex: "^zrepl_.*"
|
||||
keep_receiver:
|
||||
- type: regex
|
||||
regex: keep_
|
||||
- type: grid
|
||||
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
|
||||
regex: "^zrepl_.*"
|
||||
@@ -1,26 +0,0 @@
|
||||
jobs:
|
||||
- type: push
|
||||
name: "push"
|
||||
filesystems: {
|
||||
"<": true,
|
||||
"tmp": false
|
||||
}
|
||||
connect:
|
||||
type: tcp
|
||||
address: "backup-server.foo.bar:8888"
|
||||
snapshotting:
|
||||
type: manual
|
||||
send:
|
||||
encrypted: false
|
||||
pruning:
|
||||
keep_sender:
|
||||
- type: not_replicated
|
||||
- type: last_n
|
||||
count: 10
|
||||
- type: grid
|
||||
grid: 1x1h(keep=all) | 24x1h | 14x1d
|
||||
regex: "^zrepl_.*"
|
||||
keep_receiver:
|
||||
- type: grid
|
||||
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
|
||||
regex: "^zrepl_.*"
|
||||
@@ -1,13 +0,0 @@
|
||||
jobs:
|
||||
- type: sink
|
||||
name: "laptop_sink"
|
||||
root_fs: "pool2/backup_laptops"
|
||||
serve:
|
||||
type: tls
|
||||
listen: "192.168.122.189:8888"
|
||||
ca: "ca.pem"
|
||||
cert: "cert.pem"
|
||||
key: "key.pem"
|
||||
client_cns:
|
||||
- "laptop1"
|
||||
- "homeserver"
|
||||
@@ -1,14 +0,0 @@
|
||||
jobs:
|
||||
- name: snapjob
|
||||
type: snap
|
||||
filesystems: {
|
||||
"tank/frequently_changed<": true,
|
||||
}
|
||||
snapshotting:
|
||||
type: periodic
|
||||
interval: 2m
|
||||
prefix: zrepl_snapjob_
|
||||
pruning:
|
||||
keep:
|
||||
- type: last_n
|
||||
count: 60
|
||||
@@ -1,17 +0,0 @@
|
||||
jobs:
|
||||
- name: pull_source
|
||||
type: source
|
||||
serve:
|
||||
type: tcp
|
||||
listen: "0.0.0.0:8888"
|
||||
clients: {
|
||||
"192.168.122.123" : "client1"
|
||||
}
|
||||
filesystems: {
|
||||
"<": true,
|
||||
"secret": false
|
||||
}
|
||||
snapshotting:
|
||||
type: periodic
|
||||
interval: 10m
|
||||
prefix: zrepl_
|
||||
@@ -1,17 +0,0 @@
|
||||
jobs:
|
||||
- name: pull_source
|
||||
type: source
|
||||
serve:
|
||||
type: stdinserver
|
||||
client_identities:
|
||||
- "client1"
|
||||
- "client2"
|
||||
filesystems: {
|
||||
"<": true,
|
||||
"secret": false
|
||||
}
|
||||
snapshotting:
|
||||
type: periodic
|
||||
interval: 10m
|
||||
prefix: zrepl_
|
||||
|
||||
@@ -1,285 +0,0 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/job"
|
||||
"github.com/zrepl/zrepl/daemon/nethelpers"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/util/envconst"
|
||||
"github.com/zrepl/zrepl/version"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
"github.com/zrepl/zrepl/zfs/zfscmd"
|
||||
)
|
||||
|
||||
type controlJob struct {
|
||||
sockaddr *net.UnixAddr
|
||||
jobs *jobs
|
||||
}
|
||||
|
||||
func newControlJob(sockpath string, jobs *jobs) (j *controlJob, err error) {
|
||||
j = &controlJob{jobs: jobs}
|
||||
|
||||
j.sockaddr, err = net.ResolveUnixAddr("unix", sockpath)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "cannot resolve unix address")
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (j *controlJob) Name() string { return jobNameControl }
|
||||
|
||||
func (j *controlJob) Status() *job.Status { return &job.Status{Type: job.TypeInternal} }
|
||||
|
||||
func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
|
||||
|
||||
func (j *controlJob) SenderConfig() *endpoint.SenderConfig { return nil }
|
||||
|
||||
var promControl struct {
|
||||
requestBegin *prometheus.CounterVec
|
||||
requestFinished *prometheus.HistogramVec
|
||||
}
|
||||
|
||||
func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) {
|
||||
promControl.requestBegin = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "control",
|
||||
Name: "request_begin",
|
||||
Help: "number of request we started to handle",
|
||||
}, []string{"endpoint"})
|
||||
|
||||
promControl.requestFinished = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "control",
|
||||
Name: "request_finished",
|
||||
Help: "time it took a request to finish",
|
||||
Buckets: []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3, 10e-3, 100e-3, 200e-3, 400e-3, 800e-3, 1, 10, 20},
|
||||
}, []string{"endpoint"})
|
||||
registerer.MustRegister(promControl.requestBegin)
|
||||
registerer.MustRegister(promControl.requestFinished)
|
||||
}
|
||||
|
||||
const (
|
||||
ControlJobEndpointPProf string = "/debug/pprof"
|
||||
ControlJobEndpointVersion string = "/version"
|
||||
ControlJobEndpointStatus string = "/status"
|
||||
ControlJobEndpointSignal string = "/signal"
|
||||
)
|
||||
|
||||
func (j *controlJob) Run(ctx context.Context) {
|
||||
|
||||
log := job.GetLogger(ctx)
|
||||
defer log.Info("control job finished")
|
||||
|
||||
l, err := nethelpers.ListenUnixPrivate(j.sockaddr)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("error listening")
|
||||
return
|
||||
}
|
||||
|
||||
pprofServer := NewPProfServer(ctx)
|
||||
if listen := envconst.String("ZREPL_DAEMON_AUTOSTART_PPROF_SERVER", ""); listen != "" {
|
||||
pprofServer.Control(PprofServerControlMsg{
|
||||
Run: true,
|
||||
HttpListenAddress: listen,
|
||||
})
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle(ControlJobEndpointPProf,
|
||||
requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) {
|
||||
var msg PprofServerControlMsg
|
||||
err := decoder(&msg)
|
||||
if err != nil {
|
||||
return nil, errors.Errorf("decode failed")
|
||||
}
|
||||
pprofServer.Control(msg)
|
||||
return struct{}{}, nil
|
||||
}}})
|
||||
|
||||
mux.Handle(ControlJobEndpointVersion,
|
||||
requestLogger{log: log, handler: jsonResponder{log, func() (interface{}, error) {
|
||||
return version.NewZreplVersionInformation(), nil
|
||||
}}})
|
||||
|
||||
mux.Handle(ControlJobEndpointStatus,
|
||||
// don't log requests to status endpoint, too spammy
|
||||
jsonResponder{log, func() (interface{}, error) {
|
||||
jobs := j.jobs.status()
|
||||
globalZFS := zfscmd.GetReport()
|
||||
s := Status{Jobs: jobs, Global: GlobalStatus{ZFSCmds: globalZFS}}
|
||||
return s, nil
|
||||
}})
|
||||
|
||||
mux.Handle(ControlJobEndpointSignal,
|
||||
requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) {
|
||||
type reqT struct {
|
||||
Name string
|
||||
Op string
|
||||
}
|
||||
var req reqT
|
||||
if decoder(&req) != nil {
|
||||
return nil, errors.Errorf("decode failed")
|
||||
}
|
||||
|
||||
var err error
|
||||
switch req.Op {
|
||||
case "wakeup":
|
||||
err = j.jobs.wakeup(req.Name)
|
||||
case "reset":
|
||||
err = j.jobs.reset(req.Name)
|
||||
default:
|
||||
err = fmt.Errorf("operation %q is invalid", req.Op)
|
||||
}
|
||||
|
||||
return struct{}{}, err
|
||||
}}})
|
||||
server := http.Server{
|
||||
Handler: mux,
|
||||
// control socket is local, 1s timeout should be more than sufficient, even on a loaded system
|
||||
WriteTimeout: 1 * time.Second,
|
||||
ReadTimeout: 1 * time.Second,
|
||||
}
|
||||
|
||||
outer:
|
||||
for {
|
||||
|
||||
served := make(chan error)
|
||||
go func() {
|
||||
served <- server.Serve(l)
|
||||
close(served)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.WithError(ctx.Err()).Info("context done")
|
||||
err := server.Shutdown(context.Background())
|
||||
if err != nil {
|
||||
log.WithError(err).Error("cannot shutdown server")
|
||||
}
|
||||
break outer
|
||||
case err = <-served:
|
||||
if err != nil {
|
||||
log.WithError(err).Error("error serving")
|
||||
break outer
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
type jsonResponder struct {
|
||||
log Logger
|
||||
producer func() (interface{}, error)
|
||||
}
|
||||
|
||||
func (j jsonResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
logIoErr := func(err error) {
|
||||
if err != nil {
|
||||
j.log.WithError(err).Error("control handler io error")
|
||||
}
|
||||
}
|
||||
res, err := j.producer()
|
||||
if err != nil {
|
||||
j.log.WithError(err).Error("control handler error")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, err = io.WriteString(w, err.Error())
|
||||
logIoErr(err)
|
||||
return
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
err = json.NewEncoder(&buf).Encode(res)
|
||||
if err != nil {
|
||||
j.log.WithError(err).Error("control handler json marshal error")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, err = io.WriteString(w, err.Error())
|
||||
} else {
|
||||
_, err = io.Copy(w, &buf)
|
||||
}
|
||||
logIoErr(err)
|
||||
}
|
||||
|
||||
type jsonDecoder = func(interface{}) error
|
||||
|
||||
type jsonRequestResponder struct {
|
||||
log Logger
|
||||
producer func(decoder jsonDecoder) (interface{}, error)
|
||||
}
|
||||
|
||||
func (j jsonRequestResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
logIoErr := func(err error) {
|
||||
if err != nil {
|
||||
j.log.WithError(err).Error("control handler io error")
|
||||
}
|
||||
}
|
||||
|
||||
var decodeError error
|
||||
decoder := func(i interface{}) error {
|
||||
err := json.NewDecoder(r.Body).Decode(&i)
|
||||
decodeError = err
|
||||
return err
|
||||
}
|
||||
res, producerErr := j.producer(decoder)
|
||||
|
||||
//If we had a decode error ignore output of producer and return error
|
||||
if decodeError != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, err := io.WriteString(w, decodeError.Error())
|
||||
logIoErr(err)
|
||||
return
|
||||
}
|
||||
if producerErr != nil {
|
||||
j.log.WithError(producerErr).Error("control handler error")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, err := io.WriteString(w, producerErr.Error())
|
||||
logIoErr(err)
|
||||
return
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
encodeErr := json.NewEncoder(&buf).Encode(res)
|
||||
if encodeErr != nil {
|
||||
j.log.WithError(producerErr).Error("control handler json marshal error")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, err := io.WriteString(w, encodeErr.Error())
|
||||
logIoErr(err)
|
||||
} else {
|
||||
_, err := io.Copy(w, &buf)
|
||||
logIoErr(err)
|
||||
}
|
||||
}
|
||||
|
||||
type requestLogger struct {
|
||||
log logger.Logger
|
||||
handler http.Handler
|
||||
handlerFunc http.HandlerFunc
|
||||
}
|
||||
|
||||
func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
log := l.log.WithField("method", r.Method).WithField("url", r.URL)
|
||||
log.Debug("start")
|
||||
promControl.requestBegin.WithLabelValues(r.URL.Path).Inc()
|
||||
defer prometheus.NewTimer(promControl.requestFinished.WithLabelValues(r.URL.Path)).ObserveDuration()
|
||||
if l.handlerFunc != nil {
|
||||
l.handlerFunc(w, r)
|
||||
} else if l.handler != nil {
|
||||
l.handler.ServeHTTP(w, r)
|
||||
} else {
|
||||
log.Error("no handler or handlerFunc configured")
|
||||
}
|
||||
log.Debug("finish")
|
||||
}
|
||||
@@ -1,236 +0,0 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/job"
|
||||
"github.com/zrepl/zrepl/daemon/job/reset"
|
||||
"github.com/zrepl/zrepl/daemon/job/wakeup"
|
||||
"github.com/zrepl/zrepl/daemon/logging"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/version"
|
||||
"github.com/zrepl/zrepl/zfs/zfscmd"
|
||||
)
|
||||
|
||||
func Run(conf *config.Config) error {
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
defer cancel()
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
go func() {
|
||||
<-sigChan
|
||||
cancel()
|
||||
}()
|
||||
|
||||
outlets, err := logging.OutletsFromConfig(*conf.Global.Logging)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "cannot build logging from config")
|
||||
}
|
||||
|
||||
confJobs, err := job.JobsFromConfig(conf)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "cannot build jobs from config")
|
||||
}
|
||||
|
||||
log := logger.NewLogger(outlets, 1*time.Second)
|
||||
log.Info(version.NewZreplVersionInformation().String())
|
||||
|
||||
for _, job := range confJobs {
|
||||
if IsInternalJobName(job.Name()) {
|
||||
panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME
|
||||
}
|
||||
}
|
||||
|
||||
ctx = job.WithLogger(ctx, log)
|
||||
|
||||
jobs := newJobs()
|
||||
|
||||
// start control socket
|
||||
controlJob, err := newControlJob(conf.Global.Control.SockPath, jobs)
|
||||
if err != nil {
|
||||
panic(err) // FIXME
|
||||
}
|
||||
jobs.start(ctx, controlJob, true)
|
||||
|
||||
for i, jc := range conf.Global.Monitoring {
|
||||
var (
|
||||
job job.Job
|
||||
err error
|
||||
)
|
||||
switch v := jc.Ret.(type) {
|
||||
case *config.PrometheusMonitoring:
|
||||
job, err = newPrometheusJobFromConfig(v)
|
||||
default:
|
||||
return errors.Errorf("unknown monitoring job #%d (type %T)", i, v)
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "cannot build monitoring job #%d", i)
|
||||
}
|
||||
jobs.start(ctx, job, true)
|
||||
}
|
||||
|
||||
// register global (=non job-local) metrics
|
||||
zfscmd.RegisterMetrics(prometheus.DefaultRegisterer)
|
||||
|
||||
log.Info("starting daemon")
|
||||
|
||||
// start regular jobs
|
||||
for _, j := range confJobs {
|
||||
jobs.start(ctx, j, false)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-jobs.wait():
|
||||
log.Info("all jobs finished")
|
||||
case <-ctx.Done():
|
||||
log.WithError(ctx.Err()).Info("context finished")
|
||||
}
|
||||
log.Info("daemon exiting")
|
||||
return nil
|
||||
}
|
||||
|
||||
type jobs struct {
|
||||
wg sync.WaitGroup
|
||||
|
||||
// m protects all fields below it
|
||||
m sync.RWMutex
|
||||
wakeups map[string]wakeup.Func // by Job.Name
|
||||
resets map[string]reset.Func // by Job.Name
|
||||
jobs map[string]job.Job
|
||||
}
|
||||
|
||||
func newJobs() *jobs {
|
||||
return &jobs{
|
||||
wakeups: make(map[string]wakeup.Func),
|
||||
resets: make(map[string]reset.Func),
|
||||
jobs: make(map[string]job.Job),
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
logJobField string = "job"
|
||||
)
|
||||
|
||||
func (s *jobs) wait() <-chan struct{} {
|
||||
ch := make(chan struct{})
|
||||
go func() {
|
||||
s.wg.Wait()
|
||||
}()
|
||||
return ch
|
||||
}
|
||||
|
||||
type Status struct {
|
||||
Jobs map[string]*job.Status
|
||||
Global GlobalStatus
|
||||
}
|
||||
|
||||
type GlobalStatus struct {
|
||||
ZFSCmds *zfscmd.Report
|
||||
}
|
||||
|
||||
func (s *jobs) status() map[string]*job.Status {
|
||||
s.m.RLock()
|
||||
defer s.m.RUnlock()
|
||||
|
||||
type res struct {
|
||||
name string
|
||||
status *job.Status
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
c := make(chan res, len(s.jobs))
|
||||
for name, j := range s.jobs {
|
||||
wg.Add(1)
|
||||
go func(name string, j job.Job) {
|
||||
defer wg.Done()
|
||||
c <- res{name: name, status: j.Status()}
|
||||
}(name, j)
|
||||
}
|
||||
wg.Wait()
|
||||
close(c)
|
||||
ret := make(map[string]*job.Status, len(s.jobs))
|
||||
for res := range c {
|
||||
ret[res.name] = res.status
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
func (s *jobs) wakeup(job string) error {
|
||||
s.m.RLock()
|
||||
defer s.m.RUnlock()
|
||||
|
||||
wu, ok := s.wakeups[job]
|
||||
if !ok {
|
||||
return errors.Errorf("Job %s does not exist", job)
|
||||
}
|
||||
return wu()
|
||||
}
|
||||
|
||||
func (s *jobs) reset(job string) error {
|
||||
s.m.RLock()
|
||||
defer s.m.RUnlock()
|
||||
|
||||
wu, ok := s.resets[job]
|
||||
if !ok {
|
||||
return errors.Errorf("Job %s does not exist", job)
|
||||
}
|
||||
return wu()
|
||||
}
|
||||
|
||||
const (
|
||||
jobNamePrometheus = "_prometheus"
|
||||
jobNameControl = "_control"
|
||||
)
|
||||
|
||||
func IsInternalJobName(s string) bool {
|
||||
return strings.HasPrefix(s, "_")
|
||||
}
|
||||
|
||||
func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
|
||||
s.m.Lock()
|
||||
defer s.m.Unlock()
|
||||
|
||||
jobLog := job.GetLogger(ctx).
|
||||
WithField(logJobField, j.Name()).
|
||||
WithOutlet(newPrometheusLogOutlet(j.Name()), logger.Debug)
|
||||
jobName := j.Name()
|
||||
if !internal && IsInternalJobName(jobName) {
|
||||
panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName))
|
||||
}
|
||||
if internal && !IsInternalJobName(jobName) {
|
||||
panic(fmt.Sprintf("internal job does not use internal job name %s", jobName))
|
||||
}
|
||||
if _, ok := s.jobs[jobName]; ok {
|
||||
panic(fmt.Sprintf("duplicate job name %s", jobName))
|
||||
}
|
||||
|
||||
j.RegisterMetrics(prometheus.DefaultRegisterer)
|
||||
|
||||
s.jobs[jobName] = j
|
||||
ctx = job.WithLogger(ctx, jobLog)
|
||||
ctx = zfscmd.WithJobID(ctx, j.Name())
|
||||
ctx, wakeup := wakeup.Context(ctx)
|
||||
ctx, resetFunc := reset.Context(ctx)
|
||||
s.wakeups[jobName] = wakeup
|
||||
s.resets[jobName] = resetFunc
|
||||
|
||||
s.wg.Add(1)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
jobLog.Info("starting job")
|
||||
defer jobLog.Info("job exited")
|
||||
j.Run(ctx)
|
||||
}()
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
package filters
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
func TestDatasetMapFilter(t *testing.T) {
|
||||
|
||||
type testCase struct {
|
||||
name string
|
||||
filter map[string]string
|
||||
checkPass map[string]bool // each entry is checked to match the filter's `pass` return value
|
||||
}
|
||||
|
||||
tcs := []testCase{
|
||||
{
|
||||
"default_no_match",
|
||||
map[string]string{},
|
||||
map[string]bool{
|
||||
"": false,
|
||||
"foo": false,
|
||||
"zroot": false,
|
||||
},
|
||||
},
|
||||
{
|
||||
"more_specific_path_has_precedence",
|
||||
map[string]string{
|
||||
"tank<": "ok",
|
||||
"tank/tmp<": "!",
|
||||
"tank/home/x<": "!",
|
||||
"tank/home/x/1": "ok",
|
||||
},
|
||||
map[string]bool{
|
||||
"zroot": false,
|
||||
"tank": true,
|
||||
"tank/tmp": false,
|
||||
"tank/tmp/foo": false,
|
||||
"tank/home/x": false,
|
||||
"tank/home/y": true,
|
||||
"tank/home/x/1": true,
|
||||
"tank/home/x/2": false,
|
||||
},
|
||||
},
|
||||
{
|
||||
"precedence_of_specific_over_subtree_wildcard_on_same_path",
|
||||
map[string]string{
|
||||
"tank/home/bob": "ok",
|
||||
"tank/home/bob<": "!",
|
||||
},
|
||||
map[string]bool{
|
||||
"tank/home/bob": true,
|
||||
"tank/home/bob/downloads": false,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for tc := range tcs {
|
||||
t.Run(tcs[tc].name, func(t *testing.T) {
|
||||
c := tcs[tc]
|
||||
f := NewDatasetMapFilter(len(c.filter), true)
|
||||
for p, a := range c.filter {
|
||||
err := f.Add(p, a)
|
||||
if err != nil {
|
||||
t.Fatalf("incorrect filter spec: %s", err)
|
||||
}
|
||||
}
|
||||
for p, checkPass := range c.checkPass {
|
||||
zp, err := zfs.NewDatasetPath(p)
|
||||
if err != nil {
|
||||
t.Fatalf("incorrect path spec: %s", err)
|
||||
}
|
||||
pass, err := f.Filter(zp)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected filter error: %s", err)
|
||||
}
|
||||
ok := pass == checkPass
|
||||
failstr := "OK"
|
||||
if !ok {
|
||||
failstr = "FAIL"
|
||||
t.Fail()
|
||||
}
|
||||
t.Logf("%-40q %5v (exp=%v act=%v)", p, failstr, checkPass, pass)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
// Code generated by "stringer -type=Edge"; DO NOT EDIT.
|
||||
|
||||
package hooks
|
||||
|
||||
import "strconv"
|
||||
|
||||
func _() {
|
||||
// An "invalid array index" compiler error signifies that the constant values have changed.
|
||||
// Re-run the stringer command to generate them again.
|
||||
var x [1]struct{}
|
||||
_ = x[Pre-1]
|
||||
_ = x[Callback-2]
|
||||
_ = x[Post-4]
|
||||
}
|
||||
|
||||
const (
|
||||
_Edge_name_0 = "PreCallback"
|
||||
_Edge_name_1 = "Post"
|
||||
)
|
||||
|
||||
var (
|
||||
_Edge_index_0 = [...]uint8{0, 3, 11}
|
||||
)
|
||||
|
||||
func (i Edge) String() string {
|
||||
switch {
|
||||
case 1 <= i && i <= 2:
|
||||
i -= 1
|
||||
return _Edge_name_0[_Edge_index_0[i]:_Edge_index_0[i+1]]
|
||||
case i == 4:
|
||||
return _Edge_name_1
|
||||
default:
|
||||
return "Edge(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
package hooks
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type List []Hook
|
||||
|
||||
func HookFromConfig(in config.HookEnum) (Hook, error) {
|
||||
switch v := in.Ret.(type) {
|
||||
case *config.HookCommand:
|
||||
return NewCommandHook(v)
|
||||
case *config.HookPostgresCheckpoint:
|
||||
return PgChkptHookFromConfig(v)
|
||||
case *config.HookMySQLLockTables:
|
||||
return MyLockTablesFromConfig(v)
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown hook type %T", v)
|
||||
}
|
||||
}
|
||||
|
||||
func ListFromConfig(in *config.HookList) (r *List, err error) {
|
||||
hl := make(List, len(*in))
|
||||
|
||||
for i, h := range *in {
|
||||
hl[i], err = HookFromConfig(h)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create hook #%d: %s", i+1, err)
|
||||
}
|
||||
}
|
||||
|
||||
return &hl, nil
|
||||
}
|
||||
|
||||
func (l List) CopyFilteredForFilesystem(fs *zfs.DatasetPath) (ret List, err error) {
|
||||
ret = make(List, 0, len(l))
|
||||
|
||||
for _, h := range l {
|
||||
var passFilesystem bool
|
||||
if passFilesystem, err = h.Filesystems().Filter(fs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if passFilesystem {
|
||||
ret = append(ret, h)
|
||||
}
|
||||
}
|
||||
|
||||
return ret, nil
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
// Package hooks implements pre- and post snapshot hooks.
|
||||
//
|
||||
// Plan is a generic executor for ExpectStepReports before and after an activity specified in a callback.
|
||||
// It provides a reporting facility that can be polled while the plan is executing to gather progress information.
|
||||
//
|
||||
// This package also provides all supported hook type implementations and abstractions around them.
|
||||
//
|
||||
// Use For Other Kinds Of ExpectStepReports
|
||||
//
|
||||
// This package REQUIRES REFACTORING before it can be used for other activities than snapshots, e.g. pre- and post-replication:
|
||||
//
|
||||
// The Hook interface requires a hook to provide a Filesystems() filter, which doesn't make sense for
|
||||
// all kinds of activities.
|
||||
//
|
||||
// The hook implementations should move out of this package.
|
||||
// However, there is a lot of tight coupling which to untangle isn't worth it ATM.
|
||||
//
|
||||
// How This Package Is Used By Package Snapper
|
||||
//
|
||||
// Deserialize a config.List using ListFromConfig().
|
||||
// Then it MUST filter the list to only contain hooks for a particular filesystem using
|
||||
// hooksList.CopyFilteredForFilesystem(fs).
|
||||
//
|
||||
// Then create a CallbackHook using NewCallbackHookForFilesystem().
|
||||
//
|
||||
// Pass all of the above to NewPlan() which provides a Report() and Run() method:
|
||||
//
|
||||
// Plan.Run(ctx context.Context,dryRun bool) executes the plan and take a context as argument that should contain a logger added using hooks.WithLogger()).
|
||||
// The value of dryRun is passed through to the hooks' Run() method.
|
||||
// Command hooks make it available in the environment variable ZREPL_DRYRUN.
|
||||
//
|
||||
// Plan.Report() can be called while Plan.Run() is executing to give an overview of plan execution progress (future use in "zrepl status").
|
||||
//
|
||||
package hooks
|
||||
@@ -1,292 +0,0 @@
|
||||
package hooks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
// Re-export type here so that
|
||||
// every file in package hooks doesn't
|
||||
// have to import github.com/zrepl/zrepl/zfs
|
||||
type Filter zfs.DatasetFilter
|
||||
|
||||
type Hook interface {
|
||||
Filesystems() Filter
|
||||
|
||||
// If true and the Pre edge invocation of Run fails, Post edge will not run and other Pre edges will not run.
|
||||
ErrIsFatal() bool
|
||||
|
||||
// Run is invoked by HookPlan for a Pre edge.
|
||||
// If HookReport.HadError() == false, the Post edge will be invoked, too.
|
||||
Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport
|
||||
|
||||
String() string
|
||||
}
|
||||
|
||||
type Phase string
|
||||
|
||||
const (
|
||||
PhaseSnapshot = Phase("snapshot")
|
||||
PhaseTesting = Phase("testing")
|
||||
)
|
||||
|
||||
func (p Phase) String() string {
|
||||
return string(p)
|
||||
}
|
||||
|
||||
//go:generate stringer -type=Edge
|
||||
type Edge uint
|
||||
|
||||
const (
|
||||
Pre = Edge(1 << iota)
|
||||
Callback
|
||||
Post
|
||||
)
|
||||
|
||||
func (e Edge) StringForPhase(phase Phase) string {
|
||||
return fmt.Sprintf("%s_%s", e.String(), phase.String())
|
||||
}
|
||||
|
||||
//go:generate enumer -type=StepStatus -trimprefix=Step
|
||||
type StepStatus int
|
||||
|
||||
const (
|
||||
StepPending StepStatus = 1 << iota
|
||||
StepExec
|
||||
StepOk
|
||||
StepErr
|
||||
StepSkippedDueToFatalErr
|
||||
StepSkippedDueToPreErr
|
||||
)
|
||||
|
||||
type HookReport interface {
|
||||
String() string
|
||||
HadError() bool
|
||||
Error() string
|
||||
}
|
||||
|
||||
type Step struct {
|
||||
Hook Hook
|
||||
Edge Edge
|
||||
Status StepStatus
|
||||
Begin, End time.Time
|
||||
// Report may be nil
|
||||
// FIXME cannot serialize this for client status, but contains interesting info (like what error happened)
|
||||
Report HookReport
|
||||
state map[interface{}]interface{}
|
||||
}
|
||||
|
||||
func (s Step) String() (out string) {
|
||||
fatal := "~"
|
||||
if s.Hook.ErrIsFatal() && s.Edge == Pre {
|
||||
fatal = "!"
|
||||
}
|
||||
runTime := "..."
|
||||
if s.Status != StepPending {
|
||||
t := s.End.Sub(s.Begin)
|
||||
runTime = t.Round(time.Millisecond).String()
|
||||
}
|
||||
return fmt.Sprintf("[%s] [%5s] %s [%s] %s", s.Status, runTime, fatal, s.Edge, s.Hook)
|
||||
}
|
||||
|
||||
type Plan struct {
|
||||
mtx sync.RWMutex
|
||||
|
||||
steps []*Step
|
||||
pre []*Step // protected by mtx
|
||||
cb *Step
|
||||
post []*Step // not reversed, i.e. entry at index i corresponds to pre-edge in pre[i]
|
||||
|
||||
phase Phase
|
||||
env Env
|
||||
}
|
||||
|
||||
func NewPlan(hooks *List, phase Phase, cb *CallbackHook, extra Env) (*Plan, error) {
|
||||
|
||||
var pre, post []*Step
|
||||
// TODO sanity check unique name of hook?
|
||||
for _, hook := range *hooks {
|
||||
state := make(map[interface{}]interface{})
|
||||
preE := &Step{
|
||||
Hook: hook,
|
||||
Edge: Pre,
|
||||
Status: StepPending,
|
||||
state: state,
|
||||
}
|
||||
pre = append(pre, preE)
|
||||
postE := &Step{
|
||||
Hook: hook,
|
||||
Edge: Post,
|
||||
Status: StepPending,
|
||||
state: state,
|
||||
}
|
||||
post = append(post, postE)
|
||||
}
|
||||
|
||||
cbE := &Step{
|
||||
Hook: cb,
|
||||
Edge: Callback,
|
||||
Status: StepPending,
|
||||
}
|
||||
|
||||
steps := make([]*Step, 0, len(pre)+len(post)+1)
|
||||
steps = append(steps, pre...)
|
||||
steps = append(steps, cbE)
|
||||
for i := len(post) - 1; i >= 0; i-- {
|
||||
steps = append(steps, post[i])
|
||||
}
|
||||
|
||||
plan := &Plan{
|
||||
phase: phase,
|
||||
env: extra,
|
||||
steps: steps,
|
||||
pre: pre,
|
||||
post: post,
|
||||
cb: cbE,
|
||||
}
|
||||
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
type PlanReport []Step
|
||||
|
||||
func (p *Plan) Report() PlanReport {
|
||||
p.mtx.RLock()
|
||||
defer p.mtx.RUnlock()
|
||||
rep := make([]Step, len(p.steps))
|
||||
for i := range rep {
|
||||
rep[i] = *p.steps[i]
|
||||
}
|
||||
return rep
|
||||
}
|
||||
|
||||
func (r PlanReport) HadError() bool {
|
||||
for _, e := range r {
|
||||
if e.Status == StepErr {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (r PlanReport) HadFatalError() bool {
|
||||
for _, e := range r {
|
||||
if e.Status == StepSkippedDueToFatalErr {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (r PlanReport) String() string {
|
||||
stepStrings := make([]string, len(r))
|
||||
for i, e := range r {
|
||||
stepStrings[i] = fmt.Sprintf("%02d %s", i+1, e)
|
||||
}
|
||||
return strings.Join(stepStrings, "\n")
|
||||
}
|
||||
|
||||
func (p *Plan) Run(ctx context.Context, dryRun bool) {
|
||||
p.mtx.RLock()
|
||||
defer p.mtx.RUnlock()
|
||||
w := func(f func()) {
|
||||
p.mtx.RUnlock()
|
||||
defer p.mtx.RLock()
|
||||
p.mtx.Lock()
|
||||
defer p.mtx.Unlock()
|
||||
f()
|
||||
}
|
||||
runHook := func(s *Step, ctx context.Context, edge Edge) HookReport {
|
||||
w(func() { s.Status = StepExec })
|
||||
begin := time.Now()
|
||||
r := s.Hook.Run(ctx, edge, p.phase, dryRun, p.env, s.state)
|
||||
end := time.Now()
|
||||
w(func() {
|
||||
s.Report = r
|
||||
s.Status = StepOk
|
||||
if r.HadError() {
|
||||
s.Status = StepErr
|
||||
}
|
||||
s.Begin, s.End = begin, end
|
||||
})
|
||||
return r
|
||||
}
|
||||
|
||||
l := getLogger(ctx)
|
||||
|
||||
// it's a stack, execute until we reach the end of the list (last item in)
|
||||
// or fail inbetween
|
||||
l.Info("run pre-edges in configuration order")
|
||||
next := 0
|
||||
for ; next < len(p.pre); next++ {
|
||||
e := p.pre[next]
|
||||
l := l.WithField("hook", e.Hook)
|
||||
r := runHook(e, ctx, Pre)
|
||||
if r.HadError() {
|
||||
l.WithError(r).Error("hook invocation failed for pre-edge")
|
||||
if e.Hook.ErrIsFatal() {
|
||||
l.Error("the hook run was aborted due to a fatal error in this hook")
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
hadFatalErr := next != len(p.pre)
|
||||
if hadFatalErr {
|
||||
l.Error("fatal error in a pre-snapshot hook invocation")
|
||||
l.Error("no snapshot will be taken")
|
||||
l.Error("only running post-edges for successful pre-edges")
|
||||
w(func() {
|
||||
p.post[next].Status = StepSkippedDueToFatalErr
|
||||
for i := next + 1; i < len(p.pre); i++ {
|
||||
p.pre[i].Status = StepSkippedDueToFatalErr
|
||||
p.post[i].Status = StepSkippedDueToFatalErr
|
||||
}
|
||||
p.cb.Status = StepSkippedDueToFatalErr
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
l.Info("running callback")
|
||||
cbR := runHook(p.cb, ctx, Callback)
|
||||
if cbR.HadError() {
|
||||
l.WithError(cbR).Error("callback failed")
|
||||
}
|
||||
|
||||
l.Info("run post-edges for successful pre-edges in reverse configuration order")
|
||||
|
||||
// the constructor produces pre and post entries
|
||||
// post is NOT reversed
|
||||
next-- // now at index of last executed pre-edge
|
||||
for ; next >= 0; next-- {
|
||||
e := p.post[next]
|
||||
l := l.WithField("hook", e.Hook)
|
||||
|
||||
if p.pre[next].Status != StepOk {
|
||||
if p.pre[next].Status != StepErr {
|
||||
panic(fmt.Sprintf("expecting a pre-edge hook report to be either Ok or Err, got %s", p.pre[next].Status))
|
||||
}
|
||||
l.Info("skip post-edge because pre-edge failed")
|
||||
w(func() {
|
||||
e.Status = StepSkippedDueToPreErr
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
report := runHook(e, ctx, Post)
|
||||
|
||||
if report.HadError() {
|
||||
l.WithError(report).Error("hook invocation failed for post-edge")
|
||||
l.Error("subsequent post-edges run regardless of this post-edge failure")
|
||||
}
|
||||
|
||||
// ErrIsFatal is only relevant for Pre
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
package hooks
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/util/envconst"
|
||||
)
|
||||
|
||||
type contextKey int
|
||||
|
||||
const (
|
||||
contextKeyLog contextKey = 0
|
||||
)
|
||||
|
||||
type Logger = logger.Logger
|
||||
|
||||
func WithLogger(ctx context.Context, log Logger) context.Context {
|
||||
return context.WithValue(ctx, contextKeyLog, log)
|
||||
}
|
||||
|
||||
func GetLogger(ctx context.Context) Logger { return getLogger(ctx) }
|
||||
|
||||
func getLogger(ctx context.Context) Logger {
|
||||
if log, ok := ctx.Value(contextKeyLog).(Logger); ok {
|
||||
return log
|
||||
}
|
||||
return logger.NewNullLogger()
|
||||
}
|
||||
|
||||
const MAX_HOOK_LOG_SIZE_DEFAULT int = 1 << 20
|
||||
|
||||
type logWriter struct {
|
||||
/*
|
||||
Mutex prevents:
|
||||
concurrent writes to buf, scanner in Write([]byte)
|
||||
data race on scanner vs Write([]byte)
|
||||
and concurrent write to buf (call to buf.Reset())
|
||||
in Close()
|
||||
|
||||
(Also, Close() should generally block until any Write() call completes.)
|
||||
*/
|
||||
mtx *sync.Mutex
|
||||
buf bytes.Buffer
|
||||
scanner *bufio.Scanner
|
||||
logger Logger
|
||||
level logger.Level
|
||||
field string
|
||||
}
|
||||
|
||||
func NewLogWriter(mtx *sync.Mutex, logger Logger, level logger.Level, field string) *logWriter {
|
||||
w := new(logWriter)
|
||||
w.mtx = mtx
|
||||
w.scanner = bufio.NewScanner(&w.buf)
|
||||
w.logger = logger
|
||||
w.level = level
|
||||
w.field = field
|
||||
return w
|
||||
}
|
||||
|
||||
func (w *logWriter) log(line string) {
|
||||
w.logger.WithField(w.field, line).Log(w.level, "hook output")
|
||||
}
|
||||
|
||||
func (w *logWriter) logUnreadBytes() error {
|
||||
for w.scanner.Scan() {
|
||||
w.log(w.scanner.Text())
|
||||
}
|
||||
if w.buf.Cap() > envconst.Int("ZREPL_MAX_HOOK_LOG_SIZE", MAX_HOOK_LOG_SIZE_DEFAULT) {
|
||||
w.buf.Reset()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *logWriter) Write(in []byte) (int, error) {
|
||||
w.mtx.Lock()
|
||||
defer w.mtx.Unlock()
|
||||
|
||||
n, bufErr := w.buf.Write(in)
|
||||
if bufErr != nil {
|
||||
return n, bufErr
|
||||
}
|
||||
|
||||
err := w.logUnreadBytes()
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
// Always reset the scanner for the next Write
|
||||
w.scanner = bufio.NewScanner(&w.buf)
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (w *logWriter) Close() (err error) {
|
||||
w.mtx.Lock()
|
||||
defer w.mtx.Unlock()
|
||||
|
||||
return w.logUnreadBytes()
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
package hooks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type HookJobCallback func(ctx context.Context) error
|
||||
|
||||
type CallbackHook struct {
|
||||
cb HookJobCallback
|
||||
filter Filter
|
||||
displayString string
|
||||
}
|
||||
|
||||
func NewCallbackHookForFilesystem(displayString string, fs *zfs.DatasetPath, cb HookJobCallback) *CallbackHook {
|
||||
filter, _ := filters.DatasetMapFilterFromConfig(map[string]bool{fs.ToString(): true})
|
||||
return NewCallbackHook(displayString, cb, filter)
|
||||
}
|
||||
|
||||
func NewCallbackHook(displayString string, cb HookJobCallback, filter Filter) *CallbackHook {
|
||||
return &CallbackHook{
|
||||
cb: cb,
|
||||
filter: filter,
|
||||
displayString: displayString,
|
||||
}
|
||||
}
|
||||
|
||||
func (h *CallbackHook) Filesystems() Filter {
|
||||
return h.filter
|
||||
}
|
||||
|
||||
func (h *CallbackHook) ErrIsFatal() bool {
|
||||
return false // callback is by definition
|
||||
}
|
||||
|
||||
func (h *CallbackHook) String() string {
|
||||
return h.displayString
|
||||
}
|
||||
|
||||
type CallbackHookReport struct {
|
||||
Name string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (r *CallbackHookReport) String() string {
|
||||
if r.HadError() {
|
||||
return r.Error()
|
||||
}
|
||||
return r.Name
|
||||
}
|
||||
|
||||
func (r *CallbackHookReport) HadError() bool { return r.Err != nil }
|
||||
|
||||
func (r *CallbackHookReport) Error() string {
|
||||
return fmt.Sprintf("%s error: %s", r.Name, r.Err)
|
||||
}
|
||||
|
||||
func (h *CallbackHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
|
||||
err := h.cb(ctx)
|
||||
return &CallbackHookReport{h.displayString, err}
|
||||
}
|
||||
@@ -1,192 +0,0 @@
|
||||
package hooks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/util/circlog"
|
||||
"github.com/zrepl/zrepl/util/envconst"
|
||||
)
|
||||
|
||||
type HookEnvVar string
|
||||
|
||||
const (
|
||||
EnvType HookEnvVar = "ZREPL_HOOKTYPE"
|
||||
EnvDryRun HookEnvVar = "ZREPL_DRYRUN"
|
||||
EnvFS HookEnvVar = "ZREPL_FS"
|
||||
EnvSnapshot HookEnvVar = "ZREPL_SNAPNAME"
|
||||
EnvTimeout HookEnvVar = "ZREPL_TIMEOUT"
|
||||
)
|
||||
|
||||
type Env map[HookEnvVar]string
|
||||
|
||||
func NewHookEnv(edge Edge, phase Phase, dryRun bool, timeout time.Duration, extra Env) Env {
|
||||
r := Env{
|
||||
EnvTimeout: fmt.Sprintf("%.f", math.Floor(timeout.Seconds())),
|
||||
}
|
||||
|
||||
edgeString := edge.StringForPhase(phase)
|
||||
r[EnvType] = strings.ToLower(edgeString)
|
||||
|
||||
var dryRunString string
|
||||
if dryRun {
|
||||
dryRunString = "true"
|
||||
} else {
|
||||
dryRunString = ""
|
||||
}
|
||||
r[EnvDryRun] = dryRunString
|
||||
|
||||
for k, v := range extra {
|
||||
r[k] = v
|
||||
}
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
type CommandHook struct {
|
||||
edge Edge
|
||||
filter Filter
|
||||
errIsFatal bool
|
||||
command string
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
type CommandHookReport struct {
|
||||
Command string
|
||||
Args []string // currently always empty
|
||||
Env Env
|
||||
Err error
|
||||
CapturedStdoutStderrCombined []byte
|
||||
}
|
||||
|
||||
func (r *CommandHookReport) String() string {
|
||||
// Reproduces a POSIX shell-compatible command line
|
||||
var cmdLine strings.Builder
|
||||
sep := ""
|
||||
|
||||
// Make sure environment variables are always
|
||||
// printed in the same order
|
||||
var hookEnvKeys []HookEnvVar
|
||||
for k := range r.Env {
|
||||
hookEnvKeys = append(hookEnvKeys, k)
|
||||
}
|
||||
sort.Slice(hookEnvKeys, func(i, j int) bool { return string(hookEnvKeys[i]) < string(hookEnvKeys[j]) })
|
||||
|
||||
for _, k := range hookEnvKeys {
|
||||
cmdLine.WriteString(fmt.Sprintf("%s%s='%s'", sep, k, r.Env[k]))
|
||||
sep = " "
|
||||
}
|
||||
|
||||
cmdLine.WriteString(fmt.Sprintf("%s%s", sep, r.Command))
|
||||
for _, a := range r.Args {
|
||||
cmdLine.WriteString(fmt.Sprintf("%s'%s'", sep, a))
|
||||
}
|
||||
|
||||
return fmt.Sprintf("command hook invocation: \"%s\"", cmdLine.String()) // no %q to make copy-pastable
|
||||
}
|
||||
func (r *CommandHookReport) Error() string {
|
||||
if r.Err == nil {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("%s FAILED with error: %s", r.String(), r.Err)
|
||||
}
|
||||
|
||||
func (r *CommandHookReport) HadError() bool {
|
||||
return r.Err != nil
|
||||
}
|
||||
|
||||
func NewCommandHook(in *config.HookCommand) (r *CommandHook, err error) {
|
||||
r = &CommandHook{
|
||||
errIsFatal: in.ErrIsFatal,
|
||||
command: in.Path,
|
||||
timeout: in.Timeout,
|
||||
}
|
||||
|
||||
r.filter, err = filters.DatasetMapFilterFromConfig(in.Filesystems)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse filesystem filter: %s", err)
|
||||
}
|
||||
|
||||
r.edge = Pre | Post
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (h *CommandHook) Filesystems() Filter {
|
||||
return h.filter
|
||||
}
|
||||
|
||||
func (h *CommandHook) ErrIsFatal() bool {
|
||||
return h.errIsFatal
|
||||
}
|
||||
|
||||
func (h *CommandHook) String() string {
|
||||
return h.command
|
||||
}
|
||||
|
||||
func (h *CommandHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
|
||||
l := getLogger(ctx).WithField("command", h.command)
|
||||
|
||||
cmdCtx, cancel := context.WithTimeout(ctx, h.timeout)
|
||||
defer cancel()
|
||||
|
||||
cmdExec := exec.CommandContext(cmdCtx, h.command)
|
||||
|
||||
hookEnv := NewHookEnv(edge, phase, dryRun, h.timeout, extra)
|
||||
cmdEnv := os.Environ()
|
||||
for k, v := range hookEnv {
|
||||
cmdEnv = append(cmdEnv, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
cmdExec.Env = cmdEnv
|
||||
|
||||
var scanMutex sync.Mutex
|
||||
combinedOutput, err := circlog.NewCircularLog(envconst.Int("ZREPL_MAX_HOOK_LOG_SIZE", MAX_HOOK_LOG_SIZE_DEFAULT))
|
||||
if err != nil {
|
||||
return &CommandHookReport{Err: err}
|
||||
}
|
||||
logErrWriter := NewLogWriter(&scanMutex, l, logger.Warn, "stderr")
|
||||
logOutWriter := NewLogWriter(&scanMutex, l, logger.Info, "stdout")
|
||||
defer logErrWriter.Close()
|
||||
defer logOutWriter.Close()
|
||||
|
||||
cmdExec.Stderr = io.MultiWriter(logErrWriter, combinedOutput)
|
||||
cmdExec.Stdout = io.MultiWriter(logOutWriter, combinedOutput)
|
||||
|
||||
report := &CommandHookReport{
|
||||
Command: h.command,
|
||||
Env: hookEnv,
|
||||
// no report.Args
|
||||
}
|
||||
|
||||
err = cmdExec.Start()
|
||||
if err != nil {
|
||||
report.Err = err
|
||||
return report
|
||||
}
|
||||
|
||||
err = cmdExec.Wait()
|
||||
combinedOutputBytes := combinedOutput.Bytes()
|
||||
report.CapturedStdoutStderrCombined = make([]byte, len(combinedOutputBytes))
|
||||
copy(report.CapturedStdoutStderrCombined, combinedOutputBytes)
|
||||
if err != nil {
|
||||
if cmdCtx.Err() == context.DeadlineExceeded {
|
||||
report.Err = fmt.Errorf("timed out after %s: %s", h.timeout, err)
|
||||
return report
|
||||
}
|
||||
report.Err = err
|
||||
return report
|
||||
}
|
||||
|
||||
return report
|
||||
}
|
||||
@@ -1,145 +0,0 @@
|
||||
package hooks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
sqldriver "database/sql/driver"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
// https://dev.mysql.com/doc/mysql-backup-excerpt/5.7/en/backup-methods.html
|
||||
//
|
||||
// Making Backups Using a File System Snapshot:
|
||||
//
|
||||
// If you are using a Veritas file system, you can make a backup like this:
|
||||
//
|
||||
// From a client program, execute FLUSH TABLES WITH READ LOCK.
|
||||
// From another shell, execute mount vxfs snapshot.
|
||||
// From the first client, execute UNLOCK TABLES.
|
||||
// Copy files from the snapshot.
|
||||
// Unmount the snapshot.
|
||||
//
|
||||
// Similar snapshot capabilities may be available in other file systems, such as LVM or ZFS.
|
||||
type MySQLLockTables struct {
|
||||
errIsFatal bool
|
||||
connector sqldriver.Connector
|
||||
filesystems Filter
|
||||
}
|
||||
|
||||
type myLockTablesStateKey int
|
||||
|
||||
const (
|
||||
myLockTablesConnection myLockTablesStateKey = 1 + iota
|
||||
)
|
||||
|
||||
func MyLockTablesFromConfig(in *config.HookMySQLLockTables) (*MySQLLockTables, error) {
|
||||
conf, err := mysql.ParseDSN(in.DSN)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "`dsn` invalid")
|
||||
}
|
||||
cn, err := mysql.NewConnector(conf)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "`connect` invalid")
|
||||
}
|
||||
|
||||
filesystems, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "`filesystems` invalid")
|
||||
}
|
||||
|
||||
return &MySQLLockTables{
|
||||
in.ErrIsFatal,
|
||||
cn,
|
||||
filesystems,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (h *MySQLLockTables) ErrIsFatal() bool { return h.errIsFatal }
|
||||
func (h *MySQLLockTables) Filesystems() Filter { return h.filesystems }
|
||||
func (h *MySQLLockTables) String() string { return "MySQL FLUSH TABLES WITH READ LOCK" }
|
||||
|
||||
type MyLockTablesReport struct {
|
||||
What string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (r *MyLockTablesReport) HadError() bool { return r.Err != nil }
|
||||
func (r *MyLockTablesReport) Error() string { return r.String() }
|
||||
func (r *MyLockTablesReport) String() string {
|
||||
var s strings.Builder
|
||||
s.WriteString(r.What)
|
||||
if r.Err != nil {
|
||||
fmt.Fprintf(&s, ": %s", r.Err)
|
||||
}
|
||||
return s.String()
|
||||
}
|
||||
|
||||
func (h *MySQLLockTables) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
|
||||
fs, ok := extra[EnvFS]
|
||||
if !ok {
|
||||
panic(extra)
|
||||
}
|
||||
dp, err := zfs.NewDatasetPath(fs)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if pass, err := h.filesystems.Filter(dp); err != nil {
|
||||
return &MyLockTablesReport{What: "filesystem filter", Err: err}
|
||||
} else if !pass {
|
||||
getLogger(ctx).Debug("filesystem does not match filter, skipping")
|
||||
return &MyLockTablesReport{What: "filesystem filter skipped this filesystem", Err: nil}
|
||||
}
|
||||
|
||||
switch edge {
|
||||
case Pre:
|
||||
err := h.doRunPre(ctx, dp, dryRun, state)
|
||||
return &MyLockTablesReport{"FLUSH TABLES WITH READ LOCK", err}
|
||||
case Post:
|
||||
err := h.doRunPost(ctx, dp, dryRun, state)
|
||||
return &MyLockTablesReport{"UNLOCK TABLES", err}
|
||||
}
|
||||
return &MyLockTablesReport{What: "skipped this edge", Err: nil}
|
||||
}
|
||||
|
||||
func (h *MySQLLockTables) doRunPre(ctx context.Context, fs *zfs.DatasetPath, dry bool, state map[interface{}]interface{}) (err error) {
|
||||
db := sql.OpenDB(h.connector)
|
||||
defer func(err *error) {
|
||||
if *err != nil {
|
||||
db.Close()
|
||||
}
|
||||
}(&err)
|
||||
|
||||
getLogger(ctx).Debug("do FLUSH TABLES WITH READ LOCK")
|
||||
_, err = db.ExecContext(ctx, "FLUSH TABLES WITH READ LOCK")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
state[myLockTablesConnection] = db
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *MySQLLockTables) doRunPost(ctx context.Context, fs *zfs.DatasetPath, dry bool, state map[interface{}]interface{}) error {
|
||||
|
||||
db := state[myLockTablesConnection].(*sql.DB)
|
||||
defer db.Close()
|
||||
|
||||
getLogger(ctx).Debug("do UNLOCK TABLES")
|
||||
_, err := db.ExecContext(ctx, "UNLOCK TABLES")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user