Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Christian Schwarz
|
d12ab6cd57 |
@@ -1,378 +0,0 @@
|
|||||||
version: 2.1
|
|
||||||
|
|
||||||
commands:
|
|
||||||
setup-home-local-bin:
|
|
||||||
steps:
|
|
||||||
- run:
|
|
||||||
shell: /bin/bash -euo pipefail
|
|
||||||
command: |
|
|
||||||
mkdir -p "$HOME/.local/bin"
|
|
||||||
line='export PATH="$HOME/.local/bin:$PATH"'
|
|
||||||
if grep "$line" $BASH_ENV >/dev/null; then
|
|
||||||
echo "$line" >> $BASH_ENV
|
|
||||||
fi
|
|
||||||
|
|
||||||
invoke-lazy-sh:
|
|
||||||
parameters:
|
|
||||||
subcommand:
|
|
||||||
type: string
|
|
||||||
steps:
|
|
||||||
- run:
|
|
||||||
environment:
|
|
||||||
TERM: xterm
|
|
||||||
command: ./lazy.sh <<parameters.subcommand>>
|
|
||||||
|
|
||||||
apt-update-and-install-common-deps:
|
|
||||||
steps:
|
|
||||||
- run: sudo apt update && sudo apt install gawk make
|
|
||||||
|
|
||||||
restore-cache-gomod:
|
|
||||||
steps:
|
|
||||||
- restore_cache:
|
|
||||||
key: go-mod-v4-{{ checksum "go.sum" }}
|
|
||||||
save-cache-gomod:
|
|
||||||
steps:
|
|
||||||
- save_cache:
|
|
||||||
key: go-mod-v4-{{ checksum "go.sum" }}
|
|
||||||
paths:
|
|
||||||
- "/go/pkg/mod"
|
|
||||||
|
|
||||||
install-godep:
|
|
||||||
steps:
|
|
||||||
- apt-update-and-install-common-deps
|
|
||||||
- invoke-lazy-sh:
|
|
||||||
subcommand: godep
|
|
||||||
|
|
||||||
install-docdep:
|
|
||||||
steps:
|
|
||||||
- apt-update-and-install-common-deps
|
|
||||||
- run: sudo apt install python3 python3-pip libgirepository1.0-dev
|
|
||||||
- invoke-lazy-sh:
|
|
||||||
subcommand: docdep
|
|
||||||
|
|
||||||
download-and-install-minio-client:
|
|
||||||
steps:
|
|
||||||
- setup-home-local-bin
|
|
||||||
- restore_cache:
|
|
||||||
key: minio-client-v2
|
|
||||||
- run:
|
|
||||||
shell: /bin/bash -eo pipefail
|
|
||||||
command: |
|
|
||||||
if which mc; then exit 0; fi
|
|
||||||
sudo curl -sSL https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2020-08-20T00-23-01Z \
|
|
||||||
-o "$HOME/.local/bin/mc"
|
|
||||||
sudo chmod +x "$HOME/.local/bin/mc"
|
|
||||||
- save_cache:
|
|
||||||
key: minio-client-v2
|
|
||||||
paths:
|
|
||||||
- "$HOME/.local/bin/mc"
|
|
||||||
|
|
||||||
upload-minio:
|
|
||||||
parameters:
|
|
||||||
src:
|
|
||||||
type: string
|
|
||||||
dst:
|
|
||||||
type: string
|
|
||||||
steps:
|
|
||||||
- run:
|
|
||||||
shell: /bin/bash -eo pipefail
|
|
||||||
when: always
|
|
||||||
command: |
|
|
||||||
if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
|
|
||||||
echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
set -u # from now on
|
|
||||||
|
|
||||||
mc config host add --api s3v4 zrepl-minio https://minio.cschwarz.com ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
|
|
||||||
|
|
||||||
# keep in sync with set-github-minio-status
|
|
||||||
jobprefix=zrepl-ci-artifacts/${CIRCLE_SHA1}-pipeline-<<pipeline.number>>/${CIRCLE_JOB}
|
|
||||||
|
|
||||||
# Upload artifacts
|
|
||||||
mkdir -p ./artifacts
|
|
||||||
mc cp -r <<parameters.src>> "zrepl-minio/$jobprefix/<<parameters.dst>>"
|
|
||||||
|
|
||||||
set-github-minio-status:
|
|
||||||
parameters:
|
|
||||||
context:
|
|
||||||
type: string
|
|
||||||
description:
|
|
||||||
type: string
|
|
||||||
minio-dst:
|
|
||||||
type: string
|
|
||||||
steps:
|
|
||||||
- run:
|
|
||||||
shell: /bin/bash -eo pipefail
|
|
||||||
command: |
|
|
||||||
if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
|
|
||||||
echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
set -u # from now on
|
|
||||||
|
|
||||||
# keep in sync with with upload-minio command
|
|
||||||
jobprefix=zrepl-ci-artifacts/${CIRCLE_SHA1}-pipeline-<<pipeline.number>>/${CIRCLE_JOB}
|
|
||||||
# Push Artifact Link to GitHub
|
|
||||||
REPO="zrepl/zrepl"
|
|
||||||
COMMIT="${CIRCLE_SHA1}"
|
|
||||||
JOB_NAME="${CIRCLE_JOB}"
|
|
||||||
CONTEXT="<<parameters.context>>"
|
|
||||||
DESCRIPTION="<<parameters.description>>"
|
|
||||||
TARGETURL=https://minio.cschwarz.com/minio/"$jobprefix"/"<<parameters.minio-dst>>"
|
|
||||||
curl "https://api.github.com/repos/$REPO/statuses/$COMMIT" \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-H "Authorization: token $GITHUB_COMMIT_STATUS_TOKEN" \
|
|
||||||
-X POST \
|
|
||||||
-d '{"context":"'"$CONTEXT"'", "state": "success", "description":"'"$DESCRIPTION"'", "target_url":"'"$TARGETURL"'"}'
|
|
||||||
|
|
||||||
|
|
||||||
trigger-pipeline:
|
|
||||||
parameters:
|
|
||||||
body_no_shell_subst:
|
|
||||||
type: string
|
|
||||||
steps:
|
|
||||||
- run: |
|
|
||||||
curl -X POST https://circleci.com/api/v2/project/github/zrepl/zrepl/pipeline \
|
|
||||||
-H 'Content-Type: application/json' \
|
|
||||||
-H 'Accept: application/json' \
|
|
||||||
-H "Circle-Token: $ZREPL_BOT_CIRCLE_TOKEN" \
|
|
||||||
--data '<<parameters.body_no_shell_subst>>'
|
|
||||||
|
|
||||||
parameters:
|
|
||||||
do_ci:
|
|
||||||
type: boolean
|
|
||||||
default: true
|
|
||||||
|
|
||||||
do_release:
|
|
||||||
type: boolean
|
|
||||||
default: false
|
|
||||||
|
|
||||||
release_docker_baseimage_tag:
|
|
||||||
type: string
|
|
||||||
default: "1.16"
|
|
||||||
|
|
||||||
workflows:
|
|
||||||
version: 2
|
|
||||||
|
|
||||||
ci:
|
|
||||||
when: << pipeline.parameters.do_ci >>
|
|
||||||
jobs:
|
|
||||||
- quickcheck-docs
|
|
||||||
- quickcheck-go: &quickcheck-go-smoketest
|
|
||||||
name: quickcheck-go-amd64-linux-1.16
|
|
||||||
goversion: &latest-go-release "1.16"
|
|
||||||
goos: linux
|
|
||||||
goarch: amd64
|
|
||||||
- test-go-on-latest-go-release:
|
|
||||||
goversion: *latest-go-release
|
|
||||||
- quickcheck-go:
|
|
||||||
requires:
|
|
||||||
- quickcheck-go-amd64-linux-1.16 #quickcheck-go-smoketest.name
|
|
||||||
matrix: &quickcheck-go-matrix
|
|
||||||
alias: quickcheck-go-matrix
|
|
||||||
parameters:
|
|
||||||
goversion: [*latest-go-release, "1.12"]
|
|
||||||
goos: ["linux", "freebsd"]
|
|
||||||
goarch: ["amd64", "arm64"]
|
|
||||||
exclude:
|
|
||||||
# don't re-do quickcheck-go-smoketest
|
|
||||||
- goversion: *latest-go-release
|
|
||||||
goos: linux
|
|
||||||
goarch: amd64
|
|
||||||
# not supported by Go 1.12
|
|
||||||
- goversion: "1.12"
|
|
||||||
goos: freebsd
|
|
||||||
goarch: arm64
|
|
||||||
|
|
||||||
release:
|
|
||||||
when: << pipeline.parameters.do_release >>
|
|
||||||
jobs:
|
|
||||||
- release-build
|
|
||||||
- release-deb:
|
|
||||||
requires:
|
|
||||||
- release-build
|
|
||||||
- release-rpm:
|
|
||||||
requires:
|
|
||||||
- release-build
|
|
||||||
- release-upload:
|
|
||||||
requires:
|
|
||||||
- release-build
|
|
||||||
- release-deb
|
|
||||||
- release-rpm
|
|
||||||
|
|
||||||
periodic:
|
|
||||||
triggers:
|
|
||||||
- schedule:
|
|
||||||
cron: "00 17 * * *"
|
|
||||||
filters:
|
|
||||||
branches:
|
|
||||||
only:
|
|
||||||
- master
|
|
||||||
- stable
|
|
||||||
- problame/circleci-build
|
|
||||||
jobs:
|
|
||||||
- periodic-full-pipeline-run
|
|
||||||
|
|
||||||
zrepl.github.io:
|
|
||||||
jobs:
|
|
||||||
- publish-zrepl-github-io:
|
|
||||||
filters:
|
|
||||||
branches:
|
|
||||||
only:
|
|
||||||
- stable
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
quickcheck-docs:
|
|
||||||
docker:
|
|
||||||
- image: cimg/base:2020.08
|
|
||||||
steps:
|
|
||||||
- checkout
|
|
||||||
- install-docdep
|
|
||||||
- run: make docs
|
|
||||||
|
|
||||||
- store_artifacts:
|
|
||||||
path: artifacts
|
|
||||||
- download-and-install-minio-client
|
|
||||||
- upload-minio:
|
|
||||||
src: artifacts
|
|
||||||
dst: ""
|
|
||||||
- set-github-minio-status:
|
|
||||||
context: artifacts/${CIRCLE_JOB}
|
|
||||||
description: artifacts of CI job ${CIRCLE_JOB}
|
|
||||||
minio-dst: ""
|
|
||||||
|
|
||||||
quickcheck-go:
|
|
||||||
parameters:
|
|
||||||
goversion:
|
|
||||||
type: string
|
|
||||||
goos:
|
|
||||||
type: string
|
|
||||||
goarch:
|
|
||||||
type: string
|
|
||||||
docker:
|
|
||||||
- image: circleci/golang:<<parameters.goversion>>
|
|
||||||
environment:
|
|
||||||
GOOS: <<parameters.goos>>
|
|
||||||
GOARCH: <<parameters.goarch>>
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- checkout
|
|
||||||
|
|
||||||
- restore-cache-gomod
|
|
||||||
- run: go mod download
|
|
||||||
- run: cd build && go mod download
|
|
||||||
- save-cache-gomod
|
|
||||||
|
|
||||||
- install-godep
|
|
||||||
- run: make formatcheck
|
|
||||||
- run: make generate-platform-test-list
|
|
||||||
- run: make zrepl-bin test-platform-bin
|
|
||||||
- run: make vet
|
|
||||||
- run: make lint
|
|
||||||
|
|
||||||
- download-and-install-minio-client
|
|
||||||
- run: rm -f artifacts/generate-platform-test-list
|
|
||||||
- store_artifacts:
|
|
||||||
path: artifacts
|
|
||||||
- upload-minio:
|
|
||||||
src: artifacts
|
|
||||||
dst: ""
|
|
||||||
- set-github-minio-status:
|
|
||||||
context: artifacts/${CIRCLE_JOB}
|
|
||||||
description: artifacts of CI job ${CIRCLE_JOB}
|
|
||||||
minio-dst: ""
|
|
||||||
|
|
||||||
test-go-on-latest-go-release:
|
|
||||||
parameters:
|
|
||||||
goversion:
|
|
||||||
type: string
|
|
||||||
docker:
|
|
||||||
- image: circleci/golang:<<parameters.goversion>>
|
|
||||||
steps:
|
|
||||||
- checkout
|
|
||||||
- restore-cache-gomod
|
|
||||||
- run: make test-go
|
|
||||||
# don't save-cache-gomod here, test-go doesn't pull all the dependencies
|
|
||||||
|
|
||||||
release-build:
|
|
||||||
machine: true
|
|
||||||
steps:
|
|
||||||
- checkout
|
|
||||||
- run: make release-docker RELEASE_DOCKER_BASEIMAGE_TAG=<<pipeline.parameters.release_docker_baseimage_tag>>
|
|
||||||
- persist_to_workspace:
|
|
||||||
root: .
|
|
||||||
paths: [.]
|
|
||||||
release-deb:
|
|
||||||
machine: true
|
|
||||||
steps:
|
|
||||||
- attach_workspace:
|
|
||||||
at: .
|
|
||||||
- run: make debs-docker
|
|
||||||
- persist_to_workspace:
|
|
||||||
root: .
|
|
||||||
paths:
|
|
||||||
- "artifacts/*.deb"
|
|
||||||
|
|
||||||
release-rpm:
|
|
||||||
machine: true
|
|
||||||
steps:
|
|
||||||
- attach_workspace:
|
|
||||||
at: .
|
|
||||||
- run: make rpms-docker
|
|
||||||
- persist_to_workspace:
|
|
||||||
root: .
|
|
||||||
paths:
|
|
||||||
- "artifacts/*.rpm"
|
|
||||||
|
|
||||||
release-upload:
|
|
||||||
docker:
|
|
||||||
- image: cimg/base:2020.08
|
|
||||||
steps:
|
|
||||||
- attach_workspace:
|
|
||||||
at: .
|
|
||||||
- store_artifacts:
|
|
||||||
path: artifacts
|
|
||||||
- download-and-install-minio-client
|
|
||||||
- upload-minio:
|
|
||||||
src: artifacts
|
|
||||||
dst: ""
|
|
||||||
- set-github-minio-status:
|
|
||||||
context: artifacts/release
|
|
||||||
description: CI-generated release artifacts
|
|
||||||
minio-dst: ""
|
|
||||||
|
|
||||||
periodic-full-pipeline-run:
|
|
||||||
docker:
|
|
||||||
- image: cimg/base:2020.08
|
|
||||||
steps:
|
|
||||||
- trigger-pipeline:
|
|
||||||
body_no_shell_subst: '{"branch":"<<pipeline.git.branch>>", "parameters": { "do_ci": true, "do_release": true }}'
|
|
||||||
|
|
||||||
publish-zrepl-github-io:
|
|
||||||
docker:
|
|
||||||
- image: cimg/python:3.7
|
|
||||||
steps:
|
|
||||||
- checkout
|
|
||||||
- invoke-lazy-sh:
|
|
||||||
subcommand: docdep
|
|
||||||
- run:
|
|
||||||
command: |
|
|
||||||
git config --global user.email "me@cschwarz.com"
|
|
||||||
git config --global user.name "zrepl-github-io-ci"
|
|
||||||
|
|
||||||
# https://circleci.com/docs/2.0/add-ssh-key/#adding-multiple-keys-with-blank-hostnames
|
|
||||||
- run: ssh-add -D
|
|
||||||
# the default circleci ssh config only additional ssh keys for Host !github.com
|
|
||||||
- run:
|
|
||||||
command: |
|
|
||||||
cat > ~/.ssh/config \<<EOF
|
|
||||||
Host *
|
|
||||||
IdentityFile /home/circleci/.ssh/id_rsa_458e62c517f6c480e40452126ce47421
|
|
||||||
EOF
|
|
||||||
- add_ssh_keys:
|
|
||||||
fingerprints:
|
|
||||||
# deploy key for zrepl.github.io
|
|
||||||
- "45:8e:62:c5:17:f6:c4:80:e4:04:52:12:6c:e4:74:21"
|
|
||||||
|
|
||||||
- run: bash -x docs/publish.sh -c -a
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
COMMIT="$1"
|
|
||||||
GO_VERSION="$2"
|
|
||||||
|
|
||||||
curl -v -X POST https://api.github.com/repos/zrepl/debian-binary-packaging/dispatches \
|
|
||||||
-H 'Accept: application/vnd.github.v3+json' \
|
|
||||||
-H "Authorization: token $GITHUB_ACCESS_TOKEN" \
|
|
||||||
--data '{"event_type": "push", "client_payload": { "zrepl_main_repo_commit": "'"$COMMIT"'", "go_version": "'"$GO_VERSION"'" }}'
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
github: problame
|
|
||||||
patreon: zrepl
|
|
||||||
liberapay: zrepl
|
|
||||||
custom: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96
|
|
||||||
ko_fi: problame
|
|
||||||
-10
@@ -1,13 +1,3 @@
|
|||||||
# Build
|
|
||||||
artifacts/
|
|
||||||
|
|
||||||
# Golang
|
|
||||||
vendor/
|
|
||||||
|
|
||||||
# IDEs
|
|
||||||
.idea/
|
|
||||||
.vscode/
|
|
||||||
|
|
||||||
# Compiled Object files, Static and Dynamic libs (Shared Objects)
|
# Compiled Object files, Static and Dynamic libs (Shared Objects)
|
||||||
*.o
|
*.o
|
||||||
*.a
|
*.a
|
||||||
|
|||||||
@@ -0,0 +1,4 @@
|
|||||||
|
[submodule "docs/themes/docdock"]
|
||||||
|
path = docs/themes/docdock
|
||||||
|
url = git@github.com:zrepl/hugo-theme-docdock.git
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +0,0 @@
|
|||||||
linters:
|
|
||||||
enable:
|
|
||||||
- goimports
|
|
||||||
|
|
||||||
issues:
|
|
||||||
exclude-rules:
|
|
||||||
- path: _test\.go
|
|
||||||
linters:
|
|
||||||
- errcheck
|
|
||||||
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
MIT License
|
MIT License
|
||||||
|
|
||||||
Copyright (c) 2017-2018 Christian Schwarz
|
Copyright (c) 2017 Christian Schwarz
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
|||||||
@@ -1,338 +0,0 @@
|
|||||||
.PHONY: generate build test vet cover release docs docs-clean clean format lint platformtest
|
|
||||||
.PHONY: release bins-all release-noarch
|
|
||||||
.DEFAULT_GOAL := zrepl-bin
|
|
||||||
|
|
||||||
ARTIFACTDIR := artifacts
|
|
||||||
|
|
||||||
ifdef ZREPL_VERSION
|
|
||||||
_ZREPL_VERSION := $(ZREPL_VERSION)
|
|
||||||
endif
|
|
||||||
ifndef _ZREPL_VERSION
|
|
||||||
_ZREPL_VERSION := $(shell git describe --always --dirty 2>/dev/null || echo "ZREPL_BUILD_INVALID_VERSION" )
|
|
||||||
ifeq ($(_ZREPL_VERSION),ZREPL_BUILD_INVALID_VERSION) # can't use .SHELLSTATUS because Debian Stretch is still on gmake 4.1
|
|
||||||
$(error cannot infer variable ZREPL_VERSION using git and variable is not overriden by make invocation)
|
|
||||||
endif
|
|
||||||
endif
|
|
||||||
|
|
||||||
GO := go
|
|
||||||
GOOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOOS"')
|
|
||||||
GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"')
|
|
||||||
GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"')
|
|
||||||
GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"')
|
|
||||||
GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
|
|
||||||
GO_ENV_VARS := GO111MODULE=on
|
|
||||||
GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
|
|
||||||
GO_MOD_READONLY := -mod=readonly
|
|
||||||
GO_EXTRA_BUILDFLAGS :=
|
|
||||||
GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
|
|
||||||
GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
|
|
||||||
GOLANGCI_LINT := golangci-lint
|
|
||||||
GOCOVMERGE := gocovmerge
|
|
||||||
RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.16
|
|
||||||
RELEASE_DOCKER_BASEIMAGE ?= golang:$(RELEASE_DOCKER_BASEIMAGE_TAG)
|
|
||||||
|
|
||||||
ifneq ($(GOARM),)
|
|
||||||
ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM)
|
|
||||||
else
|
|
||||||
ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)
|
|
||||||
endif
|
|
||||||
|
|
||||||
.PHONY: printvars
|
|
||||||
printvars:
|
|
||||||
@echo GOOS=$(GOOS)
|
|
||||||
@echo GOARCH=$(GOARCH)
|
|
||||||
@echo GOARM=$(GOARM)
|
|
||||||
|
|
||||||
|
|
||||||
##################### PRODUCING A RELEASE #############
|
|
||||||
.PHONY: release wrapup-and-checksum check-git-clean sign clean
|
|
||||||
|
|
||||||
release: clean
|
|
||||||
# no cross-platform support for target test
|
|
||||||
$(MAKE) test-go
|
|
||||||
$(MAKE) bins-all
|
|
||||||
$(MAKE) noarch
|
|
||||||
$(MAKE) wrapup-and-checksum
|
|
||||||
$(MAKE) check-git-clean
|
|
||||||
ifeq (SIGN, 1)
|
|
||||||
$(make) sign
|
|
||||||
endif
|
|
||||||
@echo "ZREPL RELEASE ARTIFACTS AVAILABLE IN artifacts/release"
|
|
||||||
|
|
||||||
release-docker: $(ARTIFACTDIR)
|
|
||||||
sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > artifacts/release-docker.Dockerfile
|
|
||||||
docker build -t zrepl_release --pull -f artifacts/release-docker.Dockerfile .
|
|
||||||
docker run --rm -i -v $(CURDIR):/src -u $$(id -u):$$(id -g) \
|
|
||||||
zrepl_release \
|
|
||||||
make release GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
|
|
||||||
|
|
||||||
debs-docker:
|
|
||||||
$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=deb
|
|
||||||
rpms-docker:
|
|
||||||
$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=rpm
|
|
||||||
_debs_or_rpms_docker: # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
|
|
||||||
$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=amd64
|
|
||||||
$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=arm64
|
|
||||||
$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=arm GOARM=7
|
|
||||||
$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=386
|
|
||||||
|
|
||||||
rpm: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
|
|
||||||
$(eval _ZREPL_RPM_VERSION := $(subst -,.,$(_ZREPL_VERSION)))
|
|
||||||
$(eval _ZREPL_RPM_TOPDIR_ABS := $(CURDIR)/$(ARTIFACTDIR)/rpmbuild)
|
|
||||||
rm -rf "$(_ZREPL_RPM_TOPDIR_ABS)"
|
|
||||||
mkdir "$(_ZREPL_RPM_TOPDIR_ABS)"
|
|
||||||
mkdir -p "$(_ZREPL_RPM_TOPDIR_ABS)"/{SPECS,RPMS,BUILD,BUILDROOT}
|
|
||||||
sed "s/^Version:.*/Version: $(_ZREPL_RPM_VERSION)/g" \
|
|
||||||
packaging/rpm/zrepl.spec > $(_ZREPL_RPM_TOPDIR_ABS)/SPECS/zrepl.spec
|
|
||||||
|
|
||||||
# see /usr/lib/rpm/platform
|
|
||||||
ifeq ($(GOARCH),amd64)
|
|
||||||
$(eval _ZREPL_RPMBUILD_TARGET := x86_64)
|
|
||||||
else ifeq ($(GOARCH), 386)
|
|
||||||
$(eval _ZREPL_RPMBUILD_TARGET := i386)
|
|
||||||
else ifeq ($(GOARCH), arm64)
|
|
||||||
$(eval _ZREPL_RPMBUILD_TARGET := aarch64)
|
|
||||||
else ifeq ($(GOARCH), arm)
|
|
||||||
$(eval _ZREPL_RPMBUILD_TARGET := armv7hl)
|
|
||||||
else
|
|
||||||
$(eval _ZREPL_RPMBUILD_TARGET := $(GOARCH))
|
|
||||||
endif
|
|
||||||
rpmbuild \
|
|
||||||
--build-in-place \
|
|
||||||
--define "_sourcedir $(CURDIR)" \
|
|
||||||
--define "_topdir $(_ZREPL_RPM_TOPDIR_ABS)" \
|
|
||||||
--define "_zrepl_binary_filename zrepl-$(ZREPL_TARGET_TUPLE)" \
|
|
||||||
--target $(_ZREPL_RPMBUILD_TARGET) \
|
|
||||||
-bb "$(_ZREPL_RPM_TOPDIR_ABS)"/SPECS/zrepl.spec
|
|
||||||
cp "$(_ZREPL_RPM_TOPDIR_ABS)"/RPMS/$(_ZREPL_RPMBUILD_TARGET)/zrepl-$(_ZREPL_RPM_VERSION)*.rpm $(ARTIFACTDIR)/
|
|
||||||
|
|
||||||
rpm-docker:
|
|
||||||
docker build -t zrepl_rpm_pkg --pull -f packaging/rpm/Dockerfile .
|
|
||||||
docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
|
|
||||||
zrepl_rpm_pkg \
|
|
||||||
make rpm GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
|
|
||||||
|
|
||||||
deb: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
|
|
||||||
|
|
||||||
cp packaging/deb/debian/changelog.template packaging/deb/debian/changelog
|
|
||||||
sed -i 's/DATE_DASH_R_OUTPUT/$(shell date -R)/' packaging/deb/debian/changelog
|
|
||||||
VERSION="$(subst -,.,$(_ZREPL_VERSION))"; \
|
|
||||||
export VERSION="$${VERSION#v}"; \
|
|
||||||
sed -i 's/VERSION/'"$$VERSION"'/' packaging/deb/debian/changelog
|
|
||||||
|
|
||||||
ifeq ($(GOARCH), arm)
|
|
||||||
$(eval DEB_HOST_ARCH := armhf)
|
|
||||||
else ifeq ($(GOARCH), 386)
|
|
||||||
$(eval DEB_HOST_ARCH := i386)
|
|
||||||
else
|
|
||||||
$(eval DEB_HOST_ARCH := $(GOARCH))
|
|
||||||
endif
|
|
||||||
|
|
||||||
export ZREPL_DPKG_ZREPL_BINARY_FILENAME=zrepl-$(ZREPL_TARGET_TUPLE); \
|
|
||||||
dpkg-buildpackage -b --no-sign --host-arch $(DEB_HOST_ARCH)
|
|
||||||
cp ../*.deb artifacts/
|
|
||||||
|
|
||||||
deb-docker:
|
|
||||||
docker build -t zrepl_debian_pkg --pull -f packaging/deb/Dockerfile .
|
|
||||||
docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
|
|
||||||
zrepl_debian_pkg \
|
|
||||||
make deb GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
|
|
||||||
|
|
||||||
# expects `release` target to have run before
|
|
||||||
NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
|
|
||||||
wrapup-and-checksum:
|
|
||||||
rm -f $(NOARCH_TARBALL)
|
|
||||||
tar --mtime='1970-01-01' --sort=name \
|
|
||||||
--transform 's/$(ARTIFACTDIR)/zrepl-$(_ZREPL_VERSION)-noarch/' \
|
|
||||||
--transform 's#dist#zrepl-$(_ZREPL_VERSION)-noarch/dist#' \
|
|
||||||
--transform 's#config/samples#zrepl-$(_ZREPL_VERSION)-noarch/config#' \
|
|
||||||
-acf $(NOARCH_TARBALL) \
|
|
||||||
$(ARTIFACTDIR)/docs/html \
|
|
||||||
$(ARTIFACTDIR)/bash_completion \
|
|
||||||
$(ARTIFACTDIR)/_zrepl.zsh_completion \
|
|
||||||
$(ARTIFACTDIR)/go_env.txt \
|
|
||||||
dist \
|
|
||||||
config/samples
|
|
||||||
rm -rf "$(ARTIFACTDIR)/release"
|
|
||||||
mkdir -p "$(ARTIFACTDIR)/release"
|
|
||||||
cp -l $(ARTIFACTDIR)/zrepl-* \
|
|
||||||
$(ARTIFACTDIR)/platformtest-* \
|
|
||||||
"$(ARTIFACTDIR)/release"
|
|
||||||
cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
|
|
||||||
|
|
||||||
check-git-clean:
|
|
||||||
@# note that we use ZREPL_VERSION and not _ZREPL_VERSION because we want to detect the override
|
|
||||||
@if git describe --always --dirty 2>/dev/null | grep dirty >/dev/null; then \
|
|
||||||
echo '[INFO] either git reports checkout is dirty or git is not installed or this is not a git checkout'; \
|
|
||||||
if [ "$(ZREPL_VERSION)" = "" ]; then \
|
|
||||||
echo '[WARN] git checkout is dirty and make variable ZREPL_VERSION was not used to override'; \
|
|
||||||
git status; \
|
|
||||||
echo "git diff:"; \
|
|
||||||
git diff | cat; \
|
|
||||||
exit 1; \
|
|
||||||
fi; \
|
|
||||||
fi;
|
|
||||||
|
|
||||||
sign:
|
|
||||||
gpg -u "89BC 5D89 C845 568B F578 B306 CDBD 8EC8 E27C A5FC" \
|
|
||||||
--armor \
|
|
||||||
--detach-sign $(ARTIFACTDIR)/release/sha512sum.txt
|
|
||||||
|
|
||||||
clean: docs-clean
|
|
||||||
rm -rf "$(ARTIFACTDIR)"
|
|
||||||
|
|
||||||
##################### BINARIES #####################
|
|
||||||
.PHONY: bins-all lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin generate-platform-test-list
|
|
||||||
|
|
||||||
BINS_ALL_TARGETS := zrepl-bin test-platform-bin vet lint
|
|
||||||
GO_SUPPORTS_ILLUMOS := $(shell $(GO) version | gawk -F '.' '/^go version /{split($$0, comps, " "); split(comps[3], v, "."); if (v[1] == "go1" && v[2] >= 13) { print "illumos"; } else { print "noillumos"; }}')
|
|
||||||
bins-all:
|
|
||||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd GOARCH=amd64
|
|
||||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd GOARCH=386
|
|
||||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux GOARCH=amd64
|
|
||||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux GOARCH=arm64
|
|
||||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux GOARCH=arm GOARM=7
|
|
||||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux GOARCH=386
|
|
||||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=darwin GOARCH=amd64
|
|
||||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=solaris GOARCH=amd64
|
|
||||||
ifeq ($(GO_SUPPORTS_ILLUMOS), illumos)
|
|
||||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=illumos GOARCH=amd64
|
|
||||||
else ifeq ($(GO_SUPPORTS_ILLUMOS), noillumos)
|
|
||||||
@echo "SKIPPING ILLUMOS BUILD BECAUSE GO VERSION DOESN'T SUPPORT IT"
|
|
||||||
else
|
|
||||||
@echo "CANNOT DETERMINE WHETHER GO VERSION SUPPORTS GOOS=illumos"; exit 1
|
|
||||||
endif
|
|
||||||
|
|
||||||
lint:
|
|
||||||
$(GO_ENV_VARS) $(GOLANGCI_LINT) run ./...
|
|
||||||
|
|
||||||
vet:
|
|
||||||
$(GO_ENV_VARS) $(GO) vet $(GO_BUILDFLAGS) ./...
|
|
||||||
|
|
||||||
test-go: $(ARTIFACTDIR)
|
|
||||||
rm -f "$(ARTIFACTDIR)/gotest.cover"
|
|
||||||
ifeq ($(COVER),1)
|
|
||||||
$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
|
|
||||||
-coverpkg github.com/zrepl/zrepl/... \
|
|
||||||
-covermode atomic \
|
|
||||||
-coverprofile "$(ARTIFACTDIR)/gotest.cover" \
|
|
||||||
./...
|
|
||||||
else
|
|
||||||
$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
|
|
||||||
./...
|
|
||||||
endif
|
|
||||||
|
|
||||||
zrepl-bin:
|
|
||||||
$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(ZREPL_TARGET_TUPLE)"
|
|
||||||
|
|
||||||
generate-platform-test-list:
|
|
||||||
$(GO_BUILD) -o $(ARTIFACTDIR)/generate-platform-test-list ./platformtest/tests/gen
|
|
||||||
|
|
||||||
COVER_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-cover-$(ZREPL_TARGET_TUPLE)
|
|
||||||
cover-platform-bin:
|
|
||||||
$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
|
|
||||||
-c -o "$(COVER_PLATFORM_BIN_PATH)" \
|
|
||||||
-covermode=atomic -cover -coverpkg github.com/zrepl/zrepl/... \
|
|
||||||
./platformtest/harness
|
|
||||||
cover-platform:
|
|
||||||
# do not track dependency on cover-platform-bin to allow build of binary outside of test VM
|
|
||||||
export _TEST_PLATFORM_CMD="$(COVER_PLATFORM_BIN_PATH) \
|
|
||||||
-test.coverprofile \"$(ARTIFACTDIR)/platformtest.cover\" \
|
|
||||||
-test.v \
|
|
||||||
__DEVEL--i-heard-you-like-tests"; \
|
|
||||||
$(MAKE) _test-or-cover-platform-impl
|
|
||||||
|
|
||||||
TEST_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)
|
|
||||||
test-platform-bin:
|
|
||||||
$(GO_BUILD) -o "$(TEST_PLATFORM_BIN_PATH)" ./platformtest/harness
|
|
||||||
test-platform:
|
|
||||||
export _TEST_PLATFORM_CMD="\"$(TEST_PLATFORM_BIN_PATH)\""; \
|
|
||||||
$(MAKE) _test-or-cover-platform-impl
|
|
||||||
|
|
||||||
ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
|
|
||||||
ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
|
|
||||||
ZREPL_PLATFORMTEST_MOUNTPOINT := /tmp/zreplplatformtest.pool
|
|
||||||
ZREPL_PLATFORMTEST_ZFS_LOG := /tmp/zreplplatformtest.zfs.log
|
|
||||||
# ZREPL_PLATFORMTEST_STOP_AND_KEEP := -failure.stop-and-keep-pool
|
|
||||||
ZREPL_PLATFORMTEST_ARGS :=
|
|
||||||
_test-or-cover-platform-impl: $(ARTIFACTDIR)
|
|
||||||
ifndef _TEST_PLATFORM_CMD
|
|
||||||
$(error _TEST_PLATFORM_CMD is undefined, caller 'cover-platform' or 'test-platform' should have defined it)
|
|
||||||
endif
|
|
||||||
rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
|
|
||||||
rm -f "$(ARTIFACTDIR)/platformtest.cover"
|
|
||||||
platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
|
|
||||||
$(_TEST_PLATFORM_CMD) \
|
|
||||||
-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
|
|
||||||
-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
|
|
||||||
-mountpoint "$(ZREPL_PLATFORMTEST_MOUNTPOINT)" \
|
|
||||||
$(ZREPL_PLATFORMTEST_STOP_AND_KEEP) \
|
|
||||||
$(ZREPL_PLATFORMTEST_ARGS)
|
|
||||||
|
|
||||||
cover-merge: $(ARTIFACTDIR)
|
|
||||||
$(GOCOVMERGE) $(ARTIFACTDIR)/platformtest.cover $(ARTIFACTDIR)/gotest.cover > $(ARTIFACTDIR)/merged.cover
|
|
||||||
cover-html: cover-merge
|
|
||||||
$(GO) tool cover -html "$(ARTIFACTDIR)/merged.cover" -o "$(ARTIFACTDIR)/merged.cover.html"
|
|
||||||
|
|
||||||
cover-full:
|
|
||||||
test "$$(id -u)" = "0" || echo "MUST RUN AS ROOT" 1>&2
|
|
||||||
$(MAKE) test-go COVER=1
|
|
||||||
$(MAKE) cover-platform-bin
|
|
||||||
$(MAKE) cover-platform
|
|
||||||
$(MAKE) cover-html
|
|
||||||
|
|
||||||
##################### DEV TARGETS #####################
|
|
||||||
# not part of the build, must do that manually
|
|
||||||
.PHONY: generate formatcheck format
|
|
||||||
|
|
||||||
generate: generate-platform-test-list
|
|
||||||
protoc -I=replication/logic/pdu --go_out=replication/logic/pdu --go-grpc_out=replication/logic/pdu replication/logic/pdu/pdu.proto
|
|
||||||
protoc -I=rpc/grpcclientidentity/example --go_out=rpc/grpcclientidentity/example/pdu --go-grpc_out=rpc/grpcclientidentity/example/pdu rpc/grpcclientidentity/example/grpcauth.proto
|
|
||||||
$(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./...
|
|
||||||
|
|
||||||
GOIMPORTS := goimports -srcdir . -local 'github.com/zrepl/zrepl'
|
|
||||||
FINDSRCFILES := find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go'
|
|
||||||
|
|
||||||
formatcheck:
|
|
||||||
@# goimports doesn't have a knob to exit with non-zero status code if formatting is needed
|
|
||||||
@# see https://go-review.googlesource.com/c/tools/+/237378
|
|
||||||
@ affectedfiles=$$($(GOIMPORTS) -l $(shell $(FINDSRCFILES)) | tee /dev/stderr | wc -l); test "$$affectedfiles" = 0
|
|
||||||
|
|
||||||
format:
|
|
||||||
@ $(GOIMPORTS) -w -d $(shell $(FINDSRCFILES))
|
|
||||||
|
|
||||||
##################### NOARCH #####################
|
|
||||||
.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
|
|
||||||
|
|
||||||
|
|
||||||
$(ARTIFACTDIR):
|
|
||||||
mkdir -p "$@"
|
|
||||||
$(ARTIFACTDIR)/docs: $(ARTIFACTDIR)
|
|
||||||
mkdir -p "$@"
|
|
||||||
|
|
||||||
noarch: $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs
|
|
||||||
# pass
|
|
||||||
|
|
||||||
$(ARTIFACTDIR)/bash_completion:
|
|
||||||
$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
|
|
||||||
artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion bash "$@"
|
|
||||||
|
|
||||||
$(ARTIFACTDIR)/_zrepl.zsh_completion:
|
|
||||||
$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
|
|
||||||
artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion zsh "$@"
|
|
||||||
|
|
||||||
$(ARTIFACTDIR)/go_env.txt:
|
|
||||||
$(GO_ENV_VARS) $(GO) env > $@
|
|
||||||
$(GO) version >> $@
|
|
||||||
|
|
||||||
docs: $(ARTIFACTDIR)/docs
|
|
||||||
# https://www.sphinx-doc.org/en/master/man/sphinx-build.html
|
|
||||||
make -C docs \
|
|
||||||
html \
|
|
||||||
BUILDDIR=../artifacts/docs \
|
|
||||||
SPHINXOPTS="-W --keep-going -n"
|
|
||||||
|
|
||||||
docs-clean:
|
|
||||||
make -C docs \
|
|
||||||
clean \
|
|
||||||
BUILDDIR=../artifacts/docs
|
|
||||||
@@ -1,134 +1,2 @@
|
|||||||
[](https://github.com/zrepl/zrepl/blob/master/LICENSE)
|
|
||||||
[](https://golang.org/)
|
|
||||||
[](https://zrepl.github.io)
|
|
||||||
[](https://patreon.com/zrepl)
|
|
||||||
[](https://github.com/sponsors/problame)
|
|
||||||
[](https://liberapay.com/zrepl/donate)
|
|
||||||
[](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
|
|
||||||
[](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl)
|
|
||||||
|
|
||||||
# zrepl
|
# zrepl
|
||||||
zrepl is a one-stop ZFS backup & replication solution.
|
ZFS dataset replication. Not a shell script.
|
||||||
|
|
||||||
## User Documentation
|
|
||||||
|
|
||||||
**User Documentation** can be found at [zrepl.github.io](https://zrepl.github.io).
|
|
||||||
|
|
||||||
## Bug Reports
|
|
||||||
|
|
||||||
1. If the issue is reproducible, enable debug logging, reproduce and capture the log.
|
|
||||||
2. Open an issue on GitHub, with logs pasted as GitHub gists / inline.
|
|
||||||
|
|
||||||
## Feature Requests
|
|
||||||
|
|
||||||
1. Does your feature request require default values / some kind of configuration?
|
|
||||||
If so, think of an expressive configuration example.
|
|
||||||
2. Think of at least one use case that generalizes from your concrete application.
|
|
||||||
3. Open an issue on GitHub with example conf & use case attached.
|
|
||||||
4. **Optional**: [Post a bounty](https://www.bountysource.com/teams/zrepl) on the issue, or [contact Christian Schwarz](https://cschwarz.com) for contract work.
|
|
||||||
|
|
||||||
The above does not apply if you already implemented everything.
|
|
||||||
Check out the *Coding Workflow* section below for details.
|
|
||||||
|
|
||||||
## Building, Releasing, Downstream-Packaging
|
|
||||||
|
|
||||||
This section provides an overview of the zrepl build & release process.
|
|
||||||
Check out `docs/installation/compile-from-source.rst` for build-from-source instructions.
|
|
||||||
|
|
||||||
### Overview
|
|
||||||
|
|
||||||
zrepl is written in [Go](https://golang.org) and uses [Go modules](https://github.com/golang/go/wiki/Modules) to manage dependencies.
|
|
||||||
The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
|
|
||||||
|
|
||||||
Install **build dependencies** using `./lazy.sh devsetup`.
|
|
||||||
`lazy.sh` uses `python3-pip` to fetch the build dependencies for the docs - you might want to use a [venv](https://docs.python.org/3/library/venv.html).
|
|
||||||
If you just want to install the Go dependencies, run `./lazy.sh godep`.
|
|
||||||
|
|
||||||
The **test suite** is split into pure **Go tests** (`make test-go`) and **platform tests** that interact with ZFS and thus generally **require root privileges** (`sudo make test-platform`).
|
|
||||||
Platform tests run on their own pool with the name `zreplplatformtest`, which is created using the file vdev in `/tmp`.
|
|
||||||
|
|
||||||
For a full **code coverage** profile, run `make test-go COVER=1 && sudo make test-platform && make cover-merge`.
|
|
||||||
An HTML report can be generated using `make cover-html`.
|
|
||||||
|
|
||||||
**Code generation** is triggered by `make generate`. Generated code is committed to the source tree.
|
|
||||||
|
|
||||||
### Build & Release Process
|
|
||||||
|
|
||||||
**The `Makefile` is catering to the needs of developers & CI, not distro packagers**.
|
|
||||||
It provides phony targets for
|
|
||||||
* local development (building, running tests, etc)
|
|
||||||
* building a release in Docker (used by the CI & release management)
|
|
||||||
* building .deb and .rpm packages out of the release artifacts.
|
|
||||||
|
|
||||||
**Build tooling & dependencies** are documented as code in `lazy.sh`.
|
|
||||||
Go dependencies are then fetched by the go command and pip dependencies are pinned through a `requirements.txt`.
|
|
||||||
|
|
||||||
**We use CircleCI for continuous integration**.
|
|
||||||
There are two workflows:
|
|
||||||
|
|
||||||
* `ci` runs for every commit / branch / tag pushed to GitHub.
|
|
||||||
It is supposed to run very fast (<5min and provides quick feedback to developers).
|
|
||||||
It runs formatting checks, lints and tests on the most important OSes / architectures.
|
|
||||||
Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
|
|
||||||
|
|
||||||
* `release` runs
|
|
||||||
* on manual triggers through the CircleCI API (in order to produce a release)
|
|
||||||
* periodically on `master`
|
|
||||||
Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
|
|
||||||
|
|
||||||
**Releases** are issued via Git tags + GitHub Releases feature.
|
|
||||||
The procedure to issue a release is as follows:
|
|
||||||
* Issue the source release:
|
|
||||||
* Git tag the release on the `master` branch.
|
|
||||||
* Push the tag.
|
|
||||||
* Run `./docs/publish.sh` to re-build & push zrepl.github.io.
|
|
||||||
* Issue the official binary release:
|
|
||||||
* Run the `release` pipeline (triggered via CircleCI API)
|
|
||||||
* Download the artifacts to the release manager's machine.
|
|
||||||
* Create a GitHub release, edit the changelog, upload all the release artifacts, including .rpm and .deb files.
|
|
||||||
* Issue the GitHub release.
|
|
||||||
* Add the .rpm and .deb files to the official zrepl repos, publish those.
|
|
||||||
|
|
||||||
**Official binary releases are not re-built when Go receives an update. If the Go update is critical to zrepl (e.g. a Go security update that affects zrepl), we'd issue a new source release**.
|
|
||||||
The rationale for this is that whereas distros provide a mechanism for this (`$zrepl_source_release-$distro_package_revision`), GitHub Releases doesn't which means we'd need to update the existing GitHub release's assets, which nobody would notice (no RSS feed updates, etc.).
|
|
||||||
Downstream packagers can read the changelog to determine whether they want to push that minor release into their distro or simply skip it.
|
|
||||||
|
|
||||||
### Additional Notes to Distro Package Maintainers
|
|
||||||
|
|
||||||
* Use `sudo make test-platform-bin && sudo make test-platform` **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
|
|
||||||
* Ship a default config that adheres to your distro's `hier` and logging system.
|
|
||||||
* Ship a service manager file and _please_ try to upstream it to this repository.
|
|
||||||
* `dist/systemd` contains a Systemd unit template.
|
|
||||||
* Ship other material provided in `./dist`, e.g. in `/usr/share/zrepl/`.
|
|
||||||
* Have a look at the `Makefile`'s `ZREPL_VERSION` variable and how it passed to Go's `ldFlags`.
|
|
||||||
This is how `zrepl version` knows what version number to show.
|
|
||||||
Your build system should set the `ldFlags` flags appropriately and add a prefix or suffix that indicates that the given zrepl binary is a distro build, not an official one.
|
|
||||||
* Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
|
|
||||||
|
|
||||||
|
|
||||||
## Contributing Code
|
|
||||||
|
|
||||||
* Open an issue when starting to hack on a new feature
|
|
||||||
* Commits should reference the issue they are related to
|
|
||||||
* Docs improvements not documenting new features do not require an issue.
|
|
||||||
|
|
||||||
### Breaking Changes
|
|
||||||
|
|
||||||
Backward-incompatible changes must be documented in the git commit message and are listed in `docs/changelog.rst`.
|
|
||||||
|
|
||||||
### Glossary & Naming Inconsistencies
|
|
||||||
|
|
||||||
In ZFS, *dataset* refers to the objects *filesystem*, *ZVOL* and *snapshot*. <br />
|
|
||||||
However, we need a word for *filesystem* & *ZVOL* but not a snapshot, bookmark, etc.
|
|
||||||
|
|
||||||
Toward the user, the following terminology is used:
|
|
||||||
|
|
||||||
* **filesystem**: a ZFS filesystem or a ZVOL
|
|
||||||
* **filesystem version**: a ZFS snapshot or a bookmark
|
|
||||||
|
|
||||||
Sadly, the zrepl implementation is inconsistent in its use of these words:
|
|
||||||
variables and types are often named *dataset* when they in fact refer to a *filesystem*.
|
|
||||||
|
|
||||||
There will not be a big refactoring (an attempt was made, but it's destroying too much history without much gain).
|
|
||||||
|
|
||||||
However, new contributions & patches should fix naming without further notice in the commit message.
|
|
||||||
|
|||||||
@@ -1,30 +0,0 @@
|
|||||||
FROM !SUBSTITUTED_BY_MAKEFILE
|
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y \
|
|
||||||
python3-pip \
|
|
||||||
unzip \
|
|
||||||
gawk
|
|
||||||
|
|
||||||
ADD build.installprotoc.bash ./
|
|
||||||
RUN bash build.installprotoc.bash
|
|
||||||
|
|
||||||
ADD lazy.sh /tmp/lazy.sh
|
|
||||||
ADD docs/requirements.txt /tmp/requirements.txt
|
|
||||||
ENV ZREPL_LAZY_DOCS_REQPATH=/tmp/requirements.txt
|
|
||||||
RUN /tmp/lazy.sh docdep
|
|
||||||
|
|
||||||
# prepare volume mount of git checkout to /zrepl
|
|
||||||
RUN mkdir -p /src/github.com/zrepl/zrepl
|
|
||||||
RUN mkdir -p /.cache && chmod -R 0777 /.cache
|
|
||||||
|
|
||||||
# $GOPATH is /go
|
|
||||||
# Go 1.12 doesn't use modules within GOPATH, but 1.13 and later do
|
|
||||||
# => store source outside of GOPATH
|
|
||||||
WORKDIR /src
|
|
||||||
|
|
||||||
# Install build tools (e.g. protobuf generator, stringer) into $GOPATH/bin
|
|
||||||
ADD build/ /tmp/build
|
|
||||||
RUN /tmp/lazy.sh godep
|
|
||||||
|
|
||||||
RUN chmod -R 0777 /go
|
|
||||||
|
|
||||||
@@ -1,25 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
set -x
|
|
||||||
|
|
||||||
MACH=$(uname -m)
|
|
||||||
MACH="${MACH/aarch64/aarch_64}"
|
|
||||||
|
|
||||||
VERSION=3.6.1
|
|
||||||
FILENAME=protoc-"$VERSION"-linux-"$MACH".zip
|
|
||||||
|
|
||||||
if [ -e "$FILENAME" ]; then
|
|
||||||
echo "$FILENAME" already exists 1>&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
wget https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
|
|
||||||
|
|
||||||
stat "$FILENAME"
|
|
||||||
|
|
||||||
sha256sum -c --ignore-missing <<EOF
|
|
||||||
6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807 protoc-3.6.1-linux-x86_64.zip
|
|
||||||
af8e5aaaf39ddec62ec8dd2be1b8d9602c6da66564883a16393ade5f71170922 protoc-3.6.1-linux-aarch_64.zip
|
|
||||||
EOF
|
|
||||||
|
|
||||||
unzip -d /usr "$FILENAME"
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
module github.com/zrepl/zrepl/build
|
|
||||||
|
|
||||||
go 1.12
|
|
||||||
|
|
||||||
require (
|
|
||||||
github.com/alvaroloes/enumer v1.1.1
|
|
||||||
github.com/golangci/golangci-lint v1.35.2
|
|
||||||
github.com/golangci/misspell v0.3.4 // indirect
|
|
||||||
github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 // indirect
|
|
||||||
github.com/spf13/afero v1.2.2 // indirect
|
|
||||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
|
||||||
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad
|
|
||||||
golang.org/x/tools v0.0.0-20210105210202-9ed45478a130
|
|
||||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 // indirect
|
|
||||||
google.golang.org/protobuf v1.25.0
|
|
||||||
)
|
|
||||||
-698
@@ -1,698 +0,0 @@
|
|||||||
4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a h1:wFEQiK85fRsEVF0CRrPAos5LoAryUsIX1kPW/WrIqFw=
|
|
||||||
4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a/go.mod h1:wfdC5ZjKSPr7CybKEcgJhUOgeAQW1+7WcyK8OvUilfo=
|
|
||||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
|
||||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
|
||||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
|
||||||
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
|
|
||||||
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
|
|
||||||
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
|
|
||||||
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
|
|
||||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
|
||||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
|
||||||
cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
|
|
||||||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
|
||||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
|
||||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
|
||||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
|
||||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
|
||||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
|
||||||
github.com/Djarvur/go-err113 v0.0.0-20200511133814-5174e21577d5 h1:XTrzB+F8+SpRmbhAH8HLxhiiG6nYNwaBZjrFps1oWEk=
|
|
||||||
github.com/Djarvur/go-err113 v0.0.0-20200511133814-5174e21577d5/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs=
|
|
||||||
github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
|
|
||||||
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
|
|
||||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
|
||||||
github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us=
|
|
||||||
github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM=
|
|
||||||
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
|
||||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
|
||||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
|
||||||
github.com/alvaroloes/enumer v1.1.1 h1:v+1scNqEhSFAPb9tLwblQegeTXJhnw8hf9O0amTbOvQ=
|
|
||||||
github.com/alvaroloes/enumer v1.1.1/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
|
|
||||||
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
|
|
||||||
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
|
|
||||||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
|
||||||
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
|
||||||
github.com/ashanbrown/forbidigo v1.0.0 h1:QdNXBduDUopc3GW+YVYZn8jzmIMklQiCfdN2N5+dQeE=
|
|
||||||
github.com/ashanbrown/forbidigo v1.0.0/go.mod h1:PH+zMRWE15yW69fYfe7Kn8nYR6yYyafc3ntEGh2BBAg=
|
|
||||||
github.com/ashanbrown/makezero v0.0.0-20201205152432-7b7cdbb3025a h1:/U9tbJzDRof4fOR51vwzWdIBsIH6R2yU0KG1MBRM2Js=
|
|
||||||
github.com/ashanbrown/makezero v0.0.0-20201205152432-7b7cdbb3025a/go.mod h1:oG9Dnez7/ESBqc4EdrdNlryeo7d0KcW1ftXHm7nU/UU=
|
|
||||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
|
||||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
|
||||||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
|
||||||
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
|
|
||||||
github.com/bombsimon/wsl/v3 v3.1.0 h1:E5SRssoBgtVFPcYWUOFJEcgaySgdtTNYzsSKDOY7ss8=
|
|
||||||
github.com/bombsimon/wsl/v3 v3.1.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=
|
|
||||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
|
||||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
|
||||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
|
||||||
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
|
|
||||||
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
|
||||||
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
|
||||||
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
|
||||||
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
|
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
|
||||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
|
||||||
github.com/daixiang0/gci v0.2.8 h1:1mrIGMBQsBu0P7j7m1M8Lb+ZeZxsZL+jyGX4YoMJJpg=
|
|
||||||
github.com/daixiang0/gci v0.2.8/go.mod h1:+4dZ7TISfSmqfAGv59ePaHfNzgGtIkHAhhdKggP1JAc=
|
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
|
||||||
github.com/denis-tingajkin/go-header v0.4.2 h1:jEeSF4sdv8/3cT/WY8AgDHUoItNSoEZ7qg9dX7pc218=
|
|
||||||
github.com/denis-tingajkin/go-header v0.4.2/go.mod h1:eLRHAVXzE5atsKAnNRDB90WHCFFnBUn4RN0nRcs1LJA=
|
|
||||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
|
||||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
|
||||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
|
||||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
|
||||||
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
|
|
||||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
|
||||||
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
|
|
||||||
github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
|
|
||||||
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
|
|
||||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
|
||||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
|
||||||
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
|
|
||||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
|
||||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
|
||||||
github.com/go-critic/go-critic v0.5.3 h1:xQEweNxzBNpSqI3wotXZAixRarETng3PTG4pkcrLCOA=
|
|
||||||
github.com/go-critic/go-critic v0.5.3/go.mod h1:2Lrs1m4jtOnnG/EdezbSpAoL0F2pRW+9HWJUZ+QaktY=
|
|
||||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
|
||||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
|
||||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
|
||||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
|
||||||
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
|
|
||||||
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
|
||||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
|
||||||
github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=
|
|
||||||
github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
|
|
||||||
github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8=
|
|
||||||
github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
|
|
||||||
github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ=
|
|
||||||
github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
|
|
||||||
github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k=
|
|
||||||
github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
|
|
||||||
github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
|
|
||||||
github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg=
|
|
||||||
github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
|
|
||||||
github.com/go-toolsmith/pkgload v1.0.0 h1:4DFWWMXVfbcN5So1sBNW9+yeiMqLFGl1wFLTL5R0Tgg=
|
|
||||||
github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
|
|
||||||
github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4=
|
|
||||||
github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
|
|
||||||
github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA=
|
|
||||||
github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
|
|
||||||
github.com/go-toolsmith/typep v1.0.2 h1:8xdsa1+FSIH/RhEkgnD1j2CJOy5mNllW1Q9tRiYwvlk=
|
|
||||||
github.com/go-toolsmith/typep v1.0.2/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
|
|
||||||
github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b h1:khEcpUM4yFcxg4/FHQWkvVRmgijNXRfzkIDHh23ggEo=
|
|
||||||
github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM=
|
|
||||||
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
|
|
||||||
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
|
|
||||||
github.com/gofrs/flock v0.8.0 h1:MSdYClljsF3PbENUUEx85nkWfJSGfzYI9yEBZOJz6CY=
|
|
||||||
github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
|
|
||||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
|
||||||
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
|
|
||||||
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
|
||||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
|
||||||
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
|
||||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
|
||||||
github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
|
|
||||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
|
||||||
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
|
|
||||||
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
|
||||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
|
||||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
|
||||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
|
||||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
|
||||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
|
||||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
|
||||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
|
||||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
|
||||||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
|
||||||
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
|
|
||||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
|
||||||
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0=
|
|
||||||
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
|
|
||||||
github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
|
|
||||||
github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
|
|
||||||
github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w=
|
|
||||||
github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
|
|
||||||
github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw=
|
|
||||||
github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
|
|
||||||
github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d h1:pXTK/gkVNs7Zyy7WKgLXmpQ5bHTrq5GDsp8R9Qs67g0=
|
|
||||||
github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
|
|
||||||
github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=
|
|
||||||
github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
|
|
||||||
github.com/golangci/golangci-lint v1.35.2 h1:hD1999/sq3tCPXhhI4UpunxpAAdH9pK7kDIObqoGuWA=
|
|
||||||
github.com/golangci/golangci-lint v1.35.2/go.mod h1:Sg5fFp5oLLI1B8gXfUVUSePju8XF0uWefMkuZuGIHUo=
|
|
||||||
github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
|
|
||||||
github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
|
|
||||||
github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA=
|
|
||||||
github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
|
|
||||||
github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
|
|
||||||
github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
|
|
||||||
github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
|
|
||||||
github.com/golangci/misspell v0.3.4 h1:kAD5J0611Zo2VirUn9KFP15RjEqkmfEfnFxyIVxZCYg=
|
|
||||||
github.com/golangci/misspell v0.3.4/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
|
|
||||||
github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us=
|
|
||||||
github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
|
|
||||||
github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
|
|
||||||
github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 h1:XQKc8IYQOeRwVs36tDrEmTgDgP88d5iEURwpmtiAlOM=
|
|
||||||
github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
|
|
||||||
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
|
|
||||||
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
|
|
||||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
|
||||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
|
||||||
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
|
|
||||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
|
||||||
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
|
|
||||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
|
||||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
|
||||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
|
||||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
|
||||||
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
|
||||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
|
||||||
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
|
|
||||||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
|
||||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
|
||||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
|
||||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
|
||||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
|
||||||
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
|
||||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
|
||||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
|
||||||
github.com/gookit/color v1.3.1/go.mod h1:R3ogXq2B9rTbXoSHJ1HyUVAZ3poOJHpd9nQmyGZsfvQ=
|
|
||||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
|
|
||||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
|
||||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
|
||||||
github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw=
|
|
||||||
github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
|
|
||||||
github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
|
|
||||||
github.com/gostaticanalysis/analysisutil v0.1.0/go.mod h1:dMhHRU9KTiDcuLGdy87/2gTR8WruwYZrKdRq9m1O6uw=
|
|
||||||
github.com/gostaticanalysis/analysisutil v0.4.1 h1:/7clKqrVfiVwiBQLM0Uke4KvXnO6JcCTS7HwF2D6wG8=
|
|
||||||
github.com/gostaticanalysis/analysisutil v0.4.1/go.mod h1:18U/DLpRgIUd459wGxVHE0fRgmo1UgHDcbw7F5idXu0=
|
|
||||||
github.com/gostaticanalysis/comment v1.3.0/go.mod h1:xMicKDx7XRXYdVwY9f9wQpDJVnqWxw9wCauCMKp+IBI=
|
|
||||||
github.com/gostaticanalysis/comment v1.4.1 h1:xHopR5L2lRz6OsjH4R2HG5wRhW9ySl3FsHIvi5pcXwc=
|
|
||||||
github.com/gostaticanalysis/comment v1.4.1/go.mod h1:ih6ZxzTHLdadaiSnF5WY3dxUoXfXAlTaRzuaNDlSado=
|
|
||||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
|
||||||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
|
|
||||||
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
|
|
||||||
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
|
|
||||||
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
|
|
||||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
|
||||||
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
|
||||||
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
|
||||||
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
|
|
||||||
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
|
|
||||||
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
|
|
||||||
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
|
|
||||||
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
|
|
||||||
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
|
||||||
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
|
||||||
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
|
|
||||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
|
||||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
|
||||||
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
|
||||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
|
||||||
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
|
|
||||||
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
|
|
||||||
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
|
||||||
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
|
|
||||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
|
||||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
|
||||||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
|
|
||||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
|
||||||
github.com/jgautheron/goconst v0.0.0-20201117150253-ccae5bf973f3 h1:7nkB9fLPMwtn/R6qfPcHileL/x9ydlhw8XyDrLI1ZXg=
|
|
||||||
github.com/jgautheron/goconst v0.0.0-20201117150253-ccae5bf973f3/go.mod h1:aAosetZ5zaeC/2EfMeRswtxUFBpe2Hr7HzkgX4fanO4=
|
|
||||||
github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a h1:GmsqmapfzSJkm28dhRoHz2tLRbJmqhU86IPgBtN3mmk=
|
|
||||||
github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s=
|
|
||||||
github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3 h1:jNYPNLe3d8smommaoQlK7LOA5ESyUJJ+Wf79ZtA7Vp4=
|
|
||||||
github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0=
|
|
||||||
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
|
|
||||||
github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
|
|
||||||
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
|
|
||||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
|
||||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
|
||||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
|
||||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
|
||||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
|
||||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
|
||||||
github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
|
|
||||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
|
||||||
github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
|
||||||
github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
|
||||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
|
||||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
|
||||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
|
||||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
|
||||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
|
||||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
|
||||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
|
||||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
|
||||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
|
||||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
|
||||||
github.com/kulti/thelper v0.2.1 h1:H4rSHiB3ALx//SXr+k9OPqKoOw2cAZpIQwVNH1RL5T4=
|
|
||||||
github.com/kulti/thelper v0.2.1/go.mod h1:vMu2Cizjy/grP+jmsvOFDx1kYP6+PD1lqg4Yu5exl2U=
|
|
||||||
github.com/kunwardeep/paralleltest v1.0.2 h1:/jJRv0TiqPoEy/Y8dQxCFJhD56uS/pnvtatgTZBHokU=
|
|
||||||
github.com/kunwardeep/paralleltest v1.0.2/go.mod h1:ZPqNm1fVHPllh5LPVujzbVz1JN2GhLxSfY+oqUsvG30=
|
|
||||||
github.com/kyoh86/exportloopref v0.1.8 h1:5Ry/at+eFdkX9Vsdw3qU4YkvGtzuVfzT4X7S77LoN/M=
|
|
||||||
github.com/kyoh86/exportloopref v0.1.8/go.mod h1:1tUcJeiioIs7VWe5gcOObrux3lb66+sBqGZrRkMwPgg=
|
|
||||||
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
|
||||||
github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
|
|
||||||
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
|
|
||||||
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
|
||||||
github.com/maratori/testpackage v1.0.1 h1:QtJ5ZjqapShm0w5DosRjg0PRlSdAdlx+W6cCKoALdbQ=
|
|
||||||
github.com/maratori/testpackage v1.0.1/go.mod h1:ddKdw+XG0Phzhx8BFDTKgpWP4i7MpApTE5fXSKAqwDU=
|
|
||||||
github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE=
|
|
||||||
github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
|
|
||||||
github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
|
|
||||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
|
||||||
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
|
||||||
github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
|
|
||||||
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
|
||||||
github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
|
|
||||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
|
||||||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
|
||||||
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
|
|
||||||
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
|
|
||||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
|
||||||
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
|
||||||
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
|
|
||||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
|
||||||
github.com/mbilski/exhaustivestruct v1.1.0 h1:4ykwscnAFeHJruT+EY3M3vdeP8uXMh0VV2E61iR7XD8=
|
|
||||||
github.com/mbilski/exhaustivestruct v1.1.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aksxSUOUy+nvtVEfzXc=
|
|
||||||
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
|
||||||
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
|
|
||||||
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
|
||||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
|
||||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
|
||||||
github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
|
|
||||||
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
|
|
||||||
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
|
|
||||||
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
|
|
||||||
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
|
||||||
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
|
|
||||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
|
||||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
|
||||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
|
||||||
github.com/moricho/tparallel v0.2.1 h1:95FytivzT6rYzdJLdtfn6m1bfFJylOJK41+lgv/EHf4=
|
|
||||||
github.com/moricho/tparallel v0.2.1/go.mod h1:fXEIZxG2vdfl0ZF8b42f5a78EhjjD5mX8qUplsoSU4k=
|
|
||||||
github.com/mozilla/tls-observatory v0.0.0-20200317151703-4fa42e1c2dee/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
|
|
||||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
|
||||||
github.com/nakabonne/nestif v0.3.0 h1:+yOViDGhg8ygGrmII72nV9B/zGxY188TYpfolntsaPw=
|
|
||||||
github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c=
|
|
||||||
github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E=
|
|
||||||
github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
|
|
||||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
|
|
||||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
|
||||||
github.com/nishanths/exhaustive v0.1.0 h1:kVlMw8h2LHPMGUVqUj6230oQjjTMFjwcZrnkhXzFfl8=
|
|
||||||
github.com/nishanths/exhaustive v0.1.0/go.mod h1:S1j9110vxV1ECdCudXRkeMnFQ/DQk9ajLT0Uf2MYZQQ=
|
|
||||||
github.com/nishanths/predeclared v0.2.1 h1:1TXtjmy4f3YCFjTxRd8zcFHOmoUir+gp0ESzjFzG2sw=
|
|
||||||
github.com/nishanths/predeclared v0.2.1/go.mod h1:HvkGJcA3naj4lOwnFXFDkFxVtSqQMB9sbB1usJ+xjQE=
|
|
||||||
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
|
|
||||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
|
||||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
|
||||||
github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
|
|
||||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
|
||||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
|
||||||
github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4=
|
|
||||||
github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
|
|
||||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
|
||||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
|
||||||
github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs=
|
|
||||||
github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
|
||||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
|
||||||
github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 h1:/I3lTljEEDNYLho3/FUB7iD/oc2cEFgVmbHzV+O0PtU=
|
|
||||||
github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ=
|
|
||||||
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
|
|
||||||
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
|
|
||||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
|
||||||
github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d h1:CdDQnGF8Nq9ocOS/xlSptM1N3BbrA6/kmaep5ggwaIA=
|
|
||||||
github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw=
|
|
||||||
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
|
|
||||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
|
||||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
|
||||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
|
||||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
|
||||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
|
||||||
github.com/polyfloyd/go-errorlint v0.0.0-20201127212506-19bd8db6546f h1:xAw10KgJqG5NJDfmRqJ05Z0IFblKumjtMeyiOLxj3+4=
|
|
||||||
github.com/polyfloyd/go-errorlint v0.0.0-20201127212506-19bd8db6546f/go.mod h1:wi9BfjxjF/bwiZ701TzmfKu6UKC357IOAtNr0Td0Lvw=
|
|
||||||
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
|
|
||||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
|
||||||
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
|
|
||||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
|
||||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
|
||||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
|
||||||
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
|
||||||
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
|
||||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
|
||||||
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
|
||||||
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
|
||||||
github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
|
|
||||||
github.com/quasilyte/go-ruleguard v0.2.1-0.20201030093329-408e96760278 h1:5gcJ7tORNCNB2QjOJF+MYjzS9aiWpxhP3gntf7RVrOQ=
|
|
||||||
github.com/quasilyte/go-ruleguard v0.2.1-0.20201030093329-408e96760278/go.mod h1:2RT/tf0Ce0UDj5y243iWKosQogJd8+1G3Rs2fxmlYnw=
|
|
||||||
github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 h1:L8QM9bvf68pVdQ3bCFZMDmnt9yqcMBro1pC7F+IPYMY=
|
|
||||||
github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0=
|
|
||||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
|
||||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
|
||||||
github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
|
||||||
github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
|
||||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
|
||||||
github.com/ryancurrah/gomodguard v1.2.0 h1:YWfhGOrXwLGiqcC/u5EqG6YeS8nh+1fw0HEc85CVZro=
|
|
||||||
github.com/ryancurrah/gomodguard v1.2.0/go.mod h1:rNqbC4TOIdUDcVMSIpNNAzTbzXAZa6W5lnUepvuMMgQ=
|
|
||||||
github.com/ryanrolds/sqlclosecheck v0.3.0 h1:AZx+Bixh8zdUBxUA1NxbxVAS78vTPq4rCb8OUZI9xFw=
|
|
||||||
github.com/ryanrolds/sqlclosecheck v0.3.0/go.mod h1:1gREqxyTGR3lVtpngyFo3hZAgk0KCtEdgEkHwDbigdA=
|
|
||||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
|
||||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
|
||||||
github.com/securego/gosec/v2 v2.5.0 h1:kjfXLeKdk98gBe2+eYRFMpC4+mxmQQtbidpiiOQ69Qc=
|
|
||||||
github.com/securego/gosec/v2 v2.5.0/go.mod h1:L/CDXVntIff5ypVHIkqPXbtRpJiNCh6c6Amn68jXDjo=
|
|
||||||
github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU=
|
|
||||||
github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs=
|
|
||||||
github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
|
|
||||||
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
|
|
||||||
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM=
|
|
||||||
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
|
|
||||||
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc=
|
|
||||||
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
|
|
||||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
|
||||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
|
||||||
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
|
|
||||||
github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
|
|
||||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
|
||||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
|
|
||||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
|
||||||
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
|
|
||||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
|
||||||
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
|
|
||||||
github.com/sonatard/noctx v0.0.1 h1:VC1Qhl6Oxx9vvWo3UDgrGXYCeKCe3Wbw7qAWL6FrmTY=
|
|
||||||
github.com/sonatard/noctx v0.0.1/go.mod h1:9D2D/EoULe8Yy2joDHJj7bv3sZoq9AaSb8B4lqBjiZI=
|
|
||||||
github.com/sourcegraph/go-diff v0.6.1 h1:hmA1LzxW0n1c3Q4YbrFgg4P99GSnebYa3x8gr0HZqLQ=
|
|
||||||
github.com/sourcegraph/go-diff v0.6.1/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs=
|
|
||||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
|
||||||
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
|
||||||
github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
|
|
||||||
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
|
|
||||||
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
|
|
||||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
|
||||||
github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
|
|
||||||
github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
|
|
||||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
|
||||||
github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
|
|
||||||
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
|
|
||||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
|
||||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
|
||||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
|
||||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
|
||||||
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
|
|
||||||
github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
|
|
||||||
github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
|
|
||||||
github.com/ssgreg/nlreturn/v2 v2.1.0 h1:6/s4Rc49L6Uo6RLjhWZGBpWWjfzk2yrf1nIW8m4wgVA=
|
|
||||||
github.com/ssgreg/nlreturn/v2 v2.1.0/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I=
|
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
|
||||||
github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
|
|
||||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
|
||||||
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
|
|
||||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
|
||||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
|
||||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
|
||||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
|
||||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
|
||||||
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
|
|
||||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
|
||||||
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
|
|
||||||
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
|
||||||
github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2 h1:Xr9gkxfOP0KQWXKNqmwe8vEeSUiUj4Rlee9CMVX2ZUQ=
|
|
||||||
github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
|
|
||||||
github.com/tetafro/godot v1.3.2 h1:HzWC3XjadkyeuBZxkfAFNY20UVvle0YD51I6zf6RKlU=
|
|
||||||
github.com/tetafro/godot v1.3.2/go.mod h1:ah7jjYmOMnIjS9ku2krapvGQrFNtTLo9Z/qB3dGU1eU=
|
|
||||||
github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q=
|
|
||||||
github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
|
|
||||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
|
||||||
github.com/tomarrell/wrapcheck v0.0.0-20200807122107-df9e8bcb914d h1:3EZyvNUMsGD1QA8cu0STNn1L7I77rvhf2IhOcHYQhSw=
|
|
||||||
github.com/tomarrell/wrapcheck v0.0.0-20200807122107-df9e8bcb914d/go.mod h1:yiFB6fFoV7saXirUGfuK+cPtUh4NX/Hf5y2WC2lehu0=
|
|
||||||
github.com/tommy-muehle/go-mnd v1.3.1-0.20201008215730-16041ac3fe65 h1:Y0bLA422kvb32uZI4fy/Plop/Tbld0l9pSzl+j1FWok=
|
|
||||||
github.com/tommy-muehle/go-mnd v1.3.1-0.20201008215730-16041ac3fe65/go.mod h1:T22e7iRN4LsFPZGyRLRXeF+DWVXFuV9thsyO7NjbbTI=
|
|
||||||
github.com/ultraware/funlen v0.0.3 h1:5ylVWm8wsNwH5aWo9438pwvsK0QiqVuUrt9bn7S/iLA=
|
|
||||||
github.com/ultraware/funlen v0.0.3/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
|
|
||||||
github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg=
|
|
||||||
github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
|
|
||||||
github.com/uudashr/gocognit v1.0.1 h1:MoG2fZ0b/Eo7NXoIwCVFLG5JED3qgQz5/NEE+rOsjPs=
|
|
||||||
github.com/uudashr/gocognit v1.0.1/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM=
|
|
||||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
|
||||||
github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
|
|
||||||
github.com/valyala/quicktemplate v1.6.3/go.mod h1:fwPzK2fHuYEODzJ9pkw0ipCPNHZ2tD5KW4lOuSdPKzY=
|
|
||||||
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
|
|
||||||
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad h1:W0LEBv82YCGEtcmPA3uNZBI33/qF//HAAs3MawDjRa0=
|
|
||||||
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad/go.mod h1:Hy8o65+MXnS6EwGElrSRjUzQDLXreJlzYLlWiHtt8hM=
|
|
||||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
|
||||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
|
||||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
|
||||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
|
||||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
|
||||||
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
|
||||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
|
||||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
|
||||||
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
|
||||||
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
|
|
||||||
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
|
|
||||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
|
||||||
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
|
||||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
|
||||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
|
||||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
|
||||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
|
||||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
|
||||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
|
||||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
|
||||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
|
||||||
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
|
|
||||||
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
|
|
||||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
|
||||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
|
||||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
|
||||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
|
||||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
|
||||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
|
||||||
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
|
||||||
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
|
||||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
|
||||||
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
|
||||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
|
||||||
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
|
|
||||||
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
|
|
||||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
|
||||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
|
||||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
|
||||||
golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8=
|
|
||||||
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
|
||||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
|
||||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
|
||||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
|
||||||
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
|
||||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
|
||||||
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
|
||||||
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
|
||||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
|
||||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
|
||||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
|
||||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
|
||||||
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
|
||||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
|
||||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
|
||||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
|
||||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
|
||||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
|
||||||
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
|
||||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
|
||||||
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
|
||||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
|
|
||||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
|
||||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
|
||||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
|
||||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
|
||||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
|
||||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
|
||||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
|
||||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
|
||||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
|
|
||||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
|
||||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
|
||||||
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
|
||||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
|
||||||
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
|
||||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
|
||||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
|
||||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
|
||||||
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
|
||||||
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
|
||||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
|
||||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
|
||||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
|
||||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 h1:bNEHhJCnrwMKNMmOx3yAynp5vs5/gRy+XWFtZFu7NBM=
|
|
||||||
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
|
|
||||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
|
||||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
|
||||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
|
||||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
|
||||||
golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
|
|
||||||
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
|
||||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
|
||||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
|
||||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
|
||||||
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
|
||||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
|
||||||
golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
|
||||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
|
||||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
|
||||||
golang.org/x/tools v0.0.0-20190307163923-6a08e3108db3/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4=
|
|
||||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
|
||||||
golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
|
||||||
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
|
||||||
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
|
||||||
golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
|
||||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
|
||||||
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
|
||||||
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
|
||||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
|
||||||
golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b h1:iEAPfYPbYbxG/2lNN4cMOHkmgKNsCuUwkxlDCK46UlU=
|
|
||||||
golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
|
||||||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
|
||||||
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
|
||||||
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
|
||||||
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
|
||||||
golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
|
||||||
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
|
||||||
golang.org/x/tools v0.0.0-20190916130336-e45ffcd953cc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
|
||||||
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
|
||||||
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
|
||||||
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
|
||||||
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
|
||||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
|
||||||
golang.org/x/tools v0.0.0-20200117220505-0cba7a3a9ee9/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
|
||||||
golang.org/x/tools v0.0.0-20200324003944-a576cf524670/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
|
|
||||||
golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
|
|
||||||
golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
|
||||||
golang.org/x/tools v0.0.0-20200414032229-332987a829c3/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
|
||||||
golang.org/x/tools v0.0.0-20200422022333-3d57cf2e726e/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
|
||||||
golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
|
||||||
golang.org/x/tools v0.0.0-20200622203043-20e05c1c8ffa/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
|
||||||
golang.org/x/tools v0.0.0-20200624225443-88f3c62a19ff/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
|
||||||
golang.org/x/tools v0.0.0-20200625211823-6506e20df31f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
|
||||||
golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
|
||||||
golang.org/x/tools v0.0.0-20200812195022-5ae4c3c160a0/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
|
||||||
golang.org/x/tools v0.0.0-20200820010801-b793a1359eac/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
|
||||||
golang.org/x/tools v0.0.0-20200831203904-5a2aa26beb65/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
|
|
||||||
golang.org/x/tools v0.0.0-20201001104356-43ebab892c4c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
|
|
||||||
golang.org/x/tools v0.0.0-20201002184944-ecd9fd270d5d/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
|
|
||||||
golang.org/x/tools v0.0.0-20201007032633-0806396f153e/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
|
|
||||||
golang.org/x/tools v0.0.0-20201011145850-ed2f50202694/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
|
|
||||||
golang.org/x/tools v0.0.0-20201030010431-2feb2bb1ff51/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
|
||||||
golang.org/x/tools v0.0.0-20201114224030-61ea331ec02b/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
|
||||||
golang.org/x/tools v0.0.0-20201118003311-bd56c0adb394/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
|
||||||
golang.org/x/tools v0.0.0-20201230224404-63754364767c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
|
||||||
golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
|
||||||
golang.org/x/tools v0.0.0-20210105210202-9ed45478a130 h1:8qSBr5nyKsEgkP918Pu5FFDZpTtLIjXSo6mrtdVOFfk=
|
|
||||||
golang.org/x/tools v0.0.0-20210105210202-9ed45478a130/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
|
||||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
|
||||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
|
||||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
|
||||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
|
|
||||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
|
||||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
|
||||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
|
||||||
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
|
||||||
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
|
||||||
google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
|
||||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
|
||||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
|
||||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
|
||||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
|
||||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
|
||||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
|
||||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
|
||||||
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
|
||||||
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
|
||||||
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
|
||||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
|
||||||
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
|
|
||||||
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
|
||||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
|
||||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
|
||||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
|
||||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
|
||||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
|
||||||
google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=
|
|
||||||
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
|
||||||
google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8=
|
|
||||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE=
|
|
||||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
|
|
||||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
|
||||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
|
||||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
|
||||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
|
||||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
|
||||||
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
|
||||||
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
|
|
||||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
|
||||||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
|
||||||
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
|
|
||||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
|
||||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
|
||||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
|
||||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
|
||||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
|
||||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
|
|
||||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
|
||||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
|
||||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
|
||||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
|
||||||
gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
|
|
||||||
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
|
||||||
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
|
|
||||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
|
||||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
|
||||||
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
|
|
||||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|
||||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
|
||||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|
||||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|
||||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|
||||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|
||||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
|
||||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
|
||||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
|
||||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
|
||||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
|
||||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
|
||||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
|
||||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
|
||||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
|
||||||
honnef.co/go/tools v0.0.1-2020.1.6 h1:W18jzjh8mfPez+AwGLxmOImucz/IFjpNlrKVnaj2YVc=
|
|
||||||
honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzEYLhQB2YY=
|
|
||||||
mvdan.cc/gofumpt v0.1.0 h1:hsVv+Y9UsZ/mFZTxJZuHVI6shSQCtzZ11h1JEFPAZLw=
|
|
||||||
mvdan.cc/gofumpt v0.1.0/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48=
|
|
||||||
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
|
|
||||||
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
|
|
||||||
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=
|
|
||||||
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
|
|
||||||
mvdan.cc/unparam v0.0.0-20200501210554-b37ab49443f7 h1:kAREL6MPwpsk1/PQPFD3Eg7WAQR5mPTWZJaBiG5LDbY=
|
|
||||||
mvdan.cc/unparam v0.0.0-20200501210554-b37ab49443f7/go.mod h1:HGC5lll35J70Y5v7vCGb9oLhHoScFwkHDJm/05RdSTc=
|
|
||||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
// +build tools
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
// the lines are parsed by lazy.sh, do not edit
|
|
||||||
import (
|
|
||||||
_ "github.com/alvaroloes/enumer"
|
|
||||||
_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
|
|
||||||
_ "github.com/wadey/gocovmerge"
|
|
||||||
_ "golang.org/x/tools/cmd/goimports"
|
|
||||||
_ "golang.org/x/tools/cmd/stringer"
|
|
||||||
_ "google.golang.org/grpc/cmd/protoc-gen-go-grpc"
|
|
||||||
_ "google.golang.org/protobuf/cmd/protoc-gen-go"
|
|
||||||
)
|
|
||||||
-155
@@ -1,155 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
|
||||||
"github.com/spf13/pflag"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
var rootArgs struct {
|
|
||||||
configPath string
|
|
||||||
}
|
|
||||||
|
|
||||||
var rootCmd = &cobra.Command{
|
|
||||||
Use: "zrepl",
|
|
||||||
Short: "One-stop ZFS replication solution",
|
|
||||||
}
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
|
|
||||||
}
|
|
||||||
|
|
||||||
var genCompletionCmd = &cobra.Command{
|
|
||||||
Use: "gencompletion",
|
|
||||||
Short: "generate shell auto-completions",
|
|
||||||
}
|
|
||||||
|
|
||||||
type completionCmdInfo struct {
|
|
||||||
genFunc func(outpath string) error
|
|
||||||
help string
|
|
||||||
}
|
|
||||||
|
|
||||||
var completionCmdMap = map[string]completionCmdInfo{
|
|
||||||
"zsh": {
|
|
||||||
rootCmd.GenZshCompletionFile,
|
|
||||||
" save to file `_zrepl` in your zsh's $fpath",
|
|
||||||
},
|
|
||||||
"bash": {
|
|
||||||
rootCmd.GenBashCompletionFile,
|
|
||||||
" save to a path and source that path in your .bashrc",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
for sh, info := range completionCmdMap {
|
|
||||||
sh, info := sh, info
|
|
||||||
genCompletionCmd.AddCommand(&cobra.Command{
|
|
||||||
Use: fmt.Sprintf("%s path/to/out/file", sh),
|
|
||||||
Short: fmt.Sprintf("generate %s completions", sh),
|
|
||||||
Example: info.help,
|
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
|
||||||
if len(args) != 1 {
|
|
||||||
fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
|
|
||||||
err := cmd.Usage()
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
if err := info.genFunc(args[0]); err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "error generating %s completion: %s", sh, err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
rootCmd.AddCommand(genCompletionCmd)
|
|
||||||
}
|
|
||||||
|
|
||||||
type Subcommand struct {
|
|
||||||
Use string
|
|
||||||
Short string
|
|
||||||
Example string
|
|
||||||
NoRequireConfig bool
|
|
||||||
Run func(ctx context.Context, subcommand *Subcommand, args []string) error
|
|
||||||
SetupFlags func(f *pflag.FlagSet)
|
|
||||||
SetupSubcommands func() []*Subcommand
|
|
||||||
|
|
||||||
config *config.Config
|
|
||||||
configErr error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Subcommand) ConfigParsingError() error {
|
|
||||||
return s.configErr
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Subcommand) Config() *config.Config {
|
|
||||||
if !s.NoRequireConfig && s.config == nil {
|
|
||||||
panic("command that requires config is running and has no config set")
|
|
||||||
}
|
|
||||||
return s.config
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Subcommand) run(cmd *cobra.Command, args []string) {
|
|
||||||
s.tryParseConfig()
|
|
||||||
ctx := context.Background()
|
|
||||||
endTask := trace.WithTaskFromStackUpdateCtx(&ctx)
|
|
||||||
defer endTask()
|
|
||||||
err := s.Run(ctx, s, args)
|
|
||||||
endTask()
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "%s\n", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Subcommand) tryParseConfig() {
|
|
||||||
config, err := config.ParseConfig(rootArgs.configPath)
|
|
||||||
s.configErr = err
|
|
||||||
if err != nil {
|
|
||||||
if s.NoRequireConfig {
|
|
||||||
// doesn't matter
|
|
||||||
return
|
|
||||||
} else {
|
|
||||||
fmt.Fprintf(os.Stderr, "could not parse config: %s\n", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
s.config = config
|
|
||||||
}
|
|
||||||
|
|
||||||
func AddSubcommand(s *Subcommand) {
|
|
||||||
addSubcommandToCobraCmd(rootCmd, s)
|
|
||||||
}
|
|
||||||
|
|
||||||
func addSubcommandToCobraCmd(c *cobra.Command, s *Subcommand) {
|
|
||||||
cmd := cobra.Command{
|
|
||||||
Use: s.Use,
|
|
||||||
Short: s.Short,
|
|
||||||
Example: s.Example,
|
|
||||||
}
|
|
||||||
if s.SetupSubcommands == nil {
|
|
||||||
cmd.Run = s.run
|
|
||||||
} else {
|
|
||||||
for _, sub := range s.SetupSubcommands() {
|
|
||||||
addSubcommandToCobraCmd(&cmd, sub)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if s.SetupFlags != nil {
|
|
||||||
s.SetupFlags(cmd.Flags())
|
|
||||||
}
|
|
||||||
c.AddCommand(&cmd)
|
|
||||||
}
|
|
||||||
|
|
||||||
func Run() {
|
|
||||||
if err := rootCmd.Execute(); err != nil {
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,121 +0,0 @@
|
|||||||
package client
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/kr/pretty"
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
"github.com/spf13/pflag"
|
|
||||||
"github.com/zrepl/yaml-config"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/cli"
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
"github.com/zrepl/zrepl/daemon/job"
|
|
||||||
"github.com/zrepl/zrepl/daemon/logging"
|
|
||||||
"github.com/zrepl/zrepl/logger"
|
|
||||||
)
|
|
||||||
|
|
||||||
var configcheckArgs struct {
|
|
||||||
format string
|
|
||||||
what string
|
|
||||||
}
|
|
||||||
|
|
||||||
var ConfigcheckCmd = &cli.Subcommand{
|
|
||||||
Use: "configcheck",
|
|
||||||
Short: "check if config can be parsed without errors",
|
|
||||||
SetupFlags: func(f *pflag.FlagSet) {
|
|
||||||
f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]")
|
|
||||||
f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]")
|
|
||||||
},
|
|
||||||
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
|
||||||
formatMap := map[string]func(interface{}){
|
|
||||||
"": func(i interface{}) {},
|
|
||||||
"pretty": func(i interface{}) {
|
|
||||||
if _, err := pretty.Println(i); err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"json": func(i interface{}) {
|
|
||||||
if err := json.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"yaml": func(i interface{}) {
|
|
||||||
if err := yaml.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
formatter, ok := formatMap[configcheckArgs.format]
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("unsupported --format %q", configcheckArgs.format)
|
|
||||||
}
|
|
||||||
|
|
||||||
var hadErr bool
|
|
||||||
// further: try to build jobs
|
|
||||||
confJobs, err := job.JobsFromConfig(subcommand.Config())
|
|
||||||
if err != nil {
|
|
||||||
err := errors.Wrap(err, "cannot build jobs from config")
|
|
||||||
if configcheckArgs.what == "jobs" {
|
|
||||||
return err
|
|
||||||
} else {
|
|
||||||
fmt.Fprintf(os.Stderr, "%s\n", err)
|
|
||||||
confJobs = nil
|
|
||||||
hadErr = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// further: try to build logging outlets
|
|
||||||
outlets, err := logging.OutletsFromConfig(*subcommand.Config().Global.Logging)
|
|
||||||
if err != nil {
|
|
||||||
err := errors.Wrap(err, "cannot build logging from config")
|
|
||||||
if configcheckArgs.what == "logging" {
|
|
||||||
return err
|
|
||||||
} else {
|
|
||||||
fmt.Fprintf(os.Stderr, "%s\n", err)
|
|
||||||
outlets = nil
|
|
||||||
hadErr = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
whatMap := map[string]func(){
|
|
||||||
"all": func() {
|
|
||||||
o := struct {
|
|
||||||
config *config.Config
|
|
||||||
jobs []job.Job
|
|
||||||
logging *logger.Outlets
|
|
||||||
}{
|
|
||||||
subcommand.Config(),
|
|
||||||
confJobs,
|
|
||||||
outlets,
|
|
||||||
}
|
|
||||||
formatter(o)
|
|
||||||
},
|
|
||||||
"config": func() {
|
|
||||||
formatter(subcommand.Config())
|
|
||||||
},
|
|
||||||
"jobs": func() {
|
|
||||||
formatter(confJobs)
|
|
||||||
},
|
|
||||||
"logging": func() {
|
|
||||||
formatter(outlets)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
wf, ok := whatMap[configcheckArgs.what]
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("unsupported --format %q", configcheckArgs.what)
|
|
||||||
}
|
|
||||||
wf()
|
|
||||||
|
|
||||||
if hadErr {
|
|
||||||
return fmt.Errorf("config parsing failed")
|
|
||||||
} else {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
@@ -1,49 +0,0 @@
|
|||||||
package client
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
)
|
|
||||||
|
|
||||||
func controlHttpClient(sockpath string) (client http.Client, err error) {
|
|
||||||
return http.Client{
|
|
||||||
Transport: &http.Transport{
|
|
||||||
DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
|
|
||||||
return net.Dial("unix", sockpath)
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func jsonRequestResponse(c http.Client, endpoint string, req interface{}, res interface{}) error {
|
|
||||||
var buf bytes.Buffer
|
|
||||||
encodeErr := json.NewEncoder(&buf).Encode(req)
|
|
||||||
if encodeErr != nil {
|
|
||||||
return encodeErr
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := c.Post("http://unix"+endpoint, "application/json", &buf)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
var msg bytes.Buffer
|
|
||||||
_, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got
|
|
||||||
return errors.Errorf("%s", msg.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
decodeError := json.NewDecoder(resp.Body).Decode(&res)
|
|
||||||
if decodeError != nil {
|
|
||||||
return decodeError
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,268 +0,0 @@
|
|||||||
package client
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/fatih/color"
|
|
||||||
"github.com/kr/pretty"
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
"github.com/spf13/pflag"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/daemon/job"
|
|
||||||
"github.com/zrepl/zrepl/endpoint"
|
|
||||||
"github.com/zrepl/zrepl/zfs"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/cli"
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
MigrateCmd = &cli.Subcommand{
|
|
||||||
Use: "migrate",
|
|
||||||
Short: "perform migration of the on-disk / zfs properties",
|
|
||||||
SetupSubcommands: func() []*cli.Subcommand {
|
|
||||||
return migrations
|
|
||||||
},
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
var migrations = []*cli.Subcommand{
|
|
||||||
&cli.Subcommand{
|
|
||||||
Use: "0.0.X:0.1:placeholder",
|
|
||||||
Run: doMigratePlaceholder0_1,
|
|
||||||
SetupFlags: func(f *pflag.FlagSet) {
|
|
||||||
f.BoolVar(&migratePlaceholder0_1Args.dryRun, "dry-run", false, "dry run")
|
|
||||||
},
|
|
||||||
},
|
|
||||||
&cli.Subcommand{
|
|
||||||
Use: "replication-cursor:v1-v2",
|
|
||||||
Run: doMigrateReplicationCursor,
|
|
||||||
SetupFlags: func(f *pflag.FlagSet) {
|
|
||||||
f.BoolVar(&migrateReplicationCursorArgs.dryRun, "dry-run", false, "dry run")
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
var migratePlaceholder0_1Args struct {
|
|
||||||
dryRun bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func doMigratePlaceholder0_1(ctx context.Context, sc *cli.Subcommand, args []string) error {
|
|
||||||
if len(args) != 0 {
|
|
||||||
return fmt.Errorf("migration does not take arguments, got %v", args)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg := sc.Config()
|
|
||||||
|
|
||||||
allFSS, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrap(err, "cannot list filesystems")
|
|
||||||
}
|
|
||||||
|
|
||||||
type workItem struct {
|
|
||||||
jobName string
|
|
||||||
rootFS *zfs.DatasetPath
|
|
||||||
fss []*zfs.DatasetPath
|
|
||||||
}
|
|
||||||
var wis []workItem
|
|
||||||
for i, j := range cfg.Jobs {
|
|
||||||
var rfsS string
|
|
||||||
switch job := j.Ret.(type) {
|
|
||||||
case *config.SinkJob:
|
|
||||||
rfsS = job.RootFS
|
|
||||||
case *config.PullJob:
|
|
||||||
rfsS = job.RootFS
|
|
||||||
default:
|
|
||||||
fmt.Printf("ignoring job %q (%d/%d, type %T)\n", j.Name(), i, len(cfg.Jobs), j.Ret)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
rfs, err := zfs.NewDatasetPath(rfsS)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrapf(err, "root fs for job %q is not a valid dataset path", j.Name())
|
|
||||||
}
|
|
||||||
var fss []*zfs.DatasetPath
|
|
||||||
for _, fs := range allFSS {
|
|
||||||
if fs.HasPrefix(rfs) {
|
|
||||||
fss = append(fss, fs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
wis = append(wis, workItem{j.Name(), rfs, fss})
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, wi := range wis {
|
|
||||||
fmt.Printf("job %q => migrate filesystems below root_fs %q\n", wi.jobName, wi.rootFS.ToString())
|
|
||||||
if len(wi.fss) == 0 {
|
|
||||||
fmt.Printf("\tno filesystems\n")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, fs := range wi.fss {
|
|
||||||
fmt.Printf("\t%q ... ", fs.ToString())
|
|
||||||
r, err := zfs.ZFSMigrateHashBasedPlaceholderToCurrent(ctx, fs, migratePlaceholder0_1Args.dryRun)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("error: %s\n", err)
|
|
||||||
} else if !r.NeedsModification {
|
|
||||||
fmt.Printf("unchanged (placeholder=%v)\n", r.OriginalState.IsPlaceholder)
|
|
||||||
} else {
|
|
||||||
fmt.Printf("migrate (placeholder=%v) (old value = %q)\n",
|
|
||||||
r.OriginalState.IsPlaceholder, r.OriginalState.RawLocalPropertyValue)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var migrateReplicationCursorArgs struct {
|
|
||||||
dryRun bool
|
|
||||||
}
|
|
||||||
|
|
||||||
var bold = color.New(color.Bold)
|
|
||||||
var succ = color.New(color.FgGreen)
|
|
||||||
var fail = color.New(color.FgRed)
|
|
||||||
|
|
||||||
var migrateReplicationCursorSkipSentinel = fmt.Errorf("skipping this filesystem")
|
|
||||||
|
|
||||||
func doMigrateReplicationCursor(ctx context.Context, sc *cli.Subcommand, args []string) error {
|
|
||||||
if len(args) != 0 {
|
|
||||||
return fmt.Errorf("migration does not take arguments, got %v", args)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg := sc.Config()
|
|
||||||
jobs, err := job.JobsFromConfig(cfg)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("cannot parse config:\n%s\n\n", err)
|
|
||||||
fmt.Printf("NOTE: this migration was released together with a change in job name requirements.\n")
|
|
||||||
return fmt.Errorf("exiting migration after error")
|
|
||||||
}
|
|
||||||
|
|
||||||
v1cursorJobs := make([]job.Job, 0, len(cfg.Jobs))
|
|
||||||
for i, j := range cfg.Jobs {
|
|
||||||
if jobs[i].Name() != j.Name() {
|
|
||||||
panic("implementation error")
|
|
||||||
}
|
|
||||||
switch j.Ret.(type) {
|
|
||||||
case *config.PushJob:
|
|
||||||
v1cursorJobs = append(v1cursorJobs, jobs[i])
|
|
||||||
case *config.SourceJob:
|
|
||||||
v1cursorJobs = append(v1cursorJobs, jobs[i])
|
|
||||||
default:
|
|
||||||
fmt.Printf("ignoring job %q (%d/%d, type %T), not supposed to create v1 replication cursors\n", j.Name(), i, len(cfg.Jobs), j.Ret)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// scan all filesystems for v1 replication cursors
|
|
||||||
|
|
||||||
fss, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrap(err, "list filesystems")
|
|
||||||
}
|
|
||||||
|
|
||||||
var hadError bool
|
|
||||||
for _, fs := range fss {
|
|
||||||
|
|
||||||
bold.Printf("INSPECT FILESYSTEM %q\n", fs.ToString())
|
|
||||||
|
|
||||||
err := doMigrateReplicationCursorFS(ctx, v1cursorJobs, fs)
|
|
||||||
if err == migrateReplicationCursorSkipSentinel {
|
|
||||||
bold.Printf("FILESYSTEM SKIPPED\n")
|
|
||||||
} else if err != nil {
|
|
||||||
hadError = true
|
|
||||||
fail.Printf("MIGRATION FAILED: %s\n", err)
|
|
||||||
} else {
|
|
||||||
succ.Printf("FILESYSTEM %q COMPLETE\n", fs.ToString())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if hadError {
|
|
||||||
fail.Printf("\n\none or more filesystems could not be migrated, please inspect output and or re-run migration")
|
|
||||||
return errors.Errorf("")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func doMigrateReplicationCursorFS(ctx context.Context, v1CursorJobs []job.Job, fs *zfs.DatasetPath) error {
|
|
||||||
|
|
||||||
var owningJob job.Job = nil
|
|
||||||
for _, job := range v1CursorJobs {
|
|
||||||
conf := job.SenderConfig()
|
|
||||||
if conf == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
pass, err := conf.FSF.Filter(fs)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrapf(err, "filesystem filter error in job %q for fs %q", job.Name(), fs.ToString())
|
|
||||||
}
|
|
||||||
if !pass {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if owningJob != nil {
|
|
||||||
return errors.Errorf("jobs %q and %q both match %q\ncannot attribute replication cursor to either one", owningJob.Name(), job.Name(), fs)
|
|
||||||
}
|
|
||||||
owningJob = job
|
|
||||||
}
|
|
||||||
if owningJob == nil {
|
|
||||||
fmt.Printf("no job's Filesystems filter matches\n")
|
|
||||||
return migrateReplicationCursorSkipSentinel
|
|
||||||
}
|
|
||||||
fmt.Printf("identified owning job %q\n", owningJob.Name())
|
|
||||||
|
|
||||||
bookmarks, err := zfs.ZFSListFilesystemVersions(ctx, fs, zfs.ListFilesystemVersionsOptions{
|
|
||||||
Types: zfs.Bookmarks,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrapf(err, "list filesystem versions of %q", fs.ToString())
|
|
||||||
}
|
|
||||||
|
|
||||||
var oldCursor *zfs.FilesystemVersion
|
|
||||||
for i, fsv := range bookmarks {
|
|
||||||
_, _, err := endpoint.ParseReplicationCursorBookmarkName(fsv.ToAbsPath(fs))
|
|
||||||
if err != endpoint.ErrV1ReplicationCursor {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if oldCursor != nil {
|
|
||||||
fmt.Printf("unexpected v1 replication cursor candidate: %q", fsv.ToAbsPath(fs))
|
|
||||||
return errors.Wrap(err, "multiple filesystem versions identified as v1 replication cursors")
|
|
||||||
}
|
|
||||||
|
|
||||||
oldCursor = &bookmarks[i]
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
if oldCursor == nil {
|
|
||||||
bold.Printf("no v1 replication cursor found for filesystem %q\n", fs.ToString())
|
|
||||||
return migrateReplicationCursorSkipSentinel
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Printf("found v1 replication cursor:\n%s\n", pretty.Sprint(oldCursor))
|
|
||||||
|
|
||||||
mostRecentNew, err := endpoint.GetMostRecentReplicationCursorOfJob(ctx, fs.ToString(), owningJob.SenderConfig().JobID)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrapf(err, "get most recent v2 replication cursor")
|
|
||||||
}
|
|
||||||
|
|
||||||
if mostRecentNew == nil {
|
|
||||||
return errors.Errorf("no v2 replication cursor found for job %q on filesystem %q", owningJob.SenderConfig().JobID, fs.ToString())
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Printf("most recent v2 replication cursor:\n%#v", oldCursor)
|
|
||||||
|
|
||||||
if !(mostRecentNew.CreateTXG >= oldCursor.CreateTXG) {
|
|
||||||
return errors.Errorf("v1 replication cursor createtxg is higher than v2 cursor's, skipping this filesystem")
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Printf("determined that v2 cursor is bookmark of same or newer version than v1 cursor\n")
|
|
||||||
fmt.Printf("destroying v1 cursor %q\n", oldCursor.ToAbsPath(fs))
|
|
||||||
|
|
||||||
if migrateReplicationCursorArgs.dryRun {
|
|
||||||
succ.Printf("DRY RUN\n")
|
|
||||||
} else {
|
|
||||||
if err := zfs.ZFSDestroyFilesystemVersion(ctx, fs, oldCursor); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
package client
|
|
||||||
|
|
||||||
import "testing"
|
|
||||||
|
|
||||||
func TestMigrationsUnambiguousNames(t *testing.T) {
|
|
||||||
names := make(map[string]bool)
|
|
||||||
for _, mig := range migrations {
|
|
||||||
if _, ok := names[mig.Use]; ok {
|
|
||||||
t.Errorf("duplicate migration name %q", mig.Use)
|
|
||||||
t.FailNow()
|
|
||||||
return
|
|
||||||
} else {
|
|
||||||
names[mig.Use] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
package client
|
|
||||||
|
|
||||||
import "github.com/zrepl/zrepl/cli"
|
|
||||||
|
|
||||||
var PprofCmd = &cli.Subcommand{
|
|
||||||
Use: "pprof",
|
|
||||||
SetupSubcommands: func() []*cli.Subcommand {
|
|
||||||
return []*cli.Subcommand{PprofListenCmd, pprofActivityTraceCmd}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
@@ -1,44 +0,0 @@
|
|||||||
package client
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"io"
|
|
||||||
"log"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"golang.org/x/net/websocket"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/cli"
|
|
||||||
)
|
|
||||||
|
|
||||||
var pprofActivityTraceCmd = &cli.Subcommand{
|
|
||||||
Use: "activity-trace ZREPL_PPROF_HOST:ZREPL_PPROF_PORT",
|
|
||||||
Short: "attach to zrepl daemon with activated pprof listener and dump an activity-trace to stdout",
|
|
||||||
Run: runPProfActivityTrace,
|
|
||||||
}
|
|
||||||
|
|
||||||
func runPProfActivityTrace(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
|
||||||
log := log.New(os.Stderr, "", 0)
|
|
||||||
|
|
||||||
die := func() {
|
|
||||||
log.Printf("exiting after error")
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(args) != 1 {
|
|
||||||
log.Printf("exactly one positional argument is required")
|
|
||||||
die()
|
|
||||||
}
|
|
||||||
|
|
||||||
url := "ws://" + args[0] + "/debug/zrepl/activity-trace" // FIXME dont' repeat that
|
|
||||||
|
|
||||||
log.Printf("attaching to activity trace stream %s", url)
|
|
||||||
ws, err := websocket.Dial(url, "", url)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error: %s", err)
|
|
||||||
die()
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = io.Copy(os.Stdout, ws)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
package client
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"log"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/cli"
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
"github.com/zrepl/zrepl/daemon"
|
|
||||||
)
|
|
||||||
|
|
||||||
var pprofListenCmd struct {
|
|
||||||
daemon.PprofServerControlMsg
|
|
||||||
}
|
|
||||||
|
|
||||||
var PprofListenCmd = &cli.Subcommand{
|
|
||||||
Use: "listen off | [on TCP_LISTEN_ADDRESS]",
|
|
||||||
Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
|
|
||||||
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
|
||||||
if len(args) < 1 {
|
|
||||||
goto enargs
|
|
||||||
}
|
|
||||||
switch args[0] {
|
|
||||||
case "on":
|
|
||||||
pprofListenCmd.Run = true
|
|
||||||
if len(args) != 2 {
|
|
||||||
return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
|
|
||||||
}
|
|
||||||
pprofListenCmd.HttpListenAddress = args[1]
|
|
||||||
case "off":
|
|
||||||
if len(args) != 1 {
|
|
||||||
goto enargs
|
|
||||||
}
|
|
||||||
pprofListenCmd.Run = false
|
|
||||||
}
|
|
||||||
|
|
||||||
RunPProf(subcommand.Config())
|
|
||||||
return nil
|
|
||||||
enargs:
|
|
||||||
return errors.New("invalid number of positional arguments")
|
|
||||||
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
func RunPProf(conf *config.Config) {
|
|
||||||
log := log.New(os.Stderr, "", 0)
|
|
||||||
|
|
||||||
die := func() {
|
|
||||||
log.Printf("exiting after error")
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Printf("connecting to zrepl daemon")
|
|
||||||
|
|
||||||
httpc, err := controlHttpClient(conf.Global.Control.SockPath)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error creating http client: %s", err)
|
|
||||||
die()
|
|
||||||
}
|
|
||||||
err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofListenCmd.PprofServerControlMsg, struct{}{})
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("error sending control message: %s", err)
|
|
||||||
die()
|
|
||||||
}
|
|
||||||
log.Printf("finished")
|
|
||||||
}
|
|
||||||
@@ -1,42 +0,0 @@
|
|||||||
package client
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/cli"
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
"github.com/zrepl/zrepl/daemon"
|
|
||||||
)
|
|
||||||
|
|
||||||
var SignalCmd = &cli.Subcommand{
|
|
||||||
Use: "signal [wakeup|reset] JOB",
|
|
||||||
Short: "wake up a job from wait state or abort its current invocation",
|
|
||||||
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
|
||||||
return runSignalCmd(subcommand.Config(), args)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
func runSignalCmd(config *config.Config, args []string) error {
|
|
||||||
if len(args) != 2 {
|
|
||||||
return errors.Errorf("Expected 2 arguments: [wakeup|reset] JOB")
|
|
||||||
}
|
|
||||||
|
|
||||||
httpc, err := controlHttpClient(config.Global.Control.SockPath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = jsonRequestResponse(httpc, daemon.ControlJobEndpointSignal,
|
|
||||||
struct {
|
|
||||||
Name string
|
|
||||||
Op string
|
|
||||||
}{
|
|
||||||
Name: args[1],
|
|
||||||
Op: args[0],
|
|
||||||
},
|
|
||||||
struct{}{},
|
|
||||||
)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
@@ -1,816 +0,0 @@
|
|||||||
package client
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"math"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
// tcell is the termbox-compatible library for abstracting away escape sequences, etc.
|
|
||||||
// as of tcell#252, the number of default distributed terminals is relatively limited
|
|
||||||
// additional terminal definitions can be included via side-effect import
|
|
||||||
// See https://github.com/gdamore/tcell/blob/master/terminfo/base/base.go
|
|
||||||
// See https://github.com/gdamore/tcell/issues/252#issuecomment-533836078
|
|
||||||
"github.com/gdamore/tcell/termbox"
|
|
||||||
_ "github.com/gdamore/tcell/terminfo/s/screen" // tmux on FreeBSD 11 & 12 without ncurses
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
"github.com/spf13/pflag"
|
|
||||||
"github.com/zrepl/yaml-config"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/cli"
|
|
||||||
"github.com/zrepl/zrepl/daemon"
|
|
||||||
"github.com/zrepl/zrepl/daemon/job"
|
|
||||||
"github.com/zrepl/zrepl/daemon/pruner"
|
|
||||||
"github.com/zrepl/zrepl/daemon/snapper"
|
|
||||||
"github.com/zrepl/zrepl/replication/report"
|
|
||||||
)
|
|
||||||
|
|
||||||
type byteProgressMeasurement struct {
|
|
||||||
time time.Time
|
|
||||||
val int64
|
|
||||||
}
|
|
||||||
|
|
||||||
type bytesProgressHistory struct {
|
|
||||||
last *byteProgressMeasurement // pointer as poor man's optional
|
|
||||||
changeCount int
|
|
||||||
lastChange time.Time
|
|
||||||
bpsAvg float64
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *bytesProgressHistory) Update(currentVal int64) (bytesPerSecondAvg int64, changeCount int) {
|
|
||||||
|
|
||||||
if p.last == nil {
|
|
||||||
p.last = &byteProgressMeasurement{
|
|
||||||
time: time.Now(),
|
|
||||||
val: currentVal,
|
|
||||||
}
|
|
||||||
return 0, 0
|
|
||||||
}
|
|
||||||
|
|
||||||
if p.last.val != currentVal {
|
|
||||||
p.changeCount++
|
|
||||||
p.lastChange = time.Now()
|
|
||||||
}
|
|
||||||
|
|
||||||
if time.Since(p.lastChange) > 3*time.Second {
|
|
||||||
p.last = nil
|
|
||||||
return 0, 0
|
|
||||||
}
|
|
||||||
|
|
||||||
deltaV := currentVal - p.last.val
|
|
||||||
deltaT := time.Since(p.last.time)
|
|
||||||
rate := float64(deltaV) / deltaT.Seconds()
|
|
||||||
|
|
||||||
factor := 0.3
|
|
||||||
p.bpsAvg = (1-factor)*p.bpsAvg + factor*rate
|
|
||||||
|
|
||||||
p.last.time = time.Now()
|
|
||||||
p.last.val = currentVal
|
|
||||||
|
|
||||||
return int64(p.bpsAvg), p.changeCount
|
|
||||||
}
|
|
||||||
|
|
||||||
type tui struct {
|
|
||||||
x, y int
|
|
||||||
indent int
|
|
||||||
|
|
||||||
lock sync.Mutex //For report and error
|
|
||||||
report map[string]*job.Status
|
|
||||||
err error
|
|
||||||
|
|
||||||
jobFilter string
|
|
||||||
|
|
||||||
replicationProgress map[string]*bytesProgressHistory // by job name
|
|
||||||
}
|
|
||||||
|
|
||||||
func newTui() tui {
|
|
||||||
return tui{
|
|
||||||
replicationProgress: make(map[string]*bytesProgressHistory),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const INDENT_MULTIPLIER = 4
|
|
||||||
|
|
||||||
func (t *tui) moveLine(dl int, col int) {
|
|
||||||
t.y += dl
|
|
||||||
t.x = t.indent*INDENT_MULTIPLIER + col
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tui) write(text string) {
|
|
||||||
for _, c := range text {
|
|
||||||
if c == '\n' {
|
|
||||||
t.newline()
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
termbox.SetCell(t.x, t.y, c, termbox.ColorDefault, termbox.ColorDefault)
|
|
||||||
t.x += 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tui) printf(text string, a ...interface{}) {
|
|
||||||
t.write(fmt.Sprintf(text, a...))
|
|
||||||
}
|
|
||||||
|
|
||||||
func wrap(s string, width int) string {
|
|
||||||
var b strings.Builder
|
|
||||||
for len(s) > 0 {
|
|
||||||
rem := width
|
|
||||||
if rem > len(s) {
|
|
||||||
rem = len(s)
|
|
||||||
}
|
|
||||||
if idx := strings.IndexAny(s, "\n\r"); idx != -1 && idx < rem {
|
|
||||||
rem = idx + 1
|
|
||||||
}
|
|
||||||
untilNewline := strings.TrimRight(s[:rem], "\n\r")
|
|
||||||
s = s[rem:]
|
|
||||||
if len(untilNewline) == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
b.WriteString(untilNewline)
|
|
||||||
b.WriteString("\n")
|
|
||||||
}
|
|
||||||
return strings.TrimRight(b.String(), "\n\r")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tui) printfDrawIndentedAndWrappedIfMultiline(format string, a ...interface{}) {
|
|
||||||
whole := fmt.Sprintf(format, a...)
|
|
||||||
width, _ := termbox.Size()
|
|
||||||
if !strings.ContainsAny(whole, "\n\r") && t.x+len(whole) <= width {
|
|
||||||
t.printf(format, a...)
|
|
||||||
} else {
|
|
||||||
t.addIndent(1)
|
|
||||||
t.newline()
|
|
||||||
t.write(wrap(whole, width-INDENT_MULTIPLIER*t.indent))
|
|
||||||
t.addIndent(-1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tui) newline() {
|
|
||||||
t.moveLine(1, 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tui) setIndent(indent int) {
|
|
||||||
t.indent = indent
|
|
||||||
t.moveLine(0, 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tui) addIndent(indent int) {
|
|
||||||
t.indent += indent
|
|
||||||
t.moveLine(0, 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
var statusFlags struct {
|
|
||||||
Raw bool
|
|
||||||
Job string
|
|
||||||
}
|
|
||||||
|
|
||||||
var StatusCmd = &cli.Subcommand{
|
|
||||||
Use: "status",
|
|
||||||
Short: "show job activity or dump as JSON for monitoring",
|
|
||||||
SetupFlags: func(f *pflag.FlagSet) {
|
|
||||||
f.BoolVar(&statusFlags.Raw, "raw", false, "dump raw status description from zrepl daemon")
|
|
||||||
f.StringVar(&statusFlags.Job, "job", "", "only dump specified job")
|
|
||||||
},
|
|
||||||
Run: runStatus,
|
|
||||||
}
|
|
||||||
|
|
||||||
func runStatus(ctx context.Context, s *cli.Subcommand, args []string) error {
|
|
||||||
httpc, err := controlHttpClient(s.Config().Global.Control.SockPath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if statusFlags.Raw {
|
|
||||||
resp, err := httpc.Get("http://unix" + daemon.ControlJobEndpointStatus)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
fmt.Fprintf(os.Stderr, "Received error response:\n")
|
|
||||||
_, err := io.CopyN(os.Stderr, resp.Body, 4096)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return errors.Errorf("exit")
|
|
||||||
}
|
|
||||||
if _, err := io.Copy(os.Stdout, resp.Body); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
t := newTui()
|
|
||||||
t.lock.Lock()
|
|
||||||
t.err = errors.New("Got no report yet")
|
|
||||||
t.lock.Unlock()
|
|
||||||
t.jobFilter = statusFlags.Job
|
|
||||||
|
|
||||||
err = termbox.Init()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer termbox.Close()
|
|
||||||
|
|
||||||
update := func() {
|
|
||||||
var m daemon.Status
|
|
||||||
|
|
||||||
err2 := jsonRequestResponse(httpc, daemon.ControlJobEndpointStatus,
|
|
||||||
struct{}{},
|
|
||||||
&m,
|
|
||||||
)
|
|
||||||
|
|
||||||
t.lock.Lock()
|
|
||||||
t.err = err2
|
|
||||||
t.report = m.Jobs
|
|
||||||
t.lock.Unlock()
|
|
||||||
t.draw()
|
|
||||||
}
|
|
||||||
update()
|
|
||||||
|
|
||||||
ticker := time.NewTicker(500 * time.Millisecond)
|
|
||||||
defer ticker.Stop()
|
|
||||||
go func() {
|
|
||||||
for range ticker.C {
|
|
||||||
update()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
termbox.HideCursor()
|
|
||||||
termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
|
|
||||||
|
|
||||||
loop:
|
|
||||||
for {
|
|
||||||
switch ev := termbox.PollEvent(); ev.Type {
|
|
||||||
case termbox.EventKey:
|
|
||||||
switch ev.Key {
|
|
||||||
case termbox.KeyEsc:
|
|
||||||
break loop
|
|
||||||
case termbox.KeyCtrlC:
|
|
||||||
break loop
|
|
||||||
}
|
|
||||||
case termbox.EventResize:
|
|
||||||
t.draw()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tui) getReplicationProgressHistory(jobName string) *bytesProgressHistory {
|
|
||||||
p, ok := t.replicationProgress[jobName]
|
|
||||||
if !ok {
|
|
||||||
p = &bytesProgressHistory{}
|
|
||||||
t.replicationProgress[jobName] = p
|
|
||||||
}
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tui) draw() {
|
|
||||||
t.lock.Lock()
|
|
||||||
defer t.lock.Unlock()
|
|
||||||
|
|
||||||
termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
|
|
||||||
t.x = 0
|
|
||||||
t.y = 0
|
|
||||||
t.indent = 0
|
|
||||||
|
|
||||||
if t.err != nil {
|
|
||||||
t.write(t.err.Error())
|
|
||||||
} else {
|
|
||||||
//Iterate over map in alphabetical order
|
|
||||||
keys := make([]string, 0, len(t.report))
|
|
||||||
for k := range t.report {
|
|
||||||
if len(k) == 0 || daemon.IsInternalJobName(k) { //Internal job
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if t.jobFilter != "" && k != t.jobFilter {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
keys = append(keys, k)
|
|
||||||
}
|
|
||||||
sort.Strings(keys)
|
|
||||||
|
|
||||||
if len(keys) == 0 {
|
|
||||||
t.setIndent(0)
|
|
||||||
t.printf("no jobs to display")
|
|
||||||
t.newline()
|
|
||||||
termbox.Flush()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, k := range keys {
|
|
||||||
v := t.report[k]
|
|
||||||
|
|
||||||
t.setIndent(0)
|
|
||||||
|
|
||||||
t.printf("Job: %s", k)
|
|
||||||
t.setIndent(1)
|
|
||||||
t.newline()
|
|
||||||
t.printf("Type: %s", v.Type)
|
|
||||||
t.setIndent(1)
|
|
||||||
t.newline()
|
|
||||||
|
|
||||||
if v.Type == job.TypePush || v.Type == job.TypePull {
|
|
||||||
activeStatus, ok := v.JobSpecific.(*job.ActiveSideStatus)
|
|
||||||
if !ok || activeStatus == nil {
|
|
||||||
t.printf("ActiveSideStatus is null")
|
|
||||||
t.newline()
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
t.printf("Replication:")
|
|
||||||
t.newline()
|
|
||||||
t.addIndent(1)
|
|
||||||
t.renderReplicationReport(activeStatus.Replication, t.getReplicationProgressHistory(k))
|
|
||||||
t.addIndent(-1)
|
|
||||||
|
|
||||||
t.printf("Pruning Sender:")
|
|
||||||
t.newline()
|
|
||||||
t.addIndent(1)
|
|
||||||
t.renderPrunerReport(activeStatus.PruningSender)
|
|
||||||
t.addIndent(-1)
|
|
||||||
|
|
||||||
t.printf("Pruning Receiver:")
|
|
||||||
t.newline()
|
|
||||||
t.addIndent(1)
|
|
||||||
t.renderPrunerReport(activeStatus.PruningReceiver)
|
|
||||||
t.addIndent(-1)
|
|
||||||
|
|
||||||
if v.Type == job.TypePush {
|
|
||||||
t.printf("Snapshotting:")
|
|
||||||
t.newline()
|
|
||||||
t.addIndent(1)
|
|
||||||
t.renderSnapperReport(activeStatus.Snapshotting)
|
|
||||||
t.addIndent(-1)
|
|
||||||
}
|
|
||||||
|
|
||||||
} else if v.Type == job.TypeSnap {
|
|
||||||
snapStatus, ok := v.JobSpecific.(*job.SnapJobStatus)
|
|
||||||
if !ok || snapStatus == nil {
|
|
||||||
t.printf("SnapJobStatus is null")
|
|
||||||
t.newline()
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
t.printf("Pruning snapshots:")
|
|
||||||
t.newline()
|
|
||||||
t.addIndent(1)
|
|
||||||
t.renderPrunerReport(snapStatus.Pruning)
|
|
||||||
t.addIndent(-1)
|
|
||||||
t.printf("Snapshotting:")
|
|
||||||
t.newline()
|
|
||||||
t.addIndent(1)
|
|
||||||
t.renderSnapperReport(snapStatus.Snapshotting)
|
|
||||||
t.addIndent(-1)
|
|
||||||
} else if v.Type == job.TypeSource {
|
|
||||||
|
|
||||||
st := v.JobSpecific.(*job.PassiveStatus)
|
|
||||||
t.printf("Snapshotting:\n")
|
|
||||||
t.addIndent(1)
|
|
||||||
t.renderSnapperReport(st.Snapper)
|
|
||||||
t.addIndent(-1)
|
|
||||||
|
|
||||||
} else {
|
|
||||||
t.printf("No status representation for job type '%s', dumping as YAML", v.Type)
|
|
||||||
t.newline()
|
|
||||||
asYaml, err := yaml.Marshal(v.JobSpecific)
|
|
||||||
if err != nil {
|
|
||||||
t.printf("Error marshaling status to YAML: %s", err)
|
|
||||||
t.newline()
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
t.write(string(asYaml))
|
|
||||||
t.newline()
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
termbox.Flush()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tui) renderReplicationReport(rep *report.Report, history *bytesProgressHistory) {
|
|
||||||
if rep == nil {
|
|
||||||
t.printf("...\n")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if rep.WaitReconnectError != nil {
|
|
||||||
t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: %s", rep.WaitReconnectError)
|
|
||||||
t.newline()
|
|
||||||
}
|
|
||||||
if !rep.WaitReconnectSince.IsZero() {
|
|
||||||
delta := time.Until(rep.WaitReconnectUntil).Round(time.Second)
|
|
||||||
if rep.WaitReconnectUntil.IsZero() || delta > 0 {
|
|
||||||
var until string
|
|
||||||
if rep.WaitReconnectUntil.IsZero() {
|
|
||||||
until = "waiting indefinitely"
|
|
||||||
} else {
|
|
||||||
until = fmt.Sprintf("hard fail in %s @ %s", delta, rep.WaitReconnectUntil)
|
|
||||||
}
|
|
||||||
t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnecting with exponential backoff (since %s) (%s)",
|
|
||||||
rep.WaitReconnectSince, until)
|
|
||||||
} else {
|
|
||||||
t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnects reached hard-fail timeout @ %s", rep.WaitReconnectUntil)
|
|
||||||
}
|
|
||||||
t.newline()
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO visualize more than the latest attempt by folding all attempts into one
|
|
||||||
if len(rep.Attempts) == 0 {
|
|
||||||
t.printf("no attempts made yet")
|
|
||||||
return
|
|
||||||
} else {
|
|
||||||
t.printf("Attempt #%d", len(rep.Attempts))
|
|
||||||
if len(rep.Attempts) > 1 {
|
|
||||||
t.printf(". Previous attempts failed with the following statuses:")
|
|
||||||
t.newline()
|
|
||||||
t.addIndent(1)
|
|
||||||
for i, a := range rep.Attempts[:len(rep.Attempts)-1] {
|
|
||||||
t.printfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt))
|
|
||||||
t.newline()
|
|
||||||
}
|
|
||||||
t.addIndent(-1)
|
|
||||||
} else {
|
|
||||||
t.newline()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
latest := rep.Attempts[len(rep.Attempts)-1]
|
|
||||||
sort.Slice(latest.Filesystems, func(i, j int) bool {
|
|
||||||
return latest.Filesystems[i].Info.Name < latest.Filesystems[j].Info.Name
|
|
||||||
})
|
|
||||||
|
|
||||||
t.printf("Status: %s", latest.State)
|
|
||||||
t.newline()
|
|
||||||
if latest.State == report.AttemptPlanningError {
|
|
||||||
t.printf("Problem: ")
|
|
||||||
t.printfDrawIndentedAndWrappedIfMultiline("%s", latest.PlanError)
|
|
||||||
t.newline()
|
|
||||||
} else if latest.State == report.AttemptFanOutError {
|
|
||||||
t.printf("Problem: one or more of the filesystems encountered errors")
|
|
||||||
t.newline()
|
|
||||||
}
|
|
||||||
|
|
||||||
if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
|
|
||||||
// Draw global progress bar
|
|
||||||
// Progress: [---------------]
|
|
||||||
expected, replicated, containsInvalidSizeEstimates := latest.BytesSum()
|
|
||||||
rate, changeCount := history.Update(replicated)
|
|
||||||
eta := time.Duration(0)
|
|
||||||
if rate > 0 {
|
|
||||||
eta = time.Duration((expected-replicated)/rate) * time.Second
|
|
||||||
}
|
|
||||||
t.write("Progress: ")
|
|
||||||
t.drawBar(50, replicated, expected, changeCount)
|
|
||||||
t.write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinary(replicated), ByteCountBinary(expected), ByteCountBinary(rate)))
|
|
||||||
if eta != 0 {
|
|
||||||
t.write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
|
|
||||||
}
|
|
||||||
t.newline()
|
|
||||||
if containsInvalidSizeEstimates {
|
|
||||||
t.write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
|
|
||||||
t.newline()
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(latest.Filesystems) == 0 {
|
|
||||||
t.write("NOTE: no filesystems were considered for replication!")
|
|
||||||
t.newline()
|
|
||||||
}
|
|
||||||
|
|
||||||
var maxFSLen int
|
|
||||||
for _, fs := range latest.Filesystems {
|
|
||||||
if len(fs.Info.Name) > maxFSLen {
|
|
||||||
maxFSLen = len(fs.Info.Name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, fs := range latest.Filesystems {
|
|
||||||
t.printFilesystemStatus(fs, false, maxFSLen) // FIXME bring 'active' flag back
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tui) renderPrunerReport(r *pruner.Report) {
|
|
||||||
if r == nil {
|
|
||||||
t.printf("...\n")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
state, err := pruner.StateString(r.State)
|
|
||||||
if err != nil {
|
|
||||||
t.printf("Status: %q (parse error: %q)\n", r.State, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
t.printf("Status: %s", state)
|
|
||||||
t.newline()
|
|
||||||
|
|
||||||
if r.Error != "" {
|
|
||||||
t.printf("Error: %s\n", r.Error)
|
|
||||||
}
|
|
||||||
|
|
||||||
type commonFS struct {
|
|
||||||
*pruner.FSReport
|
|
||||||
completed bool
|
|
||||||
}
|
|
||||||
all := make([]commonFS, 0, len(r.Pending)+len(r.Completed))
|
|
||||||
for i := range r.Pending {
|
|
||||||
all = append(all, commonFS{&r.Pending[i], false})
|
|
||||||
}
|
|
||||||
for i := range r.Completed {
|
|
||||||
all = append(all, commonFS{&r.Completed[i], true})
|
|
||||||
}
|
|
||||||
|
|
||||||
switch state {
|
|
||||||
case pruner.Plan:
|
|
||||||
fallthrough
|
|
||||||
case pruner.PlanErr:
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(all) == 0 {
|
|
||||||
t.printf("nothing to do\n")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
var totalDestroyCount, completedDestroyCount int
|
|
||||||
var maxFSname int
|
|
||||||
for _, fs := range all {
|
|
||||||
totalDestroyCount += len(fs.DestroyList)
|
|
||||||
if fs.completed {
|
|
||||||
completedDestroyCount += len(fs.DestroyList)
|
|
||||||
}
|
|
||||||
if maxFSname < len(fs.Filesystem) {
|
|
||||||
maxFSname = len(fs.Filesystem)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// global progress bar
|
|
||||||
progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
|
|
||||||
t.write("Progress: ")
|
|
||||||
t.write("[")
|
|
||||||
t.write(times("=", progress))
|
|
||||||
t.write(">")
|
|
||||||
t.write(times("-", 80-progress))
|
|
||||||
t.write("]")
|
|
||||||
t.printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
|
|
||||||
t.newline()
|
|
||||||
|
|
||||||
sort.SliceStable(all, func(i, j int) bool {
|
|
||||||
return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
|
|
||||||
})
|
|
||||||
|
|
||||||
// Draw a table-like representation of 'all'
|
|
||||||
for _, fs := range all {
|
|
||||||
t.write(rightPad(fs.Filesystem, maxFSname, " "))
|
|
||||||
t.write(" ")
|
|
||||||
if !fs.SkipReason.NotSkipped() {
|
|
||||||
t.printf("skipped: %s\n", fs.SkipReason)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if fs.LastError != "" {
|
|
||||||
if strings.ContainsAny(fs.LastError, "\r\n") {
|
|
||||||
t.printf("ERROR:")
|
|
||||||
t.printfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError)
|
|
||||||
} else {
|
|
||||||
t.printfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError)
|
|
||||||
}
|
|
||||||
t.newline()
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
pruneRuleActionStr := fmt.Sprintf("(destroy %d of %d snapshots)",
|
|
||||||
len(fs.DestroyList), len(fs.SnapshotList))
|
|
||||||
|
|
||||||
if fs.completed {
|
|
||||||
t.printf("Completed %s\n", pruneRuleActionStr)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
t.write("Pending ") // whitespace is padding 10
|
|
||||||
if len(fs.DestroyList) == 1 {
|
|
||||||
t.write(fs.DestroyList[0].Name)
|
|
||||||
} else {
|
|
||||||
t.write(pruneRuleActionStr)
|
|
||||||
}
|
|
||||||
t.newline()
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tui) renderSnapperReport(r *snapper.Report) {
|
|
||||||
if r == nil {
|
|
||||||
t.printf("<snapshot type does not have a report>\n")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
t.printf("Status: %s", r.State)
|
|
||||||
t.newline()
|
|
||||||
|
|
||||||
if r.Error != "" {
|
|
||||||
t.printf("Error: %s\n", r.Error)
|
|
||||||
}
|
|
||||||
if !r.SleepUntil.IsZero() {
|
|
||||||
t.printf("Sleep until: %s\n", r.SleepUntil)
|
|
||||||
}
|
|
||||||
|
|
||||||
sort.Slice(r.Progress, func(i, j int) bool {
|
|
||||||
return strings.Compare(r.Progress[i].Path, r.Progress[j].Path) == -1
|
|
||||||
})
|
|
||||||
|
|
||||||
t.addIndent(1)
|
|
||||||
defer t.addIndent(-1)
|
|
||||||
dur := func(d time.Duration) string {
|
|
||||||
return d.Round(100 * time.Millisecond).String()
|
|
||||||
}
|
|
||||||
|
|
||||||
type row struct {
|
|
||||||
path, state, duration, remainder, hookReport string
|
|
||||||
}
|
|
||||||
var widths struct {
|
|
||||||
path, state, duration int
|
|
||||||
}
|
|
||||||
rows := make([]*row, len(r.Progress))
|
|
||||||
for i, fs := range r.Progress {
|
|
||||||
r := &row{
|
|
||||||
path: fs.Path,
|
|
||||||
state: fs.State.String(),
|
|
||||||
}
|
|
||||||
if fs.HooksHadError {
|
|
||||||
r.hookReport = fs.Hooks // FIXME render here, not in daemon
|
|
||||||
}
|
|
||||||
switch fs.State {
|
|
||||||
case snapper.SnapPending:
|
|
||||||
r.duration = "..."
|
|
||||||
r.remainder = ""
|
|
||||||
case snapper.SnapStarted:
|
|
||||||
r.duration = dur(time.Since(fs.StartAt))
|
|
||||||
r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
|
|
||||||
case snapper.SnapDone:
|
|
||||||
fallthrough
|
|
||||||
case snapper.SnapError:
|
|
||||||
r.duration = dur(fs.DoneAt.Sub(fs.StartAt))
|
|
||||||
r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
|
|
||||||
}
|
|
||||||
rows[i] = r
|
|
||||||
if len(r.path) > widths.path {
|
|
||||||
widths.path = len(r.path)
|
|
||||||
}
|
|
||||||
if len(r.state) > widths.state {
|
|
||||||
widths.state = len(r.state)
|
|
||||||
}
|
|
||||||
if len(r.duration) > widths.duration {
|
|
||||||
widths.duration = len(r.duration)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, r := range rows {
|
|
||||||
path := rightPad(r.path, widths.path, " ")
|
|
||||||
state := rightPad(r.state, widths.state, " ")
|
|
||||||
duration := rightPad(r.duration, widths.duration, " ")
|
|
||||||
t.printf("%s %s %s", path, state, duration)
|
|
||||||
t.printfDrawIndentedAndWrappedIfMultiline(" %s", r.remainder)
|
|
||||||
if r.hookReport != "" {
|
|
||||||
t.printfDrawIndentedAndWrappedIfMultiline("%s", r.hookReport)
|
|
||||||
}
|
|
||||||
t.newline()
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func times(str string, n int) (out string) {
|
|
||||||
for i := 0; i < n; i++ {
|
|
||||||
out += str
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
func rightPad(str string, length int, pad string) string {
|
|
||||||
if len(str) > length {
|
|
||||||
return str[:length]
|
|
||||||
}
|
|
||||||
return str + strings.Repeat(pad, length-len(str))
|
|
||||||
}
|
|
||||||
|
|
||||||
var arrowPositions = `>\|/`
|
|
||||||
|
|
||||||
// changeCount = 0 indicates stall / no progress
|
|
||||||
func (t *tui) drawBar(length int, bytes, totalBytes int64, changeCount int) {
|
|
||||||
var completedLength int
|
|
||||||
if totalBytes > 0 {
|
|
||||||
completedLength = int(int64(length) * bytes / totalBytes)
|
|
||||||
if completedLength > length {
|
|
||||||
completedLength = length
|
|
||||||
}
|
|
||||||
} else if totalBytes == bytes {
|
|
||||||
completedLength = length
|
|
||||||
}
|
|
||||||
|
|
||||||
t.write("[")
|
|
||||||
t.write(times("=", completedLength))
|
|
||||||
t.write(string(arrowPositions[changeCount%len(arrowPositions)]))
|
|
||||||
t.write(times("-", length-completedLength))
|
|
||||||
t.write("]")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *tui) printFilesystemStatus(rep *report.FilesystemReport, active bool, maxFS int) {
|
|
||||||
|
|
||||||
expected, replicated, containsInvalidSizeEstimates := rep.BytesSum()
|
|
||||||
sizeEstimationImpreciseNotice := ""
|
|
||||||
if containsInvalidSizeEstimates {
|
|
||||||
sizeEstimationImpreciseNotice = " (some steps lack size estimation)"
|
|
||||||
}
|
|
||||||
if rep.CurrentStep < len(rep.Steps) && rep.Steps[rep.CurrentStep].Info.BytesExpected == 0 {
|
|
||||||
sizeEstimationImpreciseNotice = " (step lacks size estimation)"
|
|
||||||
}
|
|
||||||
|
|
||||||
status := fmt.Sprintf("%s (step %d/%d, %s/%s)%s",
|
|
||||||
strings.ToUpper(string(rep.State)),
|
|
||||||
rep.CurrentStep, len(rep.Steps),
|
|
||||||
ByteCountBinary(replicated), ByteCountBinary(expected),
|
|
||||||
sizeEstimationImpreciseNotice,
|
|
||||||
)
|
|
||||||
|
|
||||||
activeIndicator := " "
|
|
||||||
if active {
|
|
||||||
activeIndicator = "*"
|
|
||||||
}
|
|
||||||
t.printf("%s %s %s ",
|
|
||||||
activeIndicator,
|
|
||||||
rightPad(rep.Info.Name, maxFS, " "),
|
|
||||||
status)
|
|
||||||
|
|
||||||
next := ""
|
|
||||||
if err := rep.Error(); err != nil {
|
|
||||||
next = err.Err
|
|
||||||
} else if rep.State != report.FilesystemDone {
|
|
||||||
if nextStep := rep.NextStep(); nextStep != nil {
|
|
||||||
if nextStep.IsIncremental() {
|
|
||||||
next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
|
|
||||||
} else {
|
|
||||||
next = fmt.Sprintf("next: full send %s", nextStep.Info.To)
|
|
||||||
}
|
|
||||||
attribs := []string{}
|
|
||||||
|
|
||||||
if nextStep.Info.Resumed {
|
|
||||||
attribs = append(attribs, "resumed")
|
|
||||||
}
|
|
||||||
|
|
||||||
attribs = append(attribs, fmt.Sprintf("encrypted=%s", nextStep.Info.Encrypted))
|
|
||||||
|
|
||||||
next += fmt.Sprintf(" (%s)", strings.Join(attribs, ", "))
|
|
||||||
} else {
|
|
||||||
next = "" // individual FSes may still be in planning state
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
t.printfDrawIndentedAndWrappedIfMultiline("%s", next)
|
|
||||||
|
|
||||||
t.newline()
|
|
||||||
}
|
|
||||||
|
|
||||||
func ByteCountBinary(b int64) string {
|
|
||||||
const unit = 1024
|
|
||||||
if b < unit {
|
|
||||||
return fmt.Sprintf("%d B", b)
|
|
||||||
}
|
|
||||||
div, exp := int64(unit), 0
|
|
||||||
for n := b / unit; n >= unit; n /= unit {
|
|
||||||
div *= unit
|
|
||||||
exp++
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
|
|
||||||
}
|
|
||||||
|
|
||||||
func humanizeDuration(duration time.Duration) string {
|
|
||||||
days := int64(duration.Hours() / 24)
|
|
||||||
hours := int64(math.Mod(duration.Hours(), 24))
|
|
||||||
minutes := int64(math.Mod(duration.Minutes(), 60))
|
|
||||||
seconds := int64(math.Mod(duration.Seconds(), 60))
|
|
||||||
|
|
||||||
var parts []string
|
|
||||||
|
|
||||||
force := false
|
|
||||||
chunks := []int64{days, hours, minutes, seconds}
|
|
||||||
for i, chunk := range chunks {
|
|
||||||
if force || chunk > 0 {
|
|
||||||
padding := 0
|
|
||||||
if force {
|
|
||||||
padding = 2
|
|
||||||
}
|
|
||||||
parts = append(parts, fmt.Sprintf("%*d%c", padding, chunk, "dhms"[i]))
|
|
||||||
force = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return strings.Join(parts, " ")
|
|
||||||
}
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
package client
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/problame/go-netssh"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/cli"
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"log"
|
|
||||||
"path"
|
|
||||||
)
|
|
||||||
|
|
||||||
var StdinserverCmd = &cli.Subcommand{
|
|
||||||
Use: "stdinserver CLIENT_IDENTITY",
|
|
||||||
Short: "stdinserver transport mode (started from authorized_keys file as forced command)",
|
|
||||||
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
|
||||||
return runStdinserver(subcommand.Config(), args)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
func runStdinserver(config *config.Config, args []string) error {
|
|
||||||
|
|
||||||
// NOTE: the netssh proxying protocol requires exiting with non-zero status if anything goes wrong
|
|
||||||
defer os.Exit(1)
|
|
||||||
|
|
||||||
log := log.New(os.Stderr, "", log.LUTC|log.Ldate|log.Ltime)
|
|
||||||
|
|
||||||
if len(args) != 1 || args[0] == "" {
|
|
||||||
err := errors.New("must specify client_identity as positional argument")
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
identity := args[0]
|
|
||||||
unixaddr := path.Join(config.Global.Serve.StdinServer.SockDir, identity)
|
|
||||||
|
|
||||||
log.Printf("proxying client identity '%s' to zrepl daemon '%s'", identity, unixaddr)
|
|
||||||
|
|
||||||
ctx := netssh.ContextWithLog(context.TODO(), log)
|
|
||||||
|
|
||||||
err := netssh.Proxy(ctx, unixaddr)
|
|
||||||
if err == nil {
|
|
||||||
log.Print("proxying finished successfully, exiting with status 0")
|
|
||||||
os.Exit(0)
|
|
||||||
}
|
|
||||||
log.Printf("error proxying: %s", err)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,212 +0,0 @@
|
|||||||
package client
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
"github.com/spf13/pflag"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/cli"
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
"github.com/zrepl/zrepl/daemon/filters"
|
|
||||||
"github.com/zrepl/zrepl/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
var TestCmd = &cli.Subcommand{
|
|
||||||
Use: "test",
|
|
||||||
SetupSubcommands: func() []*cli.Subcommand {
|
|
||||||
return []*cli.Subcommand{testFilter, testPlaceholder, testDecodeResumeToken}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
var testFilterArgs struct {
|
|
||||||
job string
|
|
||||||
all bool
|
|
||||||
input string
|
|
||||||
}
|
|
||||||
|
|
||||||
var testFilter = &cli.Subcommand{
|
|
||||||
Use: "filesystems --job JOB [--all | --input INPUT]",
|
|
||||||
Short: "test filesystems filter specified in push or source job",
|
|
||||||
SetupFlags: func(f *pflag.FlagSet) {
|
|
||||||
f.StringVar(&testFilterArgs.job, "job", "", "the name of the push or source job")
|
|
||||||
f.StringVar(&testFilterArgs.input, "input", "", "a filesystem name to test against the job's filters")
|
|
||||||
f.BoolVar(&testFilterArgs.all, "all", false, "test all local filesystems")
|
|
||||||
},
|
|
||||||
Run: runTestFilterCmd,
|
|
||||||
}
|
|
||||||
|
|
||||||
func runTestFilterCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
|
||||||
|
|
||||||
if testFilterArgs.job == "" {
|
|
||||||
return fmt.Errorf("must specify --job flag")
|
|
||||||
}
|
|
||||||
if !(testFilterArgs.all != (testFilterArgs.input != "")) { // xor
|
|
||||||
return fmt.Errorf("must set one: --all or --input")
|
|
||||||
}
|
|
||||||
|
|
||||||
conf := subcommand.Config()
|
|
||||||
|
|
||||||
var confFilter config.FilesystemsFilter
|
|
||||||
job, err := conf.Job(testFilterArgs.job)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
switch j := job.Ret.(type) {
|
|
||||||
case *config.SourceJob:
|
|
||||||
confFilter = j.Filesystems
|
|
||||||
case *config.PushJob:
|
|
||||||
confFilter = j.Filesystems
|
|
||||||
case *config.SnapJob:
|
|
||||||
confFilter = j.Filesystems
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("job type %T does not have filesystems filter", j)
|
|
||||||
}
|
|
||||||
|
|
||||||
f, err := filters.DatasetMapFilterFromConfig(confFilter)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("filter invalid: %s", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var fsnames []string
|
|
||||||
if testFilterArgs.input != "" {
|
|
||||||
fsnames = []string{testFilterArgs.input}
|
|
||||||
} else {
|
|
||||||
out, err := zfs.ZFSList(ctx, []string{"name"})
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("could not list ZFS filesystems: %s", err)
|
|
||||||
}
|
|
||||||
for _, row := range out {
|
|
||||||
|
|
||||||
fsnames = append(fsnames, row[0])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fspaths := make([]*zfs.DatasetPath, len(fsnames))
|
|
||||||
for i, fsname := range fsnames {
|
|
||||||
path, err := zfs.NewDatasetPath(fsname)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fspaths[i] = path
|
|
||||||
}
|
|
||||||
|
|
||||||
hadFilterErr := false
|
|
||||||
for _, in := range fspaths {
|
|
||||||
var res string
|
|
||||||
var errStr string
|
|
||||||
pass, err := f.Filter(in)
|
|
||||||
if err != nil {
|
|
||||||
res = "ERROR"
|
|
||||||
errStr = err.Error()
|
|
||||||
hadFilterErr = true
|
|
||||||
} else if pass {
|
|
||||||
res = "ACCEPT"
|
|
||||||
} else {
|
|
||||||
res = "REJECT"
|
|
||||||
}
|
|
||||||
fmt.Printf("%s\t%s\t%s\n", res, in.ToString(), errStr)
|
|
||||||
}
|
|
||||||
|
|
||||||
if hadFilterErr {
|
|
||||||
return fmt.Errorf("filter errors occurred")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var testPlaceholderArgs struct {
|
|
||||||
ds string
|
|
||||||
all bool
|
|
||||||
}
|
|
||||||
|
|
||||||
var testPlaceholder = &cli.Subcommand{
|
|
||||||
Use: "placeholder [--all | --dataset DATASET]",
|
|
||||||
Short: fmt.Sprintf("list received placeholder filesystems (zfs property %q)", zfs.PlaceholderPropertyName),
|
|
||||||
Example: `
|
|
||||||
placeholder --all
|
|
||||||
placeholder --dataset path/to/sink/clientident/fs`,
|
|
||||||
NoRequireConfig: true,
|
|
||||||
SetupFlags: func(f *pflag.FlagSet) {
|
|
||||||
f.StringVar(&testPlaceholderArgs.ds, "dataset", "", "dataset path (not required to exist)")
|
|
||||||
f.BoolVar(&testPlaceholderArgs.all, "all", false, "list tab-separated placeholder status of all filesystems")
|
|
||||||
},
|
|
||||||
Run: runTestPlaceholder,
|
|
||||||
}
|
|
||||||
|
|
||||||
func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
|
||||||
|
|
||||||
var checkDPs []*zfs.DatasetPath
|
|
||||||
|
|
||||||
// all actions first
|
|
||||||
if testPlaceholderArgs.all {
|
|
||||||
out, err := zfs.ZFSList(ctx, []string{"name"})
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrap(err, "could not list ZFS filesystems")
|
|
||||||
}
|
|
||||||
for _, row := range out {
|
|
||||||
dp, err := zfs.NewDatasetPath(row[0])
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
checkDPs = append(checkDPs, dp)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
dp, err := zfs.NewDatasetPath(testPlaceholderArgs.ds)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if dp.Empty() {
|
|
||||||
return fmt.Errorf("must specify --dataset DATASET or --all")
|
|
||||||
}
|
|
||||||
checkDPs = append(checkDPs, dp)
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Printf("IS_PLACEHOLDER\tDATASET\tzrepl:placeholder\n")
|
|
||||||
for _, dp := range checkDPs {
|
|
||||||
ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, dp)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrap(err, "cannot get placeholder state")
|
|
||||||
}
|
|
||||||
if !ph.FSExists {
|
|
||||||
panic("placeholder state inconsistent: filesystem " + ph.FS + " must exist in this context")
|
|
||||||
}
|
|
||||||
is := "yes"
|
|
||||||
if !ph.IsPlaceholder {
|
|
||||||
is = "no"
|
|
||||||
}
|
|
||||||
fmt.Printf("%s\t%s\t%s\n", is, dp.ToString(), ph.RawLocalPropertyValue)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var testDecodeResumeTokenArgs struct {
|
|
||||||
token string
|
|
||||||
}
|
|
||||||
|
|
||||||
var testDecodeResumeToken = &cli.Subcommand{
|
|
||||||
Use: "decoderesumetoken --token TOKEN",
|
|
||||||
Short: "decode resume token",
|
|
||||||
SetupFlags: func(f *pflag.FlagSet) {
|
|
||||||
f.StringVar(&testDecodeResumeTokenArgs.token, "token", "", "the resume token obtained from the receive_resume_token property")
|
|
||||||
},
|
|
||||||
Run: runTestDecodeResumeTokenCmd,
|
|
||||||
}
|
|
||||||
|
|
||||||
func runTestDecodeResumeTokenCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
|
||||||
if testDecodeResumeTokenArgs.token == "" {
|
|
||||||
return fmt.Errorf("token argument must be specified")
|
|
||||||
}
|
|
||||||
token, err := zfs.ParseResumeToken(ctx, testDecodeResumeTokenArgs.token)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
enc := json.NewEncoder(os.Stdout)
|
|
||||||
enc.SetIndent("", " ")
|
|
||||||
if err := enc.Encode(&token); err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,75 +0,0 @@
|
|||||||
package client
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/spf13/pflag"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/cli"
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
"github.com/zrepl/zrepl/daemon"
|
|
||||||
"github.com/zrepl/zrepl/version"
|
|
||||||
)
|
|
||||||
|
|
||||||
var versionArgs struct {
|
|
||||||
Show string
|
|
||||||
Config *config.Config
|
|
||||||
ConfigErr error
|
|
||||||
}
|
|
||||||
|
|
||||||
var VersionCmd = &cli.Subcommand{
|
|
||||||
Use: "version",
|
|
||||||
Short: "print version of zrepl binary and running daemon",
|
|
||||||
NoRequireConfig: true,
|
|
||||||
SetupFlags: func(f *pflag.FlagSet) {
|
|
||||||
f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)")
|
|
||||||
},
|
|
||||||
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
|
||||||
versionArgs.Config = subcommand.Config()
|
|
||||||
versionArgs.ConfigErr = subcommand.ConfigParsingError()
|
|
||||||
return runVersionCmd()
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
func runVersionCmd() error {
|
|
||||||
args := versionArgs
|
|
||||||
|
|
||||||
if args.Show != "daemon" && args.Show != "client" && args.Show != "" {
|
|
||||||
return fmt.Errorf("show flag must be 'client' or 'server' or be left empty")
|
|
||||||
}
|
|
||||||
|
|
||||||
var clientVersion, daemonVersion *version.ZreplVersionInformation
|
|
||||||
if args.Show == "client" || args.Show == "" {
|
|
||||||
clientVersion = version.NewZreplVersionInformation()
|
|
||||||
fmt.Printf("client: %s\n", clientVersion.String())
|
|
||||||
}
|
|
||||||
if args.Show == "daemon" || args.Show == "" {
|
|
||||||
|
|
||||||
if args.ConfigErr != nil {
|
|
||||||
return fmt.Errorf("config parsing error: %s", args.ConfigErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
httpc, err := controlHttpClient(args.Config.Global.Control.SockPath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("server: error: %s\n", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var info version.ZreplVersionInformation
|
|
||||||
err = jsonRequestResponse(httpc, daemon.ControlJobEndpointVersion, "", &info)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("server: error: %s\n", err)
|
|
||||||
}
|
|
||||||
daemonVersion = &info
|
|
||||||
fmt.Printf("server: %s\n", daemonVersion.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
if args.Show == "" {
|
|
||||||
if clientVersion.Version != daemonVersion.Version {
|
|
||||||
fmt.Fprintf(os.Stderr, "WARNING: client version != daemon version, restart zrepl daemon\n")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,147 +0,0 @@
|
|||||||
package client
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/spf13/pflag"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/cli"
|
|
||||||
"github.com/zrepl/zrepl/daemon/filters"
|
|
||||||
"github.com/zrepl/zrepl/endpoint"
|
|
||||||
"github.com/zrepl/zrepl/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
ZFSAbstractionsCmd = &cli.Subcommand{
|
|
||||||
Use: "zfs-abstraction",
|
|
||||||
Short: "manage abstractions that zrepl builds on top of ZFS",
|
|
||||||
SetupSubcommands: func() []*cli.Subcommand {
|
|
||||||
return []*cli.Subcommand{
|
|
||||||
zabsCmdList,
|
|
||||||
zabsCmdReleaseAll,
|
|
||||||
zabsCmdReleaseStale,
|
|
||||||
zabsCmdCreate,
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
// a common set of CLI flags that map to the fields of an
|
|
||||||
// endpoint.ListZFSHoldsAndBookmarksQuery
|
|
||||||
type zabsFilterFlags struct {
|
|
||||||
Filesystems FilesystemsFilterFlag
|
|
||||||
Job JobIDFlag
|
|
||||||
Types AbstractionTypesFlag
|
|
||||||
Concurrency int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// produce a query from the CLI flags
|
|
||||||
func (f zabsFilterFlags) Query() (endpoint.ListZFSHoldsAndBookmarksQuery, error) {
|
|
||||||
q := endpoint.ListZFSHoldsAndBookmarksQuery{
|
|
||||||
FS: f.Filesystems.FlagValue(),
|
|
||||||
What: f.Types.FlagValue(),
|
|
||||||
JobID: f.Job.FlagValue(),
|
|
||||||
Concurrency: f.Concurrency,
|
|
||||||
}
|
|
||||||
return q, q.Validate()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *zabsFilterFlags) registerZabsFilterFlags(s *pflag.FlagSet, verb string) {
|
|
||||||
// Note: the default value is defined in the .FlagValue methods
|
|
||||||
s.Var(&f.Filesystems, "fs", fmt.Sprintf("only %s holds on the specified filesystem [default: all filesystems] [comma-separated list of <dataset-pattern>:<ok|!> pairs]", verb))
|
|
||||||
s.Var(&f.Job, "job", fmt.Sprintf("only %s holds created by the specified job [default: any job]", verb))
|
|
||||||
|
|
||||||
variants := make([]string, 0, len(endpoint.AbstractionTypesAll))
|
|
||||||
for v := range endpoint.AbstractionTypesAll {
|
|
||||||
variants = append(variants, string(v))
|
|
||||||
}
|
|
||||||
variants = sort.StringSlice(variants)
|
|
||||||
variantsJoined := strings.Join(variants, "|")
|
|
||||||
s.Var(&f.Types, "type", fmt.Sprintf("only %s holds of the specified type [default: all] [comma-separated list of %s]", verb, variantsJoined))
|
|
||||||
|
|
||||||
s.Int64VarP(&f.Concurrency, "concurrency", "p", 1, "number of concurrently queried filesystems")
|
|
||||||
}
|
|
||||||
|
|
||||||
type JobIDFlag struct{ J *endpoint.JobID }
|
|
||||||
|
|
||||||
func (f *JobIDFlag) Set(s string) error {
|
|
||||||
if len(s) == 0 {
|
|
||||||
*f = JobIDFlag{J: nil}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
jobID, err := endpoint.MakeJobID(s)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
*f = JobIDFlag{J: &jobID}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
func (f JobIDFlag) Type() string { return "job-ID" }
|
|
||||||
func (f JobIDFlag) String() string { return fmt.Sprint(f.J) }
|
|
||||||
func (f JobIDFlag) FlagValue() *endpoint.JobID { return f.J }
|
|
||||||
|
|
||||||
type AbstractionTypesFlag map[endpoint.AbstractionType]bool
|
|
||||||
|
|
||||||
func (f *AbstractionTypesFlag) Set(s string) error {
|
|
||||||
ats, err := endpoint.AbstractionTypeSetFromStrings(strings.Split(s, ","))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
*f = AbstractionTypesFlag(ats)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
func (f AbstractionTypesFlag) Type() string { return "abstraction-type" }
|
|
||||||
func (f AbstractionTypesFlag) String() string {
|
|
||||||
return endpoint.AbstractionTypeSet(f).String()
|
|
||||||
}
|
|
||||||
func (f AbstractionTypesFlag) FlagValue() map[endpoint.AbstractionType]bool {
|
|
||||||
if len(f) > 0 {
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
return endpoint.AbstractionTypesAll
|
|
||||||
}
|
|
||||||
|
|
||||||
type FilesystemsFilterFlag struct {
|
|
||||||
F endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter
|
|
||||||
}
|
|
||||||
|
|
||||||
func (flag *FilesystemsFilterFlag) Set(s string) error {
|
|
||||||
mappings := strings.Split(s, ",")
|
|
||||||
if len(mappings) == 1 && !strings.Contains(mappings[0], ":") {
|
|
||||||
flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
|
|
||||||
FS: &mappings[0],
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
f := filters.NewDatasetMapFilter(len(mappings), true)
|
|
||||||
for _, m := range mappings {
|
|
||||||
thisMappingErr := fmt.Errorf("expecting comma-separated list of <dataset-pattern>:<ok|!> pairs, got %q", m)
|
|
||||||
lhsrhs := strings.SplitN(m, ":", 2)
|
|
||||||
if len(lhsrhs) != 2 {
|
|
||||||
return thisMappingErr
|
|
||||||
}
|
|
||||||
err := f.Add(lhsrhs[0], lhsrhs[1])
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("%s: %s", thisMappingErr, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
|
|
||||||
Filter: f,
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
func (flag FilesystemsFilterFlag) Type() string { return "filesystem filter spec" }
|
|
||||||
func (flag FilesystemsFilterFlag) String() string {
|
|
||||||
return fmt.Sprintf("%v", flag.F)
|
|
||||||
}
|
|
||||||
func (flag FilesystemsFilterFlag) FlagValue() endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter {
|
|
||||||
var z FilesystemsFilterFlag
|
|
||||||
if flag == z {
|
|
||||||
return endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{Filter: zfs.NoFilter()}
|
|
||||||
}
|
|
||||||
return flag.F
|
|
||||||
}
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
package client
|
|
||||||
|
|
||||||
import "github.com/zrepl/zrepl/cli"
|
|
||||||
|
|
||||||
var zabsCmdCreate = &cli.Subcommand{
|
|
||||||
Use: "create",
|
|
||||||
NoRequireConfig: true,
|
|
||||||
Short: `create zrepl ZFS abstractions (mostly useful for debugging & development, users should not need to use this command)`,
|
|
||||||
SetupSubcommands: func() []*cli.Subcommand {
|
|
||||||
return []*cli.Subcommand{
|
|
||||||
zabsCmdCreateStepHold,
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
@@ -1,58 +0,0 @@
|
|||||||
package client
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
"github.com/spf13/pflag"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/cli"
|
|
||||||
"github.com/zrepl/zrepl/endpoint"
|
|
||||||
"github.com/zrepl/zrepl/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
var zabsCreateStepHoldFlags struct {
|
|
||||||
target string
|
|
||||||
jobid JobIDFlag
|
|
||||||
}
|
|
||||||
|
|
||||||
var zabsCmdCreateStepHold = &cli.Subcommand{
|
|
||||||
Use: "step",
|
|
||||||
Run: doZabsCreateStep,
|
|
||||||
NoRequireConfig: true,
|
|
||||||
Short: `create a step hold or bookmark`,
|
|
||||||
SetupFlags: func(f *pflag.FlagSet) {
|
|
||||||
f.StringVarP(&zabsCreateStepHoldFlags.target, "target", "t", "", "snapshot to be held / bookmark to be held")
|
|
||||||
f.VarP(&zabsCreateStepHoldFlags.jobid, "jobid", "j", "jobid for which the hold is installed")
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
func doZabsCreateStep(ctx context.Context, sc *cli.Subcommand, args []string) error {
|
|
||||||
if len(args) > 0 {
|
|
||||||
return errors.New("subcommand takes no arguments")
|
|
||||||
}
|
|
||||||
|
|
||||||
f := &zabsCreateStepHoldFlags
|
|
||||||
|
|
||||||
fs, _, _, err := zfs.DecomposeVersionString(f.target)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrapf(err, "%q invalid target", f.target)
|
|
||||||
}
|
|
||||||
|
|
||||||
if f.jobid.FlagValue() == nil {
|
|
||||||
return errors.Errorf("jobid must be set")
|
|
||||||
}
|
|
||||||
|
|
||||||
v, err := zfs.ZFSGetFilesystemVersion(ctx, f.target)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrapf(err, "get info about target %q", f.target)
|
|
||||||
}
|
|
||||||
|
|
||||||
step, err := endpoint.HoldStep(ctx, fs, v, *f.jobid.FlagValue())
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrap(err, "create step hold")
|
|
||||||
}
|
|
||||||
fmt.Println(step.String())
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,99 +0,0 @@
|
|||||||
package client
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
"github.com/fatih/color"
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
"github.com/spf13/pflag"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/cli"
|
|
||||||
"github.com/zrepl/zrepl/endpoint"
|
|
||||||
"github.com/zrepl/zrepl/util/chainlock"
|
|
||||||
)
|
|
||||||
|
|
||||||
var zabsListFlags struct {
|
|
||||||
Filter zabsFilterFlags
|
|
||||||
Json bool
|
|
||||||
}
|
|
||||||
|
|
||||||
var zabsCmdList = &cli.Subcommand{
|
|
||||||
Use: "list",
|
|
||||||
Short: `list zrepl ZFS abstractions`,
|
|
||||||
Run: doZabsList,
|
|
||||||
NoRequireConfig: true,
|
|
||||||
SetupFlags: func(f *pflag.FlagSet) {
|
|
||||||
zabsListFlags.Filter.registerZabsFilterFlags(f, "list")
|
|
||||||
f.BoolVar(&zabsListFlags.Json, "json", false, "emit JSON")
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
func doZabsList(ctx context.Context, sc *cli.Subcommand, args []string) error {
|
|
||||||
var err error
|
|
||||||
|
|
||||||
if len(args) > 0 {
|
|
||||||
return errors.New("this subcommand takes no positional arguments")
|
|
||||||
}
|
|
||||||
|
|
||||||
q, err := zabsListFlags.Filter.Query()
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrap(err, "invalid filter specification on command line")
|
|
||||||
}
|
|
||||||
|
|
||||||
abstractions, errors, err := endpoint.ListAbstractionsStreamed(ctx, q)
|
|
||||||
if err != nil {
|
|
||||||
return err // context clear by invocation of command
|
|
||||||
}
|
|
||||||
|
|
||||||
var line chainlock.L
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
defer wg.Wait()
|
|
||||||
|
|
||||||
// print results
|
|
||||||
wg.Add(1)
|
|
||||||
go func() {
|
|
||||||
defer wg.Done()
|
|
||||||
enc := json.NewEncoder(os.Stdout)
|
|
||||||
for a := range abstractions {
|
|
||||||
func() {
|
|
||||||
defer line.Lock().Unlock()
|
|
||||||
if zabsListFlags.Json {
|
|
||||||
enc.SetIndent("", " ")
|
|
||||||
if err := enc.Encode(a); err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
fmt.Println()
|
|
||||||
} else {
|
|
||||||
fmt.Println(a)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
// print errors to stderr
|
|
||||||
errorColor := color.New(color.FgRed)
|
|
||||||
var errorsSlice []endpoint.ListAbstractionsError
|
|
||||||
wg.Add(1)
|
|
||||||
go func() {
|
|
||||||
defer wg.Done()
|
|
||||||
for err := range errors {
|
|
||||||
func() {
|
|
||||||
defer line.Lock().Unlock()
|
|
||||||
errorsSlice = append(errorsSlice, err)
|
|
||||||
errorColor.Fprintf(os.Stderr, "%s\n", err)
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
wg.Wait()
|
|
||||||
if len(errorsSlice) > 0 {
|
|
||||||
errorColor.Add(color.Bold).Fprintf(os.Stderr, "there were errors in listing the abstractions")
|
|
||||||
return fmt.Errorf("")
|
|
||||||
} else {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,143 +0,0 @@
|
|||||||
package client
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/fatih/color"
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
"github.com/spf13/pflag"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/cli"
|
|
||||||
"github.com/zrepl/zrepl/endpoint"
|
|
||||||
)
|
|
||||||
|
|
||||||
// shared between release-all and release-step
|
|
||||||
var zabsReleaseFlags struct {
|
|
||||||
Filter zabsFilterFlags
|
|
||||||
Json bool
|
|
||||||
DryRun bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func registerZabsReleaseFlags(s *pflag.FlagSet) {
|
|
||||||
zabsReleaseFlags.Filter.registerZabsFilterFlags(s, "release")
|
|
||||||
s.BoolVar(&zabsReleaseFlags.Json, "json", false, "emit json instead of pretty-printed")
|
|
||||||
s.BoolVar(&zabsReleaseFlags.DryRun, "dry-run", false, "do a dry-run")
|
|
||||||
}
|
|
||||||
|
|
||||||
var zabsCmdReleaseAll = &cli.Subcommand{
|
|
||||||
Use: "release-all",
|
|
||||||
Run: doZabsReleaseAll,
|
|
||||||
NoRequireConfig: true,
|
|
||||||
Short: `(DANGEROUS) release ALL zrepl ZFS abstractions (mostly useful for uninstalling zrepl completely or for "de-zrepl-ing" a filesystem)`,
|
|
||||||
SetupFlags: registerZabsReleaseFlags,
|
|
||||||
}
|
|
||||||
|
|
||||||
var zabsCmdReleaseStale = &cli.Subcommand{
|
|
||||||
Use: "release-stale",
|
|
||||||
Run: doZabsReleaseStale,
|
|
||||||
NoRequireConfig: true,
|
|
||||||
Short: `release stale zrepl ZFS abstractions (useful if zrepl has a bug and does not do it by itself)`,
|
|
||||||
SetupFlags: registerZabsReleaseFlags,
|
|
||||||
}
|
|
||||||
|
|
||||||
func doZabsReleaseAll(ctx context.Context, sc *cli.Subcommand, args []string) error {
|
|
||||||
var err error
|
|
||||||
|
|
||||||
if len(args) > 0 {
|
|
||||||
return errors.New("this subcommand takes no positional arguments")
|
|
||||||
}
|
|
||||||
|
|
||||||
q, err := zabsReleaseFlags.Filter.Query()
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrap(err, "invalid filter specification on command line")
|
|
||||||
}
|
|
||||||
|
|
||||||
abstractions, listErrors, err := endpoint.ListAbstractions(ctx, q)
|
|
||||||
if err != nil {
|
|
||||||
return err // context clear by invocation of command
|
|
||||||
}
|
|
||||||
if len(listErrors) > 0 {
|
|
||||||
color.New(color.FgRed).Fprintf(os.Stderr, "there were errors in listing the abstractions:\n%s\n", listErrors)
|
|
||||||
// proceed anyways with rest of abstractions
|
|
||||||
}
|
|
||||||
|
|
||||||
return doZabsRelease_Common(ctx, abstractions)
|
|
||||||
}
|
|
||||||
|
|
||||||
func doZabsReleaseStale(ctx context.Context, sc *cli.Subcommand, args []string) error {
|
|
||||||
|
|
||||||
var err error
|
|
||||||
|
|
||||||
if len(args) > 0 {
|
|
||||||
return errors.New("this subcommand takes no positional arguments")
|
|
||||||
}
|
|
||||||
|
|
||||||
q, err := zabsReleaseFlags.Filter.Query()
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrap(err, "invalid filter specification on command line")
|
|
||||||
}
|
|
||||||
|
|
||||||
stalenessInfo, err := endpoint.ListStale(ctx, q)
|
|
||||||
if err != nil {
|
|
||||||
return err // context clear by invocation of command
|
|
||||||
}
|
|
||||||
|
|
||||||
return doZabsRelease_Common(ctx, stalenessInfo.Stale)
|
|
||||||
}
|
|
||||||
|
|
||||||
func doZabsRelease_Common(ctx context.Context, destroy []endpoint.Abstraction) error {
|
|
||||||
|
|
||||||
if zabsReleaseFlags.DryRun {
|
|
||||||
if zabsReleaseFlags.Json {
|
|
||||||
m, err := json.MarshalIndent(destroy, "", " ")
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
if _, err := os.Stdout.Write(m); err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
fmt.Println()
|
|
||||||
} else {
|
|
||||||
for _, a := range destroy {
|
|
||||||
fmt.Printf("would destroy %s\n", a)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
outcome := endpoint.BatchDestroy(ctx, destroy)
|
|
||||||
hadErr := false
|
|
||||||
|
|
||||||
enc := json.NewEncoder(os.Stdout)
|
|
||||||
enc.SetIndent("", " ")
|
|
||||||
colorErr := color.New(color.FgRed)
|
|
||||||
printfSuccess := color.New(color.FgGreen).FprintfFunc()
|
|
||||||
printfSection := color.New(color.Bold).FprintfFunc()
|
|
||||||
|
|
||||||
for res := range outcome {
|
|
||||||
hadErr = hadErr || res.DestroyErr != nil
|
|
||||||
if zabsReleaseFlags.Json {
|
|
||||||
err := enc.Encode(res)
|
|
||||||
if err != nil {
|
|
||||||
colorErr.Fprintf(os.Stderr, "cannot marshal there were errors in destroying the abstractions")
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
printfSection(os.Stdout, "destroy %s ...", res.Abstraction)
|
|
||||||
if res.DestroyErr != nil {
|
|
||||||
colorErr.Fprintf(os.Stdout, " failed:\n%s\n", res.DestroyErr)
|
|
||||||
} else {
|
|
||||||
printfSuccess(os.Stdout, " OK\n")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if hadErr {
|
|
||||||
colorErr.Add(color.Bold).Fprintf(os.Stderr, "there were errors in destroying the abstractions")
|
|
||||||
return fmt.Errorf("")
|
|
||||||
} else {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,91 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"github.com/zrepl/zrepl/jobrun"
|
||||||
|
"github.com/zrepl/zrepl/zfs"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
var AutosnapCmd = &cobra.Command{
|
||||||
|
Use: "autosnap",
|
||||||
|
Short: "perform automatic snapshotting",
|
||||||
|
Run: cmdAutosnap,
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
RootCmd.AddCommand(AutosnapCmd)
|
||||||
|
}
|
||||||
|
|
||||||
|
func cmdAutosnap(cmd *cobra.Command, args []string) {
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
runner.Start()
|
||||||
|
}()
|
||||||
|
|
||||||
|
if len(args) < 1 {
|
||||||
|
log.Printf("must specify exactly one job as positional argument")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
snap, ok := conf.Autosnaps[args[0]]
|
||||||
|
if !ok {
|
||||||
|
log.Printf("could not find autosnap job: %s", args[0])
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
job := jobrun.Job{
|
||||||
|
Name: snap.JobName,
|
||||||
|
RepeatStrategy: snap.Interval,
|
||||||
|
RunFunc: func(log jobrun.Logger) error {
|
||||||
|
log.Printf("doing autosnap: %v", snap)
|
||||||
|
ctx := AutosnapContext{snap}
|
||||||
|
return doAutosnap(ctx, log)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
runner.AddJob(job)
|
||||||
|
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
type AutosnapContext struct {
|
||||||
|
Autosnap *Autosnap
|
||||||
|
}
|
||||||
|
|
||||||
|
func doAutosnap(ctx AutosnapContext, log Logger) (err error) {
|
||||||
|
|
||||||
|
snap := ctx.Autosnap
|
||||||
|
|
||||||
|
filesystems, err := zfs.ZFSListMapping(snap.DatasetFilter)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot filter datasets: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
|
||||||
|
snapname := fmt.Sprintf("%s%s", snap.Prefix, suffix)
|
||||||
|
|
||||||
|
hadError := false
|
||||||
|
|
||||||
|
for _, fs := range filesystems { // optimization: use recursive snapshots / channel programs here
|
||||||
|
log.Printf("snapshotting filesystem %s@%s", fs, snapname)
|
||||||
|
err := zfs.ZFSSnapshot(fs, snapname, false)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("error snapshotting %s: %s", fs, err)
|
||||||
|
hadError = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if hadError {
|
||||||
|
err = fmt.Errorf("errors occurred during autosnap, check logs for details")
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
}
|
||||||
+730
@@ -0,0 +1,730 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"github.com/jinzhu/copier"
|
||||||
|
"github.com/mitchellh/mapstructure"
|
||||||
|
"github.com/zrepl/zrepl/jobrun"
|
||||||
|
"github.com/zrepl/zrepl/rpc"
|
||||||
|
"github.com/zrepl/zrepl/sshbytestream"
|
||||||
|
. "github.com/zrepl/zrepl/util"
|
||||||
|
"github.com/zrepl/zrepl/zfs"
|
||||||
|
yaml "gopkg.in/yaml.v2"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
JobSectionPush string = "push"
|
||||||
|
JobSectionPull string = "pull"
|
||||||
|
JobSectionPrune string = "prune"
|
||||||
|
JobSectionAutosnap string = "autosnap"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Remote struct {
|
||||||
|
Name string
|
||||||
|
Transport Transport
|
||||||
|
}
|
||||||
|
|
||||||
|
type Transport interface {
|
||||||
|
Connect(rpcLog Logger) (rpc.RPCRequester, error)
|
||||||
|
}
|
||||||
|
type LocalTransport struct {
|
||||||
|
Handler rpc.RPCHandler
|
||||||
|
}
|
||||||
|
type SSHTransport struct {
|
||||||
|
Host string
|
||||||
|
User string
|
||||||
|
Port uint16
|
||||||
|
IdentityFile string `mapstructure:"identity_file"`
|
||||||
|
TransportOpenCommand []string `mapstructure:"transport_open_command"`
|
||||||
|
SSHCommand string `mapstructure:"ssh_command"`
|
||||||
|
Options []string
|
||||||
|
ConnLogReadFile string `mapstructure:"connlog_read_file"`
|
||||||
|
ConnLogWriteFile string `mapstructure:"connlog_write_file"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Push struct {
|
||||||
|
JobName string // for use with jobrun package
|
||||||
|
To *Remote
|
||||||
|
Filter zfs.DatasetFilter
|
||||||
|
InitialReplPolicy rpc.InitialReplPolicy
|
||||||
|
RepeatStrategy jobrun.RepeatStrategy
|
||||||
|
}
|
||||||
|
type Pull struct {
|
||||||
|
JobName string // for use with jobrun package
|
||||||
|
From *Remote
|
||||||
|
Mapping DatasetMapFilter
|
||||||
|
InitialReplPolicy rpc.InitialReplPolicy
|
||||||
|
RepeatStrategy jobrun.RepeatStrategy
|
||||||
|
}
|
||||||
|
|
||||||
|
type Prune struct {
|
||||||
|
JobName string // for use with jobrun package
|
||||||
|
DatasetFilter zfs.DatasetFilter
|
||||||
|
SnapshotFilter zfs.FilesystemVersionFilter
|
||||||
|
RetentionPolicy *RetentionGrid // TODO abstract interface to support future policies?
|
||||||
|
}
|
||||||
|
|
||||||
|
type Autosnap struct {
|
||||||
|
JobName string // for use with jobrun package
|
||||||
|
Prefix string
|
||||||
|
Interval jobrun.RepeatStrategy
|
||||||
|
DatasetFilter zfs.DatasetFilter
|
||||||
|
}
|
||||||
|
|
||||||
|
type Config struct {
|
||||||
|
Remotes map[string]*Remote
|
||||||
|
Pushs map[string]*Push // job name -> job
|
||||||
|
Pulls map[string]*Pull // job name -> job
|
||||||
|
Sinks map[string]DatasetMapFilter // client identity -> mapping
|
||||||
|
PullACLs map[string]DatasetMapFilter // client identity -> filter
|
||||||
|
Prunes map[string]*Prune // job name -> job
|
||||||
|
Autosnaps map[string]*Autosnap // job name -> job
|
||||||
|
}
|
||||||
|
|
||||||
|
func ParseConfig(path string) (config Config, err error) {
|
||||||
|
|
||||||
|
c := make(map[string]interface{}, 0)
|
||||||
|
|
||||||
|
var bytes []byte
|
||||||
|
|
||||||
|
if bytes, err = ioutil.ReadFile(path); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = yaml.Unmarshal(bytes, &c); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
return parseMain(c)
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseMain(root map[string]interface{}) (c Config, err error) {
|
||||||
|
if c.Remotes, err = parseRemotes(root["remotes"]); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
remoteLookup := func(name string) (remote *Remote, err error) {
|
||||||
|
remote = c.Remotes[name]
|
||||||
|
if remote == nil {
|
||||||
|
err = fmt.Errorf("remote '%s' not defined", name)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.Pushs, err = parsePushs(root["pushs"], remoteLookup); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if c.Pulls, err = parsePulls(root["pulls"], remoteLookup); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if c.Sinks, err = parseSinks(root["sinks"]); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if c.PullACLs, err = parsePullACLs(root["pull_acls"]); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if c.Prunes, err = parsePrunes(root["prune"]); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if c.Autosnaps, err = parseAutosnaps(root["autosnap"]); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func fullJobName(section, name string) (full string, err error) {
|
||||||
|
if len(name) < 1 {
|
||||||
|
err = fmt.Errorf("job name not set")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
full = fmt.Sprintf("%s.%s", section, name)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseRemotes(v interface{}) (remotes map[string]*Remote, err error) {
|
||||||
|
|
||||||
|
asMap := make(map[string]struct {
|
||||||
|
Transport map[string]interface{}
|
||||||
|
}, 0)
|
||||||
|
if err = mapstructure.Decode(v, &asMap); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
remotes = make(map[string]*Remote, len(asMap))
|
||||||
|
for name, p := range asMap {
|
||||||
|
|
||||||
|
if name == rpc.LOCAL_TRANSPORT_IDENTITY {
|
||||||
|
err = errors.New(fmt.Sprintf("remote name '%s' reserved for local pulls", rpc.LOCAL_TRANSPORT_IDENTITY))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var transport Transport
|
||||||
|
if transport, err = parseTransport(p.Transport); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
remotes[name] = &Remote{
|
||||||
|
Name: name,
|
||||||
|
Transport: transport,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseTransport(it map[string]interface{}) (t Transport, err error) {
|
||||||
|
|
||||||
|
if len(it) != 1 {
|
||||||
|
err = errors.New("ambiguous transport type")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for key, val := range it {
|
||||||
|
switch key {
|
||||||
|
case "ssh":
|
||||||
|
t := SSHTransport{}
|
||||||
|
if err = mapstructure.Decode(val, &t); err != nil {
|
||||||
|
err = errors.New(fmt.Sprintf("could not parse ssh transport: %s", err))
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return t, nil
|
||||||
|
default:
|
||||||
|
return nil, errors.New(fmt.Sprintf("unknown transport type '%s'\n", key))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return // unreachable
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
type remoteLookup func(name string) (*Remote, error)
|
||||||
|
|
||||||
|
func parsePushs(v interface{}, rl remoteLookup) (p map[string]*Push, err error) {
|
||||||
|
|
||||||
|
asMap := make(map[string]struct {
|
||||||
|
To string
|
||||||
|
Filter map[string]string
|
||||||
|
InitialReplPolicy string
|
||||||
|
Repeat map[string]string
|
||||||
|
}, 0)
|
||||||
|
|
||||||
|
if err = mapstructure.Decode(v, &asMap); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
p = make(map[string]*Push, len(asMap))
|
||||||
|
|
||||||
|
for name, e := range asMap {
|
||||||
|
|
||||||
|
var toRemote *Remote
|
||||||
|
if toRemote, err = rl(e.To); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
push := &Push{
|
||||||
|
To: toRemote,
|
||||||
|
}
|
||||||
|
|
||||||
|
if push.JobName, err = fullJobName(JobSectionPush, name); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if push.Filter, err = parseDatasetMapFilter(e.Filter, true); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if push.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, rpc.DEFAULT_INITIAL_REPL_POLICY); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if push.RepeatStrategy, err = parseRepeatStrategy(e.Repeat); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
p[name] = push
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func parsePulls(v interface{}, rl remoteLookup) (p map[string]*Pull, err error) {
|
||||||
|
|
||||||
|
asMap := make(map[string]struct {
|
||||||
|
From string
|
||||||
|
Mapping map[string]string
|
||||||
|
InitialReplPolicy string
|
||||||
|
Repeat map[string]string
|
||||||
|
}, 0)
|
||||||
|
|
||||||
|
if err = mapstructure.Decode(v, &asMap); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
p = make(map[string]*Pull, len(asMap))
|
||||||
|
|
||||||
|
for name, e := range asMap {
|
||||||
|
|
||||||
|
if len(e.From) < 1 {
|
||||||
|
err = fmt.Errorf("source not set ('from' attribute is empty)")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var fromRemote *Remote
|
||||||
|
|
||||||
|
if e.From == rpc.LOCAL_TRANSPORT_IDENTITY {
|
||||||
|
fromRemote = &Remote{
|
||||||
|
Name: rpc.LOCAL_TRANSPORT_IDENTITY,
|
||||||
|
Transport: LocalTransport{},
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if fromRemote, err = rl(e.From); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pull := &Pull{
|
||||||
|
From: fromRemote,
|
||||||
|
}
|
||||||
|
if pull.JobName, err = fullJobName(JobSectionPull, name); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if pull.Mapping, err = parseDatasetMapFilter(e.Mapping, false); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if pull.InitialReplPolicy, err = parseInitialReplPolicy(e.InitialReplPolicy, rpc.DEFAULT_INITIAL_REPL_POLICY); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if pull.RepeatStrategy, err = parseRepeatStrategy(e.Repeat); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
p[name] = pull
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseInitialReplPolicy(v interface{}, defaultPolicy rpc.InitialReplPolicy) (p rpc.InitialReplPolicy, err error) {
|
||||||
|
s, ok := v.(string)
|
||||||
|
if !ok {
|
||||||
|
goto err
|
||||||
|
}
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case s == "":
|
||||||
|
p = defaultPolicy
|
||||||
|
case s == "most_recent":
|
||||||
|
p = rpc.InitialReplPolicyMostRecent
|
||||||
|
case s == "all":
|
||||||
|
p = rpc.InitialReplPolicyAll
|
||||||
|
default:
|
||||||
|
goto err
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
err:
|
||||||
|
err = errors.New(fmt.Sprintf("expected InitialReplPolicy, got %#v", v))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseRepeatStrategy(r map[string]string) (s jobrun.RepeatStrategy, err error) {
|
||||||
|
|
||||||
|
if r == nil {
|
||||||
|
return jobrun.NoRepeatStrategy{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if repeatStr, ok := r["interval"]; ok {
|
||||||
|
d, err := parseDuration(repeatStr)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
s = &jobrun.PeriodicRepeatStrategy{d}
|
||||||
|
return s, err
|
||||||
|
} else {
|
||||||
|
return nil, fmt.Errorf("attribute 'interval' not found but required in repeat specification")
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func expectList(v interface{}) (asList []interface{}, err error) {
|
||||||
|
var ok bool
|
||||||
|
if asList, ok = v.([]interface{}); !ok {
|
||||||
|
err = errors.New("expected list")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseSinks(v interface{}) (m map[string]DatasetMapFilter, err error) {
|
||||||
|
var asMap map[string]map[string]interface{}
|
||||||
|
if err = mapstructure.Decode(v, &asMap); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
m = make(map[string]DatasetMapFilter, len(asMap))
|
||||||
|
|
||||||
|
for identity, entry := range asMap {
|
||||||
|
parseSink := func() (mapping DatasetMapFilter, err error) {
|
||||||
|
mappingMap, ok := entry["mapping"]
|
||||||
|
if !ok {
|
||||||
|
err = fmt.Errorf("no mapping specified")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
mapping, err = parseDatasetMapFilter(mappingMap, false)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
mapping, sinkErr := parseSink()
|
||||||
|
if sinkErr != nil {
|
||||||
|
err = fmt.Errorf("cannot parse sink for identity '%s': %s", identity, sinkErr)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
m[identity] = mapping
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func parsePullACLs(v interface{}) (m map[string]DatasetMapFilter, err error) {
|
||||||
|
var asMap map[string]map[string]interface{}
|
||||||
|
if err = mapstructure.Decode(v, &asMap); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
m = make(map[string]DatasetMapFilter, len(asMap))
|
||||||
|
|
||||||
|
for identity, entry := range asMap {
|
||||||
|
parsePullACL := func() (filter DatasetMapFilter, err error) {
|
||||||
|
filterMap, ok := entry["filter"]
|
||||||
|
if !ok {
|
||||||
|
err = fmt.Errorf("no filter specified")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
filter, err = parseDatasetMapFilter(filterMap, true)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
filter, filterErr := parsePullACL()
|
||||||
|
if filterErr != nil {
|
||||||
|
err = fmt.Errorf("cannot parse pull-ACL for identity '%s': %s", identity, filterErr)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
m[identity] = filter
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseDatasetMapFilter(mi interface{}, filterOnly bool) (f DatasetMapFilter, err error) {
|
||||||
|
|
||||||
|
var m map[string]string
|
||||||
|
if err = mapstructure.Decode(mi, &m); err != nil {
|
||||||
|
err = fmt.Errorf("maps / filters must be specified as map[string]string: %s", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
f = NewDatasetMapFilter(len(m), filterOnly)
|
||||||
|
for pathPattern, mapping := range m {
|
||||||
|
if err = f.Add(pathPattern, mapping); err != nil {
|
||||||
|
err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t SSHTransport) Connect(rpcLog Logger) (r rpc.RPCRequester, err error) {
|
||||||
|
var stream io.ReadWriteCloser
|
||||||
|
var rpcTransport sshbytestream.SSHTransport
|
||||||
|
if err = copier.Copy(&rpcTransport, t); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if stream, err = sshbytestream.Outgoing(rpcTransport); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
stream, err = NewReadWriteCloserLogger(stream, t.ConnLogReadFile, t.ConnLogWriteFile)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
return rpc.ConnectByteStreamRPC(stream, rpcLog)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t LocalTransport) Connect(rpcLog Logger) (r rpc.RPCRequester, err error) {
|
||||||
|
if t.Handler == nil {
|
||||||
|
panic("local transport with uninitialized handler")
|
||||||
|
}
|
||||||
|
return rpc.ConnectLocalRPC(t.Handler), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *LocalTransport) SetHandler(handler rpc.RPCHandler) {
|
||||||
|
t.Handler = handler
|
||||||
|
}
|
||||||
|
|
||||||
|
func parsePrunes(m interface{}) (rets map[string]*Prune, err error) {
|
||||||
|
|
||||||
|
asList := make(map[string]map[string]interface{}, 0)
|
||||||
|
if err = mapstructure.Decode(m, &asList); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
rets = make(map[string]*Prune, len(asList))
|
||||||
|
|
||||||
|
for name, e := range asList {
|
||||||
|
var prune *Prune
|
||||||
|
if prune, err = parsePrune(e, name); err != nil {
|
||||||
|
err = fmt.Errorf("cannot parse prune job %s: %s", name, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
rets[name] = prune
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func parsePrune(e map[string]interface{}, name string) (prune *Prune, err error) {
|
||||||
|
|
||||||
|
// Only support grid policy for now
|
||||||
|
policyName, ok := e["policy"]
|
||||||
|
if !ok || policyName != "grid" {
|
||||||
|
err = fmt.Errorf("prune job with unimplemented policy '%s'", policyName)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var i struct {
|
||||||
|
Grid string
|
||||||
|
DatasetFilter map[string]string `mapstructure:"dataset_filter"`
|
||||||
|
SnapshotFilter map[string]string `mapstructure:"snapshot_filter"`
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = mapstructure.Decode(e, &i); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
prune = &Prune{}
|
||||||
|
|
||||||
|
if prune.JobName, err = fullJobName(JobSectionPrune, name); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse grid policy
|
||||||
|
intervals, err := parseRetentionGridIntervalsString(i.Grid)
|
||||||
|
if err != nil {
|
||||||
|
err = fmt.Errorf("cannot parse retention grid: %s", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Assert intervals are of increasing length (not necessarily required, but indicates config mistake)
|
||||||
|
lastDuration := time.Duration(0)
|
||||||
|
for i := range intervals {
|
||||||
|
if intervals[i].Length < lastDuration {
|
||||||
|
err = fmt.Errorf("retention grid interval length must be monotonically increasing:"+
|
||||||
|
"interval %d is shorter than %d", i+1, i)
|
||||||
|
return
|
||||||
|
} else {
|
||||||
|
lastDuration = intervals[i].Length
|
||||||
|
}
|
||||||
|
}
|
||||||
|
prune.RetentionPolicy = NewRetentionGrid(intervals)
|
||||||
|
|
||||||
|
// Parse filters
|
||||||
|
if prune.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil {
|
||||||
|
err = fmt.Errorf("cannot parse dataset filter: %s", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if prune.SnapshotFilter, err = parseSnapshotFilter(i.SnapshotFilter); err != nil {
|
||||||
|
err = fmt.Errorf("cannot parse snapshot filter: %s", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
|
||||||
|
|
||||||
|
func parseDuration(e string) (d time.Duration, err error) {
|
||||||
|
comps := durationStringRegex.FindStringSubmatch(e)
|
||||||
|
if len(comps) != 3 {
|
||||||
|
err = fmt.Errorf("does not match regex: %s %#v", e, comps)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var durationUnit time.Duration
|
||||||
|
switch comps[2] {
|
||||||
|
case "s":
|
||||||
|
durationUnit = time.Second
|
||||||
|
case "m":
|
||||||
|
durationUnit = time.Minute
|
||||||
|
case "h":
|
||||||
|
durationUnit = time.Hour
|
||||||
|
case "d":
|
||||||
|
durationUnit = 24 * time.Hour
|
||||||
|
case "w":
|
||||||
|
durationUnit = 24 * 7 * time.Hour
|
||||||
|
default:
|
||||||
|
err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
d = time.Duration(durationFactor) * durationUnit
|
||||||
|
return
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
|
||||||
|
|
||||||
|
func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {
|
||||||
|
|
||||||
|
comps := retentionStringIntervalRegex.FindStringSubmatch(e)
|
||||||
|
if comps == nil {
|
||||||
|
err = fmt.Errorf("retention string does not match expected format")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
times, err := strconv.Atoi(comps[1])
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
} else if times <= 0 {
|
||||||
|
return nil, fmt.Errorf("contains factor <= 0")
|
||||||
|
}
|
||||||
|
|
||||||
|
duration, err := parseDuration(comps[2])
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
keepCount := 1
|
||||||
|
if comps[3] != "" {
|
||||||
|
// Decompose key=value, comma separated
|
||||||
|
// For now, only keep_count is supported
|
||||||
|
re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
|
||||||
|
res := re.FindStringSubmatch(comps[4])
|
||||||
|
if res == nil || len(res) != 2 {
|
||||||
|
err = fmt.Errorf("interval parameter contains unknown parameters")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if res[1] == "all" {
|
||||||
|
keepCount = RetentionGridKeepCountAll
|
||||||
|
} else {
|
||||||
|
keepCount, err = strconv.Atoi(res[1])
|
||||||
|
if err != nil {
|
||||||
|
err = fmt.Errorf("cannot parse keep_count value")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
intervals = make([]RetentionInterval, times)
|
||||||
|
for i := range intervals {
|
||||||
|
intervals[i] = RetentionInterval{
|
||||||
|
Length: duration,
|
||||||
|
KeepCount: keepCount,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseRetentionGridIntervalsString(s string) (intervals []RetentionInterval, err error) {
|
||||||
|
|
||||||
|
ges := strings.Split(s, "|")
|
||||||
|
intervals = make([]RetentionInterval, 0, 7*len(ges))
|
||||||
|
|
||||||
|
for intervalIdx, e := range ges {
|
||||||
|
parsed, err := parseRetentionGridIntervalString(e)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
|
||||||
|
}
|
||||||
|
intervals = append(intervals, parsed...)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
type prefixSnapshotFilter struct {
|
||||||
|
prefix string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f prefixSnapshotFilter) Filter(fsv zfs.FilesystemVersion) (accept bool, err error) {
|
||||||
|
return fsv.Type == zfs.Snapshot && strings.HasPrefix(fsv.Name, f.prefix), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseSnapshotFilter(fm map[string]string) (snapFilter zfs.FilesystemVersionFilter, err error) {
|
||||||
|
prefix, ok := fm["prefix"]
|
||||||
|
if !ok {
|
||||||
|
err = fmt.Errorf("unsupported snapshot filter")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
snapFilter = prefixSnapshotFilter{prefix}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseAutosnaps(m interface{}) (snaps map[string]*Autosnap, err error) {
|
||||||
|
|
||||||
|
asMap := make(map[string]interface{}, 0)
|
||||||
|
if err = mapstructure.Decode(m, &asMap); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
snaps = make(map[string]*Autosnap, len(asMap))
|
||||||
|
|
||||||
|
for name, e := range asMap {
|
||||||
|
var snap *Autosnap
|
||||||
|
if snap, err = parseAutosnap(e, name); err != nil {
|
||||||
|
err = fmt.Errorf("cannot parse autonsap job %s: %s", name, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
snaps[name] = snap
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseAutosnap(m interface{}, name string) (a *Autosnap, err error) {
|
||||||
|
|
||||||
|
var i struct {
|
||||||
|
Prefix string
|
||||||
|
Interval string
|
||||||
|
DatasetFilter map[string]string `mapstructure:"dataset_filter"`
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = mapstructure.Decode(m, &i); err != nil {
|
||||||
|
err = fmt.Errorf("structure unfit: %s", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
a = &Autosnap{}
|
||||||
|
|
||||||
|
if a.JobName, err = fullJobName(JobSectionAutosnap, name); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(i.Prefix) < 1 {
|
||||||
|
err = fmt.Errorf("prefix must not be empty")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.Prefix = i.Prefix
|
||||||
|
|
||||||
|
var interval time.Duration
|
||||||
|
if interval, err = parseDuration(i.Interval); err != nil {
|
||||||
|
err = fmt.Errorf("cannot parse interval: %s", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.Interval = &jobrun.PeriodicRepeatStrategy{interval}
|
||||||
|
|
||||||
|
if len(i.DatasetFilter) == 0 {
|
||||||
|
err = fmt.Errorf("dataset_filter not specified")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if a.DatasetFilter, err = parseDatasetMapFilter(i.DatasetFilter, true); err != nil {
|
||||||
|
err = fmt.Errorf("cannot parse dataset filter: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/zrepl/zrepl/zfs"
|
||||||
|
)
|
||||||
|
|
||||||
|
type DatasetMapFilter struct {
|
||||||
|
entries []datasetMapFilterEntry
|
||||||
|
|
||||||
|
// if set, only valid filter entries can be added using Add()
|
||||||
|
// and Map() will always return an error
|
||||||
|
filterOnly bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type datasetMapFilterEntry struct {
|
||||||
|
path *zfs.DatasetPath
|
||||||
|
// the mapping. since this datastructure acts as both mapping and filter
|
||||||
|
// we have to convert it to the desired rep dynamically
|
||||||
|
mapping string
|
||||||
|
subtreeMatch bool
|
||||||
|
}
|
||||||
|
|
||||||
|
var NoMatchError error = errors.New("no match found in mapping")
|
||||||
|
|
||||||
|
func NewDatasetMapFilter(capacity int, filterOnly bool) DatasetMapFilter {
|
||||||
|
return DatasetMapFilter{
|
||||||
|
entries: make([]datasetMapFilterEntry, 0, capacity),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *DatasetMapFilter) Add(pathPattern, mapping string) (err error) {
|
||||||
|
|
||||||
|
if m.filterOnly {
|
||||||
|
if _, err = parseDatasetFilterResult(mapping); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// assert path glob adheres to spec
|
||||||
|
const SUBTREE_PATTERN string = "<"
|
||||||
|
patternCount := strings.Count(pathPattern, SUBTREE_PATTERN)
|
||||||
|
switch {
|
||||||
|
case patternCount > 1:
|
||||||
|
case patternCount == 1 && !strings.HasSuffix(pathPattern, SUBTREE_PATTERN):
|
||||||
|
err = fmt.Errorf("pattern invalid: only one '<' at end of string allowed")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
pathStr := strings.TrimSuffix(pathPattern, SUBTREE_PATTERN)
|
||||||
|
path, err := zfs.NewDatasetPath(pathStr)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("pattern is not a dataset path: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
entry := datasetMapFilterEntry{
|
||||||
|
path: path,
|
||||||
|
mapping: mapping,
|
||||||
|
subtreeMatch: patternCount > 0,
|
||||||
|
}
|
||||||
|
m.entries = append(m.entries, entry)
|
||||||
|
return
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// find the most specific prefix mapping we have
|
||||||
|
//
|
||||||
|
// longer prefix wins over shorter prefix, direct wins over glob
|
||||||
|
func (m DatasetMapFilter) mostSpecificPrefixMapping(path *zfs.DatasetPath) (idx int, found bool) {
|
||||||
|
lcp, lcp_entry_idx := -1, -1
|
||||||
|
direct_idx := -1
|
||||||
|
for e := range m.entries {
|
||||||
|
entry := m.entries[e]
|
||||||
|
ep := m.entries[e].path
|
||||||
|
lep := ep.Length()
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case !entry.subtreeMatch && ep.Equal(path):
|
||||||
|
direct_idx = e
|
||||||
|
continue
|
||||||
|
case entry.subtreeMatch && path.HasPrefix(ep) && lep > lcp:
|
||||||
|
lcp = lep
|
||||||
|
lcp_entry_idx = e
|
||||||
|
default:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if lcp_entry_idx >= 0 || direct_idx >= 0 {
|
||||||
|
found = true
|
||||||
|
switch {
|
||||||
|
case direct_idx >= 0:
|
||||||
|
idx = direct_idx
|
||||||
|
case lcp_entry_idx >= 0:
|
||||||
|
idx = lcp_entry_idx
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error) {
|
||||||
|
|
||||||
|
if m.filterOnly {
|
||||||
|
err = fmt.Errorf("using a filter for mapping simply does not work")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
mi, hasMapping := m.mostSpecificPrefixMapping(source)
|
||||||
|
if !hasMapping {
|
||||||
|
err = NoMatchError
|
||||||
|
return
|
||||||
|
}
|
||||||
|
me := m.entries[mi]
|
||||||
|
|
||||||
|
target, err = zfs.NewDatasetPath(me.mapping)
|
||||||
|
if err != nil {
|
||||||
|
err = fmt.Errorf("mapping target is not a dataset path: %s", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if m.entries[mi].subtreeMatch {
|
||||||
|
// strip common prefix
|
||||||
|
extendComps := source.Copy()
|
||||||
|
if me.path.Empty() {
|
||||||
|
// special case: trying to map the root => strip first component
|
||||||
|
extendComps.TrimNPrefixComps(1)
|
||||||
|
} else {
|
||||||
|
extendComps.TrimPrefix(me.path)
|
||||||
|
}
|
||||||
|
target.Extend(extendComps)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
|
||||||
|
mi, hasMapping := m.mostSpecificPrefixMapping(p)
|
||||||
|
if !hasMapping {
|
||||||
|
pass = false
|
||||||
|
return
|
||||||
|
}
|
||||||
|
me := m.entries[mi]
|
||||||
|
pass, err = parseDatasetFilterResult(me.mapping)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse a dataset filter result
|
||||||
|
func parseDatasetFilterResult(result string) (pass bool, err error) {
|
||||||
|
l := strings.ToLower(result)
|
||||||
|
switch strings.ToLower(l) {
|
||||||
|
case "ok":
|
||||||
|
pass = true
|
||||||
|
return
|
||||||
|
case "omit":
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
err = fmt.Errorf("'%s' is not a valid filter result", result)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/zrepl/zrepl/util"
|
||||||
|
"github.com/zrepl/zrepl/zfs"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSampleConfigFileIsParsedWithoutErrors(t *testing.T) {
|
||||||
|
_, err := ParseConfig("./sampleconf/zrepl.yml")
|
||||||
|
assert.Nil(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseRetentionGridStringParsing(t *testing.T) {
|
||||||
|
|
||||||
|
intervals, err := parseRetentionGridIntervalsString("2x10m(keep=2) | 1x1h | 3x1w")
|
||||||
|
|
||||||
|
assert.Nil(t, err)
|
||||||
|
assert.Len(t, intervals, 6)
|
||||||
|
proto := util.RetentionInterval{
|
||||||
|
KeepCount: 2,
|
||||||
|
Length: 10 * time.Minute,
|
||||||
|
}
|
||||||
|
assert.EqualValues(t, proto, intervals[0])
|
||||||
|
assert.EqualValues(t, proto, intervals[1])
|
||||||
|
|
||||||
|
proto.KeepCount = 1
|
||||||
|
proto.Length = 1 * time.Hour
|
||||||
|
assert.EqualValues(t, proto, intervals[2])
|
||||||
|
|
||||||
|
proto.Length = 7 * 24 * time.Hour
|
||||||
|
assert.EqualValues(t, proto, intervals[3])
|
||||||
|
assert.EqualValues(t, proto, intervals[4])
|
||||||
|
assert.EqualValues(t, proto, intervals[5])
|
||||||
|
|
||||||
|
intervals, err = parseRetentionGridIntervalsString("|")
|
||||||
|
assert.Error(t, err)
|
||||||
|
intervals, err = parseRetentionGridIntervalsString("2x10m")
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
intervals, err = parseRetentionGridIntervalsString("1x10m(keep=all)")
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Len(t, intervals, 1)
|
||||||
|
assert.EqualValues(t, util.RetentionGridKeepCountAll, intervals[0].KeepCount)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetMapFilter(t *testing.T) {
|
||||||
|
|
||||||
|
expectMapping := func(m map[string]string, from, to string) {
|
||||||
|
dmf, err := parseDatasetMapFilter(m, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Logf("expect test map to be valid: %s", err)
|
||||||
|
t.FailNow()
|
||||||
|
}
|
||||||
|
fromPath, err := zfs.NewDatasetPath(from)
|
||||||
|
if err != nil {
|
||||||
|
t.Logf("expect test from path to be valid: %s", err)
|
||||||
|
t.FailNow()
|
||||||
|
}
|
||||||
|
toPath, err := zfs.NewDatasetPath(to)
|
||||||
|
if err != nil {
|
||||||
|
t.Logf("expect test to path to be valid: %s", err)
|
||||||
|
t.FailNow()
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := dmf.Map(fromPath)
|
||||||
|
t.Logf("%s => %s", fromPath.ToString(), res.ToString())
|
||||||
|
assert.Nil(t, err)
|
||||||
|
assert.True(t, res.Equal(toPath))
|
||||||
|
}
|
||||||
|
|
||||||
|
expectFilter := func(m map[string]string, path string, pass bool) {
|
||||||
|
dmf, err := parseDatasetMapFilter(m, true)
|
||||||
|
if err != nil {
|
||||||
|
t.Logf("expect test filter to be valid: %s", err)
|
||||||
|
t.FailNow()
|
||||||
|
}
|
||||||
|
p, err := zfs.NewDatasetPath(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Logf("expect test path to be valid: %s", err)
|
||||||
|
t.FailNow()
|
||||||
|
}
|
||||||
|
res, err := dmf.Filter(p)
|
||||||
|
assert.Nil(t, err)
|
||||||
|
assert.Equal(t, pass, res)
|
||||||
|
}
|
||||||
|
|
||||||
|
map1 := map[string]string{
|
||||||
|
"a/b/c<": "root1",
|
||||||
|
"a/b<": "root2",
|
||||||
|
"<": "root3/b/c",
|
||||||
|
"q<": "root4/1/2",
|
||||||
|
}
|
||||||
|
|
||||||
|
expectMapping(map1, "a/b/c", "root1")
|
||||||
|
expectMapping(map1, "a/b/c/d", "root1/d")
|
||||||
|
expectMapping(map1, "a/b/e", "root2/e")
|
||||||
|
expectMapping(map1, "a/b", "root2")
|
||||||
|
expectMapping(map1, "x", "root3/b/c")
|
||||||
|
expectMapping(map1, "x/y", "root3/b/c/y")
|
||||||
|
expectMapping(map1, "q", "root4/1/2")
|
||||||
|
expectMapping(map1, "q/r", "root4/1/2/r")
|
||||||
|
|
||||||
|
filter1 := map[string]string{
|
||||||
|
"<": "omit",
|
||||||
|
"a<": "ok",
|
||||||
|
"a/b<": "omit",
|
||||||
|
}
|
||||||
|
|
||||||
|
expectFilter(filter1, "b", false)
|
||||||
|
expectFilter(filter1, "a", true)
|
||||||
|
expectFilter(filter1, "a/d", true)
|
||||||
|
expectFilter(filter1, "a/b", false)
|
||||||
|
expectFilter(filter1, "a/b/c", false)
|
||||||
|
|
||||||
|
filter2 := map[string]string{}
|
||||||
|
expectFilter(filter2, "foo", false) // default to omit
|
||||||
|
|
||||||
|
}
|
||||||
+177
@@ -0,0 +1,177 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"github.com/zrepl/zrepl/rpc"
|
||||||
|
"github.com/zrepl/zrepl/zfs"
|
||||||
|
"io"
|
||||||
|
)
|
||||||
|
|
||||||
|
type DatasetMapping interface {
|
||||||
|
Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type Handler struct {
|
||||||
|
Logger Logger
|
||||||
|
PullACL zfs.DatasetFilter
|
||||||
|
SinkMappingFunc func(clientIdentity string) (mapping DatasetMapping, err error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h Handler) HandleFilesystemRequest(r rpc.FilesystemRequest) (roots []*zfs.DatasetPath, err error) {
|
||||||
|
|
||||||
|
h.Logger.Printf("handling fsr: %#v", r)
|
||||||
|
|
||||||
|
h.Logger.Printf("using PullACL: %#v", h.PullACL)
|
||||||
|
|
||||||
|
if roots, err = zfs.ZFSListMapping(h.PullACL); err != nil {
|
||||||
|
h.Logger.Printf("handle fsr err: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
h.Logger.Printf("returning: %#v", roots)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h Handler) HandleFilesystemVersionsRequest(r rpc.FilesystemVersionsRequest) (versions []zfs.FilesystemVersion, err error) {
|
||||||
|
|
||||||
|
h.Logger.Printf("handling filesystem versions request: %#v", r)
|
||||||
|
|
||||||
|
// allowed to request that?
|
||||||
|
if h.pullACLCheck(r.Filesystem); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// find our versions
|
||||||
|
if versions, err = zfs.ZFSListFilesystemVersions(r.Filesystem, nil); err != nil {
|
||||||
|
h.Logger.Printf("our versions error: %#v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
h.Logger.Printf("our versions: %#v\n", versions)
|
||||||
|
return
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h Handler) HandleInitialTransferRequest(r rpc.InitialTransferRequest) (stream io.Reader, err error) {
|
||||||
|
|
||||||
|
h.Logger.Printf("handling initial transfer request: %#v", r)
|
||||||
|
if err = h.pullACLCheck(r.Filesystem); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
h.Logger.Printf("invoking zfs send")
|
||||||
|
|
||||||
|
if stream, err = zfs.ZFSSend(r.Filesystem, &r.FilesystemVersion, nil); err != nil {
|
||||||
|
h.Logger.Printf("error sending filesystem: %#v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h Handler) HandleIncrementalTransferRequest(r rpc.IncrementalTransferRequest) (stream io.Reader, err error) {
|
||||||
|
|
||||||
|
h.Logger.Printf("handling incremental transfer request: %#v", r)
|
||||||
|
if err = h.pullACLCheck(r.Filesystem); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
h.Logger.Printf("invoking zfs send")
|
||||||
|
|
||||||
|
if stream, err = zfs.ZFSSend(r.Filesystem, &r.From, &r.To); err != nil {
|
||||||
|
h.Logger.Printf("error sending filesystem: %#v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h Handler) HandleResumeTransferRequest(r rpc.ResumeTransferRequest) (stream io.Reader, err error) {
|
||||||
|
// decode receive_resume_token: zfs send -nvP <token>
|
||||||
|
// A) use exit code to determine if could send (exit != 0 means could not send)
|
||||||
|
// B) check ACL if the filesystem in toname field (it is a snapshot so strip its snapshot name first)
|
||||||
|
// is allowed for the given client
|
||||||
|
/*
|
||||||
|
|
||||||
|
# zfs send -nvt will print nvlist contents on ZoL and FreeBSD, should be good enough
|
||||||
|
# will print regardless of whether it can send or not -> good for us
|
||||||
|
# need clean way to extract that...
|
||||||
|
# expect 'resume token contents:\nnvlist version: 0\n'
|
||||||
|
# parse everything after that is tab-indented as key value pairs separated by = sign
|
||||||
|
# => return dict
|
||||||
|
|
||||||
|
zfs send -nv -t 1-c6491acbe-c8-789c636064000310a500c4ec50360710e72765a526973030b0419460caa7a515a79630c001489e0d493ea9b224b5182451885d7f497e7a69660a0343f79b1b9a8a2b3db65b20c97382e5f312735319188a4bf28b12d353f5931293b34b0b8af5ab8a520b72f4d3f2f31d8a53f35220660300c1091dbe
|
||||||
|
resume token contents:
|
||||||
|
nvlist version: 0
|
||||||
|
object = 0x6
|
||||||
|
offset = 0x0
|
||||||
|
bytes = 0x7100
|
||||||
|
toguid = 0xb748a92129d8ec8b
|
||||||
|
toname = storage/backups/zrepl/foo@send
|
||||||
|
cannot resume send: 'storage/backups/zrepl/foo@send' used in the initial send no longer exists
|
||||||
|
|
||||||
|
zfs send -nvt 1-ebbbbea7e-f0-789c636064000310a501c49c50360710a715e5e7a69766a6304041f79b1b9a8a2b3db62b00d9ec48eaf293b252934b181858a0ea30e4d3d28a534b18e00024cf86249f5459925a0ca44fc861d75f920f71c59c5fdf6f7b3eea32b24092e704cbe725e6a632301497e41725a6a7ea27252667971614eb5715a516e4e8a7e5e73b14a7e6a51881cd06005a222749
|
||||||
|
resume token contents:
|
||||||
|
nvlist version: 0
|
||||||
|
fromguid = 0xb748a92129d8ec8b #NOTE the fromguid field which is only in this one, so don't hardcode
|
||||||
|
object = 0x4
|
||||||
|
offset = 0x0
|
||||||
|
bytes = 0x1ec8
|
||||||
|
toguid = 0x328ae249dbf7fa9c
|
||||||
|
toname = storage/backups/zrepl/foo@send2
|
||||||
|
send from storage/backups/zrepl/foo@send to storage/backups/zrepl/foo@send2 estimated size is 1.02M
|
||||||
|
|
||||||
|
*/
|
||||||
|
panic("not implemented")
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h Handler) HandlePullMeRequest(r rpc.PullMeRequest, clientIdentity string, client rpc.RPCRequester) (err error) {
|
||||||
|
|
||||||
|
// Check if we have a sink for this request
|
||||||
|
// Use that mapping to do what happens in doPull
|
||||||
|
|
||||||
|
h.Logger.Printf("handling PullMeRequest: %#v", r)
|
||||||
|
|
||||||
|
var sinkMapping DatasetMapping
|
||||||
|
sinkMapping, err = h.SinkMappingFunc(clientIdentity)
|
||||||
|
if err != nil {
|
||||||
|
h.Logger.Printf("no sink mapping for client identity '%s', denying PullMeRequest", clientIdentity)
|
||||||
|
err = fmt.Errorf("no sink for client identity '%s'", clientIdentity)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
h.Logger.Printf("doing pull...")
|
||||||
|
|
||||||
|
err = doPull(PullContext{
|
||||||
|
Remote: client,
|
||||||
|
Log: h.Logger,
|
||||||
|
Mapping: sinkMapping,
|
||||||
|
InitialReplPolicy: r.InitialReplPolicy,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
h.Logger.Printf("PullMeRequest failed with error: %s", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
h.Logger.Printf("finished handling PullMeRequest: %#v", r)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h Handler) pullACLCheck(p *zfs.DatasetPath) (err error) {
|
||||||
|
var allowed bool
|
||||||
|
allowed, err = h.PullACL.Filter(p)
|
||||||
|
if err != nil {
|
||||||
|
err = fmt.Errorf("error evaluating ACL: %s", err)
|
||||||
|
h.Logger.Printf(err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !allowed {
|
||||||
|
err = fmt.Errorf("ACL prohibits access to %s", p.ToString())
|
||||||
|
h.Logger.Printf(err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
+104
@@ -0,0 +1,104 @@
|
|||||||
|
// zrepl replicates ZFS filesystems & volumes between pools
|
||||||
|
//
|
||||||
|
// Code Organization
|
||||||
|
//
|
||||||
|
// The cmd package uses github.com/spf13/cobra for its CLI.
|
||||||
|
//
|
||||||
|
// It combines the other packages in the zrepl project to implement zrepl functionality.
|
||||||
|
//
|
||||||
|
// Each subcommand's code is in the corresponding *.go file.
|
||||||
|
// All other *.go files contain code shared by the subcommands.
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"github.com/zrepl/zrepl/jobrun"
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
"io"
|
||||||
|
golog "log"
|
||||||
|
"net/http"
|
||||||
|
_ "net/http/pprof"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Logger interface {
|
||||||
|
Printf(format string, v ...interface{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// global state / facilities
|
||||||
|
var (
|
||||||
|
conf Config
|
||||||
|
runner *jobrun.JobRunner
|
||||||
|
logFlags int = golog.LUTC | golog.Ldate | golog.Ltime
|
||||||
|
logOut io.Writer
|
||||||
|
log Logger
|
||||||
|
)
|
||||||
|
|
||||||
|
var RootCmd = &cobra.Command{
|
||||||
|
Use: "zrepl",
|
||||||
|
Short: "ZFS dataset replication",
|
||||||
|
Long: `Replicate ZFS filesystems & volumes between pools:
|
||||||
|
|
||||||
|
- push & pull mode
|
||||||
|
- automatic snapshot creation & pruning
|
||||||
|
- local / over the network
|
||||||
|
- ACLs instead of blank SSH access`,
|
||||||
|
}
|
||||||
|
|
||||||
|
var rootArgs struct {
|
||||||
|
configFile string
|
||||||
|
stderrFile string
|
||||||
|
httpPprof string
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
cobra.OnInitialize(initConfig)
|
||||||
|
RootCmd.PersistentFlags().StringVar(&rootArgs.configFile, "config", "", "config file path")
|
||||||
|
RootCmd.PersistentFlags().StringVar(&rootArgs.stderrFile, "stderrFile", "-", "redirect stderr to given path")
|
||||||
|
RootCmd.PersistentFlags().StringVar(&rootArgs.httpPprof, "debug.pprof.http", "", "run pprof http server on given port")
|
||||||
|
}
|
||||||
|
|
||||||
|
func initConfig() {
|
||||||
|
|
||||||
|
// Logging & stderr redirection
|
||||||
|
if rootArgs.stderrFile != "-" {
|
||||||
|
file, err := os.OpenFile(rootArgs.stderrFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = unix.Dup2(int(file.Fd()), int(os.Stderr.Fd())); err != nil {
|
||||||
|
file.WriteString(fmt.Sprintf("error redirecting stderr file %s: %s\n", rootArgs.stderrFile, err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
logOut = file
|
||||||
|
} else {
|
||||||
|
logOut = os.Stderr
|
||||||
|
}
|
||||||
|
|
||||||
|
log = golog.New(logOut, "", logFlags)
|
||||||
|
|
||||||
|
// CPU profiling
|
||||||
|
if rootArgs.httpPprof != "" {
|
||||||
|
go func() {
|
||||||
|
http.ListenAndServe(rootArgs.httpPprof, nil)
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config
|
||||||
|
if rootArgs.configFile == "" {
|
||||||
|
log.Printf("config file not set")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
if conf, err = ParseConfig(rootArgs.configFile); err != nil {
|
||||||
|
log.Printf("error parsing config: %s", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
jobrunLogger := golog.New(os.Stderr, "jobrun ", logFlags)
|
||||||
|
runner = jobrun.NewJobRunner(jobrunLogger)
|
||||||
|
return
|
||||||
|
|
||||||
|
}
|
||||||
+139
@@ -0,0 +1,139 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"github.com/zrepl/zrepl/util"
|
||||||
|
"github.com/zrepl/zrepl/zfs"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
var pruneArgs struct {
|
||||||
|
job string
|
||||||
|
dryRun bool
|
||||||
|
}
|
||||||
|
|
||||||
|
var PruneCmd = &cobra.Command{
|
||||||
|
Use: "prune",
|
||||||
|
Short: "perform pruning",
|
||||||
|
Run: cmdPrune,
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
PruneCmd.Flags().StringVar(&pruneArgs.job, "job", "", "job to run")
|
||||||
|
PruneCmd.Flags().BoolVarP(&pruneArgs.dryRun, "dryrun", "n", false, "dry run")
|
||||||
|
RootCmd.AddCommand(PruneCmd)
|
||||||
|
}
|
||||||
|
|
||||||
|
func cmdPrune(cmd *cobra.Command, args []string) {
|
||||||
|
|
||||||
|
if len(args) < 1 {
|
||||||
|
log.Printf("must specify exactly one job as positional argument")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
job, ok := conf.Prunes[args[0]]
|
||||||
|
if !ok {
|
||||||
|
log.Printf("could not find prune job: %s", args[0])
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Beginning prune job:\n%s", job)
|
||||||
|
ctx := PruneContext{job, time.Now(), pruneArgs.dryRun}
|
||||||
|
err := doPrune(ctx, log)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Prune job failed with error: %s", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
type PruneContext struct {
|
||||||
|
Prune *Prune
|
||||||
|
Now time.Time
|
||||||
|
DryRun bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type retentionGridAdaptor struct {
|
||||||
|
zfs.FilesystemVersion
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a retentionGridAdaptor) Date() time.Time {
|
||||||
|
return a.Creation
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a retentionGridAdaptor) LessThan(b util.RetentionGridEntry) bool {
|
||||||
|
return a.CreateTXG < b.(retentionGridAdaptor).CreateTXG
|
||||||
|
}
|
||||||
|
|
||||||
|
func doPrune(ctx PruneContext, log Logger) error {
|
||||||
|
|
||||||
|
if ctx.DryRun {
|
||||||
|
log.Printf("doing dry run")
|
||||||
|
}
|
||||||
|
|
||||||
|
prune := ctx.Prune
|
||||||
|
|
||||||
|
// ZFSListSnapsFiltered --> todo can extend fsfilter or need new? Have already something per fs
|
||||||
|
// Dedicated snapshot object? Adaptor object to FilesystemVersion?
|
||||||
|
|
||||||
|
filesystems, err := zfs.ZFSListMapping(prune.DatasetFilter)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error applying filesystem filter: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, fs := range filesystems {
|
||||||
|
|
||||||
|
fsversions, err := zfs.ZFSListFilesystemVersions(fs, prune.SnapshotFilter)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error listing filesytem versions of %s: %s", fs, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(fsversions) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
adaptors := make([]util.RetentionGridEntry, len(fsversions))
|
||||||
|
for fsv := range fsversions {
|
||||||
|
adaptors[fsv] = retentionGridAdaptor{fsversions[fsv]}
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.SliceStable(adaptors, func(i, j int) bool {
|
||||||
|
return adaptors[i].LessThan(adaptors[j])
|
||||||
|
})
|
||||||
|
ctx.Now = adaptors[len(adaptors)-1].Date()
|
||||||
|
|
||||||
|
describe := func(a retentionGridAdaptor) string {
|
||||||
|
timeSince := a.Creation.Sub(ctx.Now)
|
||||||
|
const day time.Duration = 24 * time.Hour
|
||||||
|
days := timeSince / day
|
||||||
|
remainder := timeSince % day
|
||||||
|
return fmt.Sprintf("%s@%dd%s from now", a.ToAbsPath(fs), days, remainder)
|
||||||
|
}
|
||||||
|
|
||||||
|
keep, remove := prune.RetentionPolicy.FitEntries(ctx.Now, adaptors)
|
||||||
|
for _, a := range remove {
|
||||||
|
r := a.(retentionGridAdaptor)
|
||||||
|
log.Printf("remove %s", describe(r))
|
||||||
|
// do echo what we'll do and exec zfs destroy if not dry run
|
||||||
|
// special handling for EBUSY (zfs hold)
|
||||||
|
// error handling for clones? just echo to cli, skip over, and exit with non-zero status code (we're idempotent)
|
||||||
|
if !ctx.DryRun {
|
||||||
|
err := zfs.ZFSDestroy(r.ToAbsPath(fs))
|
||||||
|
if err != nil {
|
||||||
|
// handle
|
||||||
|
log.Printf("error: %s", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, a := range keep {
|
||||||
|
r := a.(retentionGridAdaptor)
|
||||||
|
log.Printf("would keep %s", describe(r))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,587 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"github.com/zrepl/zrepl/jobrun"
|
||||||
|
"github.com/zrepl/zrepl/rpc"
|
||||||
|
"github.com/zrepl/zrepl/util"
|
||||||
|
"github.com/zrepl/zrepl/zfs"
|
||||||
|
)
|
||||||
|
|
||||||
|
var runArgs struct {
|
||||||
|
job string
|
||||||
|
once bool
|
||||||
|
}
|
||||||
|
|
||||||
|
var RunCmd = &cobra.Command{
|
||||||
|
Use: "run",
|
||||||
|
Short: "run push & pull replication",
|
||||||
|
Run: cmdRun,
|
||||||
|
}
|
||||||
|
|
||||||
|
var PushCmd = &cobra.Command{
|
||||||
|
Use: "push",
|
||||||
|
Short: "run push job (first positional argument)",
|
||||||
|
Run: cmdPush,
|
||||||
|
}
|
||||||
|
|
||||||
|
var PullCmd = &cobra.Command{
|
||||||
|
Use: "pull",
|
||||||
|
Short: "run pull job (first positional argument)",
|
||||||
|
Run: cmdPull,
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
RootCmd.AddCommand(RunCmd)
|
||||||
|
RunCmd.Flags().BoolVar(&runArgs.once, "once", false, "run jobs only once, regardless of configured repeat behavior")
|
||||||
|
RunCmd.Flags().StringVar(&runArgs.job, "job", "", "run only the given job")
|
||||||
|
|
||||||
|
RootCmd.AddCommand(PushCmd)
|
||||||
|
RootCmd.AddCommand(PullCmd)
|
||||||
|
}
|
||||||
|
|
||||||
|
func cmdPush(cmd *cobra.Command, args []string) {
|
||||||
|
|
||||||
|
if len(args) != 1 {
|
||||||
|
log.Printf("must specify exactly one job as positional argument")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
job, ok := conf.Pushs[args[0]]
|
||||||
|
if !ok {
|
||||||
|
log.Printf("could not find push job %s", args[0])
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
if err := jobPush(job, log); err != nil {
|
||||||
|
log.Printf("error doing push: %s", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func cmdPull(cmd *cobra.Command, args []string) {
|
||||||
|
|
||||||
|
if len(args) != 1 {
|
||||||
|
log.Printf("must specify exactly one job as positional argument")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
job, ok := conf.Pulls[args[0]]
|
||||||
|
if !ok {
|
||||||
|
log.Printf("could not find pull job %s", args[0])
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := jobPull(job, log); err != nil {
|
||||||
|
log.Printf("error doing pull: %s", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func cmdRun(cmd *cobra.Command, args []string) {
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
runner.Start()
|
||||||
|
}()
|
||||||
|
|
||||||
|
jobs := make([]jobrun.Job, len(conf.Pulls)+len(conf.Pushs))
|
||||||
|
i := 0
|
||||||
|
for _, pull := range conf.Pulls {
|
||||||
|
jobs[i] = jobrun.Job{
|
||||||
|
Name: fmt.Sprintf("pull.%d", i),
|
||||||
|
RepeatStrategy: pull.RepeatStrategy,
|
||||||
|
RunFunc: func(log jobrun.Logger) error {
|
||||||
|
log.Printf("doing pull: %v", pull)
|
||||||
|
return jobPull(pull, log)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
for _, push := range conf.Pushs {
|
||||||
|
jobs[i] = jobrun.Job{
|
||||||
|
Name: fmt.Sprintf("push.%d", i),
|
||||||
|
RepeatStrategy: push.RepeatStrategy,
|
||||||
|
RunFunc: func(log jobrun.Logger) error {
|
||||||
|
log.Printf("doing push: %v", push)
|
||||||
|
return jobPush(push, log)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, j := range jobs {
|
||||||
|
if runArgs.once {
|
||||||
|
j.RepeatStrategy = jobrun.NoRepeatStrategy{}
|
||||||
|
}
|
||||||
|
if runArgs.job != "" {
|
||||||
|
if runArgs.job == j.Name {
|
||||||
|
runner.AddJob(j)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
runner.AddJob(j)
|
||||||
|
}
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case job := <-runner.NotificationChan():
|
||||||
|
log.Printf("job %s reported error: %v\n", job.Name, job.LastError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
type localPullACL struct{}
|
||||||
|
|
||||||
|
func (a localPullACL) Filter(p *zfs.DatasetPath) (pass bool, err error) {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func jobPull(pull *Pull, log jobrun.Logger) (err error) {
|
||||||
|
|
||||||
|
if lt, ok := pull.From.Transport.(LocalTransport); ok {
|
||||||
|
|
||||||
|
lt.SetHandler(Handler{
|
||||||
|
Logger: log,
|
||||||
|
// Allow access to any dataset since we control what mapping
|
||||||
|
// is passed to the pull routine.
|
||||||
|
// All local datasets will be passed to its Map() function,
|
||||||
|
// but only those for which a mapping exists will actually be pulled.
|
||||||
|
// We can pay this small performance penalty for now.
|
||||||
|
PullACL: localPullACL{},
|
||||||
|
})
|
||||||
|
pull.From.Transport = lt
|
||||||
|
log.Printf("fixing up local transport: %#v", pull.From.Transport)
|
||||||
|
}
|
||||||
|
|
||||||
|
var remote rpc.RPCRequester
|
||||||
|
|
||||||
|
if remote, err = pull.From.Transport.Connect(log); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
defer closeRPCWithTimeout(log, remote, time.Second*10, "")
|
||||||
|
|
||||||
|
return doPull(PullContext{remote, log, pull.Mapping, pull.InitialReplPolicy})
|
||||||
|
}
|
||||||
|
|
||||||
|
func jobPush(push *Push, log jobrun.Logger) (err error) {
|
||||||
|
|
||||||
|
if _, ok := push.To.Transport.(LocalTransport); ok {
|
||||||
|
panic("no support for local pushs")
|
||||||
|
}
|
||||||
|
|
||||||
|
var remote rpc.RPCRequester
|
||||||
|
if remote, err = push.To.Transport.Connect(log); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
defer closeRPCWithTimeout(log, remote, time.Second*10, "")
|
||||||
|
|
||||||
|
log.Printf("building handler for PullMeRequest")
|
||||||
|
handler := Handler{
|
||||||
|
Logger: log,
|
||||||
|
PullACL: push.Filter,
|
||||||
|
SinkMappingFunc: nil, // no need for that in the handler for PullMe
|
||||||
|
}
|
||||||
|
log.Printf("handler: %#v", handler)
|
||||||
|
|
||||||
|
r := rpc.PullMeRequest{
|
||||||
|
InitialReplPolicy: push.InitialReplPolicy,
|
||||||
|
}
|
||||||
|
log.Printf("doing PullMeRequest: %#v", r)
|
||||||
|
|
||||||
|
if err = remote.PullMeRequest(r, handler); err != nil {
|
||||||
|
log.Printf("PullMeRequest failed: %s", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("push job finished")
|
||||||
|
return
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func closeRPCWithTimeout(log Logger, remote rpc.RPCRequester, timeout time.Duration, goodbye string) {
|
||||||
|
log.Printf("closing rpc connection")
|
||||||
|
|
||||||
|
ch := make(chan error)
|
||||||
|
go func() {
|
||||||
|
ch <- remote.CloseRequest(rpc.CloseRequest{goodbye})
|
||||||
|
close(ch)
|
||||||
|
}()
|
||||||
|
|
||||||
|
var err error
|
||||||
|
select {
|
||||||
|
case <-time.After(timeout):
|
||||||
|
err = fmt.Errorf("timeout exceeded (%s)", timeout)
|
||||||
|
case closeRequestErr := <-ch:
|
||||||
|
err = closeRequestErr
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("error closing connection: %s", err)
|
||||||
|
err = remote.ForceClose()
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("error force-closing connection: %s", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
type PullContext struct {
|
||||||
|
Remote rpc.RPCRequester
|
||||||
|
Log Logger
|
||||||
|
Mapping DatasetMapping
|
||||||
|
InitialReplPolicy rpc.InitialReplPolicy
|
||||||
|
}
|
||||||
|
|
||||||
|
func doPull(pull PullContext) (err error) {
|
||||||
|
|
||||||
|
remote := pull.Remote
|
||||||
|
log := pull.Log
|
||||||
|
|
||||||
|
log.Printf("requesting remote filesystem list")
|
||||||
|
fsr := rpc.FilesystemRequest{}
|
||||||
|
var remoteFilesystems []*zfs.DatasetPath
|
||||||
|
if remoteFilesystems, err = remote.FilesystemRequest(fsr); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("map remote filesystems to local paths and determine order for per-filesystem sync")
|
||||||
|
type RemoteLocalMapping struct {
|
||||||
|
Remote *zfs.DatasetPath
|
||||||
|
Local *zfs.DatasetPath
|
||||||
|
}
|
||||||
|
replMapping := make(map[string]RemoteLocalMapping, len(remoteFilesystems))
|
||||||
|
localTraversal := zfs.NewDatasetPathForest()
|
||||||
|
for fs := range remoteFilesystems {
|
||||||
|
var err error
|
||||||
|
var localFs *zfs.DatasetPath
|
||||||
|
localFs, err = pull.Mapping.Map(remoteFilesystems[fs])
|
||||||
|
if err != nil {
|
||||||
|
if err != NoMatchError {
|
||||||
|
err := fmt.Errorf("error mapping %s: %s", remoteFilesystems[fs], err)
|
||||||
|
log.Printf("%s", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
log.Printf("%s => %s", remoteFilesystems[fs].ToString(), localFs.ToString())
|
||||||
|
m := RemoteLocalMapping{remoteFilesystems[fs], localFs}
|
||||||
|
replMapping[m.Local.ToString()] = m
|
||||||
|
localTraversal.Add(m.Local)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("build cache for already present local filesystem state")
|
||||||
|
localFilesystemState, err := zfs.ZFSListFilesystemState()
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("error requesting local filesystem state: %s", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("start per-filesystem sync")
|
||||||
|
localTraversal.WalkTopDown(func(v zfs.DatasetPathVisit) bool {
|
||||||
|
|
||||||
|
if v.FilledIn {
|
||||||
|
if _, exists := localFilesystemState[v.Path.ToString()]; exists {
|
||||||
|
// No need to verify if this is a placeholder or not. It is sufficient
|
||||||
|
// to know we can add child filesystems to it
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
log.Printf("creating placeholder filesystem %s", v.Path.ToString())
|
||||||
|
err = zfs.ZFSCreatePlaceholderFilesystem(v.Path)
|
||||||
|
if err != nil {
|
||||||
|
err = fmt.Errorf("aborting, cannot create placeholder filesystem %s: %s", v.Path, err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
m, ok := replMapping[v.Path.ToString()]
|
||||||
|
if !ok {
|
||||||
|
panic("internal inconsistency: replMapping should contain mapping for any path that was not filled in by WalkTopDown()")
|
||||||
|
}
|
||||||
|
|
||||||
|
log := func(format string, args ...interface{}) {
|
||||||
|
log.Printf("[%s => %s]: %s", m.Remote.ToString(), m.Local.ToString(), fmt.Sprintf(format, args...))
|
||||||
|
}
|
||||||
|
|
||||||
|
log("examing local filesystem state")
|
||||||
|
localState, localExists := localFilesystemState[m.Local.ToString()]
|
||||||
|
var versions []zfs.FilesystemVersion
|
||||||
|
switch {
|
||||||
|
case !localExists:
|
||||||
|
log("local filesystem does not exist")
|
||||||
|
case localState.Placeholder:
|
||||||
|
log("local filesystem is marked as placeholder")
|
||||||
|
case localState.ResumeToken != "":
|
||||||
|
|
||||||
|
log("local filesystem has receive_resume_token")
|
||||||
|
if false { // TODO Check if managed resume is disabled (explain information leakage in docs!)
|
||||||
|
log("managed resume tokens are disabled via zrepl config, assuming config change or external administrative action")
|
||||||
|
log("policy forbids aborting the partial recv automatically, skipping this filesystem")
|
||||||
|
log("for zrepl to resume replication for this dataset, administrators should finishing the recv manually or abort the recv via `zfs recv -A %s`", m.Local.ToString())
|
||||||
|
return true // TODO right choice to allow children? ... should be, since the fs exists...
|
||||||
|
}
|
||||||
|
|
||||||
|
log("decoding receive_resume_token")
|
||||||
|
// TODO, see handler comments
|
||||||
|
// TODO override logger with fromguid toguid
|
||||||
|
|
||||||
|
log("requesting resume of transfer")
|
||||||
|
r := rpc.ResumeTransferRequest{m.Remote, localState.ResumeToken}
|
||||||
|
stream, err := remote.ResumeTransferRequest(r)
|
||||||
|
if err != nil {
|
||||||
|
|
||||||
|
log("resume transfer request failed: %s", err)
|
||||||
|
rre, ok := err.(*rpc.ResumeTransferError)
|
||||||
|
if !ok {
|
||||||
|
log("skipping this filesystem, could be temporary issue")
|
||||||
|
return true // TODO right choice to allow children
|
||||||
|
}
|
||||||
|
|
||||||
|
// Determine if we should clear the resume token
|
||||||
|
clearAndUseSnaps := false
|
||||||
|
switch rre.Reason {
|
||||||
|
case rpc.ResumeTransferErrorReasonNotImplemented:
|
||||||
|
fallthrough
|
||||||
|
case rpc.ResumeTransferErrorReasonDisabled:
|
||||||
|
fallthrough
|
||||||
|
case rpc.ResumeTransferErrorReasonZFSErrorPermanent:
|
||||||
|
clearAndUseSnaps = true
|
||||||
|
|
||||||
|
case rpc.ResumeTransferErrorReasonZFSErrorMaybeTemporary:
|
||||||
|
fallthrough
|
||||||
|
default:
|
||||||
|
clearAndUseSnaps = false
|
||||||
|
}
|
||||||
|
|
||||||
|
if !clearAndUseSnaps {
|
||||||
|
log("skipping this filesystem, error identified as temporary")
|
||||||
|
return true // TODO right choice to allow children
|
||||||
|
}
|
||||||
|
|
||||||
|
log("clearing local receive_resume_token")
|
||||||
|
if err := zfs.ZFSRecvAbort(m.Local); err != nil {
|
||||||
|
log("error clearing receive_resume_token: %s", err)
|
||||||
|
return true // TODO right choice to allow children? the filesystem exists, so it should be ok
|
||||||
|
}
|
||||||
|
// TODO go back to top of function (put all this into separate function, then tail recursive call)
|
||||||
|
// TODO return this_function()
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO warning code duplication, see below. need to unify this
|
||||||
|
log("invoking zfs receive")
|
||||||
|
watcher := util.IOProgressWatcher{Reader: stream}
|
||||||
|
watcher.KickOff(1*time.Second, func(p util.IOProgress) {
|
||||||
|
log("progress on receive operation: %v bytes received", p.TotalRX)
|
||||||
|
})
|
||||||
|
|
||||||
|
recvArgs := []string{"-u"}
|
||||||
|
if localState.Placeholder {
|
||||||
|
log("receive with forced rollback to replace placeholder filesystem")
|
||||||
|
recvArgs = append(recvArgs, "-F")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = zfs.ZFSRecv(m.Local, &watcher, recvArgs...); err != nil {
|
||||||
|
log("error receiving stream: %s", err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
log("finished receiving stream, %v bytes total", watcher.Progress().TotalRX)
|
||||||
|
|
||||||
|
// TODO further problem property handling code must be duplicated here...
|
||||||
|
|
||||||
|
// TODO tail recursive call to this function, we might not be dony syncing yet
|
||||||
|
// TODO return this_function()
|
||||||
|
return true
|
||||||
|
|
||||||
|
default:
|
||||||
|
log("local filesystem exists")
|
||||||
|
log("requesting local filesystem versions")
|
||||||
|
if versions, err = zfs.ZFSListFilesystemVersions(m.Local, nil); err != nil {
|
||||||
|
log("cannot get local filesystem versions: %s", err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log("requesting remote filesystem versions")
|
||||||
|
var theirVersions []zfs.FilesystemVersion
|
||||||
|
theirVersions, err = remote.FilesystemVersionsRequest(rpc.FilesystemVersionsRequest{
|
||||||
|
Filesystem: m.Remote,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log("error requesting remote filesystem versions: %s", err)
|
||||||
|
log("stopping replication for all filesystems mapped as children of %s", m.Local.ToString())
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
log("computing diff between remote and local filesystem versions")
|
||||||
|
diff := zfs.MakeFilesystemDiff(versions, theirVersions)
|
||||||
|
log("%s", diff)
|
||||||
|
|
||||||
|
if localState.Placeholder && diff.Conflict != zfs.ConflictAllRight {
|
||||||
|
panic("internal inconsistency: local placeholder implies ConflictAllRight")
|
||||||
|
}
|
||||||
|
|
||||||
|
switch diff.Conflict {
|
||||||
|
case zfs.ConflictAllRight:
|
||||||
|
|
||||||
|
log("performing initial sync, following policy: '%s'", pull.InitialReplPolicy)
|
||||||
|
|
||||||
|
if pull.InitialReplPolicy != rpc.InitialReplPolicyMostRecent {
|
||||||
|
panic(fmt.Sprintf("policy '%s' not implemented", pull.InitialReplPolicy))
|
||||||
|
}
|
||||||
|
|
||||||
|
snapsOnly := make([]zfs.FilesystemVersion, 0, len(diff.MRCAPathRight))
|
||||||
|
for s := range diff.MRCAPathRight {
|
||||||
|
if diff.MRCAPathRight[s].Type == zfs.Snapshot {
|
||||||
|
snapsOnly = append(snapsOnly, diff.MRCAPathRight[s])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(snapsOnly) < 1 {
|
||||||
|
log("cannot perform initial sync: no remote snapshots. stopping...")
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
r := rpc.InitialTransferRequest{
|
||||||
|
Filesystem: m.Remote,
|
||||||
|
FilesystemVersion: snapsOnly[len(snapsOnly)-1],
|
||||||
|
}
|
||||||
|
|
||||||
|
log("requesting snapshot stream for %s", r.FilesystemVersion)
|
||||||
|
|
||||||
|
var stream io.Reader
|
||||||
|
if stream, err = remote.InitialTransferRequest(r); err != nil {
|
||||||
|
log("error requesting initial transfer: %s", err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
log("received initial transfer request response")
|
||||||
|
|
||||||
|
log("invoking zfs receive")
|
||||||
|
watcher := util.IOProgressWatcher{Reader: stream}
|
||||||
|
watcher.KickOff(1*time.Second, func(p util.IOProgress) {
|
||||||
|
log("progress on receive operation: %v bytes received", p.TotalRX)
|
||||||
|
})
|
||||||
|
|
||||||
|
recvArgs := []string{"-u"}
|
||||||
|
if localState.Placeholder {
|
||||||
|
log("receive with forced rollback to replace placeholder filesystem")
|
||||||
|
recvArgs = append(recvArgs, "-F")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = zfs.ZFSRecv(m.Local, &watcher, recvArgs...); err != nil {
|
||||||
|
log("error receiving stream: %s", err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
log("finished receiving stream, %v bytes total", watcher.Progress().TotalRX)
|
||||||
|
|
||||||
|
log("configuring properties of received filesystem")
|
||||||
|
if err = zfs.ZFSSet(m.Local, "readonly", "on"); err != nil {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
log("finished initial transfer")
|
||||||
|
return true
|
||||||
|
|
||||||
|
case zfs.ConflictIncremental:
|
||||||
|
|
||||||
|
if len(diff.IncrementalPath) < 2 {
|
||||||
|
log("remote and local are in sync")
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
log("following incremental path from diff")
|
||||||
|
var pathRx uint64
|
||||||
|
|
||||||
|
for i := 0; i < len(diff.IncrementalPath)-1; i++ {
|
||||||
|
|
||||||
|
from, to := diff.IncrementalPath[i], diff.IncrementalPath[i+1]
|
||||||
|
|
||||||
|
log := func(format string, args ...interface{}) {
|
||||||
|
log("[%v/%v][%s => %s]: %s", i+1, len(diff.IncrementalPath)-1,
|
||||||
|
from.Name, to.Name, fmt.Sprintf(format, args...))
|
||||||
|
}
|
||||||
|
|
||||||
|
log("requesting incremental snapshot stream")
|
||||||
|
r := rpc.IncrementalTransferRequest{
|
||||||
|
Filesystem: m.Remote,
|
||||||
|
From: from,
|
||||||
|
To: to,
|
||||||
|
}
|
||||||
|
var stream io.Reader
|
||||||
|
if stream, err = remote.IncrementalTransferRequest(r); err != nil {
|
||||||
|
log("error requesting incremental snapshot stream: %s", err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
log("invoking zfs receive")
|
||||||
|
watcher := util.IOProgressWatcher{Reader: stream}
|
||||||
|
watcher.KickOff(1*time.Second, func(p util.IOProgress) {
|
||||||
|
log("progress on receive operation: %v bytes received", p.TotalRX)
|
||||||
|
})
|
||||||
|
|
||||||
|
if err = zfs.ZFSRecv(m.Local, &watcher); err != nil {
|
||||||
|
log("error receiving stream: %s", err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
totalRx := watcher.Progress().TotalRX
|
||||||
|
pathRx += totalRx
|
||||||
|
log("finished incremental transfer, %v bytes total", totalRx)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
log("finished following incremental path, %v bytes total", pathRx)
|
||||||
|
return true
|
||||||
|
|
||||||
|
case zfs.ConflictNoCommonAncestor:
|
||||||
|
|
||||||
|
log("remote and local filesystem have snapshots, but no common one")
|
||||||
|
log("perform manual replication to establish a common snapshot history")
|
||||||
|
log("remote versions:")
|
||||||
|
for _, v := range diff.MRCAPathRight {
|
||||||
|
log(" %s (GUID %v)", v, v.Guid)
|
||||||
|
}
|
||||||
|
log("local versions:")
|
||||||
|
for _, v := range diff.MRCAPathLeft {
|
||||||
|
log(" %s (GUID %v)", v, v.Guid)
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
|
||||||
|
case zfs.ConflictDiverged:
|
||||||
|
|
||||||
|
log("remote and local filesystem share a history but have diverged")
|
||||||
|
log("perform manual replication or delete snapshots on the receiving" +
|
||||||
|
"side to establish an incremental replication parse")
|
||||||
|
log("remote-only versions:")
|
||||||
|
for _, v := range diff.MRCAPathRight {
|
||||||
|
log(" %s (GUID %v)", v, v.Guid)
|
||||||
|
}
|
||||||
|
log("local-only versions:")
|
||||||
|
for _, v := range diff.MRCAPathLeft {
|
||||||
|
log(" %s (GUID %v)", v, v.Guid)
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
panic("implementation error: this should not be reached")
|
||||||
|
return false
|
||||||
|
|
||||||
|
})
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,109 @@
|
|||||||
|
remotes:
|
||||||
|
offsite_backups:
|
||||||
|
transport:
|
||||||
|
ssh:
|
||||||
|
host: 192.168.122.6
|
||||||
|
user: root
|
||||||
|
port: 22
|
||||||
|
identity_file: /etc/zrepl/identities/offsite_backups
|
||||||
|
|
||||||
|
pushs:
|
||||||
|
|
||||||
|
offsite:
|
||||||
|
to: offsite_backups
|
||||||
|
filter: {
|
||||||
|
# like in pull_acls
|
||||||
|
"tank/var/db<": ok,
|
||||||
|
"tank/usr/home<": ok,
|
||||||
|
}
|
||||||
|
|
||||||
|
pulls:
|
||||||
|
|
||||||
|
offsite:
|
||||||
|
from: offsite_backups
|
||||||
|
mapping: {
|
||||||
|
# like in sinks
|
||||||
|
}
|
||||||
|
|
||||||
|
# local replication, only allowed in pull mode
|
||||||
|
# the from name 'local' is reserved for this purpose
|
||||||
|
homemirror:
|
||||||
|
from: local
|
||||||
|
repeat:
|
||||||
|
interval: 15m
|
||||||
|
mapping: {
|
||||||
|
"tank/usr/home":"mirrorpool/foo/bar"
|
||||||
|
}
|
||||||
|
|
||||||
|
sinks:
|
||||||
|
|
||||||
|
db1:
|
||||||
|
mapping: {
|
||||||
|
# direct mapping
|
||||||
|
"ssdpool/var/db/postgresql9.6":"zroot/backups/db1/pg_data"
|
||||||
|
}
|
||||||
|
|
||||||
|
mirror1:
|
||||||
|
mapping: {
|
||||||
|
# "<" subtree wildcard matches the dataset left of < and all its children
|
||||||
|
"tank/foo/bar<":"zroot/backups/mirror1"
|
||||||
|
}
|
||||||
|
|
||||||
|
mirror2:
|
||||||
|
# more specific path patterns win over less specific ones
|
||||||
|
# direct mappings win over subtree wildcards
|
||||||
|
# detailed rule precedence: check unit tests & docs for exact behavior
|
||||||
|
# TODO subcommand to test a mapping & filter
|
||||||
|
mapping: {
|
||||||
|
"tank<": "zroot/backups/mirror1/tank1",
|
||||||
|
"tank/cdn/root<": "storage/cdn/root",
|
||||||
|
"tank/legacydb": "legacypool/backups/legacydb",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
pull_acls:
|
||||||
|
|
||||||
|
office_backup:
|
||||||
|
filter: {
|
||||||
|
# valid filter results (right hand side): ok, omit
|
||||||
|
# default is to omit
|
||||||
|
# rule precedence is same as for mappings
|
||||||
|
"tank<": omit,
|
||||||
|
"tank/usr/home": ok,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
prune:
|
||||||
|
|
||||||
|
clean_backups:
|
||||||
|
policy: grid
|
||||||
|
grid: 6x10m | 24x1h | 7x1d | 5 x 1w | 4 x 5w
|
||||||
|
dataset_filter: {
|
||||||
|
"tank/backups/legacyscript<": omit,
|
||||||
|
"tank/backups<": ok,
|
||||||
|
}
|
||||||
|
snapshot_filter: {
|
||||||
|
prefix: zrepl_
|
||||||
|
}
|
||||||
|
|
||||||
|
hfbak_prune: # cleans up after hfbak autosnap job
|
||||||
|
policy: grid
|
||||||
|
grid: 1x1m(keep=all)
|
||||||
|
dataset_filter: {
|
||||||
|
"pool1<": ok
|
||||||
|
}
|
||||||
|
snapshot_filter: {
|
||||||
|
prefix: zrepl_hfbak_
|
||||||
|
}
|
||||||
|
|
||||||
|
autosnap:
|
||||||
|
|
||||||
|
hfbak:
|
||||||
|
prefix: zrepl_hfbak_
|
||||||
|
interval: 1s
|
||||||
|
dataset_filter: {
|
||||||
|
"pool1<": ok
|
||||||
|
}
|
||||||
|
# prune: hfbak_prune
|
||||||
|
# future versions may inline the retention policy here, but for now,
|
||||||
|
# pruning has to be triggered manually (it's safe to run autosnap + prune in parallel)
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"github.com/zrepl/zrepl/rpc"
|
||||||
|
"github.com/zrepl/zrepl/sshbytestream"
|
||||||
|
"io"
|
||||||
|
golog "log"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
var StdinserverCmd = &cobra.Command{
|
||||||
|
Use: "stdinserver CLIENT_IDENTITY",
|
||||||
|
Short: "start in stdin server mode (from authorized_keys file)",
|
||||||
|
Run: cmdStdinServer,
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
RootCmd.AddCommand(StdinserverCmd)
|
||||||
|
}
|
||||||
|
|
||||||
|
func cmdStdinServer(cmd *cobra.Command, args []string) {
|
||||||
|
|
||||||
|
var err error
|
||||||
|
defer func() {
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("stdinserver exiting with error: %s", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
if len(args) != 1 || args[0] == "" {
|
||||||
|
err = fmt.Errorf("must specify client identity as positional argument")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
identity := args[0]
|
||||||
|
|
||||||
|
pullACL, ok := conf.PullACLs[identity]
|
||||||
|
if !ok {
|
||||||
|
err = fmt.Errorf("could not find PullACL for identity '%s'", identity)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var sshByteStream io.ReadWriteCloser
|
||||||
|
if sshByteStream, err = sshbytestream.Incoming(); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
sinkMapping := func(identity string) (m DatasetMapping, err error) {
|
||||||
|
sink, ok := conf.Sinks[identity]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("could not find sink for identity '%s'", identity)
|
||||||
|
}
|
||||||
|
return sink, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
sinkLogger := golog.New(logOut, fmt.Sprintf("sink[%s] ", identity), logFlags)
|
||||||
|
handler := Handler{
|
||||||
|
Logger: sinkLogger,
|
||||||
|
SinkMappingFunc: sinkMapping,
|
||||||
|
PullACL: pullACL,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = rpc.ListenByteStreamRPC(sshByteStream, identity, handler, sinkLogger); err != nil {
|
||||||
|
log.Printf("listenbytestreamerror: %#v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
}
|
||||||
@@ -1,695 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"io/ioutil"
|
|
||||||
"log/syslog"
|
|
||||||
"os"
|
|
||||||
"reflect"
|
|
||||||
"regexp"
|
|
||||||
"strconv"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
"github.com/zrepl/yaml-config"
|
|
||||||
|
|
||||||
zfsprop "github.com/zrepl/zrepl/zfs/property"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Config struct {
|
|
||||||
Jobs []JobEnum `yaml:"jobs"`
|
|
||||||
Global *Global `yaml:"global,optional,fromdefaults"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Config) Job(name string) (*JobEnum, error) {
|
|
||||||
for _, j := range c.Jobs {
|
|
||||||
if j.Name() == name {
|
|
||||||
return &j, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("job %q not defined in config", name)
|
|
||||||
}
|
|
||||||
|
|
||||||
type JobEnum struct {
|
|
||||||
Ret interface{}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j JobEnum) Name() string {
|
|
||||||
var name string
|
|
||||||
switch v := j.Ret.(type) {
|
|
||||||
case *SnapJob:
|
|
||||||
name = v.Name
|
|
||||||
case *PushJob:
|
|
||||||
name = v.Name
|
|
||||||
case *SinkJob:
|
|
||||||
name = v.Name
|
|
||||||
case *PullJob:
|
|
||||||
name = v.Name
|
|
||||||
case *SourceJob:
|
|
||||||
name = v.Name
|
|
||||||
default:
|
|
||||||
panic(fmt.Sprintf("unknown job type %T", v))
|
|
||||||
}
|
|
||||||
return name
|
|
||||||
}
|
|
||||||
|
|
||||||
type ActiveJob struct {
|
|
||||||
Type string `yaml:"type"`
|
|
||||||
Name string `yaml:"name"`
|
|
||||||
Connect ConnectEnum `yaml:"connect"`
|
|
||||||
Pruning PruningSenderReceiver `yaml:"pruning"`
|
|
||||||
Debug JobDebugSettings `yaml:"debug,optional"`
|
|
||||||
Replication *Replication `yaml:"replication,optional,fromdefaults"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type PassiveJob struct {
|
|
||||||
Type string `yaml:"type"`
|
|
||||||
Name string `yaml:"name"`
|
|
||||||
Serve ServeEnum `yaml:"serve"`
|
|
||||||
Debug JobDebugSettings `yaml:"debug,optional"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type SnapJob struct {
|
|
||||||
Type string `yaml:"type"`
|
|
||||||
Name string `yaml:"name"`
|
|
||||||
Pruning PruningLocal `yaml:"pruning"`
|
|
||||||
Debug JobDebugSettings `yaml:"debug,optional"`
|
|
||||||
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
|
|
||||||
Filesystems FilesystemsFilter `yaml:"filesystems"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type SendOptions struct {
|
|
||||||
Encrypted bool `yaml:"encrypted,optional,default=false"`
|
|
||||||
Raw bool `yaml:"raw,optional,default=false"`
|
|
||||||
SendProperties bool `yaml:"send_properties,optional,default=false"`
|
|
||||||
BackupProperties bool `yaml:"backup_properties,optional,default=false"`
|
|
||||||
LargeBlocks bool `yaml:"large_blocks,optional,default=false"`
|
|
||||||
Compressed bool `yaml:"compressed,optional,default=false"`
|
|
||||||
EmbeddedData bool `yaml:"embbeded_data,optional,default=false"`
|
|
||||||
Saved bool `yaml:"saved,optional,default=false"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type RecvOptions struct {
|
|
||||||
// Note: we cannot enforce encrypted recv as the ZFS cli doesn't provide a mechanism for it
|
|
||||||
// Encrypted bool `yaml:"may_encrypted"`
|
|
||||||
// Future:
|
|
||||||
// Reencrypt bool `yaml:"reencrypt"`
|
|
||||||
|
|
||||||
Properties *PropertyRecvOptions `yaml:"properties,fromdefaults"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Replication struct {
|
|
||||||
Protection *ReplicationOptionsProtection `yaml:"protection,optional,fromdefaults"`
|
|
||||||
Concurrency *ReplicationOptionsConcurrency `yaml:"concurrency,optional,fromdefaults"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type ReplicationOptionsProtection struct {
|
|
||||||
Initial string `yaml:"initial,optional,default=guarantee_resumability"`
|
|
||||||
Incremental string `yaml:"incremental,optional,default=guarantee_resumability"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type ReplicationOptionsConcurrency struct {
|
|
||||||
Steps int `yaml:"steps,optional,default=1"`
|
|
||||||
SizeEstimates int `yaml:"size_estimates,optional,default=4"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *RecvOptions) SetDefault() {
|
|
||||||
*l = RecvOptions{Properties: &PropertyRecvOptions{}}
|
|
||||||
}
|
|
||||||
|
|
||||||
type PropertyRecvOptions struct {
|
|
||||||
Inherit []zfsprop.Property `yaml:"inherit,optional"`
|
|
||||||
Override map[zfsprop.Property]string `yaml:"override,optional"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type PushJob struct {
|
|
||||||
ActiveJob `yaml:",inline"`
|
|
||||||
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
|
|
||||||
Filesystems FilesystemsFilter `yaml:"filesystems"`
|
|
||||||
Send *SendOptions `yaml:"send,fromdefaults,optional"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *PushJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
|
|
||||||
func (j *PushJob) GetSendOptions() *SendOptions { return j.Send }
|
|
||||||
|
|
||||||
type PullJob struct {
|
|
||||||
ActiveJob `yaml:",inline"`
|
|
||||||
RootFS string `yaml:"root_fs"`
|
|
||||||
Interval PositiveDurationOrManual `yaml:"interval"`
|
|
||||||
Recv *RecvOptions `yaml:"recv,fromdefaults,optional"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *PullJob) GetRootFS() string { return j.RootFS }
|
|
||||||
func (j *PullJob) GetAppendClientIdentity() bool { return false }
|
|
||||||
func (j *PullJob) GetRecvOptions() *RecvOptions { return j.Recv }
|
|
||||||
|
|
||||||
type PositiveDurationOrManual struct {
|
|
||||||
Interval time.Duration
|
|
||||||
Manual bool
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ yaml.Unmarshaler = (*PositiveDurationOrManual)(nil)
|
|
||||||
|
|
||||||
func (i *PositiveDurationOrManual) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
|
||||||
var s string
|
|
||||||
if err := u(&s, true); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
switch s {
|
|
||||||
case "manual":
|
|
||||||
i.Manual = true
|
|
||||||
i.Interval = 0
|
|
||||||
case "":
|
|
||||||
return fmt.Errorf("value must not be empty")
|
|
||||||
default:
|
|
||||||
i.Manual = false
|
|
||||||
i.Interval, err = time.ParseDuration(s)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if i.Interval <= 0 {
|
|
||||||
return fmt.Errorf("value must be a positive duration, got %q", s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type SinkJob struct {
|
|
||||||
PassiveJob `yaml:",inline"`
|
|
||||||
RootFS string `yaml:"root_fs"`
|
|
||||||
Recv *RecvOptions `yaml:"recv,optional,fromdefaults"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *SinkJob) GetRootFS() string { return j.RootFS }
|
|
||||||
func (j *SinkJob) GetAppendClientIdentity() bool { return true }
|
|
||||||
func (j *SinkJob) GetRecvOptions() *RecvOptions { return j.Recv }
|
|
||||||
|
|
||||||
type SourceJob struct {
|
|
||||||
PassiveJob `yaml:",inline"`
|
|
||||||
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
|
|
||||||
Filesystems FilesystemsFilter `yaml:"filesystems"`
|
|
||||||
Send *SendOptions `yaml:"send,optional,fromdefaults"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *SourceJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
|
|
||||||
func (j *SourceJob) GetSendOptions() *SendOptions { return j.Send }
|
|
||||||
|
|
||||||
type FilesystemsFilter map[string]bool
|
|
||||||
|
|
||||||
type SnapshottingEnum struct {
|
|
||||||
Ret interface{}
|
|
||||||
}
|
|
||||||
|
|
||||||
type SnapshottingPeriodic struct {
|
|
||||||
Type string `yaml:"type"`
|
|
||||||
Prefix string `yaml:"prefix"`
|
|
||||||
Interval time.Duration `yaml:"interval,positive"`
|
|
||||||
Hooks HookList `yaml:"hooks,optional"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type SnapshottingManual struct {
|
|
||||||
Type string `yaml:"type"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type PruningSenderReceiver struct {
|
|
||||||
KeepSender []PruningEnum `yaml:"keep_sender"`
|
|
||||||
KeepReceiver []PruningEnum `yaml:"keep_receiver"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type PruningLocal struct {
|
|
||||||
Keep []PruningEnum `yaml:"keep"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type LoggingOutletEnumList []LoggingOutletEnum
|
|
||||||
|
|
||||||
func (l *LoggingOutletEnumList) SetDefault() {
|
|
||||||
def := `
|
|
||||||
type: "stdout"
|
|
||||||
time: true
|
|
||||||
level: "warn"
|
|
||||||
format: "human"
|
|
||||||
`
|
|
||||||
s := &StdoutLoggingOutlet{}
|
|
||||||
err := yaml.UnmarshalStrict([]byte(def), &s)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
*l = []LoggingOutletEnum{LoggingOutletEnum{Ret: s}}
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ yaml.Defaulter = &LoggingOutletEnumList{}
|
|
||||||
|
|
||||||
type Global struct {
|
|
||||||
Logging *LoggingOutletEnumList `yaml:"logging,optional,fromdefaults"`
|
|
||||||
Monitoring []MonitoringEnum `yaml:"monitoring,optional"`
|
|
||||||
Control *GlobalControl `yaml:"control,optional,fromdefaults"`
|
|
||||||
Serve *GlobalServe `yaml:"serve,optional,fromdefaults"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func Default(i interface{}) {
|
|
||||||
v := reflect.ValueOf(i)
|
|
||||||
if v.Kind() != reflect.Ptr {
|
|
||||||
panic(v)
|
|
||||||
}
|
|
||||||
y := `{}`
|
|
||||||
err := yaml.Unmarshal([]byte(y), v.Interface())
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type ConnectEnum struct {
|
|
||||||
Ret interface{}
|
|
||||||
}
|
|
||||||
|
|
||||||
type ConnectCommon struct {
|
|
||||||
Type string `yaml:"type"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type TCPConnect struct {
|
|
||||||
ConnectCommon `yaml:",inline"`
|
|
||||||
Address string `yaml:"address,hostport"`
|
|
||||||
DialTimeout time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type TLSConnect struct {
|
|
||||||
ConnectCommon `yaml:",inline"`
|
|
||||||
Address string `yaml:"address,hostport"`
|
|
||||||
Ca string `yaml:"ca"`
|
|
||||||
Cert string `yaml:"cert"`
|
|
||||||
Key string `yaml:"key"`
|
|
||||||
ServerCN string `yaml:"server_cn"`
|
|
||||||
DialTimeout time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type SSHStdinserverConnect struct {
|
|
||||||
ConnectCommon `yaml:",inline"`
|
|
||||||
Host string `yaml:"host"`
|
|
||||||
User string `yaml:"user"`
|
|
||||||
Port uint16 `yaml:"port"`
|
|
||||||
IdentityFile string `yaml:"identity_file"`
|
|
||||||
TransportOpenCommand []string `yaml:"transport_open_command,optional"` //TODO unused
|
|
||||||
SSHCommand string `yaml:"ssh_command,optional"` //TODO unused
|
|
||||||
Options []string `yaml:"options,optional"`
|
|
||||||
DialTimeout time.Duration `yaml:"dial_timeout,zeropositive,default=10s"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type LocalConnect struct {
|
|
||||||
ConnectCommon `yaml:",inline"`
|
|
||||||
ListenerName string `yaml:"listener_name"`
|
|
||||||
ClientIdentity string `yaml:"client_identity"`
|
|
||||||
DialTimeout time.Duration `yaml:"dial_timeout,zeropositive,default=2s"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type ServeEnum struct {
|
|
||||||
Ret interface{}
|
|
||||||
}
|
|
||||||
|
|
||||||
type ServeCommon struct {
|
|
||||||
Type string `yaml:"type"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type TCPServe struct {
|
|
||||||
ServeCommon `yaml:",inline"`
|
|
||||||
Listen string `yaml:"listen,hostport"`
|
|
||||||
ListenFreeBind bool `yaml:"listen_freebind,default=false"`
|
|
||||||
Clients map[string]string `yaml:"clients"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type TLSServe struct {
|
|
||||||
ServeCommon `yaml:",inline"`
|
|
||||||
Listen string `yaml:"listen,hostport"`
|
|
||||||
ListenFreeBind bool `yaml:"listen_freebind,default=false"`
|
|
||||||
Ca string `yaml:"ca"`
|
|
||||||
Cert string `yaml:"cert"`
|
|
||||||
Key string `yaml:"key"`
|
|
||||||
ClientCNs []string `yaml:"client_cns"`
|
|
||||||
HandshakeTimeout time.Duration `yaml:"handshake_timeout,zeropositive,default=10s"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type StdinserverServer struct {
|
|
||||||
ServeCommon `yaml:",inline"`
|
|
||||||
ClientIdentities []string `yaml:"client_identities"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type LocalServe struct {
|
|
||||||
ServeCommon `yaml:",inline"`
|
|
||||||
ListenerName string `yaml:"listener_name"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type PruningEnum struct {
|
|
||||||
Ret interface{}
|
|
||||||
}
|
|
||||||
|
|
||||||
type PruneKeepNotReplicated struct {
|
|
||||||
Type string `yaml:"type"`
|
|
||||||
KeepSnapshotAtCursor bool `yaml:"keep_snapshot_at_cursor,optional,default=true"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type PruneKeepLastN struct {
|
|
||||||
Type string `yaml:"type"`
|
|
||||||
Count int `yaml:"count"`
|
|
||||||
Regex string `yaml:"regex,optional"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type PruneKeepRegex struct { // FIXME rename to KeepRegex
|
|
||||||
Type string `yaml:"type"`
|
|
||||||
Regex string `yaml:"regex"`
|
|
||||||
Negate bool `yaml:"negate,optional,default=false"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type LoggingOutletEnum struct {
|
|
||||||
Ret interface{}
|
|
||||||
}
|
|
||||||
|
|
||||||
type LoggingOutletCommon struct {
|
|
||||||
Type string `yaml:"type"`
|
|
||||||
Level string `yaml:"level"`
|
|
||||||
Format string `yaml:"format"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type StdoutLoggingOutlet struct {
|
|
||||||
LoggingOutletCommon `yaml:",inline"`
|
|
||||||
Time bool `yaml:"time,default=true"`
|
|
||||||
Color bool `yaml:"color,default=true"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type SyslogLoggingOutlet struct {
|
|
||||||
LoggingOutletCommon `yaml:",inline"`
|
|
||||||
Facility *SyslogFacility `yaml:"facility,optional,fromdefaults"`
|
|
||||||
RetryInterval time.Duration `yaml:"retry_interval,positive,default=10s"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type TCPLoggingOutlet struct {
|
|
||||||
LoggingOutletCommon `yaml:",inline"`
|
|
||||||
Address string `yaml:"address,hostport"`
|
|
||||||
Net string `yaml:"net,default=tcp"`
|
|
||||||
RetryInterval time.Duration `yaml:"retry_interval,positive,default=10s"`
|
|
||||||
TLS *TCPLoggingOutletTLS `yaml:"tls,optional"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type TCPLoggingOutletTLS struct {
|
|
||||||
CA string `yaml:"ca"`
|
|
||||||
Cert string `yaml:"cert"`
|
|
||||||
Key string `yaml:"key"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type MonitoringEnum struct {
|
|
||||||
Ret interface{}
|
|
||||||
}
|
|
||||||
|
|
||||||
type PrometheusMonitoring struct {
|
|
||||||
Type string `yaml:"type"`
|
|
||||||
Listen string `yaml:"listen,hostport"`
|
|
||||||
ListenFreeBind bool `yaml:"listen_freebind,default=false"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type SyslogFacility syslog.Priority
|
|
||||||
|
|
||||||
func (f *SyslogFacility) SetDefault() {
|
|
||||||
*f = SyslogFacility(syslog.LOG_LOCAL0)
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ yaml.Defaulter = (*SyslogFacility)(nil)
|
|
||||||
|
|
||||||
type GlobalControl struct {
|
|
||||||
SockPath string `yaml:"sockpath,default=/var/run/zrepl/control"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type GlobalServe struct {
|
|
||||||
StdinServer *GlobalStdinServer `yaml:"stdinserver,optional,fromdefaults"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type GlobalStdinServer struct {
|
|
||||||
SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type JobDebugSettings struct {
|
|
||||||
Conn *struct {
|
|
||||||
ReadDump string `yaml:"read_dump"`
|
|
||||||
WriteDump string `yaml:"write_dump"`
|
|
||||||
} `yaml:"conn,optional"`
|
|
||||||
RPCLog bool `yaml:"rpc_log,optional,default=false"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type HookList []HookEnum
|
|
||||||
|
|
||||||
type HookEnum struct {
|
|
||||||
Ret interface{}
|
|
||||||
}
|
|
||||||
|
|
||||||
type HookCommand struct {
|
|
||||||
Path string `yaml:"path"`
|
|
||||||
Timeout time.Duration `yaml:"timeout,optional,positive,default=30s"`
|
|
||||||
Filesystems FilesystemsFilter `yaml:"filesystems,optional,default={'<': true}"`
|
|
||||||
HookSettingsCommon `yaml:",inline"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type HookPostgresCheckpoint struct {
|
|
||||||
HookSettingsCommon `yaml:",inline"`
|
|
||||||
DSN string `yaml:"dsn"`
|
|
||||||
Timeout time.Duration `yaml:"timeout,optional,positive,default=30s"`
|
|
||||||
Filesystems FilesystemsFilter `yaml:"filesystems"` // required, user should not CHECKPOINT for every FS
|
|
||||||
}
|
|
||||||
|
|
||||||
type HookMySQLLockTables struct {
|
|
||||||
HookSettingsCommon `yaml:",inline"`
|
|
||||||
DSN string `yaml:"dsn"`
|
|
||||||
Timeout time.Duration `yaml:"timeout,optional,positive,default=30s"`
|
|
||||||
Filesystems FilesystemsFilter `yaml:"filesystems"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type HookSettingsCommon struct {
|
|
||||||
Type string `yaml:"type"`
|
|
||||||
ErrIsFatal bool `yaml:"err_is_fatal,optional,default=false"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func enumUnmarshal(u func(interface{}, bool) error, types map[string]interface{}) (interface{}, error) {
|
|
||||||
var in struct {
|
|
||||||
Type string
|
|
||||||
}
|
|
||||||
if err := u(&in, true); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if in.Type == "" {
|
|
||||||
return nil, &yaml.TypeError{Errors: []string{"must specify type"}}
|
|
||||||
}
|
|
||||||
|
|
||||||
v, ok := types[in.Type]
|
|
||||||
if !ok {
|
|
||||||
return nil, &yaml.TypeError{Errors: []string{fmt.Sprintf("invalid type name %q", in.Type)}}
|
|
||||||
}
|
|
||||||
if err := u(v, false); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return v, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *JobEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
|
||||||
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
|
||||||
"snap": &SnapJob{},
|
|
||||||
"push": &PushJob{},
|
|
||||||
"sink": &SinkJob{},
|
|
||||||
"pull": &PullJob{},
|
|
||||||
"source": &SourceJob{},
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *ConnectEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
|
||||||
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
|
||||||
"tcp": &TCPConnect{},
|
|
||||||
"tls": &TLSConnect{},
|
|
||||||
"ssh+stdinserver": &SSHStdinserverConnect{},
|
|
||||||
"local": &LocalConnect{},
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
|
||||||
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
|
||||||
"tcp": &TCPServe{},
|
|
||||||
"tls": &TLSServe{},
|
|
||||||
"stdinserver": &StdinserverServer{},
|
|
||||||
"local": &LocalServe{},
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
|
||||||
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
|
||||||
"not_replicated": &PruneKeepNotReplicated{},
|
|
||||||
"last_n": &PruneKeepLastN{},
|
|
||||||
"grid": &PruneGrid{},
|
|
||||||
"regex": &PruneKeepRegex{},
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *SnapshottingEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
|
||||||
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
|
||||||
"periodic": &SnapshottingPeriodic{},
|
|
||||||
"manual": &SnapshottingManual{},
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *LoggingOutletEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
|
||||||
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
|
||||||
"stdout": &StdoutLoggingOutlet{},
|
|
||||||
"syslog": &SyslogLoggingOutlet{},
|
|
||||||
"tcp": &TCPLoggingOutlet{},
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *MonitoringEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
|
||||||
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
|
||||||
"prometheus": &PrometheusMonitoring{},
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *SyslogFacility) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
|
||||||
var s string
|
|
||||||
if err := u(&s, true); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
var level syslog.Priority
|
|
||||||
switch s {
|
|
||||||
case "kern":
|
|
||||||
level = syslog.LOG_KERN
|
|
||||||
case "user":
|
|
||||||
level = syslog.LOG_USER
|
|
||||||
case "mail":
|
|
||||||
level = syslog.LOG_MAIL
|
|
||||||
case "daemon":
|
|
||||||
level = syslog.LOG_DAEMON
|
|
||||||
case "auth":
|
|
||||||
level = syslog.LOG_AUTH
|
|
||||||
case "syslog":
|
|
||||||
level = syslog.LOG_SYSLOG
|
|
||||||
case "lpr":
|
|
||||||
level = syslog.LOG_LPR
|
|
||||||
case "news":
|
|
||||||
level = syslog.LOG_NEWS
|
|
||||||
case "uucp":
|
|
||||||
level = syslog.LOG_UUCP
|
|
||||||
case "cron":
|
|
||||||
level = syslog.LOG_CRON
|
|
||||||
case "authpriv":
|
|
||||||
level = syslog.LOG_AUTHPRIV
|
|
||||||
case "ftp":
|
|
||||||
level = syslog.LOG_FTP
|
|
||||||
case "local0":
|
|
||||||
level = syslog.LOG_LOCAL0
|
|
||||||
case "local1":
|
|
||||||
level = syslog.LOG_LOCAL1
|
|
||||||
case "local2":
|
|
||||||
level = syslog.LOG_LOCAL2
|
|
||||||
case "local3":
|
|
||||||
level = syslog.LOG_LOCAL3
|
|
||||||
case "local4":
|
|
||||||
level = syslog.LOG_LOCAL4
|
|
||||||
case "local5":
|
|
||||||
level = syslog.LOG_LOCAL5
|
|
||||||
case "local6":
|
|
||||||
level = syslog.LOG_LOCAL6
|
|
||||||
case "local7":
|
|
||||||
level = syslog.LOG_LOCAL7
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("invalid syslog level: %q", s)
|
|
||||||
}
|
|
||||||
*t = SyslogFacility(level)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *HookEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
|
||||||
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
|
||||||
"command": &HookCommand{},
|
|
||||||
"postgres-checkpoint": &HookPostgresCheckpoint{},
|
|
||||||
"mysql-lock-tables": &HookMySQLLockTables{},
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
var ConfigFileDefaultLocations = []string{
|
|
||||||
"/etc/zrepl/zrepl.yml",
|
|
||||||
"/usr/local/etc/zrepl/zrepl.yml",
|
|
||||||
}
|
|
||||||
|
|
||||||
func ParseConfig(path string) (i *Config, err error) {
|
|
||||||
|
|
||||||
if path == "" {
|
|
||||||
// Try default locations
|
|
||||||
for _, l := range ConfigFileDefaultLocations {
|
|
||||||
stat, statErr := os.Stat(l)
|
|
||||||
if statErr != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !stat.Mode().IsRegular() {
|
|
||||||
err = errors.Errorf("file at default location is not a regular file: %s", l)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
path = l
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var bytes []byte
|
|
||||||
|
|
||||||
if bytes, err = ioutil.ReadFile(path); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
return ParseConfigBytes(bytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
func ParseConfigBytes(bytes []byte) (*Config, error) {
|
|
||||||
var c *Config
|
|
||||||
if err := yaml.UnmarshalStrict(bytes, &c); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if c == nil {
|
|
||||||
return nil, fmt.Errorf("config is empty or only consists of comments")
|
|
||||||
}
|
|
||||||
return c, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
|
|
||||||
|
|
||||||
func parsePositiveDuration(e string) (d time.Duration, err error) {
|
|
||||||
comps := durationStringRegex.FindStringSubmatch(e)
|
|
||||||
if len(comps) != 3 {
|
|
||||||
err = fmt.Errorf("does not match regex: %s %#v", e, comps)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
if durationFactor <= 0 {
|
|
||||||
return 0, errors.New("duration must be positive integer")
|
|
||||||
}
|
|
||||||
|
|
||||||
var durationUnit time.Duration
|
|
||||||
switch comps[2] {
|
|
||||||
case "s":
|
|
||||||
durationUnit = time.Second
|
|
||||||
case "m":
|
|
||||||
durationUnit = time.Minute
|
|
||||||
case "h":
|
|
||||||
durationUnit = time.Hour
|
|
||||||
case "d":
|
|
||||||
durationUnit = 24 * time.Hour
|
|
||||||
case "w":
|
|
||||||
durationUnit = 24 * 7 * time.Hour
|
|
||||||
default:
|
|
||||||
err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
d = time.Duration(durationFactor) * durationUnit
|
|
||||||
return
|
|
||||||
}
|
|
||||||
@@ -1,90 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"github.com/zrepl/yaml-config"
|
|
||||||
)
|
|
||||||
|
|
||||||
type A struct {
|
|
||||||
B *B `yaml:"b,optional,fromdefaults"`
|
|
||||||
A1 string `yaml:"a1,optional"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type B struct {
|
|
||||||
C *C `yaml:"c,optional,fromdefaults"`
|
|
||||||
D string `yaml:"d,default=ddd"`
|
|
||||||
E string `yaml:"e,optional"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type C struct {
|
|
||||||
Q string `yaml:"q,optional"`
|
|
||||||
R string `yaml:"r,default=r"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDepFromDefaults(t *testing.T) {
|
|
||||||
|
|
||||||
type testcase struct {
|
|
||||||
name string
|
|
||||||
yaml string
|
|
||||||
expect *A
|
|
||||||
}
|
|
||||||
|
|
||||||
tcs := []testcase{
|
|
||||||
{
|
|
||||||
name: "empty",
|
|
||||||
yaml: `{}`,
|
|
||||||
expect: &A{
|
|
||||||
B: &B{
|
|
||||||
C: &C{
|
|
||||||
R: "r",
|
|
||||||
},
|
|
||||||
D: "ddd",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a1 set",
|
|
||||||
yaml: `{"a1":"blah"}`,
|
|
||||||
expect: &A{
|
|
||||||
A1: "blah",
|
|
||||||
B: &B{
|
|
||||||
C: &C{
|
|
||||||
R: "r",
|
|
||||||
},
|
|
||||||
D: "ddd",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "D set",
|
|
||||||
yaml: `
|
|
||||||
b:
|
|
||||||
d: 4d
|
|
||||||
`,
|
|
||||||
expect: &A{
|
|
||||||
B: &B{
|
|
||||||
D: "4d",
|
|
||||||
C: &C{
|
|
||||||
R: "r",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for tci := range tcs {
|
|
||||||
t.Run(fmt.Sprintf("%d-%s", tci, tcs[tci].name), func(t *testing.T) {
|
|
||||||
tc := tcs[tci]
|
|
||||||
|
|
||||||
var a A
|
|
||||||
err := yaml.UnmarshalStrict([]byte(tc.yaml), &a)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
require.Equal(t, tc.expect, &a)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,115 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"log/syslog"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"github.com/zrepl/yaml-config"
|
|
||||||
)
|
|
||||||
|
|
||||||
func testValidGlobalSection(t *testing.T, s string) *Config {
|
|
||||||
jobdef := `
|
|
||||||
jobs:
|
|
||||||
- name: dummyjob
|
|
||||||
type: sink
|
|
||||||
serve:
|
|
||||||
type: tcp
|
|
||||||
listen: ":2342"
|
|
||||||
clients: {
|
|
||||||
"10.0.0.1":"foo"
|
|
||||||
}
|
|
||||||
root_fs: zroot/foo
|
|
||||||
`
|
|
||||||
_, err := ParseConfigBytes([]byte(jobdef))
|
|
||||||
require.NoError(t, err)
|
|
||||||
return testValidConfig(t, s+jobdef)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOutletTypes(t *testing.T) {
|
|
||||||
conf := testValidGlobalSection(t, `
|
|
||||||
global:
|
|
||||||
logging:
|
|
||||||
- type: stdout
|
|
||||||
level: debug
|
|
||||||
format: human
|
|
||||||
- type: syslog
|
|
||||||
level: info
|
|
||||||
retry_interval: 20s
|
|
||||||
format: human
|
|
||||||
- type: tcp
|
|
||||||
level: debug
|
|
||||||
format: json
|
|
||||||
address: logserver.example.com:1234
|
|
||||||
- type: tcp
|
|
||||||
level: debug
|
|
||||||
format: json
|
|
||||||
address: encryptedlogserver.example.com:1234
|
|
||||||
retry_interval: 20s
|
|
||||||
tls:
|
|
||||||
ca: /etc/zrepl/log/ca.crt
|
|
||||||
cert: /etc/zrepl/log/key.pem
|
|
||||||
key: /etc/zrepl/log/cert.pem
|
|
||||||
`)
|
|
||||||
assert.Equal(t, 4, len(*conf.Global.Logging))
|
|
||||||
assert.NotNil(t, (*conf.Global.Logging)[3].Ret.(*TCPLoggingOutlet).TLS)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDefaultLoggingOutlet(t *testing.T) {
|
|
||||||
conf := testValidGlobalSection(t, "")
|
|
||||||
assert.Equal(t, 1, len(*conf.Global.Logging))
|
|
||||||
o := (*conf.Global.Logging)[0].Ret.(*StdoutLoggingOutlet)
|
|
||||||
assert.Equal(t, "warn", o.Level)
|
|
||||||
assert.Equal(t, "human", o.Format)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPrometheusMonitoring(t *testing.T) {
|
|
||||||
conf := testValidGlobalSection(t, `
|
|
||||||
global:
|
|
||||||
monitoring:
|
|
||||||
- type: prometheus
|
|
||||||
listen: ':9091'
|
|
||||||
`)
|
|
||||||
assert.Equal(t, ":9091", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSyslogLoggingOutletFacility(t *testing.T) {
|
|
||||||
type SyslogFacilityPriority struct {
|
|
||||||
Facility string
|
|
||||||
Priority syslog.Priority
|
|
||||||
}
|
|
||||||
syslogFacilitiesPriorities := []SyslogFacilityPriority{
|
|
||||||
{"", syslog.LOG_LOCAL0}, // default
|
|
||||||
{"kern", syslog.LOG_KERN}, {"daemon", syslog.LOG_DAEMON}, {"auth", syslog.LOG_AUTH},
|
|
||||||
{"syslog", syslog.LOG_SYSLOG}, {"lpr", syslog.LOG_LPR}, {"news", syslog.LOG_NEWS},
|
|
||||||
{"uucp", syslog.LOG_UUCP}, {"cron", syslog.LOG_CRON}, {"authpriv", syslog.LOG_AUTHPRIV},
|
|
||||||
{"ftp", syslog.LOG_FTP}, {"local0", syslog.LOG_LOCAL0}, {"local1", syslog.LOG_LOCAL1},
|
|
||||||
{"local2", syslog.LOG_LOCAL2}, {"local3", syslog.LOG_LOCAL3}, {"local4", syslog.LOG_LOCAL4},
|
|
||||||
{"local5", syslog.LOG_LOCAL5}, {"local6", syslog.LOG_LOCAL6}, {"local7", syslog.LOG_LOCAL7},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, sFP := range syslogFacilitiesPriorities {
|
|
||||||
logcfg := fmt.Sprintf(`
|
|
||||||
global:
|
|
||||||
logging:
|
|
||||||
- type: syslog
|
|
||||||
level: info
|
|
||||||
format: human
|
|
||||||
facility: %s
|
|
||||||
`, sFP.Facility)
|
|
||||||
conf := testValidGlobalSection(t, logcfg)
|
|
||||||
assert.Equal(t, 1, len(*conf.Global.Logging))
|
|
||||||
assert.True(t, SyslogFacility(sFP.Priority) == *(*conf.Global.Logging)[0].Ret.(*SyslogLoggingOutlet).Facility)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLoggingOutletEnumList_SetDefaults(t *testing.T) {
|
|
||||||
e := &LoggingOutletEnumList{}
|
|
||||||
var i yaml.Defaulter = e
|
|
||||||
require.NotPanics(t, func() {
|
|
||||||
i.SetDefault()
|
|
||||||
assert.Equal(t, "warn", (*e)[0].Ret.(*StdoutLoggingOutlet).Level)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
package config
|
|
||||||
@@ -1,40 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestConfigEmptyFails(t *testing.T) {
|
|
||||||
conf, err := testConfig(t, "\n")
|
|
||||||
assert.Nil(t, conf)
|
|
||||||
assert.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestJobsOnlyWorks(t *testing.T) {
|
|
||||||
testValidConfig(t, `
|
|
||||||
jobs:
|
|
||||||
- name: push
|
|
||||||
type: push
|
|
||||||
# snapshot the filesystems matched by the left-hand-side of the mapping
|
|
||||||
# every 10m with zrepl_ as prefix
|
|
||||||
connect:
|
|
||||||
type: tcp
|
|
||||||
address: localhost:2342
|
|
||||||
filesystems: {
|
|
||||||
"pool1/var/db<": true,
|
|
||||||
"pool1/usr/home<": true,
|
|
||||||
"pool1/usr/home/paranoid": false, #don't backup paranoid user
|
|
||||||
"pool1/poudriere/ports<": false #don't backup the ports trees
|
|
||||||
}
|
|
||||||
snapshotting:
|
|
||||||
type: manual
|
|
||||||
pruning:
|
|
||||||
keep_sender:
|
|
||||||
- type: not_replicated
|
|
||||||
keep_receiver:
|
|
||||||
- type: last_n
|
|
||||||
count: 1
|
|
||||||
`)
|
|
||||||
}
|
|
||||||
@@ -1,41 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/zrepl/yaml-config"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestPositiveDurationOrManual(t *testing.T) {
|
|
||||||
cases := []struct {
|
|
||||||
Comment, Input string
|
|
||||||
Result *PositiveDurationOrManual
|
|
||||||
}{
|
|
||||||
{"empty is error", "", nil},
|
|
||||||
{"negative is error", "-1s", nil},
|
|
||||||
{"zero seconds is error", "0s", nil},
|
|
||||||
{"zero is error", "0", nil},
|
|
||||||
{"non-manual is error", "something", nil},
|
|
||||||
{"positive seconds works", "1s", &PositiveDurationOrManual{Manual: false, Interval: 1 * time.Second}},
|
|
||||||
{"manual works", "manual", &PositiveDurationOrManual{Manual: true, Interval: 0}},
|
|
||||||
}
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.Comment, func(t *testing.T) {
|
|
||||||
var out struct {
|
|
||||||
FieldName PositiveDurationOrManual `yaml:"fieldname"`
|
|
||||||
}
|
|
||||||
input := fmt.Sprintf("\nfieldname: %s\n", tc.Input)
|
|
||||||
err := yaml.UnmarshalStrict([]byte(input), &out)
|
|
||||||
if tc.Result == nil {
|
|
||||||
assert.Error(t, err)
|
|
||||||
t.Logf("%#v", out)
|
|
||||||
} else {
|
|
||||||
assert.Equal(t, *tc.Result, out.FieldName)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,134 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
|
|
||||||
zfsprop "github.com/zrepl/zrepl/zfs/property"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestRecvOptions(t *testing.T) {
|
|
||||||
tmpl := `
|
|
||||||
jobs:
|
|
||||||
- name: foo
|
|
||||||
type: pull
|
|
||||||
connect:
|
|
||||||
type: local
|
|
||||||
listener_name: foo
|
|
||||||
client_identity: bar
|
|
||||||
root_fs: "zreplplatformtest"
|
|
||||||
%s
|
|
||||||
interval: manual
|
|
||||||
pruning:
|
|
||||||
keep_sender:
|
|
||||||
- type: last_n
|
|
||||||
count: 10
|
|
||||||
keep_receiver:
|
|
||||||
- type: last_n
|
|
||||||
count: 10
|
|
||||||
|
|
||||||
`
|
|
||||||
|
|
||||||
recv_properties_empty := `
|
|
||||||
recv:
|
|
||||||
properties:
|
|
||||||
`
|
|
||||||
|
|
||||||
recv_inherit_empty := `
|
|
||||||
recv:
|
|
||||||
properties:
|
|
||||||
inherit:
|
|
||||||
`
|
|
||||||
|
|
||||||
recv_inherit := `
|
|
||||||
recv:
|
|
||||||
properties:
|
|
||||||
inherit:
|
|
||||||
- testprop
|
|
||||||
`
|
|
||||||
|
|
||||||
recv_override_empty := `
|
|
||||||
recv:
|
|
||||||
properties:
|
|
||||||
override:
|
|
||||||
`
|
|
||||||
|
|
||||||
recv_override := `
|
|
||||||
recv:
|
|
||||||
properties:
|
|
||||||
override:
|
|
||||||
testprop2: "test123"
|
|
||||||
`
|
|
||||||
|
|
||||||
recv_override_and_inherit := `
|
|
||||||
recv:
|
|
||||||
properties:
|
|
||||||
inherit:
|
|
||||||
- testprop
|
|
||||||
override:
|
|
||||||
testprop2: "test123"
|
|
||||||
`
|
|
||||||
|
|
||||||
recv_empty := `
|
|
||||||
recv: {}
|
|
||||||
`
|
|
||||||
|
|
||||||
recv_not_specified := `
|
|
||||||
`
|
|
||||||
|
|
||||||
fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
|
|
||||||
|
|
||||||
t.Run("recv_inherit_empty", func(t *testing.T) {
|
|
||||||
c := testValidConfig(t, fill(recv_inherit_empty))
|
|
||||||
assert.NotNil(t, c)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("recv_inherit", func(t *testing.T) {
|
|
||||||
c := testValidConfig(t, fill(recv_inherit))
|
|
||||||
inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
|
|
||||||
assert.NotEmpty(t, inherit)
|
|
||||||
assert.Contains(t, inherit, zfsprop.Property("testprop"))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("recv_override_empty", func(t *testing.T) {
|
|
||||||
c := testValidConfig(t, fill(recv_override_empty))
|
|
||||||
assert.NotNil(t, c)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("recv_override", func(t *testing.T) {
|
|
||||||
c := testValidConfig(t, fill(recv_override))
|
|
||||||
override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
|
|
||||||
require.Len(t, override, 1)
|
|
||||||
require.Equal(t, "test123", override["testprop2"])
|
|
||||||
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("recv_override_and_inherit", func(t *testing.T) {
|
|
||||||
c := testValidConfig(t, fill(recv_override_and_inherit))
|
|
||||||
inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
|
|
||||||
override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
|
|
||||||
assert.NotEmpty(t, inherit)
|
|
||||||
assert.Contains(t, inherit, zfsprop.Property("testprop"))
|
|
||||||
assert.NotEmpty(t, override)
|
|
||||||
assert.Equal(t, "test123", override["testprop2"])
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("recv_properties_empty", func(t *testing.T) {
|
|
||||||
c := testValidConfig(t, fill(recv_properties_empty))
|
|
||||||
assert.NotNil(t, c)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("recv_empty", func(t *testing.T) {
|
|
||||||
c := testValidConfig(t, fill(recv_empty))
|
|
||||||
assert.NotNil(t, c)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("send_not_specified", func(t *testing.T) {
|
|
||||||
c := testValidConfig(t, fill(recv_not_specified))
|
|
||||||
assert.NotNil(t, c)
|
|
||||||
})
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,146 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestSendOptions(t *testing.T) {
|
|
||||||
tmpl := `
|
|
||||||
jobs:
|
|
||||||
- name: foo
|
|
||||||
type: push
|
|
||||||
connect:
|
|
||||||
type: local
|
|
||||||
listener_name: foo
|
|
||||||
client_identity: bar
|
|
||||||
filesystems: {"<": true}
|
|
||||||
%s
|
|
||||||
snapshotting:
|
|
||||||
type: manual
|
|
||||||
pruning:
|
|
||||||
keep_sender:
|
|
||||||
- type: last_n
|
|
||||||
count: 10
|
|
||||||
keep_receiver:
|
|
||||||
- type: last_n
|
|
||||||
count: 10
|
|
||||||
`
|
|
||||||
encrypted_false := `
|
|
||||||
send:
|
|
||||||
encrypted: false
|
|
||||||
`
|
|
||||||
|
|
||||||
encrypted_true := `
|
|
||||||
send:
|
|
||||||
encrypted: true
|
|
||||||
`
|
|
||||||
|
|
||||||
raw_true := `
|
|
||||||
send:
|
|
||||||
raw: true
|
|
||||||
|
|
||||||
`
|
|
||||||
|
|
||||||
raw_false := `
|
|
||||||
send:
|
|
||||||
raw: false
|
|
||||||
|
|
||||||
`
|
|
||||||
|
|
||||||
raw_and_encrypted := `
|
|
||||||
send:
|
|
||||||
encrypted: true
|
|
||||||
raw: true
|
|
||||||
`
|
|
||||||
|
|
||||||
properties_and_encrypted := `
|
|
||||||
send:
|
|
||||||
encrypted: true
|
|
||||||
send_properties: true
|
|
||||||
`
|
|
||||||
|
|
||||||
properties_true := `
|
|
||||||
send:
|
|
||||||
send_properties: true
|
|
||||||
`
|
|
||||||
|
|
||||||
properties_false := `
|
|
||||||
send:
|
|
||||||
send_properties: false
|
|
||||||
`
|
|
||||||
|
|
||||||
send_empty := `
|
|
||||||
send: {}
|
|
||||||
`
|
|
||||||
|
|
||||||
send_not_specified := `
|
|
||||||
`
|
|
||||||
|
|
||||||
fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
|
|
||||||
|
|
||||||
t.Run("encrypted_false", func(t *testing.T) {
|
|
||||||
c := testValidConfig(t, fill(encrypted_false))
|
|
||||||
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
|
|
||||||
assert.Equal(t, false, encrypted)
|
|
||||||
})
|
|
||||||
t.Run("encrypted_true", func(t *testing.T) {
|
|
||||||
c := testValidConfig(t, fill(encrypted_true))
|
|
||||||
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
|
|
||||||
assert.Equal(t, true, encrypted)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("send_empty", func(t *testing.T) {
|
|
||||||
c := testValidConfig(t, fill(send_empty))
|
|
||||||
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
|
|
||||||
assert.Equal(t, false, encrypted)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("properties_and_encrypted", func(t *testing.T) {
|
|
||||||
c := testValidConfig(t, fill(properties_and_encrypted))
|
|
||||||
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
|
|
||||||
properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
|
|
||||||
assert.Equal(t, true, encrypted)
|
|
||||||
assert.Equal(t, true, properties)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("properties_false", func(t *testing.T) {
|
|
||||||
c := testValidConfig(t, fill(properties_false))
|
|
||||||
properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
|
|
||||||
assert.Equal(t, false, properties)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("properties_true", func(t *testing.T) {
|
|
||||||
c := testValidConfig(t, fill(properties_true))
|
|
||||||
properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
|
|
||||||
assert.Equal(t, true, properties)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("raw_true", func(t *testing.T) {
|
|
||||||
c := testValidConfig(t, fill(raw_true))
|
|
||||||
raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
|
|
||||||
assert.Equal(t, true, raw)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("raw_false", func(t *testing.T) {
|
|
||||||
c := testValidConfig(t, fill(raw_false))
|
|
||||||
raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
|
|
||||||
assert.Equal(t, false, raw)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("raw_and_encrypted", func(t *testing.T) {
|
|
||||||
c := testValidConfig(t, fill(raw_and_encrypted))
|
|
||||||
raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
|
|
||||||
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
|
|
||||||
assert.Equal(t, true, raw)
|
|
||||||
assert.Equal(t, true, encrypted)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("send_not_specified", func(t *testing.T) {
|
|
||||||
c := testValidConfig(t, fill(send_not_specified))
|
|
||||||
assert.NotNil(t, c)
|
|
||||||
})
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,90 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestSnapshotting(t *testing.T) {
|
|
||||||
tmpl := `
|
|
||||||
jobs:
|
|
||||||
- name: foo
|
|
||||||
type: push
|
|
||||||
connect:
|
|
||||||
type: local
|
|
||||||
listener_name: foo
|
|
||||||
client_identity: bar
|
|
||||||
filesystems: {"<": true}
|
|
||||||
%s
|
|
||||||
pruning:
|
|
||||||
keep_sender:
|
|
||||||
- type: last_n
|
|
||||||
count: 10
|
|
||||||
keep_receiver:
|
|
||||||
- type: last_n
|
|
||||||
count: 10
|
|
||||||
`
|
|
||||||
manual := `
|
|
||||||
snapshotting:
|
|
||||||
type: manual
|
|
||||||
`
|
|
||||||
periodic := `
|
|
||||||
snapshotting:
|
|
||||||
type: periodic
|
|
||||||
prefix: zrepl_
|
|
||||||
interval: 10m
|
|
||||||
`
|
|
||||||
|
|
||||||
hooks := `
|
|
||||||
snapshotting:
|
|
||||||
type: periodic
|
|
||||||
prefix: zrepl_
|
|
||||||
interval: 10m
|
|
||||||
hooks:
|
|
||||||
- type: command
|
|
||||||
path: /tmp/path/to/command
|
|
||||||
- type: command
|
|
||||||
path: /tmp/path/to/command
|
|
||||||
filesystems: { "zroot<": true, "<": false }
|
|
||||||
- type: postgres-checkpoint
|
|
||||||
dsn: "host=localhost port=5432 user=postgres sslmode=disable"
|
|
||||||
filesystems: {
|
|
||||||
"tank/postgres/data11": true
|
|
||||||
}
|
|
||||||
- type: mysql-lock-tables
|
|
||||||
dsn: "root@tcp(localhost)/"
|
|
||||||
filesystems: {
|
|
||||||
"tank/mysql": true
|
|
||||||
}
|
|
||||||
`
|
|
||||||
|
|
||||||
fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
|
|
||||||
var c *Config
|
|
||||||
|
|
||||||
t.Run("manual", func(t *testing.T) {
|
|
||||||
c = testValidConfig(t, fillSnapshotting(manual))
|
|
||||||
snm := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingManual)
|
|
||||||
assert.Equal(t, "manual", snm.Type)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("periodic", func(t *testing.T) {
|
|
||||||
c = testValidConfig(t, fillSnapshotting(periodic))
|
|
||||||
snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
|
|
||||||
assert.Equal(t, "periodic", snp.Type)
|
|
||||||
assert.Equal(t, 10*time.Minute, snp.Interval)
|
|
||||||
assert.Equal(t, "zrepl_", snp.Prefix)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("hooks", func(t *testing.T) {
|
|
||||||
c = testValidConfig(t, fillSnapshotting(hooks))
|
|
||||||
hs := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic).Hooks
|
|
||||||
assert.Equal(t, hs[0].Ret.(*HookCommand).Filesystems["<"], true)
|
|
||||||
assert.Equal(t, hs[1].Ret.(*HookCommand).Filesystems["zroot<"], true)
|
|
||||||
assert.Equal(t, hs[2].Ret.(*HookPostgresCheckpoint).Filesystems["tank/postgres/data11"], true)
|
|
||||||
assert.Equal(t, hs[3].Ret.(*HookMySQLLockTables).Filesystems["tank/mysql"], true)
|
|
||||||
})
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,88 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"bytes"
|
|
||||||
"fmt"
|
|
||||||
"path"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"text/template"
|
|
||||||
|
|
||||||
"github.com/kr/pretty"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
|
|
||||||
paths, err := filepath.Glob("./samples/*")
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("glob failed: %+v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, p := range paths {
|
|
||||||
|
|
||||||
if path.Ext(p) != ".yml" {
|
|
||||||
t.Logf("skipping file %s", p)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run(p, func(t *testing.T) {
|
|
||||||
c, err := ParseConfig(p)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("error parsing %s:\n%+v", p, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Logf("file: %s", p)
|
|
||||||
t.Log(pretty.Sprint(c))
|
|
||||||
})
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
// template must be a template/text template with a single '{{ . }}' as placeholder for val
|
|
||||||
//nolint[:deadcode,unused]
|
|
||||||
func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
|
|
||||||
tmp, err := template.New("master").Parse(tmpl)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
var buf bytes.Buffer
|
|
||||||
err = tmp.Execute(&buf, val)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
return testValidConfig(t, buf.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
func testValidConfig(t *testing.T, input string) *Config {
|
|
||||||
t.Helper()
|
|
||||||
conf, err := testConfig(t, input)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, conf)
|
|
||||||
return conf
|
|
||||||
}
|
|
||||||
|
|
||||||
func testConfig(t *testing.T, input string) (*Config, error) {
|
|
||||||
t.Helper()
|
|
||||||
return ParseConfigBytes([]byte(input))
|
|
||||||
}
|
|
||||||
|
|
||||||
func trimSpaceEachLineAndPad(s, pad string) string {
|
|
||||||
var out strings.Builder
|
|
||||||
scan := bufio.NewScanner(strings.NewReader(s))
|
|
||||||
for scan.Scan() {
|
|
||||||
fmt.Fprintf(&out, "%s%s\n", pad, bytes.TrimSpace(scan.Bytes()))
|
|
||||||
}
|
|
||||||
return out.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTrimSpaceEachLineAndPad(t *testing.T) {
|
|
||||||
foo := `
|
|
||||||
foo
|
|
||||||
bar baz
|
|
||||||
`
|
|
||||||
assert.Equal(t, " \n foo\n bar baz\n \n", trimSpaceEachLineAndPad(foo, " "))
|
|
||||||
}
|
|
||||||
@@ -1,99 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestTransportConnect(t *testing.T) {
|
|
||||||
tmpl := `
|
|
||||||
jobs:
|
|
||||||
- name: foo
|
|
||||||
type: push
|
|
||||||
connect:
|
|
||||||
%s
|
|
||||||
filesystems: {"<": true}
|
|
||||||
snapshotting:
|
|
||||||
type: manual
|
|
||||||
pruning:
|
|
||||||
keep_sender:
|
|
||||||
- type: last_n
|
|
||||||
count: 10
|
|
||||||
keep_receiver:
|
|
||||||
- type: last_n
|
|
||||||
count: 10
|
|
||||||
`
|
|
||||||
|
|
||||||
mconf := func(s string) string { return fmt.Sprintf(tmpl, s) }
|
|
||||||
|
|
||||||
type test struct {
|
|
||||||
Name string
|
|
||||||
ExpectError bool
|
|
||||||
Connect string
|
|
||||||
}
|
|
||||||
|
|
||||||
testTable := []test{
|
|
||||||
{
|
|
||||||
Name: "tcp_with_address_and_port",
|
|
||||||
ExpectError: false,
|
|
||||||
Connect: `
|
|
||||||
type: tcp
|
|
||||||
address: 10.0.0.23:42
|
|
||||||
`,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "tls_with_host_and_port",
|
|
||||||
ExpectError: false,
|
|
||||||
Connect: `
|
|
||||||
type: tls
|
|
||||||
address: "server1.foo.bar:8888"
|
|
||||||
ca: /etc/zrepl/ca.crt
|
|
||||||
cert: /etc/zrepl/backupserver.fullchain
|
|
||||||
key: /etc/zrepl/backupserver.key
|
|
||||||
server_cn: "server1"
|
|
||||||
`,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "tcp_without_port",
|
|
||||||
ExpectError: true,
|
|
||||||
Connect: `
|
|
||||||
type: tcp
|
|
||||||
address: 10.0.0.23
|
|
||||||
`,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "tls_without_port",
|
|
||||||
ExpectError: true,
|
|
||||||
Connect: `
|
|
||||||
type: tls
|
|
||||||
address: 10.0.0.23
|
|
||||||
ca: /etc/zrepl/ca.crt
|
|
||||||
cert: /etc/zrepl/backupserver.fullchain
|
|
||||||
key: /etc/zrepl/backupserver.key
|
|
||||||
server_cn: "server1"
|
|
||||||
`,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range testTable {
|
|
||||||
t.Run(tc.Name, func(t *testing.T) {
|
|
||||||
require.NotEmpty(t, tc.Connect)
|
|
||||||
connect := trimSpaceEachLineAndPad(tc.Connect, " ")
|
|
||||||
conf := mconf(connect)
|
|
||||||
config, err := testConfig(t, conf)
|
|
||||||
if tc.ExpectError && err == nil {
|
|
||||||
t.Errorf("expected test failure, but got valid config %v", config)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !tc.ExpectError && err != nil {
|
|
||||||
t.Errorf("not expecint test failure but got error: %s", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
t.Logf("error=%v", err)
|
|
||||||
t.Logf("config=%v", config)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,119 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"regexp"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
type RetentionIntervalList []RetentionInterval
|
|
||||||
|
|
||||||
type PruneGrid struct {
|
|
||||||
Type string `yaml:"type"`
|
|
||||||
Grid RetentionIntervalList `yaml:"grid"`
|
|
||||||
Regex string `yaml:"regex"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type RetentionInterval struct {
|
|
||||||
length time.Duration
|
|
||||||
keepCount int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (i *RetentionInterval) Length() time.Duration {
|
|
||||||
return i.length
|
|
||||||
}
|
|
||||||
|
|
||||||
func (i *RetentionInterval) KeepCount() int {
|
|
||||||
return i.keepCount
|
|
||||||
}
|
|
||||||
|
|
||||||
const RetentionGridKeepCountAll int = -1
|
|
||||||
|
|
||||||
func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
|
||||||
var in string
|
|
||||||
if err := u(&in, true); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
intervals, err := ParseRetentionIntervalSpec(in)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
*t = intervals
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var retentionStringIntervalRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*x\s*([^\(]+)\s*(\((.*)\))?\s*$`)
|
|
||||||
|
|
||||||
func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval, err error) {
|
|
||||||
|
|
||||||
comps := retentionStringIntervalRegex.FindStringSubmatch(e)
|
|
||||||
if comps == nil {
|
|
||||||
err = fmt.Errorf("retention string does not match expected format")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
times, err := strconv.Atoi(comps[1])
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
} else if times <= 0 {
|
|
||||||
return nil, fmt.Errorf("contains factor <= 0")
|
|
||||||
}
|
|
||||||
|
|
||||||
duration, err := parsePositiveDuration(comps[2])
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
keepCount := 1
|
|
||||||
if comps[3] != "" {
|
|
||||||
// Decompose key=value, comma separated
|
|
||||||
// For now, only keep_count is supported
|
|
||||||
re := regexp.MustCompile(`^\s*keep=(.+)\s*$`)
|
|
||||||
res := re.FindStringSubmatch(comps[4])
|
|
||||||
if res == nil || len(res) != 2 {
|
|
||||||
err = fmt.Errorf("interval parameter contains unknown parameters")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if res[1] == "all" {
|
|
||||||
keepCount = RetentionGridKeepCountAll
|
|
||||||
} else {
|
|
||||||
keepCount, err = strconv.Atoi(res[1])
|
|
||||||
if err != nil {
|
|
||||||
err = fmt.Errorf("cannot parse keep_count value")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
intervals = make([]RetentionInterval, times)
|
|
||||||
for i := range intervals {
|
|
||||||
intervals[i] = RetentionInterval{
|
|
||||||
length: duration,
|
|
||||||
keepCount: keepCount,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func ParseRetentionIntervalSpec(s string) (intervals []RetentionInterval, err error) {
|
|
||||||
|
|
||||||
ges := strings.Split(s, "|")
|
|
||||||
intervals = make([]RetentionInterval, 0, 7*len(ges))
|
|
||||||
|
|
||||||
for intervalIdx, e := range ges {
|
|
||||||
parsed, err := parseRetentionGridIntervalString(e)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("cannot parse interval %d of %d: %s: %s", intervalIdx+1, len(ges), err, strings.TrimSpace(e))
|
|
||||||
}
|
|
||||||
intervals = append(intervals, parsed...)
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -e
|
|
||||||
|
|
||||||
[ "$ZREPL_DRYRUN" = "true" ] && DRYRUN="echo DRYRUN: "
|
|
||||||
|
|
||||||
pre_snapshot() {
|
|
||||||
$DRYRUN date
|
|
||||||
}
|
|
||||||
|
|
||||||
post_snapshot() {
|
|
||||||
$DRYRUN date
|
|
||||||
}
|
|
||||||
|
|
||||||
case "$ZREPL_HOOKTYPE" in
|
|
||||||
pre_snapshot|post_snapshot)
|
|
||||||
"$ZREPL_HOOKTYPE"
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
printf 'Unrecognized hook type: %s\n' "$ZREPL_HOOKTYPE"
|
|
||||||
exit 255
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
|
|
||||||
jobs:
|
|
||||||
- type: sink
|
|
||||||
name: "local_sink"
|
|
||||||
root_fs: "storage/zrepl/sink"
|
|
||||||
serve:
|
|
||||||
type: local
|
|
||||||
listener_name: localsink
|
|
||||||
|
|
||||||
- type: push
|
|
||||||
name: "backup_system"
|
|
||||||
connect:
|
|
||||||
type: local
|
|
||||||
listener_name: localsink
|
|
||||||
client_identity: local_backup
|
|
||||||
filesystems: {
|
|
||||||
"system<": true,
|
|
||||||
}
|
|
||||||
snapshotting:
|
|
||||||
type: periodic
|
|
||||||
interval: 10m
|
|
||||||
prefix: zrepl_
|
|
||||||
pruning:
|
|
||||||
keep_sender:
|
|
||||||
- type: not_replicated
|
|
||||||
- type: last_n
|
|
||||||
count: 10
|
|
||||||
keep_receiver:
|
|
||||||
- type: grid
|
|
||||||
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
|
|
||||||
regex: "zrepl_.*"
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
jobs:
|
|
||||||
- name: pull_servers
|
|
||||||
type: pull
|
|
||||||
connect:
|
|
||||||
type: tls
|
|
||||||
address: "server1.foo.bar:8888"
|
|
||||||
ca: "/certs/ca.crt"
|
|
||||||
cert: "/certs/cert.crt"
|
|
||||||
key: "/certs/key.pem"
|
|
||||||
server_cn: "server1"
|
|
||||||
root_fs: "pool2/backup_servers"
|
|
||||||
interval: 10m
|
|
||||||
pruning:
|
|
||||||
keep_sender:
|
|
||||||
- type: not_replicated
|
|
||||||
- type: last_n
|
|
||||||
count: 10
|
|
||||||
- type: grid
|
|
||||||
grid: 1x1h(keep=all) | 24x1h | 14x1d
|
|
||||||
regex: "zrepl_.*"
|
|
||||||
keep_receiver:
|
|
||||||
- type: grid
|
|
||||||
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
|
|
||||||
regex: "zrepl_.*"
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
jobs:
|
|
||||||
|
|
||||||
- name: pull_servers
|
|
||||||
type: pull
|
|
||||||
connect:
|
|
||||||
type: ssh+stdinserver
|
|
||||||
host: app-srv.example.com
|
|
||||||
user: root
|
|
||||||
port: 22
|
|
||||||
identity_file: /etc/zrepl/ssh/identity
|
|
||||||
options: # optional, default [], `-o` arguments passed to ssh
|
|
||||||
- "Compression=yes"
|
|
||||||
root_fs: "pool2/backup_servers"
|
|
||||||
interval: 10m
|
|
||||||
pruning:
|
|
||||||
keep_sender:
|
|
||||||
- type: not_replicated
|
|
||||||
- type: last_n
|
|
||||||
count: 10
|
|
||||||
- type: grid
|
|
||||||
grid: 1x1h(keep=all) | 24x1h | 14x1d
|
|
||||||
regex: "^zrepl_.*"
|
|
||||||
keep_receiver:
|
|
||||||
- type: regex
|
|
||||||
regex: keep_
|
|
||||||
- type: grid
|
|
||||||
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
|
|
||||||
regex: "^zrepl_.*"
|
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
jobs:
|
|
||||||
- type: push
|
|
||||||
name: "push"
|
|
||||||
filesystems: {
|
|
||||||
"<": true,
|
|
||||||
"tmp": false
|
|
||||||
}
|
|
||||||
connect:
|
|
||||||
type: tcp
|
|
||||||
address: "backup-server.foo.bar:8888"
|
|
||||||
snapshotting:
|
|
||||||
type: manual
|
|
||||||
send:
|
|
||||||
encrypted: false
|
|
||||||
pruning:
|
|
||||||
keep_sender:
|
|
||||||
- type: not_replicated
|
|
||||||
- type: last_n
|
|
||||||
count: 10
|
|
||||||
- type: grid
|
|
||||||
grid: 1x1h(keep=all) | 24x1h | 14x1d
|
|
||||||
regex: "^zrepl_.*"
|
|
||||||
keep_receiver:
|
|
||||||
- type: grid
|
|
||||||
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
|
|
||||||
regex: "^zrepl_.*"
|
|
||||||
@@ -1,88 +0,0 @@
|
|||||||
# This config serves as an example for a local zrepl installation that
|
|
||||||
# backups the entire zpool `system` to `backuppool/zrepl/sink`
|
|
||||||
#
|
|
||||||
# The requirements covered by this setup are described in the zrepl documentation's
|
|
||||||
# quick start section which inlines this example.
|
|
||||||
#
|
|
||||||
# CUSTOMIZATIONS YOU WILL LIKELY WANT TO APPLY:
|
|
||||||
# - adjust the name of the production pool `system` in the `filesystems` filter of jobs `snapjob` and `push_to_derive`
|
|
||||||
# - adjust the name of the backup pool `backuppool` in the `backuppool_sink` job
|
|
||||||
# - adjust the occurences of `myhostname` to the name of the system you are backing up (cannot be easily changed once you start replicating)
|
|
||||||
# - make sure the `zrepl_` prefix is not being used by any other zfs tools you might have installed (it likely isn't)
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
|
|
||||||
# this job takes care of snapshot creation + pruning
|
|
||||||
- name: snapjob
|
|
||||||
type: snap
|
|
||||||
filesystems: {
|
|
||||||
"system<": true,
|
|
||||||
}
|
|
||||||
# create snapshots with prefix `zrepl_` every 15 minutes
|
|
||||||
snapshotting:
|
|
||||||
type: periodic
|
|
||||||
interval: 15m
|
|
||||||
prefix: zrepl_
|
|
||||||
pruning:
|
|
||||||
keep:
|
|
||||||
# fade-out scheme for snapshots starting with `zrepl_`
|
|
||||||
# - keep all created in the last hour
|
|
||||||
# - then destroy snapshots such that we keep 24 each 1 hour apart
|
|
||||||
# - then destroy snapshots such that we keep 14 each 1 day apart
|
|
||||||
# - then destroy all older snapshots
|
|
||||||
- type: grid
|
|
||||||
grid: 1x1h(keep=all) | 24x1h | 14x1d
|
|
||||||
regex: "^zrepl_.*"
|
|
||||||
# keep all snapshots that don't have the `zrepl_` prefix
|
|
||||||
- type: regex
|
|
||||||
negate: true
|
|
||||||
regex: "^zrepl_.*"
|
|
||||||
|
|
||||||
# This job pushes to the local sink defined in job `backuppool_sink`.
|
|
||||||
# We trigger replication manually from the command line / udev rules using
|
|
||||||
# `zrepl signal wakeup push_to_drive`
|
|
||||||
- type: push
|
|
||||||
name: push_to_drive
|
|
||||||
connect:
|
|
||||||
type: local
|
|
||||||
listener_name: backuppool_sink
|
|
||||||
client_identity: myhostname
|
|
||||||
filesystems: {
|
|
||||||
"system<": true
|
|
||||||
}
|
|
||||||
send:
|
|
||||||
encrypted: true
|
|
||||||
replication:
|
|
||||||
protection:
|
|
||||||
initial: guarantee_resumability
|
|
||||||
# Downgrade protection to guarantee_incremental which uses zfs bookmarks instead of zfs holds.
|
|
||||||
# Thus, when we yank out the backup drive during replication
|
|
||||||
# - we might not be able to resume the interrupted replication step because the partially received `to` snapshot of a `from`->`to` step may be pruned any time
|
|
||||||
# - but in exchange we get back the disk space allocated by `to` when we prune it
|
|
||||||
# - and because we still have the bookmarks created by `guarantee_incremental`, we can still do incremental replication of `from`->`to2` in the future
|
|
||||||
incremental: guarantee_incremental
|
|
||||||
snapshotting:
|
|
||||||
type: manual
|
|
||||||
pruning:
|
|
||||||
# no-op prune rule on sender (keep all snapshots), job `snapshot` takes care of this
|
|
||||||
keep_sender:
|
|
||||||
- type: regex
|
|
||||||
regex: ".*"
|
|
||||||
# retain
|
|
||||||
keep_receiver:
|
|
||||||
# longer retention on the backup drive, we have more space there
|
|
||||||
- type: grid
|
|
||||||
grid: 1x1h(keep=all) | 24x1h | 360x1d
|
|
||||||
regex: "^zrepl_.*"
|
|
||||||
# retain all non-zrepl snapshots on the backup drive
|
|
||||||
- type: regex
|
|
||||||
negate: true
|
|
||||||
regex: "^zrepl_.*"
|
|
||||||
|
|
||||||
# This job receives from job `push_to_drive` into `backuppool/zrepl/sink/myhostname`
|
|
||||||
- type: sink
|
|
||||||
name: backuppool_sink
|
|
||||||
root_fs: "backuppool/zrepl/sink"
|
|
||||||
serve:
|
|
||||||
type: local
|
|
||||||
listener_name: backuppool_sink
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
jobs:
|
|
||||||
- name: sink
|
|
||||||
type: sink
|
|
||||||
serve:
|
|
||||||
type: tls
|
|
||||||
listen: ":8888"
|
|
||||||
ca: "/etc/zrepl/prod.crt"
|
|
||||||
cert: "/etc/zrepl/backups.crt"
|
|
||||||
key: "/etc/zrepl/backups.key"
|
|
||||||
client_cns:
|
|
||||||
- "prod"
|
|
||||||
root_fs: "storage/zrepl/sink"
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
jobs:
|
|
||||||
- name: prod_to_backups
|
|
||||||
type: push
|
|
||||||
connect:
|
|
||||||
type: tls
|
|
||||||
address: "backups.example.com:8888"
|
|
||||||
ca: /etc/zrepl/backups.crt
|
|
||||||
cert: /etc/zrepl/prod.crt
|
|
||||||
key: /etc/zrepl/prod.key
|
|
||||||
server_cn: "backups"
|
|
||||||
filesystems: {
|
|
||||||
"zroot/var/db": true,
|
|
||||||
"zroot/usr/home<": true,
|
|
||||||
"zroot/usr/home/paranoid": false
|
|
||||||
}
|
|
||||||
snapshotting:
|
|
||||||
type: periodic
|
|
||||||
prefix: zrepl_
|
|
||||||
interval: 10m
|
|
||||||
pruning:
|
|
||||||
keep_sender:
|
|
||||||
- type: not_replicated
|
|
||||||
- type: last_n
|
|
||||||
count: 10
|
|
||||||
keep_receiver:
|
|
||||||
- type: grid
|
|
||||||
grid: 1x1h(keep=all) | 24x1h | 30x1d | 6x30d
|
|
||||||
regex: "^zrepl_"
|
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
jobs:
|
|
||||||
- type: sink
|
|
||||||
name: "laptop_sink"
|
|
||||||
root_fs: "pool2/backup_laptops"
|
|
||||||
serve:
|
|
||||||
type: tls
|
|
||||||
listen: "192.168.122.189:8888"
|
|
||||||
ca: "ca.pem"
|
|
||||||
cert: "cert.pem"
|
|
||||||
key: "key.pem"
|
|
||||||
client_cns:
|
|
||||||
- "laptop1"
|
|
||||||
- "homeserver"
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
jobs:
|
|
||||||
- name: snapjob
|
|
||||||
type: snap
|
|
||||||
filesystems: {
|
|
||||||
"tank/frequently_changed<": true,
|
|
||||||
}
|
|
||||||
snapshotting:
|
|
||||||
type: periodic
|
|
||||||
interval: 2m
|
|
||||||
prefix: zrepl_snapjob_
|
|
||||||
pruning:
|
|
||||||
keep:
|
|
||||||
- type: last_n
|
|
||||||
count: 60
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
jobs:
|
|
||||||
- name: pull_source
|
|
||||||
type: source
|
|
||||||
serve:
|
|
||||||
type: tcp
|
|
||||||
listen: "0.0.0.0:8888"
|
|
||||||
clients: {
|
|
||||||
"192.168.122.123" : "mysql01",
|
|
||||||
"192.168.122.42" : "mx01",
|
|
||||||
"2001:0db8:85a3::8a2e:0370:7334": "gateway",
|
|
||||||
"10.23.42.0/24": "cluster-*",
|
|
||||||
"fde4:8dba:82e1::/64": "san-*",
|
|
||||||
}
|
|
||||||
filesystems: {
|
|
||||||
"<": true,
|
|
||||||
"secret": false
|
|
||||||
}
|
|
||||||
snapshotting:
|
|
||||||
type: periodic
|
|
||||||
interval: 10m
|
|
||||||
prefix: zrepl_
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
jobs:
|
|
||||||
- name: pull_source
|
|
||||||
type: source
|
|
||||||
serve:
|
|
||||||
type: stdinserver
|
|
||||||
client_identities:
|
|
||||||
- "client1"
|
|
||||||
- "client2"
|
|
||||||
filesystems: {
|
|
||||||
"<": true,
|
|
||||||
"secret": false
|
|
||||||
}
|
|
||||||
snapshotting:
|
|
||||||
type: periodic
|
|
||||||
interval: 10m
|
|
||||||
prefix: zrepl_
|
|
||||||
|
|
||||||
@@ -1,291 +0,0 @@
|
|||||||
package daemon
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/daemon/job"
|
|
||||||
"github.com/zrepl/zrepl/daemon/nethelpers"
|
|
||||||
"github.com/zrepl/zrepl/endpoint"
|
|
||||||
"github.com/zrepl/zrepl/logger"
|
|
||||||
"github.com/zrepl/zrepl/util/envconst"
|
|
||||||
"github.com/zrepl/zrepl/version"
|
|
||||||
"github.com/zrepl/zrepl/zfs"
|
|
||||||
"github.com/zrepl/zrepl/zfs/zfscmd"
|
|
||||||
)
|
|
||||||
|
|
||||||
type controlJob struct {
|
|
||||||
sockaddr *net.UnixAddr
|
|
||||||
jobs *jobs
|
|
||||||
}
|
|
||||||
|
|
||||||
func newControlJob(sockpath string, jobs *jobs) (j *controlJob, err error) {
|
|
||||||
j = &controlJob{jobs: jobs}
|
|
||||||
|
|
||||||
j.sockaddr, err = net.ResolveUnixAddr("unix", sockpath)
|
|
||||||
if err != nil {
|
|
||||||
err = errors.Wrap(err, "cannot resolve unix address")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *controlJob) Name() string { return jobNameControl }
|
|
||||||
|
|
||||||
func (j *controlJob) Status() *job.Status { return &job.Status{Type: job.TypeInternal} }
|
|
||||||
|
|
||||||
func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
|
|
||||||
|
|
||||||
func (j *controlJob) SenderConfig() *endpoint.SenderConfig { return nil }
|
|
||||||
|
|
||||||
var promControl struct {
|
|
||||||
requestBegin *prometheus.CounterVec
|
|
||||||
requestFinished *prometheus.HistogramVec
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) {
|
|
||||||
promControl.requestBegin = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
||||||
Namespace: "zrepl",
|
|
||||||
Subsystem: "control",
|
|
||||||
Name: "request_begin",
|
|
||||||
Help: "number of request we started to handle",
|
|
||||||
}, []string{"endpoint"})
|
|
||||||
|
|
||||||
promControl.requestFinished = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
||||||
Namespace: "zrepl",
|
|
||||||
Subsystem: "control",
|
|
||||||
Name: "request_finished",
|
|
||||||
Help: "time it took a request to finish",
|
|
||||||
Buckets: []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3, 10e-3, 100e-3, 200e-3, 400e-3, 800e-3, 1, 10, 20},
|
|
||||||
}, []string{"endpoint"})
|
|
||||||
registerer.MustRegister(promControl.requestBegin)
|
|
||||||
registerer.MustRegister(promControl.requestFinished)
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
|
||||||
ControlJobEndpointPProf string = "/debug/pprof"
|
|
||||||
ControlJobEndpointVersion string = "/version"
|
|
||||||
ControlJobEndpointStatus string = "/status"
|
|
||||||
ControlJobEndpointSignal string = "/signal"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (j *controlJob) Run(ctx context.Context) {
|
|
||||||
|
|
||||||
log := job.GetLogger(ctx)
|
|
||||||
defer log.Info("control job finished")
|
|
||||||
|
|
||||||
l, err := nethelpers.ListenUnixPrivate(j.sockaddr)
|
|
||||||
if err != nil {
|
|
||||||
log.WithError(err).Error("error listening")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
pprofServer := NewPProfServer(ctx)
|
|
||||||
if listen := envconst.String("ZREPL_DAEMON_AUTOSTART_PPROF_SERVER", ""); listen != "" {
|
|
||||||
pprofServer.Control(PprofServerControlMsg{
|
|
||||||
Run: true,
|
|
||||||
HttpListenAddress: listen,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
mux := http.NewServeMux()
|
|
||||||
mux.Handle(ControlJobEndpointPProf,
|
|
||||||
requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) {
|
|
||||||
var msg PprofServerControlMsg
|
|
||||||
err := decoder(&msg)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Errorf("decode failed")
|
|
||||||
}
|
|
||||||
pprofServer.Control(msg)
|
|
||||||
return struct{}{}, nil
|
|
||||||
}}})
|
|
||||||
|
|
||||||
mux.Handle(ControlJobEndpointVersion,
|
|
||||||
requestLogger{log: log, handler: jsonResponder{log, func() (interface{}, error) {
|
|
||||||
return version.NewZreplVersionInformation(), nil
|
|
||||||
}}})
|
|
||||||
|
|
||||||
mux.Handle(ControlJobEndpointStatus,
|
|
||||||
// don't log requests to status endpoint, too spammy
|
|
||||||
jsonResponder{log, func() (interface{}, error) {
|
|
||||||
jobs := j.jobs.status()
|
|
||||||
globalZFS := zfscmd.GetReport()
|
|
||||||
envconstReport := envconst.GetReport()
|
|
||||||
s := Status{
|
|
||||||
Jobs: jobs,
|
|
||||||
Global: GlobalStatus{
|
|
||||||
ZFSCmds: globalZFS,
|
|
||||||
Envconst: envconstReport,
|
|
||||||
}}
|
|
||||||
return s, nil
|
|
||||||
}})
|
|
||||||
|
|
||||||
mux.Handle(ControlJobEndpointSignal,
|
|
||||||
requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) {
|
|
||||||
type reqT struct {
|
|
||||||
Name string
|
|
||||||
Op string
|
|
||||||
}
|
|
||||||
var req reqT
|
|
||||||
if decoder(&req) != nil {
|
|
||||||
return nil, errors.Errorf("decode failed")
|
|
||||||
}
|
|
||||||
|
|
||||||
var err error
|
|
||||||
switch req.Op {
|
|
||||||
case "wakeup":
|
|
||||||
err = j.jobs.wakeup(req.Name)
|
|
||||||
case "reset":
|
|
||||||
err = j.jobs.reset(req.Name)
|
|
||||||
default:
|
|
||||||
err = fmt.Errorf("operation %q is invalid", req.Op)
|
|
||||||
}
|
|
||||||
|
|
||||||
return struct{}{}, err
|
|
||||||
}}})
|
|
||||||
server := http.Server{
|
|
||||||
Handler: mux,
|
|
||||||
// control socket is local, 1s timeout should be more than sufficient, even on a loaded system
|
|
||||||
WriteTimeout: 1 * time.Second,
|
|
||||||
ReadTimeout: 1 * time.Second,
|
|
||||||
}
|
|
||||||
|
|
||||||
outer:
|
|
||||||
for {
|
|
||||||
|
|
||||||
served := make(chan error)
|
|
||||||
go func() {
|
|
||||||
served <- server.Serve(l)
|
|
||||||
close(served)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
log.WithError(ctx.Err()).Info("context done")
|
|
||||||
err := server.Shutdown(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
log.WithError(err).Error("cannot shutdown server")
|
|
||||||
}
|
|
||||||
break outer
|
|
||||||
case err = <-served:
|
|
||||||
if err != nil {
|
|
||||||
log.WithError(err).Error("error serving")
|
|
||||||
break outer
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
type jsonResponder struct {
|
|
||||||
log Logger
|
|
||||||
producer func() (interface{}, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j jsonResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
||||||
logIoErr := func(err error) {
|
|
||||||
if err != nil {
|
|
||||||
j.log.WithError(err).Error("control handler io error")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
res, err := j.producer()
|
|
||||||
if err != nil {
|
|
||||||
j.log.WithError(err).Error("control handler error")
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
_, err = io.WriteString(w, err.Error())
|
|
||||||
logIoErr(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var buf bytes.Buffer
|
|
||||||
err = json.NewEncoder(&buf).Encode(res)
|
|
||||||
if err != nil {
|
|
||||||
j.log.WithError(err).Error("control handler json marshal error")
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
_, err = io.WriteString(w, err.Error())
|
|
||||||
} else {
|
|
||||||
_, err = io.Copy(w, &buf)
|
|
||||||
}
|
|
||||||
logIoErr(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
type jsonDecoder = func(interface{}) error
|
|
||||||
|
|
||||||
type jsonRequestResponder struct {
|
|
||||||
log Logger
|
|
||||||
producer func(decoder jsonDecoder) (interface{}, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j jsonRequestResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
||||||
logIoErr := func(err error) {
|
|
||||||
if err != nil {
|
|
||||||
j.log.WithError(err).Error("control handler io error")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var decodeError error
|
|
||||||
decoder := func(i interface{}) error {
|
|
||||||
err := json.NewDecoder(r.Body).Decode(&i)
|
|
||||||
decodeError = err
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
res, producerErr := j.producer(decoder)
|
|
||||||
|
|
||||||
//If we had a decode error ignore output of producer and return error
|
|
||||||
if decodeError != nil {
|
|
||||||
w.WriteHeader(http.StatusBadRequest)
|
|
||||||
_, err := io.WriteString(w, decodeError.Error())
|
|
||||||
logIoErr(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if producerErr != nil {
|
|
||||||
j.log.WithError(producerErr).Error("control handler error")
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
_, err := io.WriteString(w, producerErr.Error())
|
|
||||||
logIoErr(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
encodeErr := json.NewEncoder(&buf).Encode(res)
|
|
||||||
if encodeErr != nil {
|
|
||||||
j.log.WithError(producerErr).Error("control handler json marshal error")
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
_, err := io.WriteString(w, encodeErr.Error())
|
|
||||||
logIoErr(err)
|
|
||||||
} else {
|
|
||||||
_, err := io.Copy(w, &buf)
|
|
||||||
logIoErr(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type requestLogger struct {
|
|
||||||
log logger.Logger
|
|
||||||
handler http.Handler
|
|
||||||
handlerFunc http.HandlerFunc
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l requestLogger) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
||||||
log := l.log.WithField("method", r.Method).WithField("url", r.URL)
|
|
||||||
log.Debug("start")
|
|
||||||
promControl.requestBegin.WithLabelValues(r.URL.Path).Inc()
|
|
||||||
defer prometheus.NewTimer(promControl.requestFinished.WithLabelValues(r.URL.Path)).ObserveDuration()
|
|
||||||
if l.handlerFunc != nil {
|
|
||||||
l.handlerFunc(w, r)
|
|
||||||
} else if l.handler != nil {
|
|
||||||
l.handler.ServeHTTP(w, r)
|
|
||||||
} else {
|
|
||||||
log.Error("no handler or handlerFunc configured")
|
|
||||||
}
|
|
||||||
log.Debug("finish")
|
|
||||||
}
|
|
||||||
@@ -1,257 +0,0 @@
|
|||||||
package daemon
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"math/rand"
|
|
||||||
"os"
|
|
||||||
"os/signal"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"syscall"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
|
||||||
"github.com/zrepl/zrepl/endpoint"
|
|
||||||
"github.com/zrepl/zrepl/util/envconst"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
"github.com/zrepl/zrepl/daemon/job"
|
|
||||||
"github.com/zrepl/zrepl/daemon/job/reset"
|
|
||||||
"github.com/zrepl/zrepl/daemon/job/wakeup"
|
|
||||||
"github.com/zrepl/zrepl/daemon/logging"
|
|
||||||
"github.com/zrepl/zrepl/logger"
|
|
||||||
"github.com/zrepl/zrepl/version"
|
|
||||||
"github.com/zrepl/zrepl/zfs/zfscmd"
|
|
||||||
)
|
|
||||||
|
|
||||||
func Run(ctx context.Context, conf *config.Config) error {
|
|
||||||
ctx, cancel := context.WithCancel(ctx)
|
|
||||||
|
|
||||||
defer cancel()
|
|
||||||
sigChan := make(chan os.Signal, 1)
|
|
||||||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
|
|
||||||
go func() {
|
|
||||||
<-sigChan
|
|
||||||
cancel()
|
|
||||||
}()
|
|
||||||
|
|
||||||
// The math/rand package is used presently for generating trace IDs, we
|
|
||||||
// seed it with the current time and pid so that the IDs are mostly
|
|
||||||
// unique.
|
|
||||||
rand.Seed(time.Now().UnixNano())
|
|
||||||
rand.Seed(int64(os.Getpid()))
|
|
||||||
|
|
||||||
outlets, err := logging.OutletsFromConfig(*conf.Global.Logging)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrap(err, "cannot build logging from config")
|
|
||||||
}
|
|
||||||
outlets.Add(newPrometheusLogOutlet(), logger.Debug)
|
|
||||||
|
|
||||||
confJobs, err := job.JobsFromConfig(conf)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrap(err, "cannot build jobs from config")
|
|
||||||
}
|
|
||||||
|
|
||||||
log := logger.NewLogger(outlets, 1*time.Second)
|
|
||||||
log.Info(version.NewZreplVersionInformation().String())
|
|
||||||
|
|
||||||
ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
|
|
||||||
trace.RegisterCallback(trace.Callback{
|
|
||||||
OnBegin: func(ctx context.Context) { logging.GetLogger(ctx, logging.SubsysTraceData).Debug("begin span") },
|
|
||||||
OnEnd: func(ctx context.Context, spanInfo trace.SpanInfo) {
|
|
||||||
logging.
|
|
||||||
GetLogger(ctx, logging.SubsysTraceData).
|
|
||||||
WithField("duration_s", spanInfo.EndedAt().Sub(spanInfo.StartedAt()).Seconds()).
|
|
||||||
Debug("finished span " + spanInfo.TaskAndSpanStack(trace.SpanStackKindAnnotation))
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
for _, job := range confJobs {
|
|
||||||
if IsInternalJobName(job.Name()) {
|
|
||||||
panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
jobs := newJobs()
|
|
||||||
|
|
||||||
// start control socket
|
|
||||||
controlJob, err := newControlJob(conf.Global.Control.SockPath, jobs)
|
|
||||||
if err != nil {
|
|
||||||
panic(err) // FIXME
|
|
||||||
}
|
|
||||||
jobs.start(ctx, controlJob, true)
|
|
||||||
|
|
||||||
for i, jc := range conf.Global.Monitoring {
|
|
||||||
var (
|
|
||||||
job job.Job
|
|
||||||
err error
|
|
||||||
)
|
|
||||||
switch v := jc.Ret.(type) {
|
|
||||||
case *config.PrometheusMonitoring:
|
|
||||||
job, err = newPrometheusJobFromConfig(v)
|
|
||||||
default:
|
|
||||||
return errors.Errorf("unknown monitoring job #%d (type %T)", i, v)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrapf(err, "cannot build monitoring job #%d", i)
|
|
||||||
}
|
|
||||||
jobs.start(ctx, job, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
// register global (=non job-local) metrics
|
|
||||||
version.PrometheusRegister(prometheus.DefaultRegisterer)
|
|
||||||
zfscmd.RegisterMetrics(prometheus.DefaultRegisterer)
|
|
||||||
trace.RegisterMetrics(prometheus.DefaultRegisterer)
|
|
||||||
endpoint.RegisterMetrics(prometheus.DefaultRegisterer)
|
|
||||||
|
|
||||||
log.Info("starting daemon")
|
|
||||||
|
|
||||||
// start regular jobs
|
|
||||||
for _, j := range confJobs {
|
|
||||||
jobs.start(ctx, j, false)
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-jobs.wait():
|
|
||||||
log.Info("all jobs finished")
|
|
||||||
case <-ctx.Done():
|
|
||||||
log.WithError(ctx.Err()).Info("context finished")
|
|
||||||
}
|
|
||||||
log.Info("waiting for jobs to finish")
|
|
||||||
<-jobs.wait()
|
|
||||||
log.Info("daemon exiting")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type jobs struct {
|
|
||||||
wg sync.WaitGroup
|
|
||||||
|
|
||||||
// m protects all fields below it
|
|
||||||
m sync.RWMutex
|
|
||||||
wakeups map[string]wakeup.Func // by Job.Name
|
|
||||||
resets map[string]reset.Func // by Job.Name
|
|
||||||
jobs map[string]job.Job
|
|
||||||
}
|
|
||||||
|
|
||||||
func newJobs() *jobs {
|
|
||||||
return &jobs{
|
|
||||||
wakeups: make(map[string]wakeup.Func),
|
|
||||||
resets: make(map[string]reset.Func),
|
|
||||||
jobs: make(map[string]job.Job),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *jobs) wait() <-chan struct{} {
|
|
||||||
ch := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
s.wg.Wait()
|
|
||||||
close(ch)
|
|
||||||
}()
|
|
||||||
return ch
|
|
||||||
}
|
|
||||||
|
|
||||||
type Status struct {
|
|
||||||
Jobs map[string]*job.Status
|
|
||||||
Global GlobalStatus
|
|
||||||
}
|
|
||||||
|
|
||||||
type GlobalStatus struct {
|
|
||||||
ZFSCmds *zfscmd.Report
|
|
||||||
Envconst *envconst.Report
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *jobs) status() map[string]*job.Status {
|
|
||||||
s.m.RLock()
|
|
||||||
defer s.m.RUnlock()
|
|
||||||
|
|
||||||
type res struct {
|
|
||||||
name string
|
|
||||||
status *job.Status
|
|
||||||
}
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
c := make(chan res, len(s.jobs))
|
|
||||||
for name, j := range s.jobs {
|
|
||||||
wg.Add(1)
|
|
||||||
go func(name string, j job.Job) {
|
|
||||||
defer wg.Done()
|
|
||||||
c <- res{name: name, status: j.Status()}
|
|
||||||
}(name, j)
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
close(c)
|
|
||||||
ret := make(map[string]*job.Status, len(s.jobs))
|
|
||||||
for res := range c {
|
|
||||||
ret[res.name] = res.status
|
|
||||||
}
|
|
||||||
return ret
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *jobs) wakeup(job string) error {
|
|
||||||
s.m.RLock()
|
|
||||||
defer s.m.RUnlock()
|
|
||||||
|
|
||||||
wu, ok := s.wakeups[job]
|
|
||||||
if !ok {
|
|
||||||
return errors.Errorf("Job %s does not exist", job)
|
|
||||||
}
|
|
||||||
return wu()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *jobs) reset(job string) error {
|
|
||||||
s.m.RLock()
|
|
||||||
defer s.m.RUnlock()
|
|
||||||
|
|
||||||
wu, ok := s.resets[job]
|
|
||||||
if !ok {
|
|
||||||
return errors.Errorf("Job %s does not exist", job)
|
|
||||||
}
|
|
||||||
return wu()
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
|
||||||
jobNamePrometheus = "_prometheus"
|
|
||||||
jobNameControl = "_control"
|
|
||||||
)
|
|
||||||
|
|
||||||
func IsInternalJobName(s string) bool {
|
|
||||||
return strings.HasPrefix(s, "_")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
|
|
||||||
s.m.Lock()
|
|
||||||
defer s.m.Unlock()
|
|
||||||
|
|
||||||
ctx = logging.WithInjectedField(ctx, logging.JobField, j.Name())
|
|
||||||
|
|
||||||
jobName := j.Name()
|
|
||||||
if !internal && IsInternalJobName(jobName) {
|
|
||||||
panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName))
|
|
||||||
}
|
|
||||||
if internal && !IsInternalJobName(jobName) {
|
|
||||||
panic(fmt.Sprintf("internal job does not use internal job name %s", jobName))
|
|
||||||
}
|
|
||||||
if _, ok := s.jobs[jobName]; ok {
|
|
||||||
panic(fmt.Sprintf("duplicate job name %s", jobName))
|
|
||||||
}
|
|
||||||
|
|
||||||
j.RegisterMetrics(prometheus.DefaultRegisterer)
|
|
||||||
|
|
||||||
s.jobs[jobName] = j
|
|
||||||
ctx = zfscmd.WithJobID(ctx, j.Name())
|
|
||||||
ctx, wakeup := wakeup.Context(ctx)
|
|
||||||
ctx, resetFunc := reset.Context(ctx)
|
|
||||||
s.wakeups[jobName] = wakeup
|
|
||||||
s.resets[jobName] = resetFunc
|
|
||||||
|
|
||||||
s.wg.Add(1)
|
|
||||||
go func() {
|
|
||||||
defer s.wg.Done()
|
|
||||||
job.GetLogger(ctx).Info("starting job")
|
|
||||||
defer job.GetLogger(ctx).Info("job exited")
|
|
||||||
j.Run(ctx)
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
@@ -1,275 +0,0 @@
|
|||||||
package filters
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/endpoint"
|
|
||||||
"github.com/zrepl/zrepl/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
type DatasetMapFilter struct {
|
|
||||||
entries []datasetMapFilterEntry
|
|
||||||
|
|
||||||
// if set, only valid filter entries can be added using Add()
|
|
||||||
// and Map() will always return an error
|
|
||||||
filterMode bool
|
|
||||||
}
|
|
||||||
|
|
||||||
type datasetMapFilterEntry struct {
|
|
||||||
path *zfs.DatasetPath
|
|
||||||
// the mapping. since this datastructure acts as both mapping and filter
|
|
||||||
// we have to convert it to the desired rep dynamically
|
|
||||||
mapping string
|
|
||||||
subtreeMatch bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewDatasetMapFilter(capacity int, filterMode bool) *DatasetMapFilter {
|
|
||||||
return &DatasetMapFilter{
|
|
||||||
entries: make([]datasetMapFilterEntry, 0, capacity),
|
|
||||||
filterMode: filterMode,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *DatasetMapFilter) Add(pathPattern, mapping string) (err error) {
|
|
||||||
|
|
||||||
if m.filterMode {
|
|
||||||
if _, err = m.parseDatasetFilterResult(mapping); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// assert path glob adheres to spec
|
|
||||||
const SUBTREE_PATTERN string = "<"
|
|
||||||
patternCount := strings.Count(pathPattern, SUBTREE_PATTERN)
|
|
||||||
switch {
|
|
||||||
case patternCount > 1:
|
|
||||||
case patternCount == 1 && !strings.HasSuffix(pathPattern, SUBTREE_PATTERN):
|
|
||||||
err = fmt.Errorf("pattern invalid: only one '<' at end of string allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
pathStr := strings.TrimSuffix(pathPattern, SUBTREE_PATTERN)
|
|
||||||
path, err := zfs.NewDatasetPath(pathStr)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("pattern is not a dataset path: %s", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
entry := datasetMapFilterEntry{
|
|
||||||
path: path,
|
|
||||||
mapping: mapping,
|
|
||||||
subtreeMatch: patternCount > 0,
|
|
||||||
}
|
|
||||||
m.entries = append(m.entries, entry)
|
|
||||||
return
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
// find the most specific prefix mapping we have
|
|
||||||
//
|
|
||||||
// longer prefix wins over shorter prefix, direct wins over glob
|
|
||||||
func (m DatasetMapFilter) mostSpecificPrefixMapping(path *zfs.DatasetPath) (idx int, found bool) {
|
|
||||||
lcp, lcp_entry_idx := -1, -1
|
|
||||||
direct_idx := -1
|
|
||||||
for e := range m.entries {
|
|
||||||
entry := m.entries[e]
|
|
||||||
ep := m.entries[e].path
|
|
||||||
lep := ep.Length()
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case !entry.subtreeMatch && ep.Equal(path):
|
|
||||||
direct_idx = e
|
|
||||||
continue
|
|
||||||
case entry.subtreeMatch && path.HasPrefix(ep) && lep > lcp:
|
|
||||||
lcp = lep
|
|
||||||
lcp_entry_idx = e
|
|
||||||
default:
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if lcp_entry_idx >= 0 || direct_idx >= 0 {
|
|
||||||
found = true
|
|
||||||
switch {
|
|
||||||
case direct_idx >= 0:
|
|
||||||
idx = direct_idx
|
|
||||||
case lcp_entry_idx >= 0:
|
|
||||||
idx = lcp_entry_idx
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Returns target == nil if there is no mapping
|
|
||||||
func (m DatasetMapFilter) Map(source *zfs.DatasetPath) (target *zfs.DatasetPath, err error) {
|
|
||||||
|
|
||||||
if m.filterMode {
|
|
||||||
err = fmt.Errorf("using a filter for mapping simply does not work")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
mi, hasMapping := m.mostSpecificPrefixMapping(source)
|
|
||||||
if !hasMapping {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
me := m.entries[mi]
|
|
||||||
|
|
||||||
if me.mapping == "" {
|
|
||||||
// Special case treatment: 'foo/bar<' => ''
|
|
||||||
if !me.subtreeMatch {
|
|
||||||
return nil, fmt.Errorf("mapping to '' must be a subtree match")
|
|
||||||
}
|
|
||||||
// ok...
|
|
||||||
} else {
|
|
||||||
if strings.HasPrefix("!", me.mapping) {
|
|
||||||
// reject mapping
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
target, err = zfs.NewDatasetPath(me.mapping)
|
|
||||||
if err != nil {
|
|
||||||
err = fmt.Errorf("mapping target is not a dataset path: %s", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if me.subtreeMatch {
|
|
||||||
// strip common prefix ('<' wildcards are no special case here)
|
|
||||||
extendComps := source.Copy()
|
|
||||||
extendComps.TrimPrefix(me.path)
|
|
||||||
target.Extend(extendComps)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
|
|
||||||
|
|
||||||
if !m.filterMode {
|
|
||||||
err = fmt.Errorf("using a mapping as a filter does not work")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
mi, hasMapping := m.mostSpecificPrefixMapping(p)
|
|
||||||
if !hasMapping {
|
|
||||||
pass = false
|
|
||||||
return
|
|
||||||
}
|
|
||||||
me := m.entries[mi]
|
|
||||||
pass, err = m.parseDatasetFilterResult(me.mapping)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Construct a new filter-only DatasetMapFilter from a mapping
|
|
||||||
// The new filter allows exactly those paths that were not forbidden by the mapping.
|
|
||||||
func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
|
|
||||||
|
|
||||||
if m.filterMode {
|
|
||||||
err = errors.Errorf("can only invert mappings")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
inv = &DatasetMapFilter{
|
|
||||||
make([]datasetMapFilterEntry, len(m.entries)),
|
|
||||||
true,
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, e := range m.entries {
|
|
||||||
inv.entries[i].path, err = zfs.NewDatasetPath(e.mapping)
|
|
||||||
if err != nil {
|
|
||||||
err = errors.Wrapf(err, "mapping cannot be inverted: '%s' is not a dataset path", e.mapping)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
inv.entries[i].mapping = MapFilterResultOk
|
|
||||||
inv.entries[i].subtreeMatch = e.subtreeMatch
|
|
||||||
}
|
|
||||||
|
|
||||||
return inv, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// FIXME investigate whether we can support more...
|
|
||||||
func (m DatasetMapFilter) Invert() (endpoint.FSMap, error) {
|
|
||||||
|
|
||||||
if m.filterMode {
|
|
||||||
return nil, errors.Errorf("can only invert mappings")
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(m.entries) != 1 {
|
|
||||||
return nil, errors.Errorf("inversion of complicated mappings is not implemented") // FIXME
|
|
||||||
}
|
|
||||||
|
|
||||||
e := m.entries[0]
|
|
||||||
|
|
||||||
inv := &DatasetMapFilter{
|
|
||||||
make([]datasetMapFilterEntry, len(m.entries)),
|
|
||||||
false,
|
|
||||||
}
|
|
||||||
mp, err := zfs.NewDatasetPath(e.mapping)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
inv.entries[0] = datasetMapFilterEntry{
|
|
||||||
path: mp,
|
|
||||||
mapping: e.path.ToString(),
|
|
||||||
subtreeMatch: e.subtreeMatch,
|
|
||||||
}
|
|
||||||
|
|
||||||
return inv, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Creates a new DatasetMapFilter in filter mode from a mapping
|
|
||||||
// All accepting mapping results are mapped to accepting filter results
|
|
||||||
// All rejecting mapping results are mapped to rejecting filter results
|
|
||||||
func (m DatasetMapFilter) AsFilter() endpoint.FSFilter {
|
|
||||||
|
|
||||||
f := &DatasetMapFilter{
|
|
||||||
make([]datasetMapFilterEntry, len(m.entries)),
|
|
||||||
true,
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, e := range m.entries {
|
|
||||||
var newe datasetMapFilterEntry = e
|
|
||||||
if strings.HasPrefix(newe.mapping, "!") {
|
|
||||||
newe.mapping = MapFilterResultOmit
|
|
||||||
} else {
|
|
||||||
newe.mapping = MapFilterResultOk
|
|
||||||
}
|
|
||||||
f.entries[i] = newe
|
|
||||||
}
|
|
||||||
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
|
||||||
MapFilterResultOk string = "ok"
|
|
||||||
MapFilterResultOmit string = "!"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Parse a dataset filter result
|
|
||||||
func (m DatasetMapFilter) parseDatasetFilterResult(result string) (pass bool, err error) {
|
|
||||||
l := strings.ToLower(result)
|
|
||||||
if l == MapFilterResultOk {
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
if l == MapFilterResultOmit {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
return false, fmt.Errorf("'%s' is not a valid filter result", result)
|
|
||||||
}
|
|
||||||
|
|
||||||
func DatasetMapFilterFromConfig(in map[string]bool) (f *DatasetMapFilter, err error) {
|
|
||||||
|
|
||||||
f = NewDatasetMapFilter(len(in), true)
|
|
||||||
for pathPattern, accept := range in {
|
|
||||||
mapping := MapFilterResultOmit
|
|
||||||
if accept {
|
|
||||||
mapping = MapFilterResultOk
|
|
||||||
}
|
|
||||||
if err = f.Add(pathPattern, mapping); err != nil {
|
|
||||||
err = fmt.Errorf("invalid mapping entry ['%s':'%s']: %s", pathPattern, mapping, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
@@ -1,89 +0,0 @@
|
|||||||
package filters
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestDatasetMapFilter(t *testing.T) {
|
|
||||||
|
|
||||||
type testCase struct {
|
|
||||||
name string
|
|
||||||
filter map[string]string
|
|
||||||
checkPass map[string]bool // each entry is checked to match the filter's `pass` return value
|
|
||||||
}
|
|
||||||
|
|
||||||
tcs := []testCase{
|
|
||||||
{
|
|
||||||
"default_no_match",
|
|
||||||
map[string]string{},
|
|
||||||
map[string]bool{
|
|
||||||
"": false,
|
|
||||||
"foo": false,
|
|
||||||
"zroot": false,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"more_specific_path_has_precedence",
|
|
||||||
map[string]string{
|
|
||||||
"tank<": "ok",
|
|
||||||
"tank/tmp<": "!",
|
|
||||||
"tank/home/x<": "!",
|
|
||||||
"tank/home/x/1": "ok",
|
|
||||||
},
|
|
||||||
map[string]bool{
|
|
||||||
"zroot": false,
|
|
||||||
"tank": true,
|
|
||||||
"tank/tmp": false,
|
|
||||||
"tank/tmp/foo": false,
|
|
||||||
"tank/home/x": false,
|
|
||||||
"tank/home/y": true,
|
|
||||||
"tank/home/x/1": true,
|
|
||||||
"tank/home/x/2": false,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"precedence_of_specific_over_subtree_wildcard_on_same_path",
|
|
||||||
map[string]string{
|
|
||||||
"tank/home/bob": "ok",
|
|
||||||
"tank/home/bob<": "!",
|
|
||||||
},
|
|
||||||
map[string]bool{
|
|
||||||
"tank/home/bob": true,
|
|
||||||
"tank/home/bob/downloads": false,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for tc := range tcs {
|
|
||||||
t.Run(tcs[tc].name, func(t *testing.T) {
|
|
||||||
c := tcs[tc]
|
|
||||||
f := NewDatasetMapFilter(len(c.filter), true)
|
|
||||||
for p, a := range c.filter {
|
|
||||||
err := f.Add(p, a)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("incorrect filter spec: %s", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for p, checkPass := range c.checkPass {
|
|
||||||
zp, err := zfs.NewDatasetPath(p)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("incorrect path spec: %s", err)
|
|
||||||
}
|
|
||||||
pass, err := f.Filter(zp)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected filter error: %s", err)
|
|
||||||
}
|
|
||||||
ok := pass == checkPass
|
|
||||||
failstr := "OK"
|
|
||||||
if !ok {
|
|
||||||
failstr = "FAIL"
|
|
||||||
t.Fail()
|
|
||||||
}
|
|
||||||
t.Logf("%-40q %5v (exp=%v act=%v)", p, failstr, checkPass, pass)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,35 +0,0 @@
|
|||||||
// Code generated by "stringer -type=Edge"; DO NOT EDIT.
|
|
||||||
|
|
||||||
package hooks
|
|
||||||
|
|
||||||
import "strconv"
|
|
||||||
|
|
||||||
func _() {
|
|
||||||
// An "invalid array index" compiler error signifies that the constant values have changed.
|
|
||||||
// Re-run the stringer command to generate them again.
|
|
||||||
var x [1]struct{}
|
|
||||||
_ = x[Pre-1]
|
|
||||||
_ = x[Callback-2]
|
|
||||||
_ = x[Post-4]
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
|
||||||
_Edge_name_0 = "PreCallback"
|
|
||||||
_Edge_name_1 = "Post"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
_Edge_index_0 = [...]uint8{0, 3, 11}
|
|
||||||
)
|
|
||||||
|
|
||||||
func (i Edge) String() string {
|
|
||||||
switch {
|
|
||||||
case 1 <= i && i <= 2:
|
|
||||||
i -= 1
|
|
||||||
return _Edge_name_0[_Edge_index_0[i]:_Edge_index_0[i+1]]
|
|
||||||
case i == 4:
|
|
||||||
return _Edge_name_1
|
|
||||||
default:
|
|
||||||
return "Edge(" + strconv.FormatInt(int64(i), 10) + ")"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
package hooks
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
"github.com/zrepl/zrepl/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
type List []Hook
|
|
||||||
|
|
||||||
func HookFromConfig(in config.HookEnum) (Hook, error) {
|
|
||||||
switch v := in.Ret.(type) {
|
|
||||||
case *config.HookCommand:
|
|
||||||
return NewCommandHook(v)
|
|
||||||
case *config.HookPostgresCheckpoint:
|
|
||||||
return PgChkptHookFromConfig(v)
|
|
||||||
case *config.HookMySQLLockTables:
|
|
||||||
return MyLockTablesFromConfig(v)
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("unknown hook type %T", v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ListFromConfig(in *config.HookList) (r *List, err error) {
|
|
||||||
hl := make(List, len(*in))
|
|
||||||
|
|
||||||
for i, h := range *in {
|
|
||||||
hl[i], err = HookFromConfig(h)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("create hook #%d: %s", i+1, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return &hl, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l List) CopyFilteredForFilesystem(fs *zfs.DatasetPath) (ret List, err error) {
|
|
||||||
ret = make(List, 0, len(l))
|
|
||||||
|
|
||||||
for _, h := range l {
|
|
||||||
var passFilesystem bool
|
|
||||||
if passFilesystem, err = h.Filesystems().Filter(fs); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if passFilesystem {
|
|
||||||
ret = append(ret, h)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return ret, nil
|
|
||||||
}
|
|
||||||
@@ -1,34 +0,0 @@
|
|||||||
// Package hooks implements pre- and post snapshot hooks.
|
|
||||||
//
|
|
||||||
// Plan is a generic executor for ExpectStepReports before and after an activity specified in a callback.
|
|
||||||
// It provides a reporting facility that can be polled while the plan is executing to gather progress information.
|
|
||||||
//
|
|
||||||
// This package also provides all supported hook type implementations and abstractions around them.
|
|
||||||
//
|
|
||||||
// Use For Other Kinds Of ExpectStepReports
|
|
||||||
//
|
|
||||||
// This package REQUIRES REFACTORING before it can be used for other activities than snapshots, e.g. pre- and post-replication:
|
|
||||||
//
|
|
||||||
// The Hook interface requires a hook to provide a Filesystems() filter, which doesn't make sense for
|
|
||||||
// all kinds of activities.
|
|
||||||
//
|
|
||||||
// The hook implementations should move out of this package.
|
|
||||||
// However, there is a lot of tight coupling which to untangle isn't worth it ATM.
|
|
||||||
//
|
|
||||||
// How This Package Is Used By Package Snapper
|
|
||||||
//
|
|
||||||
// Deserialize a config.List using ListFromConfig().
|
|
||||||
// Then it MUST filter the list to only contain hooks for a particular filesystem using
|
|
||||||
// hooksList.CopyFilteredForFilesystem(fs).
|
|
||||||
//
|
|
||||||
// Then create a CallbackHook using NewCallbackHookForFilesystem().
|
|
||||||
//
|
|
||||||
// Pass all of the above to NewPlan() which provides a Report() and Run() method:
|
|
||||||
//
|
|
||||||
// Plan.Run(ctx context.Context,dryRun bool) executes the plan and take a context as argument that should contain a logger added using hooks.WithLogger()).
|
|
||||||
// The value of dryRun is passed through to the hooks' Run() method.
|
|
||||||
// Command hooks make it available in the environment variable ZREPL_DRYRUN.
|
|
||||||
//
|
|
||||||
// Plan.Report() can be called while Plan.Run() is executing to give an overview of plan execution progress (future use in "zrepl status").
|
|
||||||
//
|
|
||||||
package hooks
|
|
||||||
@@ -1,292 +0,0 @@
|
|||||||
package hooks
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Re-export type here so that
|
|
||||||
// every file in package hooks doesn't
|
|
||||||
// have to import github.com/zrepl/zrepl/zfs
|
|
||||||
type Filter zfs.DatasetFilter
|
|
||||||
|
|
||||||
type Hook interface {
|
|
||||||
Filesystems() Filter
|
|
||||||
|
|
||||||
// If true and the Pre edge invocation of Run fails, Post edge will not run and other Pre edges will not run.
|
|
||||||
ErrIsFatal() bool
|
|
||||||
|
|
||||||
// Run is invoked by HookPlan for a Pre edge.
|
|
||||||
// If HookReport.HadError() == false, the Post edge will be invoked, too.
|
|
||||||
Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport
|
|
||||||
|
|
||||||
String() string
|
|
||||||
}
|
|
||||||
|
|
||||||
type Phase string
|
|
||||||
|
|
||||||
const (
|
|
||||||
PhaseSnapshot = Phase("snapshot")
|
|
||||||
PhaseTesting = Phase("testing")
|
|
||||||
)
|
|
||||||
|
|
||||||
func (p Phase) String() string {
|
|
||||||
return string(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
//go:generate stringer -type=Edge
|
|
||||||
type Edge uint
|
|
||||||
|
|
||||||
const (
|
|
||||||
Pre = Edge(1 << iota)
|
|
||||||
Callback
|
|
||||||
Post
|
|
||||||
)
|
|
||||||
|
|
||||||
func (e Edge) StringForPhase(phase Phase) string {
|
|
||||||
return fmt.Sprintf("%s_%s", e.String(), phase.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
//go:generate enumer -type=StepStatus -trimprefix=Step
|
|
||||||
type StepStatus int
|
|
||||||
|
|
||||||
const (
|
|
||||||
StepPending StepStatus = 1 << iota
|
|
||||||
StepExec
|
|
||||||
StepOk
|
|
||||||
StepErr
|
|
||||||
StepSkippedDueToFatalErr
|
|
||||||
StepSkippedDueToPreErr
|
|
||||||
)
|
|
||||||
|
|
||||||
type HookReport interface {
|
|
||||||
String() string
|
|
||||||
HadError() bool
|
|
||||||
Error() string
|
|
||||||
}
|
|
||||||
|
|
||||||
type Step struct {
|
|
||||||
Hook Hook
|
|
||||||
Edge Edge
|
|
||||||
Status StepStatus
|
|
||||||
Begin, End time.Time
|
|
||||||
// Report may be nil
|
|
||||||
// FIXME cannot serialize this for client status, but contains interesting info (like what error happened)
|
|
||||||
Report HookReport
|
|
||||||
state map[interface{}]interface{}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s Step) String() (out string) {
|
|
||||||
fatal := "~"
|
|
||||||
if s.Hook.ErrIsFatal() && s.Edge == Pre {
|
|
||||||
fatal = "!"
|
|
||||||
}
|
|
||||||
runTime := "..."
|
|
||||||
if s.Status != StepPending {
|
|
||||||
t := s.End.Sub(s.Begin)
|
|
||||||
runTime = t.Round(time.Millisecond).String()
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("[%s] [%5s] %s [%s] %s", s.Status, runTime, fatal, s.Edge, s.Hook)
|
|
||||||
}
|
|
||||||
|
|
||||||
type Plan struct {
|
|
||||||
mtx sync.RWMutex
|
|
||||||
|
|
||||||
steps []*Step
|
|
||||||
pre []*Step // protected by mtx
|
|
||||||
cb *Step
|
|
||||||
post []*Step // not reversed, i.e. entry at index i corresponds to pre-edge in pre[i]
|
|
||||||
|
|
||||||
phase Phase
|
|
||||||
env Env
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewPlan(hooks *List, phase Phase, cb *CallbackHook, extra Env) (*Plan, error) {
|
|
||||||
|
|
||||||
var pre, post []*Step
|
|
||||||
// TODO sanity check unique name of hook?
|
|
||||||
for _, hook := range *hooks {
|
|
||||||
state := make(map[interface{}]interface{})
|
|
||||||
preE := &Step{
|
|
||||||
Hook: hook,
|
|
||||||
Edge: Pre,
|
|
||||||
Status: StepPending,
|
|
||||||
state: state,
|
|
||||||
}
|
|
||||||
pre = append(pre, preE)
|
|
||||||
postE := &Step{
|
|
||||||
Hook: hook,
|
|
||||||
Edge: Post,
|
|
||||||
Status: StepPending,
|
|
||||||
state: state,
|
|
||||||
}
|
|
||||||
post = append(post, postE)
|
|
||||||
}
|
|
||||||
|
|
||||||
cbE := &Step{
|
|
||||||
Hook: cb,
|
|
||||||
Edge: Callback,
|
|
||||||
Status: StepPending,
|
|
||||||
}
|
|
||||||
|
|
||||||
steps := make([]*Step, 0, len(pre)+len(post)+1)
|
|
||||||
steps = append(steps, pre...)
|
|
||||||
steps = append(steps, cbE)
|
|
||||||
for i := len(post) - 1; i >= 0; i-- {
|
|
||||||
steps = append(steps, post[i])
|
|
||||||
}
|
|
||||||
|
|
||||||
plan := &Plan{
|
|
||||||
phase: phase,
|
|
||||||
env: extra,
|
|
||||||
steps: steps,
|
|
||||||
pre: pre,
|
|
||||||
post: post,
|
|
||||||
cb: cbE,
|
|
||||||
}
|
|
||||||
|
|
||||||
return plan, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type PlanReport []Step
|
|
||||||
|
|
||||||
func (p *Plan) Report() PlanReport {
|
|
||||||
p.mtx.RLock()
|
|
||||||
defer p.mtx.RUnlock()
|
|
||||||
rep := make([]Step, len(p.steps))
|
|
||||||
for i := range rep {
|
|
||||||
rep[i] = *p.steps[i]
|
|
||||||
}
|
|
||||||
return rep
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r PlanReport) HadError() bool {
|
|
||||||
for _, e := range r {
|
|
||||||
if e.Status == StepErr {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r PlanReport) HadFatalError() bool {
|
|
||||||
for _, e := range r {
|
|
||||||
if e.Status == StepSkippedDueToFatalErr {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r PlanReport) String() string {
|
|
||||||
stepStrings := make([]string, len(r))
|
|
||||||
for i, e := range r {
|
|
||||||
stepStrings[i] = fmt.Sprintf("%02d %s", i+1, e)
|
|
||||||
}
|
|
||||||
return strings.Join(stepStrings, "\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *Plan) Run(ctx context.Context, dryRun bool) {
|
|
||||||
p.mtx.RLock()
|
|
||||||
defer p.mtx.RUnlock()
|
|
||||||
w := func(f func()) {
|
|
||||||
p.mtx.RUnlock()
|
|
||||||
defer p.mtx.RLock()
|
|
||||||
p.mtx.Lock()
|
|
||||||
defer p.mtx.Unlock()
|
|
||||||
f()
|
|
||||||
}
|
|
||||||
runHook := func(s *Step, ctx context.Context, edge Edge) HookReport {
|
|
||||||
w(func() { s.Status = StepExec })
|
|
||||||
begin := time.Now()
|
|
||||||
r := s.Hook.Run(ctx, edge, p.phase, dryRun, p.env, s.state)
|
|
||||||
end := time.Now()
|
|
||||||
w(func() {
|
|
||||||
s.Report = r
|
|
||||||
s.Status = StepOk
|
|
||||||
if r.HadError() {
|
|
||||||
s.Status = StepErr
|
|
||||||
}
|
|
||||||
s.Begin, s.End = begin, end
|
|
||||||
})
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
|
|
||||||
l := getLogger(ctx)
|
|
||||||
|
|
||||||
// it's a stack, execute until we reach the end of the list (last item in)
|
|
||||||
// or fail inbetween
|
|
||||||
l.Info("run pre-edges in configuration order")
|
|
||||||
next := 0
|
|
||||||
for ; next < len(p.pre); next++ {
|
|
||||||
e := p.pre[next]
|
|
||||||
l := l.WithField("hook", e.Hook)
|
|
||||||
r := runHook(e, ctx, Pre)
|
|
||||||
if r.HadError() {
|
|
||||||
l.WithError(r).Error("hook invocation failed for pre-edge")
|
|
||||||
if e.Hook.ErrIsFatal() {
|
|
||||||
l.Error("the hook run was aborted due to a fatal error in this hook")
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
hadFatalErr := next != len(p.pre)
|
|
||||||
if hadFatalErr {
|
|
||||||
l.Error("fatal error in a pre-snapshot hook invocation")
|
|
||||||
l.Error("no snapshot will be taken")
|
|
||||||
l.Error("only running post-edges for successful pre-edges")
|
|
||||||
w(func() {
|
|
||||||
p.post[next].Status = StepSkippedDueToFatalErr
|
|
||||||
for i := next + 1; i < len(p.pre); i++ {
|
|
||||||
p.pre[i].Status = StepSkippedDueToFatalErr
|
|
||||||
p.post[i].Status = StepSkippedDueToFatalErr
|
|
||||||
}
|
|
||||||
p.cb.Status = StepSkippedDueToFatalErr
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Info("running callback")
|
|
||||||
cbR := runHook(p.cb, ctx, Callback)
|
|
||||||
if cbR.HadError() {
|
|
||||||
l.WithError(cbR).Error("callback failed")
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Info("run post-edges for successful pre-edges in reverse configuration order")
|
|
||||||
|
|
||||||
// the constructor produces pre and post entries
|
|
||||||
// post is NOT reversed
|
|
||||||
next-- // now at index of last executed pre-edge
|
|
||||||
for ; next >= 0; next-- {
|
|
||||||
e := p.post[next]
|
|
||||||
l := l.WithField("hook", e.Hook)
|
|
||||||
|
|
||||||
if p.pre[next].Status != StepOk {
|
|
||||||
if p.pre[next].Status != StepErr {
|
|
||||||
panic(fmt.Sprintf("expecting a pre-edge hook report to be either Ok or Err, got %s", p.pre[next].Status))
|
|
||||||
}
|
|
||||||
l.Info("skip post-edge because pre-edge failed")
|
|
||||||
w(func() {
|
|
||||||
e.Status = StepSkippedDueToPreErr
|
|
||||||
})
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
report := runHook(e, ctx, Post)
|
|
||||||
|
|
||||||
if report.HadError() {
|
|
||||||
l.WithError(report).Error("hook invocation failed for post-edge")
|
|
||||||
l.Error("subsequent post-edges run regardless of this post-edge failure")
|
|
||||||
}
|
|
||||||
|
|
||||||
// ErrIsFatal is only relevant for Pre
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,91 +0,0 @@
|
|||||||
package hooks
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/daemon/logging"
|
|
||||||
"github.com/zrepl/zrepl/logger"
|
|
||||||
"github.com/zrepl/zrepl/util/envconst"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Logger = logger.Logger
|
|
||||||
|
|
||||||
func GetLogger(ctx context.Context) Logger { return getLogger(ctx) }
|
|
||||||
|
|
||||||
func getLogger(ctx context.Context) Logger {
|
|
||||||
return logging.GetLogger(ctx, logging.SubsysHooks)
|
|
||||||
}
|
|
||||||
|
|
||||||
const MAX_HOOK_LOG_SIZE_DEFAULT int = 1 << 20
|
|
||||||
|
|
||||||
type logWriter struct {
|
|
||||||
/*
|
|
||||||
Mutex prevents:
|
|
||||||
concurrent writes to buf, scanner in Write([]byte)
|
|
||||||
data race on scanner vs Write([]byte)
|
|
||||||
and concurrent write to buf (call to buf.Reset())
|
|
||||||
in Close()
|
|
||||||
|
|
||||||
(Also, Close() should generally block until any Write() call completes.)
|
|
||||||
*/
|
|
||||||
mtx *sync.Mutex
|
|
||||||
buf bytes.Buffer
|
|
||||||
scanner *bufio.Scanner
|
|
||||||
logger Logger
|
|
||||||
level logger.Level
|
|
||||||
field string
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewLogWriter(mtx *sync.Mutex, logger Logger, level logger.Level, field string) *logWriter {
|
|
||||||
w := new(logWriter)
|
|
||||||
w.mtx = mtx
|
|
||||||
w.scanner = bufio.NewScanner(&w.buf)
|
|
||||||
w.logger = logger
|
|
||||||
w.level = level
|
|
||||||
w.field = field
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *logWriter) log(line string) {
|
|
||||||
w.logger.WithField(w.field, line).Log(w.level, "hook output")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *logWriter) logUnreadBytes() error {
|
|
||||||
for w.scanner.Scan() {
|
|
||||||
w.log(w.scanner.Text())
|
|
||||||
}
|
|
||||||
if w.buf.Cap() > envconst.Int("ZREPL_MAX_HOOK_LOG_SIZE", MAX_HOOK_LOG_SIZE_DEFAULT) {
|
|
||||||
w.buf.Reset()
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *logWriter) Write(in []byte) (int, error) {
|
|
||||||
w.mtx.Lock()
|
|
||||||
defer w.mtx.Unlock()
|
|
||||||
|
|
||||||
n, bufErr := w.buf.Write(in)
|
|
||||||
if bufErr != nil {
|
|
||||||
return n, bufErr
|
|
||||||
}
|
|
||||||
|
|
||||||
err := w.logUnreadBytes()
|
|
||||||
if err != nil {
|
|
||||||
return n, err
|
|
||||||
}
|
|
||||||
// Always reset the scanner for the next Write
|
|
||||||
w.scanner = bufio.NewScanner(&w.buf)
|
|
||||||
|
|
||||||
return n, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *logWriter) Close() (err error) {
|
|
||||||
w.mtx.Lock()
|
|
||||||
defer w.mtx.Unlock()
|
|
||||||
|
|
||||||
return w.logUnreadBytes()
|
|
||||||
}
|
|
||||||
@@ -1,65 +0,0 @@
|
|||||||
package hooks
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/daemon/filters"
|
|
||||||
"github.com/zrepl/zrepl/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
type HookJobCallback func(ctx context.Context) error
|
|
||||||
|
|
||||||
type CallbackHook struct {
|
|
||||||
cb HookJobCallback
|
|
||||||
filter Filter
|
|
||||||
displayString string
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewCallbackHookForFilesystem(displayString string, fs *zfs.DatasetPath, cb HookJobCallback) *CallbackHook {
|
|
||||||
filter, _ := filters.DatasetMapFilterFromConfig(map[string]bool{fs.ToString(): true})
|
|
||||||
return NewCallbackHook(displayString, cb, filter)
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewCallbackHook(displayString string, cb HookJobCallback, filter Filter) *CallbackHook {
|
|
||||||
return &CallbackHook{
|
|
||||||
cb: cb,
|
|
||||||
filter: filter,
|
|
||||||
displayString: displayString,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *CallbackHook) Filesystems() Filter {
|
|
||||||
return h.filter
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *CallbackHook) ErrIsFatal() bool {
|
|
||||||
return false // callback is by definition
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *CallbackHook) String() string {
|
|
||||||
return h.displayString
|
|
||||||
}
|
|
||||||
|
|
||||||
type CallbackHookReport struct {
|
|
||||||
Name string
|
|
||||||
Err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *CallbackHookReport) String() string {
|
|
||||||
if r.HadError() {
|
|
||||||
return r.Error()
|
|
||||||
}
|
|
||||||
return r.Name
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *CallbackHookReport) HadError() bool { return r.Err != nil }
|
|
||||||
|
|
||||||
func (r *CallbackHookReport) Error() string {
|
|
||||||
return fmt.Sprintf("%s error: %s", r.Name, r.Err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *CallbackHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
|
|
||||||
err := h.cb(ctx)
|
|
||||||
return &CallbackHookReport{h.displayString, err}
|
|
||||||
}
|
|
||||||
@@ -1,192 +0,0 @@
|
|||||||
package hooks
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"math"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
"github.com/zrepl/zrepl/daemon/filters"
|
|
||||||
"github.com/zrepl/zrepl/logger"
|
|
||||||
"github.com/zrepl/zrepl/util/circlog"
|
|
||||||
"github.com/zrepl/zrepl/util/envconst"
|
|
||||||
)
|
|
||||||
|
|
||||||
type HookEnvVar string
|
|
||||||
|
|
||||||
const (
|
|
||||||
EnvType HookEnvVar = "ZREPL_HOOKTYPE"
|
|
||||||
EnvDryRun HookEnvVar = "ZREPL_DRYRUN"
|
|
||||||
EnvFS HookEnvVar = "ZREPL_FS"
|
|
||||||
EnvSnapshot HookEnvVar = "ZREPL_SNAPNAME"
|
|
||||||
EnvTimeout HookEnvVar = "ZREPL_TIMEOUT"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Env map[HookEnvVar]string
|
|
||||||
|
|
||||||
func NewHookEnv(edge Edge, phase Phase, dryRun bool, timeout time.Duration, extra Env) Env {
|
|
||||||
r := Env{
|
|
||||||
EnvTimeout: fmt.Sprintf("%.f", math.Floor(timeout.Seconds())),
|
|
||||||
}
|
|
||||||
|
|
||||||
edgeString := edge.StringForPhase(phase)
|
|
||||||
r[EnvType] = strings.ToLower(edgeString)
|
|
||||||
|
|
||||||
var dryRunString string
|
|
||||||
if dryRun {
|
|
||||||
dryRunString = "true"
|
|
||||||
} else {
|
|
||||||
dryRunString = ""
|
|
||||||
}
|
|
||||||
r[EnvDryRun] = dryRunString
|
|
||||||
|
|
||||||
for k, v := range extra {
|
|
||||||
r[k] = v
|
|
||||||
}
|
|
||||||
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
|
|
||||||
type CommandHook struct {
|
|
||||||
edge Edge
|
|
||||||
filter Filter
|
|
||||||
errIsFatal bool
|
|
||||||
command string
|
|
||||||
timeout time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
type CommandHookReport struct {
|
|
||||||
Command string
|
|
||||||
Args []string // currently always empty
|
|
||||||
Env Env
|
|
||||||
Err error
|
|
||||||
CapturedStdoutStderrCombined []byte
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *CommandHookReport) String() string {
|
|
||||||
// Reproduces a POSIX shell-compatible command line
|
|
||||||
var cmdLine strings.Builder
|
|
||||||
sep := ""
|
|
||||||
|
|
||||||
// Make sure environment variables are always
|
|
||||||
// printed in the same order
|
|
||||||
var hookEnvKeys []HookEnvVar
|
|
||||||
for k := range r.Env {
|
|
||||||
hookEnvKeys = append(hookEnvKeys, k)
|
|
||||||
}
|
|
||||||
sort.Slice(hookEnvKeys, func(i, j int) bool { return string(hookEnvKeys[i]) < string(hookEnvKeys[j]) })
|
|
||||||
|
|
||||||
for _, k := range hookEnvKeys {
|
|
||||||
cmdLine.WriteString(fmt.Sprintf("%s%s='%s'", sep, k, r.Env[k]))
|
|
||||||
sep = " "
|
|
||||||
}
|
|
||||||
|
|
||||||
cmdLine.WriteString(fmt.Sprintf("%s%s", sep, r.Command))
|
|
||||||
for _, a := range r.Args {
|
|
||||||
cmdLine.WriteString(fmt.Sprintf("%s'%s'", sep, a))
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Sprintf("command hook invocation: \"%s\"", cmdLine.String()) // no %q to make copy-pastable
|
|
||||||
}
|
|
||||||
func (r *CommandHookReport) Error() string {
|
|
||||||
if r.Err == nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s FAILED with error: %s", r.String(), r.Err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *CommandHookReport) HadError() bool {
|
|
||||||
return r.Err != nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewCommandHook(in *config.HookCommand) (r *CommandHook, err error) {
|
|
||||||
r = &CommandHook{
|
|
||||||
errIsFatal: in.ErrIsFatal,
|
|
||||||
command: in.Path,
|
|
||||||
timeout: in.Timeout,
|
|
||||||
}
|
|
||||||
|
|
||||||
r.filter, err = filters.DatasetMapFilterFromConfig(in.Filesystems)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("cannot parse filesystem filter: %s", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
r.edge = Pre | Post
|
|
||||||
|
|
||||||
return r, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *CommandHook) Filesystems() Filter {
|
|
||||||
return h.filter
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *CommandHook) ErrIsFatal() bool {
|
|
||||||
return h.errIsFatal
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *CommandHook) String() string {
|
|
||||||
return h.command
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *CommandHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
|
|
||||||
l := getLogger(ctx).WithField("command", h.command)
|
|
||||||
|
|
||||||
cmdCtx, cancel := context.WithTimeout(ctx, h.timeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
cmdExec := exec.CommandContext(cmdCtx, h.command)
|
|
||||||
|
|
||||||
hookEnv := NewHookEnv(edge, phase, dryRun, h.timeout, extra)
|
|
||||||
cmdEnv := os.Environ()
|
|
||||||
for k, v := range hookEnv {
|
|
||||||
cmdEnv = append(cmdEnv, fmt.Sprintf("%s=%s", k, v))
|
|
||||||
}
|
|
||||||
cmdExec.Env = cmdEnv
|
|
||||||
|
|
||||||
var scanMutex sync.Mutex
|
|
||||||
combinedOutput, err := circlog.NewCircularLog(envconst.Int("ZREPL_MAX_HOOK_LOG_SIZE", MAX_HOOK_LOG_SIZE_DEFAULT))
|
|
||||||
if err != nil {
|
|
||||||
return &CommandHookReport{Err: err}
|
|
||||||
}
|
|
||||||
logErrWriter := NewLogWriter(&scanMutex, l, logger.Warn, "stderr")
|
|
||||||
logOutWriter := NewLogWriter(&scanMutex, l, logger.Info, "stdout")
|
|
||||||
defer logErrWriter.Close()
|
|
||||||
defer logOutWriter.Close()
|
|
||||||
|
|
||||||
cmdExec.Stderr = io.MultiWriter(logErrWriter, combinedOutput)
|
|
||||||
cmdExec.Stdout = io.MultiWriter(logOutWriter, combinedOutput)
|
|
||||||
|
|
||||||
report := &CommandHookReport{
|
|
||||||
Command: h.command,
|
|
||||||
Env: hookEnv,
|
|
||||||
// no report.Args
|
|
||||||
}
|
|
||||||
|
|
||||||
err = cmdExec.Start()
|
|
||||||
if err != nil {
|
|
||||||
report.Err = err
|
|
||||||
return report
|
|
||||||
}
|
|
||||||
|
|
||||||
err = cmdExec.Wait()
|
|
||||||
combinedOutputBytes := combinedOutput.Bytes()
|
|
||||||
report.CapturedStdoutStderrCombined = make([]byte, len(combinedOutputBytes))
|
|
||||||
copy(report.CapturedStdoutStderrCombined, combinedOutputBytes)
|
|
||||||
if err != nil {
|
|
||||||
if cmdCtx.Err() == context.DeadlineExceeded {
|
|
||||||
report.Err = fmt.Errorf("timed out after %s: %s", h.timeout, err)
|
|
||||||
return report
|
|
||||||
}
|
|
||||||
report.Err = err
|
|
||||||
return report
|
|
||||||
}
|
|
||||||
|
|
||||||
return report
|
|
||||||
}
|
|
||||||
@@ -1,145 +0,0 @@
|
|||||||
package hooks
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
sqldriver "database/sql/driver"
|
|
||||||
|
|
||||||
"github.com/go-sql-driver/mysql"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
"github.com/zrepl/zrepl/daemon/filters"
|
|
||||||
"github.com/zrepl/zrepl/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
// https://dev.mysql.com/doc/mysql-backup-excerpt/5.7/en/backup-methods.html
|
|
||||||
//
|
|
||||||
// Making Backups Using a File System Snapshot:
|
|
||||||
//
|
|
||||||
// If you are using a Veritas file system, you can make a backup like this:
|
|
||||||
//
|
|
||||||
// From a client program, execute FLUSH TABLES WITH READ LOCK.
|
|
||||||
// From another shell, execute mount vxfs snapshot.
|
|
||||||
// From the first client, execute UNLOCK TABLES.
|
|
||||||
// Copy files from the snapshot.
|
|
||||||
// Unmount the snapshot.
|
|
||||||
//
|
|
||||||
// Similar snapshot capabilities may be available in other file systems, such as LVM or ZFS.
|
|
||||||
type MySQLLockTables struct {
|
|
||||||
errIsFatal bool
|
|
||||||
connector sqldriver.Connector
|
|
||||||
filesystems Filter
|
|
||||||
}
|
|
||||||
|
|
||||||
type myLockTablesStateKey int
|
|
||||||
|
|
||||||
const (
|
|
||||||
myLockTablesConnection myLockTablesStateKey = 1 + iota
|
|
||||||
)
|
|
||||||
|
|
||||||
func MyLockTablesFromConfig(in *config.HookMySQLLockTables) (*MySQLLockTables, error) {
|
|
||||||
conf, err := mysql.ParseDSN(in.DSN)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "`dsn` invalid")
|
|
||||||
}
|
|
||||||
cn, err := mysql.NewConnector(conf)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "`connect` invalid")
|
|
||||||
}
|
|
||||||
|
|
||||||
filesystems, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "`filesystems` invalid")
|
|
||||||
}
|
|
||||||
|
|
||||||
return &MySQLLockTables{
|
|
||||||
in.ErrIsFatal,
|
|
||||||
cn,
|
|
||||||
filesystems,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *MySQLLockTables) ErrIsFatal() bool { return h.errIsFatal }
|
|
||||||
func (h *MySQLLockTables) Filesystems() Filter { return h.filesystems }
|
|
||||||
func (h *MySQLLockTables) String() string { return "MySQL FLUSH TABLES WITH READ LOCK" }
|
|
||||||
|
|
||||||
type MyLockTablesReport struct {
|
|
||||||
What string
|
|
||||||
Err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *MyLockTablesReport) HadError() bool { return r.Err != nil }
|
|
||||||
func (r *MyLockTablesReport) Error() string { return r.String() }
|
|
||||||
func (r *MyLockTablesReport) String() string {
|
|
||||||
var s strings.Builder
|
|
||||||
s.WriteString(r.What)
|
|
||||||
if r.Err != nil {
|
|
||||||
fmt.Fprintf(&s, ": %s", r.Err)
|
|
||||||
}
|
|
||||||
return s.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *MySQLLockTables) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
|
|
||||||
fs, ok := extra[EnvFS]
|
|
||||||
if !ok {
|
|
||||||
panic(extra)
|
|
||||||
}
|
|
||||||
dp, err := zfs.NewDatasetPath(fs)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
if pass, err := h.filesystems.Filter(dp); err != nil {
|
|
||||||
return &MyLockTablesReport{What: "filesystem filter", Err: err}
|
|
||||||
} else if !pass {
|
|
||||||
getLogger(ctx).Debug("filesystem does not match filter, skipping")
|
|
||||||
return &MyLockTablesReport{What: "filesystem filter skipped this filesystem", Err: nil}
|
|
||||||
}
|
|
||||||
|
|
||||||
switch edge {
|
|
||||||
case Pre:
|
|
||||||
err := h.doRunPre(ctx, dp, dryRun, state)
|
|
||||||
return &MyLockTablesReport{"FLUSH TABLES WITH READ LOCK", err}
|
|
||||||
case Post:
|
|
||||||
err := h.doRunPost(ctx, dp, dryRun, state)
|
|
||||||
return &MyLockTablesReport{"UNLOCK TABLES", err}
|
|
||||||
}
|
|
||||||
return &MyLockTablesReport{What: "skipped this edge", Err: nil}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *MySQLLockTables) doRunPre(ctx context.Context, fs *zfs.DatasetPath, dry bool, state map[interface{}]interface{}) (err error) {
|
|
||||||
db := sql.OpenDB(h.connector)
|
|
||||||
defer func(err *error) {
|
|
||||||
if *err != nil {
|
|
||||||
db.Close()
|
|
||||||
}
|
|
||||||
}(&err)
|
|
||||||
|
|
||||||
getLogger(ctx).Debug("do FLUSH TABLES WITH READ LOCK")
|
|
||||||
_, err = db.ExecContext(ctx, "FLUSH TABLES WITH READ LOCK")
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
state[myLockTablesConnection] = db
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *MySQLLockTables) doRunPost(ctx context.Context, fs *zfs.DatasetPath, dry bool, state map[interface{}]interface{}) error {
|
|
||||||
|
|
||||||
db := state[myLockTablesConnection].(*sql.DB)
|
|
||||||
defer db.Close()
|
|
||||||
|
|
||||||
getLogger(ctx).Debug("do UNLOCK TABLES")
|
|
||||||
_, err := db.ExecContext(ctx, "UNLOCK TABLES")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,98 +0,0 @@
|
|||||||
package hooks
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"fmt"
|
|
||||||
"math"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/lib/pq"
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
"github.com/zrepl/zrepl/daemon/filters"
|
|
||||||
"github.com/zrepl/zrepl/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
type PgChkptHook struct {
|
|
||||||
errIsFatal bool
|
|
||||||
connector *pq.Connector
|
|
||||||
filesystems Filter
|
|
||||||
}
|
|
||||||
|
|
||||||
func PgChkptHookFromConfig(in *config.HookPostgresCheckpoint) (*PgChkptHook, error) {
|
|
||||||
filesystems, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "`filesystems` invalid")
|
|
||||||
}
|
|
||||||
cn, err := pq.NewConnector(in.DSN)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "`dsn` invalid")
|
|
||||||
}
|
|
||||||
|
|
||||||
return &PgChkptHook{
|
|
||||||
in.ErrIsFatal,
|
|
||||||
cn,
|
|
||||||
filesystems,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *PgChkptHook) ErrIsFatal() bool { return h.errIsFatal }
|
|
||||||
func (h *PgChkptHook) Filesystems() Filter { return h.filesystems }
|
|
||||||
func (h *PgChkptHook) String() string { return "postgres checkpoint" }
|
|
||||||
|
|
||||||
type PgChkptHookReport struct{ Err error }
|
|
||||||
|
|
||||||
func (r *PgChkptHookReport) HadError() bool { return r.Err != nil }
|
|
||||||
func (r *PgChkptHookReport) Error() string { return r.Err.Error() }
|
|
||||||
func (r *PgChkptHookReport) String() string {
|
|
||||||
if r.Err != nil {
|
|
||||||
return fmt.Sprintf("postgres CHECKPOINT failed: %s", r.Err)
|
|
||||||
} else {
|
|
||||||
return "postgres CHECKPOINT completed"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *PgChkptHook) Run(ctx context.Context, edge Edge, phase Phase, dryRun bool, extra Env, state map[interface{}]interface{}) HookReport {
|
|
||||||
if edge != Pre {
|
|
||||||
return &PgChkptHookReport{nil}
|
|
||||||
}
|
|
||||||
fs, ok := extra[EnvFS]
|
|
||||||
if !ok {
|
|
||||||
panic(extra)
|
|
||||||
}
|
|
||||||
dp, err := zfs.NewDatasetPath(fs)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
err = h.doRunPre(ctx, dp, dryRun)
|
|
||||||
return &PgChkptHookReport{err}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *PgChkptHook) doRunPre(ctx context.Context, fs *zfs.DatasetPath, dry bool) error {
|
|
||||||
|
|
||||||
if pass, err := h.filesystems.Filter(fs); err != nil || !pass {
|
|
||||||
getLogger(ctx).Debug("filesystem does not match filter, skipping")
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
db := sql.OpenDB(h.connector)
|
|
||||||
defer db.Close()
|
|
||||||
dl, ok := ctx.Deadline()
|
|
||||||
if ok {
|
|
||||||
timeout := uint64(math.Floor(time.Until(dl).Seconds() * 1000)) // TODO go1.13 milliseconds
|
|
||||||
getLogger(ctx).WithField("statement_timeout", timeout).Debug("setting statement timeout for CHECKPOINT")
|
|
||||||
_, err := db.ExecContext(ctx, "SET statement_timeout TO ?", timeout)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if dry {
|
|
||||||
getLogger(ctx).Debug("dry-run - use ping instead of CHECKPOINT")
|
|
||||||
return db.PingContext(ctx)
|
|
||||||
}
|
|
||||||
getLogger(ctx).Info("execute CHECKPOINT command")
|
|
||||||
_, err := db.ExecContext(ctx, "CHECKPOINT")
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
@@ -1,489 +0,0 @@
|
|||||||
package hooks_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"regexp"
|
|
||||||
"testing"
|
|
||||||
"text/template"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
"github.com/zrepl/zrepl/daemon/hooks"
|
|
||||||
"github.com/zrepl/zrepl/daemon/logging"
|
|
||||||
"github.com/zrepl/zrepl/logger"
|
|
||||||
"github.com/zrepl/zrepl/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
type comparisonAssertionFunc func(require.TestingT, interface{}, interface{}, ...interface{})
|
|
||||||
type valueAssertionFunc func(require.TestingT, interface{}, ...interface{})
|
|
||||||
|
|
||||||
type expectStep struct {
|
|
||||||
ExpectedEdge hooks.Edge
|
|
||||||
ExpectStatus hooks.StepStatus
|
|
||||||
OutputTest valueAssertionFunc
|
|
||||||
ErrorTest valueAssertionFunc
|
|
||||||
}
|
|
||||||
|
|
||||||
type testCase struct {
|
|
||||||
Name string
|
|
||||||
Config []string
|
|
||||||
IsSlow bool
|
|
||||||
SuppressOutput bool
|
|
||||||
|
|
||||||
ExpectCallbackSkipped bool
|
|
||||||
ExpectHadFatalErr bool
|
|
||||||
ExpectHadError bool
|
|
||||||
ExpectStepReports []expectStep
|
|
||||||
}
|
|
||||||
|
|
||||||
func curryLeft(f comparisonAssertionFunc, expected interface{}) valueAssertionFunc {
|
|
||||||
return curry(f, expected, false)
|
|
||||||
}
|
|
||||||
|
|
||||||
func curryRight(f comparisonAssertionFunc, expected interface{}) valueAssertionFunc {
|
|
||||||
return curry(f, expected, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
func curry(f comparisonAssertionFunc, expected interface{}, right bool) (ret valueAssertionFunc) {
|
|
||||||
ret = func(t require.TestingT, s interface{}, v ...interface{}) {
|
|
||||||
var x interface{}
|
|
||||||
var y interface{}
|
|
||||||
if right {
|
|
||||||
x = s
|
|
||||||
y = expected
|
|
||||||
} else {
|
|
||||||
x = expected
|
|
||||||
y = s
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(v) > 0 {
|
|
||||||
f(t, x, y, v)
|
|
||||||
} else {
|
|
||||||
f(t, x, y)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHooks(t *testing.T) {
|
|
||||||
ctx, end := trace.WithTaskFromStack(context.Background())
|
|
||||||
defer end()
|
|
||||||
|
|
||||||
testFSName := "testpool/testdataset"
|
|
||||||
testSnapshotName := "testsnap"
|
|
||||||
|
|
||||||
tmpl, err := template.New("TestHooks").Parse(`
|
|
||||||
jobs:
|
|
||||||
- name: TestHooks
|
|
||||||
type: snap
|
|
||||||
filesystems: {"<": true}
|
|
||||||
snapshotting:
|
|
||||||
type: periodic
|
|
||||||
interval: 1m
|
|
||||||
prefix: zrepl_snapjob_
|
|
||||||
hooks:
|
|
||||||
{{- template "List" . }}
|
|
||||||
pruning:
|
|
||||||
keep:
|
|
||||||
- type: last_n
|
|
||||||
count: 10
|
|
||||||
`)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
regexpTest := func(s string) valueAssertionFunc {
|
|
||||||
return curryLeft(require.Regexp, regexp.MustCompile(s))
|
|
||||||
}
|
|
||||||
|
|
||||||
containsTest := func(s string) valueAssertionFunc {
|
|
||||||
return curryRight(require.Contains, s)
|
|
||||||
}
|
|
||||||
|
|
||||||
testTable := []testCase{
|
|
||||||
testCase{
|
|
||||||
Name: "no_hooks",
|
|
||||||
ExpectStepReports: []expectStep{
|
|
||||||
{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
testCase{
|
|
||||||
Name: "timeout",
|
|
||||||
IsSlow: true,
|
|
||||||
ExpectHadError: true,
|
|
||||||
Config: []string{`{type: command, path: {{.WorkDir}}/test/test-timeout.sh, timeout: 2s}`},
|
|
||||||
ExpectStepReports: []expectStep{
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Pre,
|
|
||||||
ExpectStatus: hooks.StepErr,
|
|
||||||
OutputTest: containsTest(fmt.Sprintf("TEST pre_testing %s@%s ZREPL_TIMEOUT=2", testFSName, testSnapshotName)),
|
|
||||||
ErrorTest: regexpTest(`timed out after 2(.\d+)?s`),
|
|
||||||
},
|
|
||||||
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Post,
|
|
||||||
ExpectStatus: hooks.StepSkippedDueToPreErr,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
testCase{
|
|
||||||
Name: "check_env",
|
|
||||||
Config: []string{`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`},
|
|
||||||
ExpectHadError: false,
|
|
||||||
ExpectStepReports: []expectStep{
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Pre,
|
|
||||||
ExpectStatus: hooks.StepOk,
|
|
||||||
OutputTest: containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
|
|
||||||
},
|
|
||||||
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Post,
|
|
||||||
ExpectStatus: hooks.StepOk,
|
|
||||||
OutputTest: containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
|
|
||||||
testCase{
|
|
||||||
Name: "nonfatal_pre_error_continues",
|
|
||||||
ExpectCallbackSkipped: false,
|
|
||||||
ExpectHadError: true,
|
|
||||||
Config: []string{`{type: command, path: {{.WorkDir}}/test/test-error.sh}`},
|
|
||||||
ExpectStepReports: []expectStep{
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Pre,
|
|
||||||
ExpectStatus: hooks.StepErr,
|
|
||||||
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
|
|
||||||
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
|
|
||||||
},
|
|
||||||
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Post,
|
|
||||||
ExpectStatus: hooks.StepSkippedDueToPreErr, // post-edge is not executed for failing pre-edge
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
|
|
||||||
testCase{
|
|
||||||
Name: "pre_error_fatal_skips_subsequent_pre_edges_and_callback_and_its_post_edge_and_post_edges",
|
|
||||||
ExpectCallbackSkipped: true,
|
|
||||||
ExpectHadFatalErr: true,
|
|
||||||
ExpectHadError: true,
|
|
||||||
Config: []string{
|
|
||||||
`{type: command, path: {{.WorkDir}}/test/test-error.sh, err_is_fatal: true}`,
|
|
||||||
`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`,
|
|
||||||
},
|
|
||||||
ExpectStepReports: []expectStep{
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Pre,
|
|
||||||
ExpectStatus: hooks.StepErr,
|
|
||||||
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
|
|
||||||
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
|
|
||||||
},
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Pre,
|
|
||||||
ExpectStatus: hooks.StepSkippedDueToFatalErr,
|
|
||||||
},
|
|
||||||
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepSkippedDueToFatalErr},
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Post,
|
|
||||||
ExpectStatus: hooks.StepSkippedDueToFatalErr,
|
|
||||||
},
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Post,
|
|
||||||
ExpectStatus: hooks.StepSkippedDueToFatalErr,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
|
|
||||||
testCase{
|
|
||||||
Name: "post_error_fails_are_ignored_even_if_fatal",
|
|
||||||
ExpectHadFatalErr: false, // only occurs during Post, so it's not a fatal error
|
|
||||||
ExpectHadError: true,
|
|
||||||
Config: []string{
|
|
||||||
`{type: command, path: {{.WorkDir}}/test/test-post-error.sh, err_is_fatal: true}`,
|
|
||||||
`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`,
|
|
||||||
},
|
|
||||||
ExpectStepReports: []expectStep{
|
|
||||||
expectStep{
|
|
||||||
// No-action run of test-post-error.sh
|
|
||||||
ExpectedEdge: hooks.Pre,
|
|
||||||
ExpectStatus: hooks.StepOk,
|
|
||||||
OutputTest: require.Empty,
|
|
||||||
},
|
|
||||||
expectStep{
|
|
||||||
// Pre run of test-report-env.sh
|
|
||||||
ExpectedEdge: hooks.Pre,
|
|
||||||
ExpectStatus: hooks.StepOk,
|
|
||||||
OutputTest: containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
|
|
||||||
},
|
|
||||||
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Post,
|
|
||||||
ExpectStatus: hooks.StepOk,
|
|
||||||
OutputTest: containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
|
|
||||||
},
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Post,
|
|
||||||
ExpectStatus: hooks.StepErr,
|
|
||||||
OutputTest: containsTest(fmt.Sprintf("TEST ERROR post_testing %s@%s", testFSName, testSnapshotName)),
|
|
||||||
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
|
|
||||||
testCase{
|
|
||||||
Name: "cleanup_check_env",
|
|
||||||
Config: []string{`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`},
|
|
||||||
ExpectStepReports: []expectStep{
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Pre,
|
|
||||||
ExpectStatus: hooks.StepOk,
|
|
||||||
OutputTest: containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
|
|
||||||
},
|
|
||||||
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Post,
|
|
||||||
ExpectStatus: hooks.StepOk,
|
|
||||||
OutputTest: containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
|
|
||||||
testCase{
|
|
||||||
Name: "pre_error_cancels_post",
|
|
||||||
Config: []string{`{type: command, path: {{.WorkDir}}/test/test-pre-error-post-ok.sh}`},
|
|
||||||
ExpectHadError: true,
|
|
||||||
ExpectHadFatalErr: false,
|
|
||||||
ExpectStepReports: []expectStep{
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Pre,
|
|
||||||
ExpectStatus: hooks.StepErr,
|
|
||||||
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
|
|
||||||
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
|
|
||||||
},
|
|
||||||
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Post,
|
|
||||||
ExpectStatus: hooks.StepSkippedDueToPreErr,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
|
|
||||||
testCase{
|
|
||||||
Name: "pre_error_does_not_cancel_other_posts_but_itself",
|
|
||||||
Config: []string{
|
|
||||||
`{type: command, path: {{.WorkDir}}/test/test-report-env.sh}`,
|
|
||||||
`{type: command, path: {{.WorkDir}}/test/test-pre-error-post-ok.sh}`,
|
|
||||||
},
|
|
||||||
ExpectHadError: true,
|
|
||||||
ExpectHadFatalErr: false,
|
|
||||||
ExpectStepReports: []expectStep{
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Pre,
|
|
||||||
ExpectStatus: hooks.StepOk,
|
|
||||||
OutputTest: containsTest(fmt.Sprintf("TEST pre_testing %s@%s", testFSName, testSnapshotName)),
|
|
||||||
},
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Pre,
|
|
||||||
ExpectStatus: hooks.StepErr,
|
|
||||||
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
|
|
||||||
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
|
|
||||||
},
|
|
||||||
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Post,
|
|
||||||
ExpectStatus: hooks.StepSkippedDueToPreErr,
|
|
||||||
},
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Post,
|
|
||||||
ExpectStatus: hooks.StepOk,
|
|
||||||
OutputTest: containsTest(fmt.Sprintf("TEST post_testing %s@%s", testFSName, testSnapshotName)),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
|
|
||||||
testCase{
|
|
||||||
Name: "exceed_buffer_limit",
|
|
||||||
SuppressOutput: true,
|
|
||||||
Config: []string{`{type: command, path: {{.WorkDir}}/test/test-large-stdout.sh}`},
|
|
||||||
ExpectHadError: false,
|
|
||||||
ExpectHadFatalErr: false,
|
|
||||||
ExpectStepReports: []expectStep{
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Pre,
|
|
||||||
ExpectStatus: hooks.StepOk,
|
|
||||||
OutputTest: func(t require.TestingT, s interface{}, v ...interface{}) {
|
|
||||||
require.Len(t, s, 1<<20)
|
|
||||||
},
|
|
||||||
},
|
|
||||||
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
|
|
||||||
expectStep{
|
|
||||||
// No-action run of above hook
|
|
||||||
ExpectedEdge: hooks.Post,
|
|
||||||
ExpectStatus: hooks.StepOk,
|
|
||||||
OutputTest: require.Empty,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
|
|
||||||
/*
|
|
||||||
Following not intended to test functionality of
|
|
||||||
filter package. Only to demonstrate that hook
|
|
||||||
filters are being applied. The following should
|
|
||||||
result in NO hooks running. If it does run, a
|
|
||||||
fatal hooks.RunReport will be returned.
|
|
||||||
*/
|
|
||||||
testCase{
|
|
||||||
Name: "exclude_all_filesystems",
|
|
||||||
Config: []string{
|
|
||||||
`{type: command, path: {{.WorkDir}}/test/test-error.sh, err_is_fatal: true, filesystems: {"<": false}}`,
|
|
||||||
},
|
|
||||||
ExpectStepReports: []expectStep{
|
|
||||||
expectStep{
|
|
||||||
ExpectedEdge: hooks.Callback,
|
|
||||||
ExpectStatus: hooks.StepOk,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
parseHookConfig := func(t *testing.T, in string) *config.Config {
|
|
||||||
t.Helper()
|
|
||||||
conf, err := config.ParseConfigBytes([]byte(in))
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, conf)
|
|
||||||
return conf
|
|
||||||
}
|
|
||||||
|
|
||||||
fillHooks := func(tt *testCase) string {
|
|
||||||
// make hook path absolute
|
|
||||||
cwd, err := os.Getwd()
|
|
||||||
if err != nil {
|
|
||||||
panic("os.Getwd() failed")
|
|
||||||
}
|
|
||||||
var hooksTmpl string = "\n"
|
|
||||||
for _, l := range tt.Config {
|
|
||||||
hooksTmpl += fmt.Sprintf(" - %s\n", l)
|
|
||||||
}
|
|
||||||
tmpl.New("List").Parse(hooksTmpl)
|
|
||||||
|
|
||||||
var outBytes bytes.Buffer
|
|
||||||
data := struct {
|
|
||||||
WorkDir string
|
|
||||||
}{
|
|
||||||
WorkDir: cwd,
|
|
||||||
}
|
|
||||||
if err := tmpl.Execute(&outBytes, data); err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return outBytes.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
var c *config.Config
|
|
||||||
fs, err := zfs.NewDatasetPath(testFSName)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
log := logger.NewTestLogger(t)
|
|
||||||
|
|
||||||
var cbReached bool
|
|
||||||
cb := hooks.NewCallbackHookForFilesystem("testcallback", fs, func(_ context.Context) error {
|
|
||||||
cbReached = true
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
|
|
||||||
hookEnvExtra := hooks.Env{
|
|
||||||
hooks.EnvFS: fs.ToString(),
|
|
||||||
hooks.EnvSnapshot: testSnapshotName,
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range testTable {
|
|
||||||
if testing.Short() && tt.IsSlow {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run(tt.Name, func(t *testing.T) {
|
|
||||||
c = parseHookConfig(t, fillHooks(&tt))
|
|
||||||
snp := c.Jobs[0].Ret.(*config.SnapJob).Snapshotting.Ret.(*config.SnapshottingPeriodic)
|
|
||||||
hookList, err := hooks.ListFromConfig(&snp.Hooks)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
filteredHooks, err := hookList.CopyFilteredForFilesystem(fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
plan, err := hooks.NewPlan(&filteredHooks, hooks.PhaseTesting, cb, hookEnvExtra)
|
|
||||||
require.NoError(t, err)
|
|
||||||
t.Logf("REPORT PRE EXECUTION:\n%s", plan.Report())
|
|
||||||
|
|
||||||
cbReached = false
|
|
||||||
|
|
||||||
if testing.Verbose() && !tt.SuppressOutput {
|
|
||||||
ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
|
|
||||||
}
|
|
||||||
plan.Run(ctx, false)
|
|
||||||
report := plan.Report()
|
|
||||||
|
|
||||||
t.Logf("REPORT POST EXECUTION:\n%s", report)
|
|
||||||
|
|
||||||
/*
|
|
||||||
* TEST ASSERTIONS
|
|
||||||
*/
|
|
||||||
|
|
||||||
t.Logf("len(runReports)=%v", len(report))
|
|
||||||
t.Logf("len(tt.ExpectStepReports)=%v", len(tt.ExpectStepReports))
|
|
||||||
require.Equal(t, len(tt.ExpectStepReports), len(report), "ExpectStepReports must be same length as expected number of hook runs, excluding possible Callback")
|
|
||||||
|
|
||||||
// Check if callback ran, when required
|
|
||||||
if tt.ExpectCallbackSkipped {
|
|
||||||
require.False(t, cbReached, "callback ran but should not have run")
|
|
||||||
} else {
|
|
||||||
require.True(t, cbReached, "callback should have run but did not")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if a fatal run error occurred and was expected
|
|
||||||
require.Equal(t, tt.ExpectHadFatalErr, report.HadFatalError(), "non-matching HadFatalError")
|
|
||||||
require.Equal(t, tt.ExpectHadError, report.HadError(), "non-matching HadError")
|
|
||||||
|
|
||||||
if tt.ExpectHadFatalErr {
|
|
||||||
require.True(t, tt.ExpectHadError, "ExpectHadFatalErr implies ExpectHadError")
|
|
||||||
}
|
|
||||||
if !tt.ExpectHadError {
|
|
||||||
require.False(t, tt.ExpectHadFatalErr, "!ExpectHadError implies !ExpectHadFatalErr")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Iterate through each expected hook run
|
|
||||||
for i, hook := range tt.ExpectStepReports {
|
|
||||||
t.Logf("expecting report conforming to %v", hook)
|
|
||||||
|
|
||||||
exp, act := hook.ExpectStatus, report[i].Status
|
|
||||||
require.Equal(t, exp, act, "%s != %s", exp, act)
|
|
||||||
|
|
||||||
// Check for required ExpectedEdge
|
|
||||||
require.NotZero(t, hook.ExpectedEdge, "each hook must have an ExpectedEdge")
|
|
||||||
require.Equal(t, hook.ExpectedEdge, report[i].Edge,
|
|
||||||
"incorrect edge: expected %q, actual %q", hook.ExpectedEdge.String(), report[i].Edge.String(),
|
|
||||||
)
|
|
||||||
|
|
||||||
// Check for expected output
|
|
||||||
if hook.OutputTest != nil {
|
|
||||||
require.IsType(t, (*hooks.CommandHookReport)(nil), report[i].Report)
|
|
||||||
chr := report[i].Report.(*hooks.CommandHookReport)
|
|
||||||
hook.OutputTest(t, string(chr.CapturedStdoutStderrCombined))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for expected errors
|
|
||||||
if hook.ErrorTest != nil {
|
|
||||||
hook.ErrorTest(t, string(report[i].Report.Error()))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,77 +0,0 @@
|
|||||||
// Code generated by "enumer -type=StepStatus -trimprefix=Step"; DO NOT EDIT.
|
|
||||||
|
|
||||||
//
|
|
||||||
package hooks
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
_StepStatusName_0 = "PendingExec"
|
|
||||||
_StepStatusName_1 = "Ok"
|
|
||||||
_StepStatusName_2 = "Err"
|
|
||||||
_StepStatusName_3 = "SkippedDueToFatalErr"
|
|
||||||
_StepStatusName_4 = "SkippedDueToPreErr"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
_StepStatusIndex_0 = [...]uint8{0, 7, 11}
|
|
||||||
_StepStatusIndex_1 = [...]uint8{0, 2}
|
|
||||||
_StepStatusIndex_2 = [...]uint8{0, 3}
|
|
||||||
_StepStatusIndex_3 = [...]uint8{0, 20}
|
|
||||||
_StepStatusIndex_4 = [...]uint8{0, 18}
|
|
||||||
)
|
|
||||||
|
|
||||||
func (i StepStatus) String() string {
|
|
||||||
switch {
|
|
||||||
case 1 <= i && i <= 2:
|
|
||||||
i -= 1
|
|
||||||
return _StepStatusName_0[_StepStatusIndex_0[i]:_StepStatusIndex_0[i+1]]
|
|
||||||
case i == 4:
|
|
||||||
return _StepStatusName_1
|
|
||||||
case i == 8:
|
|
||||||
return _StepStatusName_2
|
|
||||||
case i == 16:
|
|
||||||
return _StepStatusName_3
|
|
||||||
case i == 32:
|
|
||||||
return _StepStatusName_4
|
|
||||||
default:
|
|
||||||
return fmt.Sprintf("StepStatus(%d)", i)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var _StepStatusValues = []StepStatus{1, 2, 4, 8, 16, 32}
|
|
||||||
|
|
||||||
var _StepStatusNameToValueMap = map[string]StepStatus{
|
|
||||||
_StepStatusName_0[0:7]: 1,
|
|
||||||
_StepStatusName_0[7:11]: 2,
|
|
||||||
_StepStatusName_1[0:2]: 4,
|
|
||||||
_StepStatusName_2[0:3]: 8,
|
|
||||||
_StepStatusName_3[0:20]: 16,
|
|
||||||
_StepStatusName_4[0:18]: 32,
|
|
||||||
}
|
|
||||||
|
|
||||||
// StepStatusString retrieves an enum value from the enum constants string name.
|
|
||||||
// Throws an error if the param is not part of the enum.
|
|
||||||
func StepStatusString(s string) (StepStatus, error) {
|
|
||||||
if val, ok := _StepStatusNameToValueMap[s]; ok {
|
|
||||||
return val, nil
|
|
||||||
}
|
|
||||||
return 0, fmt.Errorf("%s does not belong to StepStatus values", s)
|
|
||||||
}
|
|
||||||
|
|
||||||
// StepStatusValues returns all values of the enum
|
|
||||||
func StepStatusValues() []StepStatus {
|
|
||||||
return _StepStatusValues
|
|
||||||
}
|
|
||||||
|
|
||||||
// IsAStepStatus returns "true" if the value is listed in the enum definition. "false" otherwise
|
|
||||||
func (i StepStatus) IsAStepStatus() bool {
|
|
||||||
for _, v := range _StepStatusValues {
|
|
||||||
if i == v {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
#!/bin/sh -eu
|
|
||||||
|
|
||||||
>&2 echo "TEST ERROR $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
|
|
||||||
|
|
||||||
exit 1
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
#!/bin/sh -eu
|
|
||||||
|
|
||||||
# Exceed default hook log size of 1<<20 bytes by double
|
|
||||||
# The Go test should fail if buffer size exceeds 1MB
|
|
||||||
|
|
||||||
pre_testing() {
|
|
||||||
ln=0
|
|
||||||
while :; do printf '%06d: 012345678901234567890123456789012345678901234567890123456789012345678901\n' "$ln"; ln="$(($ln + 1))"; done \
|
|
||||||
| head -c$(( 2 << 20 ))
|
|
||||||
}
|
|
||||||
|
|
||||||
case "$ZREPL_HOOKTYPE" in
|
|
||||||
pre_testing)
|
|
||||||
"$ZREPL_HOOKTYPE";;
|
|
||||||
*)
|
|
||||||
# Not handled by this script
|
|
||||||
exit 0;;
|
|
||||||
esac
|
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
#!/bin/sh -eu
|
|
||||||
|
|
||||||
post_testing() {
|
|
||||||
>&2 echo "TEST ERROR $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
|
|
||||||
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
case "$ZREPL_HOOKTYPE" in
|
|
||||||
post_testing)
|
|
||||||
"$ZREPL_HOOKTYPE";;
|
|
||||||
*)
|
|
||||||
# Not handled by this script
|
|
||||||
exit 0;;
|
|
||||||
esac
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
#!/bin/sh -eu
|
|
||||||
|
|
||||||
if [ "$ZREPL_HOOKTYPE" = "pre_testing" ]; then
|
|
||||||
>&2 echo "TEST ERROR $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
|
|
||||||
exit 1
|
|
||||||
elif [ "$ZREPL_HOOKTYPE" = "post_testing" ]; then
|
|
||||||
echo "TEST $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME"
|
|
||||||
else
|
|
||||||
printf "Unknown hook type: %s" "$ZREPL_HOOKTYPE"
|
|
||||||
exit 255
|
|
||||||
fi
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
#!/bin/sh -eu
|
|
||||||
|
|
||||||
echo "TEST $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME ${ZREPL_DRYRUN:+(dry run)}"
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
#!/bin/sh -eu
|
|
||||||
|
|
||||||
echo "TEST $ZREPL_HOOKTYPE $ZREPL_FS@$ZREPL_SNAPNAME ZREPL_TIMEOUT=$ZREPL_TIMEOUT"
|
|
||||||
|
|
||||||
exec sleep $(($ZREPL_TIMEOUT + 1))
|
|
||||||
@@ -1,545 +0,0 @@
|
|||||||
package job
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
"github.com/prometheus/common/log"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
|
||||||
"github.com/zrepl/zrepl/util/envconst"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
"github.com/zrepl/zrepl/daemon/job/reset"
|
|
||||||
"github.com/zrepl/zrepl/daemon/job/wakeup"
|
|
||||||
"github.com/zrepl/zrepl/daemon/pruner"
|
|
||||||
"github.com/zrepl/zrepl/daemon/snapper"
|
|
||||||
"github.com/zrepl/zrepl/endpoint"
|
|
||||||
"github.com/zrepl/zrepl/replication"
|
|
||||||
"github.com/zrepl/zrepl/replication/driver"
|
|
||||||
"github.com/zrepl/zrepl/replication/logic"
|
|
||||||
"github.com/zrepl/zrepl/replication/report"
|
|
||||||
"github.com/zrepl/zrepl/rpc"
|
|
||||||
"github.com/zrepl/zrepl/transport"
|
|
||||||
"github.com/zrepl/zrepl/transport/fromconfig"
|
|
||||||
"github.com/zrepl/zrepl/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
type ActiveSide struct {
|
|
||||||
mode activeMode
|
|
||||||
name endpoint.JobID
|
|
||||||
connecter transport.Connecter
|
|
||||||
|
|
||||||
replicationDriverConfig driver.Config
|
|
||||||
|
|
||||||
prunerFactory *pruner.PrunerFactory
|
|
||||||
|
|
||||||
promRepStateSecs *prometheus.HistogramVec // labels: state
|
|
||||||
promPruneSecs *prometheus.HistogramVec // labels: prune_side
|
|
||||||
promBytesReplicated *prometheus.CounterVec // labels: filesystem
|
|
||||||
promReplicationErrors prometheus.Gauge
|
|
||||||
|
|
||||||
tasksMtx sync.Mutex
|
|
||||||
tasks activeSideTasks
|
|
||||||
}
|
|
||||||
|
|
||||||
//go:generate enumer -type=ActiveSideState
|
|
||||||
type ActiveSideState int
|
|
||||||
|
|
||||||
const (
|
|
||||||
ActiveSideReplicating ActiveSideState = 1 << iota
|
|
||||||
ActiveSidePruneSender
|
|
||||||
ActiveSidePruneReceiver
|
|
||||||
ActiveSideDone // also errors
|
|
||||||
)
|
|
||||||
|
|
||||||
type activeSideTasks struct {
|
|
||||||
state ActiveSideState
|
|
||||||
|
|
||||||
// valid for state ActiveSideReplicating, ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
|
|
||||||
replicationReport driver.ReportFunc
|
|
||||||
replicationCancel context.CancelFunc
|
|
||||||
|
|
||||||
// valid for state ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
|
|
||||||
prunerSender, prunerReceiver *pruner.Pruner
|
|
||||||
|
|
||||||
// valid for state ActiveSidePruneReceiver, ActiveSideDone
|
|
||||||
prunerSenderCancel, prunerReceiverCancel context.CancelFunc
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *ActiveSide) updateTasks(u func(*activeSideTasks)) activeSideTasks {
|
|
||||||
a.tasksMtx.Lock()
|
|
||||||
defer a.tasksMtx.Unlock()
|
|
||||||
copy := a.tasks
|
|
||||||
if u == nil {
|
|
||||||
return copy
|
|
||||||
}
|
|
||||||
u(©)
|
|
||||||
a.tasks = copy
|
|
||||||
return copy
|
|
||||||
}
|
|
||||||
|
|
||||||
type activeMode interface {
|
|
||||||
ConnectEndpoints(ctx context.Context, connecter transport.Connecter)
|
|
||||||
DisconnectEndpoints()
|
|
||||||
SenderReceiver() (logic.Sender, logic.Receiver)
|
|
||||||
Type() Type
|
|
||||||
PlannerPolicy() logic.PlannerPolicy
|
|
||||||
RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{})
|
|
||||||
SnapperReport() *snapper.Report
|
|
||||||
ResetConnectBackoff()
|
|
||||||
}
|
|
||||||
|
|
||||||
type modePush struct {
|
|
||||||
setupMtx sync.Mutex
|
|
||||||
sender *endpoint.Sender
|
|
||||||
receiver *rpc.Client
|
|
||||||
senderConfig *endpoint.SenderConfig
|
|
||||||
plannerPolicy *logic.PlannerPolicy
|
|
||||||
snapper *snapper.PeriodicOrManual
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *modePush) ConnectEndpoints(ctx context.Context, connecter transport.Connecter) {
|
|
||||||
m.setupMtx.Lock()
|
|
||||||
defer m.setupMtx.Unlock()
|
|
||||||
if m.receiver != nil || m.sender != nil {
|
|
||||||
panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
|
|
||||||
}
|
|
||||||
m.sender = endpoint.NewSender(*m.senderConfig)
|
|
||||||
m.receiver = rpc.NewClient(connecter, rpc.GetLoggersOrPanic(ctx))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *modePush) DisconnectEndpoints() {
|
|
||||||
m.setupMtx.Lock()
|
|
||||||
defer m.setupMtx.Unlock()
|
|
||||||
m.receiver.Close()
|
|
||||||
m.sender = nil
|
|
||||||
m.receiver = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *modePush) SenderReceiver() (logic.Sender, logic.Receiver) {
|
|
||||||
m.setupMtx.Lock()
|
|
||||||
defer m.setupMtx.Unlock()
|
|
||||||
return m.sender, m.receiver
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *modePush) Type() Type { return TypePush }
|
|
||||||
|
|
||||||
func (m *modePush) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
|
|
||||||
|
|
||||||
func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
|
|
||||||
m.snapper.Run(ctx, wakeUpCommon)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *modePush) SnapperReport() *snapper.Report {
|
|
||||||
return m.snapper.Report()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *modePush) ResetConnectBackoff() {
|
|
||||||
m.setupMtx.Lock()
|
|
||||||
defer m.setupMtx.Unlock()
|
|
||||||
if m.receiver != nil {
|
|
||||||
m.receiver.ResetConnectBackoff()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func modePushFromConfig(g *config.Global, in *config.PushJob, jobID endpoint.JobID) (*modePush, error) {
|
|
||||||
m := &modePush{}
|
|
||||||
var err error
|
|
||||||
|
|
||||||
m.senderConfig, err = buildSenderConfig(in, jobID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "sender config")
|
|
||||||
}
|
|
||||||
|
|
||||||
replicationConfig, err := logic.ReplicationConfigFromConfig(in.Replication)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "field `replication`")
|
|
||||||
}
|
|
||||||
|
|
||||||
m.plannerPolicy = &logic.PlannerPolicy{
|
|
||||||
EncryptedSend: logic.TriFromBool(in.Send.Encrypted),
|
|
||||||
ReplicationConfig: replicationConfig,
|
|
||||||
SizeEstimationConcurrency: in.Replication.Concurrency.SizeEstimates,
|
|
||||||
}
|
|
||||||
if err := m.plannerPolicy.Validate(); err != nil {
|
|
||||||
return nil, errors.Wrap(err, "cannot build planner policy")
|
|
||||||
}
|
|
||||||
|
|
||||||
if m.snapper, err = snapper.FromConfig(g, m.senderConfig.FSF, in.Snapshotting); err != nil {
|
|
||||||
return nil, errors.Wrap(err, "cannot build snapper")
|
|
||||||
}
|
|
||||||
|
|
||||||
return m, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type modePull struct {
|
|
||||||
setupMtx sync.Mutex
|
|
||||||
receiver *endpoint.Receiver
|
|
||||||
receiverConfig endpoint.ReceiverConfig
|
|
||||||
sender *rpc.Client
|
|
||||||
plannerPolicy *logic.PlannerPolicy
|
|
||||||
interval config.PositiveDurationOrManual
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *modePull) ConnectEndpoints(ctx context.Context, connecter transport.Connecter) {
|
|
||||||
m.setupMtx.Lock()
|
|
||||||
defer m.setupMtx.Unlock()
|
|
||||||
if m.receiver != nil || m.sender != nil {
|
|
||||||
panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
|
|
||||||
}
|
|
||||||
m.receiver = endpoint.NewReceiver(m.receiverConfig)
|
|
||||||
m.sender = rpc.NewClient(connecter, rpc.GetLoggersOrPanic(ctx))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *modePull) DisconnectEndpoints() {
|
|
||||||
m.setupMtx.Lock()
|
|
||||||
defer m.setupMtx.Unlock()
|
|
||||||
m.sender.Close()
|
|
||||||
m.sender = nil
|
|
||||||
m.receiver = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *modePull) SenderReceiver() (logic.Sender, logic.Receiver) {
|
|
||||||
m.setupMtx.Lock()
|
|
||||||
defer m.setupMtx.Unlock()
|
|
||||||
return m.sender, m.receiver
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*modePull) Type() Type { return TypePull }
|
|
||||||
|
|
||||||
func (m *modePull) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
|
|
||||||
|
|
||||||
func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
|
|
||||||
if m.interval.Manual {
|
|
||||||
GetLogger(ctx).Info("manual pull configured, periodic pull disabled")
|
|
||||||
// "waiting for wakeups" is printed in common ActiveSide.do
|
|
||||||
return
|
|
||||||
}
|
|
||||||
t := time.NewTicker(m.interval.Interval)
|
|
||||||
defer t.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-t.C:
|
|
||||||
select {
|
|
||||||
case wakeUpCommon <- struct{}{}:
|
|
||||||
default:
|
|
||||||
GetLogger(ctx).
|
|
||||||
WithField("pull_interval", m.interval).
|
|
||||||
Warn("pull job took longer than pull interval")
|
|
||||||
wakeUpCommon <- struct{}{} // block anyways, to queue up the wakeup
|
|
||||||
}
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *modePull) SnapperReport() *snapper.Report {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *modePull) ResetConnectBackoff() {
|
|
||||||
m.setupMtx.Lock()
|
|
||||||
defer m.setupMtx.Unlock()
|
|
||||||
if m.sender != nil {
|
|
||||||
m.sender.ResetConnectBackoff()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func modePullFromConfig(g *config.Global, in *config.PullJob, jobID endpoint.JobID) (m *modePull, err error) {
|
|
||||||
m = &modePull{}
|
|
||||||
m.interval = in.Interval
|
|
||||||
|
|
||||||
replicationConfig, err := logic.ReplicationConfigFromConfig(in.Replication)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "field `replication`")
|
|
||||||
}
|
|
||||||
|
|
||||||
m.plannerPolicy = &logic.PlannerPolicy{
|
|
||||||
EncryptedSend: logic.DontCare,
|
|
||||||
ReplicationConfig: replicationConfig,
|
|
||||||
SizeEstimationConcurrency: in.Replication.Concurrency.SizeEstimates,
|
|
||||||
}
|
|
||||||
if err := m.plannerPolicy.Validate(); err != nil {
|
|
||||||
return nil, errors.Wrap(err, "cannot build planner policy")
|
|
||||||
}
|
|
||||||
|
|
||||||
m.receiverConfig, err = buildReceiverConfig(in, jobID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return m, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func replicationDriverConfigFromConfig(in *config.Replication) (c driver.Config, err error) {
|
|
||||||
c = driver.Config{
|
|
||||||
StepQueueConcurrency: in.Concurrency.Steps,
|
|
||||||
MaxAttempts: envconst.Int("ZREPL_REPLICATION_MAX_ATTEMPTS", 3),
|
|
||||||
ReconnectHardFailTimeout: envconst.Duration("ZREPL_REPLICATION_RECONNECT_HARD_FAIL_TIMEOUT", 10*time.Minute),
|
|
||||||
}
|
|
||||||
err = c.Validate()
|
|
||||||
return c, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}) (j *ActiveSide, err error) {
|
|
||||||
|
|
||||||
j = &ActiveSide{}
|
|
||||||
j.name, err = endpoint.MakeJobID(in.Name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "invalid job name")
|
|
||||||
}
|
|
||||||
|
|
||||||
switch v := configJob.(type) {
|
|
||||||
case *config.PushJob:
|
|
||||||
j.mode, err = modePushFromConfig(g, v, j.name) // shadow
|
|
||||||
case *config.PullJob:
|
|
||||||
j.mode, err = modePullFromConfig(g, v, j.name) // shadow
|
|
||||||
default:
|
|
||||||
panic(fmt.Sprintf("implementation error: unknown job type %T", v))
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, err // no wrapping required
|
|
||||||
}
|
|
||||||
|
|
||||||
j.promRepStateSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
||||||
Namespace: "zrepl",
|
|
||||||
Subsystem: "replication",
|
|
||||||
Name: "state_time",
|
|
||||||
Help: "seconds spent during replication",
|
|
||||||
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
|
|
||||||
}, []string{"state"})
|
|
||||||
j.promBytesReplicated = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
||||||
Namespace: "zrepl",
|
|
||||||
Subsystem: "replication",
|
|
||||||
Name: "bytes_replicated",
|
|
||||||
Help: "number of bytes replicated from sender to receiver per filesystem",
|
|
||||||
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
|
|
||||||
}, []string{"filesystem"})
|
|
||||||
|
|
||||||
j.promReplicationErrors = prometheus.NewGauge(prometheus.GaugeOpts{
|
|
||||||
Namespace: "zrepl",
|
|
||||||
Subsystem: "replication",
|
|
||||||
Name: "filesystem_errors",
|
|
||||||
Help: "number of filesystems that failed replication in the latest replication attempt, or -1 if the job failed before enumerating the filesystems",
|
|
||||||
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
|
|
||||||
})
|
|
||||||
|
|
||||||
j.connecter, err = fromconfig.ConnecterFromConfig(g, in.Connect)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "cannot build client")
|
|
||||||
}
|
|
||||||
|
|
||||||
j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
||||||
Namespace: "zrepl",
|
|
||||||
Subsystem: "pruning",
|
|
||||||
Name: "time",
|
|
||||||
Help: "seconds spent in pruner",
|
|
||||||
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
|
|
||||||
}, []string{"prune_side"})
|
|
||||||
j.prunerFactory, err = pruner.NewPrunerFactory(in.Pruning, j.promPruneSecs)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
j.replicationDriverConfig, err = replicationDriverConfigFromConfig(in.Replication)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "cannot build replication driver config")
|
|
||||||
}
|
|
||||||
|
|
||||||
return j, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) {
|
|
||||||
registerer.MustRegister(j.promRepStateSecs)
|
|
||||||
registerer.MustRegister(j.promPruneSecs)
|
|
||||||
registerer.MustRegister(j.promBytesReplicated)
|
|
||||||
registerer.MustRegister(j.promReplicationErrors)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *ActiveSide) Name() string { return j.name.String() }
|
|
||||||
|
|
||||||
type ActiveSideStatus struct {
|
|
||||||
Replication *report.Report
|
|
||||||
PruningSender, PruningReceiver *pruner.Report
|
|
||||||
Snapshotting *snapper.Report
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *ActiveSide) Status() *Status {
|
|
||||||
tasks := j.updateTasks(nil)
|
|
||||||
|
|
||||||
s := &ActiveSideStatus{}
|
|
||||||
t := j.mode.Type()
|
|
||||||
if tasks.replicationReport != nil {
|
|
||||||
s.Replication = tasks.replicationReport()
|
|
||||||
}
|
|
||||||
if tasks.prunerSender != nil {
|
|
||||||
s.PruningSender = tasks.prunerSender.Report()
|
|
||||||
}
|
|
||||||
if tasks.prunerReceiver != nil {
|
|
||||||
s.PruningReceiver = tasks.prunerReceiver.Report()
|
|
||||||
}
|
|
||||||
s.Snapshotting = j.mode.SnapperReport()
|
|
||||||
return &Status{Type: t, JobSpecific: s}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *ActiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
|
|
||||||
pull, ok := j.mode.(*modePull)
|
|
||||||
if !ok {
|
|
||||||
_ = j.mode.(*modePush) // make sure we didn't introduce a new job type
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
return pull.receiverConfig.RootWithoutClientComponent.Copy(), true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *ActiveSide) SenderConfig() *endpoint.SenderConfig {
|
|
||||||
push, ok := j.mode.(*modePush)
|
|
||||||
if !ok {
|
|
||||||
_ = j.mode.(*modePull) // make sure we didn't introduce a new job type
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return push.senderConfig
|
|
||||||
}
|
|
||||||
|
|
||||||
// The active side of a replication uses one end (sender or receiver)
|
|
||||||
// directly by method invocation, without going through a transport that
|
|
||||||
// provides a client identity.
|
|
||||||
// However, in order to avoid the need to distinguish between direct-method-invocating
|
|
||||||
// clients and RPC client, we use an invalid client identity as a sentinel value.
|
|
||||||
func FakeActiveSideDirectMethodInvocationClientIdentity(jobId endpoint.JobID) string {
|
|
||||||
return fmt.Sprintf("<local><active><job><client><identity><job=%q>", jobId.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *ActiveSide) Run(ctx context.Context) {
|
|
||||||
ctx, endTask := trace.WithTaskAndSpan(ctx, "active-side-job", j.Name())
|
|
||||||
defer endTask()
|
|
||||||
|
|
||||||
ctx = context.WithValue(ctx, endpoint.ClientIdentityKey, FakeActiveSideDirectMethodInvocationClientIdentity(j.name))
|
|
||||||
|
|
||||||
log := GetLogger(ctx)
|
|
||||||
|
|
||||||
defer log.Info("job exiting")
|
|
||||||
|
|
||||||
periodicDone := make(chan struct{})
|
|
||||||
ctx, cancel := context.WithCancel(ctx)
|
|
||||||
defer cancel()
|
|
||||||
periodicCtx, endTask := trace.WithTask(ctx, "periodic")
|
|
||||||
defer endTask()
|
|
||||||
go j.mode.RunPeriodic(periodicCtx, periodicDone)
|
|
||||||
|
|
||||||
invocationCount := 0
|
|
||||||
outer:
|
|
||||||
for {
|
|
||||||
log.Info("wait for wakeups")
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
log.WithError(ctx.Err()).Info("context")
|
|
||||||
break outer
|
|
||||||
|
|
||||||
case <-wakeup.Wait(ctx):
|
|
||||||
j.mode.ResetConnectBackoff()
|
|
||||||
case <-periodicDone:
|
|
||||||
}
|
|
||||||
invocationCount++
|
|
||||||
invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", invocationCount))
|
|
||||||
j.do(invocationCtx)
|
|
||||||
endSpan()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *ActiveSide) do(ctx context.Context) {
|
|
||||||
|
|
||||||
j.mode.ConnectEndpoints(ctx, j.connecter)
|
|
||||||
defer j.mode.DisconnectEndpoints()
|
|
||||||
|
|
||||||
// allow cancellation of an invocation (this function)
|
|
||||||
ctx, cancelThisRun := context.WithCancel(ctx)
|
|
||||||
defer cancelThisRun()
|
|
||||||
go func() {
|
|
||||||
select {
|
|
||||||
case <-reset.Wait(ctx):
|
|
||||||
log.Info("reset received, cancelling current invocation")
|
|
||||||
cancelThisRun()
|
|
||||||
case <-ctx.Done():
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
sender, receiver := j.mode.SenderReceiver()
|
|
||||||
|
|
||||||
{
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
ctx, endSpan := trace.WithSpan(ctx, "replication")
|
|
||||||
ctx, repCancel := context.WithCancel(ctx)
|
|
||||||
var repWait driver.WaitFunc
|
|
||||||
j.updateTasks(func(tasks *activeSideTasks) {
|
|
||||||
// reset it
|
|
||||||
*tasks = activeSideTasks{}
|
|
||||||
tasks.replicationCancel = func() { repCancel(); endSpan() }
|
|
||||||
tasks.replicationReport, repWait = replication.Do(
|
|
||||||
ctx, j.replicationDriverConfig, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver, j.mode.PlannerPolicy()),
|
|
||||||
)
|
|
||||||
tasks.state = ActiveSideReplicating
|
|
||||||
})
|
|
||||||
GetLogger(ctx).Info("start replication")
|
|
||||||
repWait(true) // wait blocking
|
|
||||||
repCancel() // always cancel to free up context resources
|
|
||||||
|
|
||||||
replicationReport := j.tasks.replicationReport()
|
|
||||||
j.promReplicationErrors.Set(float64(replicationReport.GetFailedFilesystemsCountInLatestAttempt()))
|
|
||||||
|
|
||||||
endSpan()
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
ctx, endSpan := trace.WithSpan(ctx, "prune_sender")
|
|
||||||
ctx, senderCancel := context.WithCancel(ctx)
|
|
||||||
tasks := j.updateTasks(func(tasks *activeSideTasks) {
|
|
||||||
tasks.prunerSender = j.prunerFactory.BuildSenderPruner(ctx, sender, sender)
|
|
||||||
tasks.prunerSenderCancel = func() { senderCancel(); endSpan() }
|
|
||||||
tasks.state = ActiveSidePruneSender
|
|
||||||
})
|
|
||||||
GetLogger(ctx).Info("start pruning sender")
|
|
||||||
tasks.prunerSender.Prune()
|
|
||||||
GetLogger(ctx).Info("finished pruning sender")
|
|
||||||
senderCancel()
|
|
||||||
endSpan()
|
|
||||||
}
|
|
||||||
{
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
ctx, endSpan := trace.WithSpan(ctx, "prune_recever")
|
|
||||||
ctx, receiverCancel := context.WithCancel(ctx)
|
|
||||||
tasks := j.updateTasks(func(tasks *activeSideTasks) {
|
|
||||||
tasks.prunerReceiver = j.prunerFactory.BuildReceiverPruner(ctx, receiver, sender)
|
|
||||||
tasks.prunerReceiverCancel = func() { receiverCancel(); endSpan() }
|
|
||||||
tasks.state = ActiveSidePruneReceiver
|
|
||||||
})
|
|
||||||
GetLogger(ctx).Info("start pruning receiver")
|
|
||||||
tasks.prunerReceiver.Prune()
|
|
||||||
GetLogger(ctx).Info("finished pruning receiver")
|
|
||||||
receiverCancel()
|
|
||||||
endSpan()
|
|
||||||
}
|
|
||||||
|
|
||||||
j.updateTasks(func(tasks *activeSideTasks) {
|
|
||||||
tasks.state = ActiveSideDone
|
|
||||||
})
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,20 +0,0 @@
|
|||||||
package job
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/endpoint"
|
|
||||||
"github.com/zrepl/zrepl/transport"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestFakeActiveSideDirectMethodInvocationClientIdentityDoesNotPassValidityTest(t *testing.T) {
|
|
||||||
jobid, err := endpoint.MakeJobID("validjobname")
|
|
||||||
require.NoError(t, err)
|
|
||||||
clientIdentity := FakeActiveSideDirectMethodInvocationClientIdentity(jobid)
|
|
||||||
t.Logf("%v", clientIdentity)
|
|
||||||
err = transport.ValidateClientIdentity(clientIdentity)
|
|
||||||
assert.Error(t, err)
|
|
||||||
}
|
|
||||||
@@ -1,67 +0,0 @@
|
|||||||
// Code generated by "enumer -type=ActiveSideState"; DO NOT EDIT.
|
|
||||||
|
|
||||||
//
|
|
||||||
package job
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
_ActiveSideStateName_0 = "ActiveSideReplicatingActiveSidePruneSender"
|
|
||||||
_ActiveSideStateName_1 = "ActiveSidePruneReceiver"
|
|
||||||
_ActiveSideStateName_2 = "ActiveSideDone"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
_ActiveSideStateIndex_0 = [...]uint8{0, 21, 42}
|
|
||||||
_ActiveSideStateIndex_1 = [...]uint8{0, 23}
|
|
||||||
_ActiveSideStateIndex_2 = [...]uint8{0, 14}
|
|
||||||
)
|
|
||||||
|
|
||||||
func (i ActiveSideState) String() string {
|
|
||||||
switch {
|
|
||||||
case 1 <= i && i <= 2:
|
|
||||||
i -= 1
|
|
||||||
return _ActiveSideStateName_0[_ActiveSideStateIndex_0[i]:_ActiveSideStateIndex_0[i+1]]
|
|
||||||
case i == 4:
|
|
||||||
return _ActiveSideStateName_1
|
|
||||||
case i == 8:
|
|
||||||
return _ActiveSideStateName_2
|
|
||||||
default:
|
|
||||||
return fmt.Sprintf("ActiveSideState(%d)", i)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ActiveSideStateValues = []ActiveSideState{1, 2, 4, 8}
|
|
||||||
|
|
||||||
var _ActiveSideStateNameToValueMap = map[string]ActiveSideState{
|
|
||||||
_ActiveSideStateName_0[0:21]: 1,
|
|
||||||
_ActiveSideStateName_0[21:42]: 2,
|
|
||||||
_ActiveSideStateName_1[0:23]: 4,
|
|
||||||
_ActiveSideStateName_2[0:14]: 8,
|
|
||||||
}
|
|
||||||
|
|
||||||
// ActiveSideStateString retrieves an enum value from the enum constants string name.
|
|
||||||
// Throws an error if the param is not part of the enum.
|
|
||||||
func ActiveSideStateString(s string) (ActiveSideState, error) {
|
|
||||||
if val, ok := _ActiveSideStateNameToValueMap[s]; ok {
|
|
||||||
return val, nil
|
|
||||||
}
|
|
||||||
return 0, fmt.Errorf("%s does not belong to ActiveSideState values", s)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ActiveSideStateValues returns all values of the enum
|
|
||||||
func ActiveSideStateValues() []ActiveSideState {
|
|
||||||
return _ActiveSideStateValues
|
|
||||||
}
|
|
||||||
|
|
||||||
// IsAActiveSideState returns "true" if the value is listed in the enum definition. "false" otherwise
|
|
||||||
func (i ActiveSideState) IsAActiveSideState() bool {
|
|
||||||
for _, v := range _ActiveSideStateValues {
|
|
||||||
if i == v {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
@@ -1,109 +0,0 @@
|
|||||||
package job
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
func JobsFromConfig(c *config.Config) ([]Job, error) {
|
|
||||||
js := make([]Job, len(c.Jobs))
|
|
||||||
for i := range c.Jobs {
|
|
||||||
j, err := buildJob(c.Global, c.Jobs[i])
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if j == nil || j.Name() == "" {
|
|
||||||
panic(fmt.Sprintf("implementation error: job builder returned nil job type %T", c.Jobs[i].Ret))
|
|
||||||
}
|
|
||||||
js[i] = j
|
|
||||||
}
|
|
||||||
|
|
||||||
// receiving-side root filesystems must not overlap
|
|
||||||
{
|
|
||||||
rfss := make([]string, 0, len(js))
|
|
||||||
for _, j := range js {
|
|
||||||
jrfs, ok := j.OwnedDatasetSubtreeRoot()
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
rfss = append(rfss, jrfs.ToString())
|
|
||||||
}
|
|
||||||
if err := validateReceivingSidesDoNotOverlap(rfss); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return js, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildJob(c *config.Global, in config.JobEnum) (j Job, err error) {
|
|
||||||
cannotBuildJob := func(e error, name string) (Job, error) {
|
|
||||||
return nil, errors.Wrapf(e, "cannot build job %q", name)
|
|
||||||
}
|
|
||||||
// FIXME prettify this
|
|
||||||
switch v := in.Ret.(type) {
|
|
||||||
case *config.SinkJob:
|
|
||||||
j, err = passiveSideFromConfig(c, &v.PassiveJob, v)
|
|
||||||
if err != nil {
|
|
||||||
return cannotBuildJob(err, v.Name)
|
|
||||||
}
|
|
||||||
case *config.SourceJob:
|
|
||||||
j, err = passiveSideFromConfig(c, &v.PassiveJob, v)
|
|
||||||
if err != nil {
|
|
||||||
return cannotBuildJob(err, v.Name)
|
|
||||||
}
|
|
||||||
case *config.SnapJob:
|
|
||||||
j, err = snapJobFromConfig(c, v)
|
|
||||||
if err != nil {
|
|
||||||
return cannotBuildJob(err, v.Name)
|
|
||||||
}
|
|
||||||
case *config.PushJob:
|
|
||||||
j, err = activeSide(c, &v.ActiveJob, v)
|
|
||||||
if err != nil {
|
|
||||||
return cannotBuildJob(err, v.Name)
|
|
||||||
}
|
|
||||||
case *config.PullJob:
|
|
||||||
j, err = activeSide(c, &v.ActiveJob, v)
|
|
||||||
if err != nil {
|
|
||||||
return cannotBuildJob(err, v.Name)
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
panic(fmt.Sprintf("implementation error: unknown job type %T", v))
|
|
||||||
}
|
|
||||||
return j, nil
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateReceivingSidesDoNotOverlap(receivingRootFSs []string) error {
|
|
||||||
if len(receivingRootFSs) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
rfss := make([]string, len(receivingRootFSs))
|
|
||||||
copy(rfss, receivingRootFSs)
|
|
||||||
sort.Slice(rfss, func(i, j int) bool {
|
|
||||||
return strings.Compare(rfss[i], rfss[j]) == -1
|
|
||||||
})
|
|
||||||
// add tailing slash because of hierarchy-simulation
|
|
||||||
// rootfs/ is not root of rootfs2/
|
|
||||||
for i := 0; i < len(rfss); i++ {
|
|
||||||
rfss[i] += "/"
|
|
||||||
}
|
|
||||||
// idea:
|
|
||||||
// no path in rfss must be prefix of another
|
|
||||||
//
|
|
||||||
// rfss is now lexicographically sorted, which means that
|
|
||||||
// if i is prefix of j, i < j (in lexicographical order)
|
|
||||||
// thus,
|
|
||||||
// if any i is prefix of i+n (n >= 1), there is overlap
|
|
||||||
for i := 0; i < len(rfss)-1; i++ {
|
|
||||||
if strings.HasPrefix(rfss[i+1], rfss[i]) {
|
|
||||||
return fmt.Errorf("receiving jobs with overlapping root filesystems are forbidden")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,69 +0,0 @@
|
|||||||
package job
|
|
||||||
|
|
||||||
import (
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
"github.com/zrepl/zrepl/daemon/filters"
|
|
||||||
"github.com/zrepl/zrepl/endpoint"
|
|
||||||
"github.com/zrepl/zrepl/util/nodefault"
|
|
||||||
"github.com/zrepl/zrepl/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
type SendingJobConfig interface {
|
|
||||||
GetFilesystems() config.FilesystemsFilter
|
|
||||||
GetSendOptions() *config.SendOptions // must not be nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildSenderConfig(in SendingJobConfig, jobID endpoint.JobID) (*endpoint.SenderConfig, error) {
|
|
||||||
|
|
||||||
fsf, err := filters.DatasetMapFilterFromConfig(in.GetFilesystems())
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "cannot build filesystem filter")
|
|
||||||
}
|
|
||||||
sendOpts := in.GetSendOptions()
|
|
||||||
return &endpoint.SenderConfig{
|
|
||||||
FSF: fsf,
|
|
||||||
JobID: jobID,
|
|
||||||
|
|
||||||
Encrypt: &nodefault.Bool{B: sendOpts.Encrypted},
|
|
||||||
SendRaw: sendOpts.Raw,
|
|
||||||
SendProperties: sendOpts.SendProperties,
|
|
||||||
SendBackupProperties: sendOpts.BackupProperties,
|
|
||||||
SendLargeBlocks: sendOpts.LargeBlocks,
|
|
||||||
SendCompressed: sendOpts.Compressed,
|
|
||||||
SendEmbeddedData: sendOpts.EmbeddedData,
|
|
||||||
SendSaved: sendOpts.Saved,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type ReceivingJobConfig interface {
|
|
||||||
GetRootFS() string
|
|
||||||
GetAppendClientIdentity() bool
|
|
||||||
GetRecvOptions() *config.RecvOptions
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildReceiverConfig(in ReceivingJobConfig, jobID endpoint.JobID) (rc endpoint.ReceiverConfig, err error) {
|
|
||||||
rootFs, err := zfs.NewDatasetPath(in.GetRootFS())
|
|
||||||
if err != nil {
|
|
||||||
return rc, errors.New("root_fs is not a valid zfs filesystem path")
|
|
||||||
}
|
|
||||||
if rootFs.Length() <= 0 {
|
|
||||||
return rc, errors.New("root_fs must not be empty") // duplicates error check of receiver
|
|
||||||
}
|
|
||||||
|
|
||||||
recvOpts := in.GetRecvOptions()
|
|
||||||
rc = endpoint.ReceiverConfig{
|
|
||||||
JobID: jobID,
|
|
||||||
RootWithoutClientComponent: rootFs,
|
|
||||||
AppendClientIdentity: in.GetAppendClientIdentity(),
|
|
||||||
|
|
||||||
InheritProperties: recvOpts.Properties.Inherit,
|
|
||||||
OverrideProperties: recvOpts.Properties.Override,
|
|
||||||
}
|
|
||||||
if err := rc.Validate(); err != nil {
|
|
||||||
return rc, errors.Wrap(err, "cannot build receiver config")
|
|
||||||
}
|
|
||||||
|
|
||||||
return rc, nil
|
|
||||||
}
|
|
||||||
@@ -1,256 +0,0 @@
|
|||||||
package job
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"path"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/kr/pretty"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
"github.com/zrepl/zrepl/transport/tls"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
|
|
||||||
type testCase struct {
|
|
||||||
err bool
|
|
||||||
input []string
|
|
||||||
}
|
|
||||||
tcs := []testCase{
|
|
||||||
{false, nil},
|
|
||||||
{false, []string{}},
|
|
||||||
{false, []string{""}}, // not our job to determine valid paths
|
|
||||||
{false, []string{"a"}},
|
|
||||||
{false, []string{"some/path"}},
|
|
||||||
{false, []string{"zroot/sink1", "zroot/sink2", "zroot/sink3"}},
|
|
||||||
{false, []string{"zroot/foo", "zroot/foobar"}},
|
|
||||||
{true, []string{"zroot/b", "zroot/b"}},
|
|
||||||
{true, []string{"zroot/foo", "zroot/foo/bar", "zroot/baz"}},
|
|
||||||
{false, []string{"a/x", "b/x"}},
|
|
||||||
{false, []string{"a", "b"}},
|
|
||||||
{true, []string{"a", "a"}},
|
|
||||||
{true, []string{"a/x/y", "a/x"}},
|
|
||||||
{true, []string{"a/x", "a/x/y"}},
|
|
||||||
{true, []string{"a/x", "b/x", "a/x/y"}},
|
|
||||||
{true, []string{"a", "a/b", "a/c", "a/b"}},
|
|
||||||
{true, []string{"a/b", "a/c", "a/b", "a/d", "a/c"}},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tcs {
|
|
||||||
t.Logf("input: %v", tc.input)
|
|
||||||
err := validateReceivingSidesDoNotOverlap(tc.input)
|
|
||||||
if tc.err {
|
|
||||||
assert.Error(t, err)
|
|
||||||
} else {
|
|
||||||
assert.NoError(t, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestJobIDErrorHandling(t *testing.T) {
|
|
||||||
tmpl := `
|
|
||||||
jobs:
|
|
||||||
- name: %s
|
|
||||||
type: push
|
|
||||||
connect:
|
|
||||||
type: local
|
|
||||||
listener_name: foo
|
|
||||||
client_identity: bar
|
|
||||||
filesystems: {"<": true}
|
|
||||||
snapshotting:
|
|
||||||
type: manual
|
|
||||||
pruning:
|
|
||||||
keep_sender:
|
|
||||||
- type: last_n
|
|
||||||
count: 10
|
|
||||||
keep_receiver:
|
|
||||||
- type: last_n
|
|
||||||
count: 10
|
|
||||||
`
|
|
||||||
fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
|
|
||||||
|
|
||||||
type Case struct {
|
|
||||||
jobName string
|
|
||||||
valid bool
|
|
||||||
}
|
|
||||||
cases := []Case{
|
|
||||||
{"validjobname", true},
|
|
||||||
{"valid with spaces", true},
|
|
||||||
{"invalid\twith\ttabs", false},
|
|
||||||
{"invalid#withdelimiter", false},
|
|
||||||
{"invalid@withdelimiter", false},
|
|
||||||
{"withnewline\\nmiddle", false},
|
|
||||||
{"withnewline\\n", false},
|
|
||||||
{"withslash/", false},
|
|
||||||
{"withslash/inthemiddle", false},
|
|
||||||
{"/", false},
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := range cases {
|
|
||||||
t.Run(cases[i].jobName, func(t *testing.T) {
|
|
||||||
c := cases[i]
|
|
||||||
|
|
||||||
conf, err := config.ParseConfigBytes([]byte(fill(c.jobName)))
|
|
||||||
require.NoError(t, err, "not expecting yaml-config to know about job ids")
|
|
||||||
require.NotNil(t, conf)
|
|
||||||
jobs, err := JobsFromConfig(conf)
|
|
||||||
|
|
||||||
if c.valid {
|
|
||||||
assert.NoError(t, err)
|
|
||||||
require.Len(t, jobs, 1)
|
|
||||||
assert.Equal(t, c.jobName, jobs[0].Name())
|
|
||||||
} else {
|
|
||||||
t.Logf("error: %s", err)
|
|
||||||
assert.Error(t, err)
|
|
||||||
assert.Nil(t, jobs)
|
|
||||||
}
|
|
||||||
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSampleConfigsAreBuiltWithoutErrors(t *testing.T) {
|
|
||||||
paths, err := filepath.Glob("../../config/samples/*")
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("glob failed: %+v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, p := range paths {
|
|
||||||
|
|
||||||
if path.Ext(p) != ".yml" {
|
|
||||||
t.Logf("skipping file %s", p)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run(p, func(t *testing.T) {
|
|
||||||
c, err := config.ParseConfig(p)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("error parsing %s:\n%+v", p, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Logf("file: %s", p)
|
|
||||||
t.Log(pretty.Sprint(c))
|
|
||||||
|
|
||||||
tls.FakeCertificateLoading(t)
|
|
||||||
jobs, err := JobsFromConfig(c)
|
|
||||||
t.Logf("jobs: %#v", jobs)
|
|
||||||
assert.NoError(t, err)
|
|
||||||
})
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReplicationOptions(t *testing.T) {
|
|
||||||
tmpl := `
|
|
||||||
jobs:
|
|
||||||
- name: foo
|
|
||||||
type: push
|
|
||||||
connect:
|
|
||||||
type: local
|
|
||||||
listener_name: foo
|
|
||||||
client_identity: bar
|
|
||||||
filesystems: {"<": true}
|
|
||||||
%s
|
|
||||||
snapshotting:
|
|
||||||
type: manual
|
|
||||||
pruning:
|
|
||||||
keep_sender:
|
|
||||||
- type: last_n
|
|
||||||
count: 10
|
|
||||||
keep_receiver:
|
|
||||||
- type: last_n
|
|
||||||
count: 10
|
|
||||||
`
|
|
||||||
|
|
||||||
type Test struct {
|
|
||||||
name string
|
|
||||||
input string
|
|
||||||
expectOk func(t *testing.T, a *ActiveSide, m *modePush)
|
|
||||||
expectError bool
|
|
||||||
}
|
|
||||||
|
|
||||||
tests := []Test{
|
|
||||||
{
|
|
||||||
name: "defaults",
|
|
||||||
input: `
|
|
||||||
replication: {}
|
|
||||||
`,
|
|
||||||
expectOk: func(t *testing.T, a *ActiveSide, m *modePush) {},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "steps_zero",
|
|
||||||
input: `
|
|
||||||
replication:
|
|
||||||
concurrency:
|
|
||||||
steps: 0
|
|
||||||
`,
|
|
||||||
expectError: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "size_estimates_zero",
|
|
||||||
input: `
|
|
||||||
replication:
|
|
||||||
concurrency:
|
|
||||||
size_estimates: 0
|
|
||||||
`,
|
|
||||||
expectError: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "custom_values",
|
|
||||||
input: `
|
|
||||||
replication:
|
|
||||||
concurrency:
|
|
||||||
steps: 23
|
|
||||||
size_estimates: 42
|
|
||||||
`,
|
|
||||||
expectOk: func(t *testing.T, a *ActiveSide, m *modePush) {
|
|
||||||
assert.Equal(t, 23, a.replicationDriverConfig.StepQueueConcurrency)
|
|
||||||
assert.Equal(t, 42, m.plannerPolicy.SizeEstimationConcurrency)
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "negative_values_forbidden",
|
|
||||||
input: `
|
|
||||||
replication:
|
|
||||||
concurrency:
|
|
||||||
steps: -23
|
|
||||||
size_estimates: -42
|
|
||||||
`,
|
|
||||||
expectError: true,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
|
|
||||||
|
|
||||||
for _, ts := range tests {
|
|
||||||
t.Run(ts.name, func(t *testing.T) {
|
|
||||||
assert.True(t, (ts.expectError) != (ts.expectOk != nil))
|
|
||||||
|
|
||||||
cstr := fill(ts.input)
|
|
||||||
t.Logf("testing config:\n%s", cstr)
|
|
||||||
c, err := config.ParseConfigBytes([]byte(cstr))
|
|
||||||
require.NoError(t, err)
|
|
||||||
jobs, err := JobsFromConfig(c)
|
|
||||||
if ts.expectOk != nil {
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, c)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, jobs, 1)
|
|
||||||
a := jobs[0].(*ActiveSide)
|
|
||||||
m := a.mode.(*modePush)
|
|
||||||
ts.expectOk(t, a, m)
|
|
||||||
} else if ts.expectError {
|
|
||||||
require.Error(t, err)
|
|
||||||
} else {
|
|
||||||
t.Fatalf("test must define expectOk or expectError")
|
|
||||||
}
|
|
||||||
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,108 +0,0 @@
|
|||||||
package job
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/daemon/logging"
|
|
||||||
"github.com/zrepl/zrepl/endpoint"
|
|
||||||
"github.com/zrepl/zrepl/logger"
|
|
||||||
"github.com/zrepl/zrepl/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Logger = logger.Logger
|
|
||||||
|
|
||||||
func GetLogger(ctx context.Context) Logger {
|
|
||||||
return logging.GetLogger(ctx, logging.SubsysJob)
|
|
||||||
}
|
|
||||||
|
|
||||||
type Job interface {
|
|
||||||
Name() string
|
|
||||||
Run(ctx context.Context)
|
|
||||||
Status() *Status
|
|
||||||
RegisterMetrics(registerer prometheus.Registerer)
|
|
||||||
// Jobs that return a subtree of the dataset hierarchy
|
|
||||||
// must return the root of that subtree as rfs and ok = true
|
|
||||||
OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool)
|
|
||||||
SenderConfig() *endpoint.SenderConfig
|
|
||||||
}
|
|
||||||
|
|
||||||
type Type string
|
|
||||||
|
|
||||||
const (
|
|
||||||
TypeInternal Type = "internal"
|
|
||||||
TypeSnap Type = "snap"
|
|
||||||
TypePush Type = "push"
|
|
||||||
TypeSink Type = "sink"
|
|
||||||
TypePull Type = "pull"
|
|
||||||
TypeSource Type = "source"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Status struct {
|
|
||||||
Type Type
|
|
||||||
JobSpecific interface{}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Status) MarshalJSON() ([]byte, error) {
|
|
||||||
typeJson, err := json.Marshal(s.Type)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
jobJSON, err := json.Marshal(s.JobSpecific)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
m := map[string]json.RawMessage{
|
|
||||||
"type": typeJson,
|
|
||||||
string(s.Type): jobJSON,
|
|
||||||
}
|
|
||||||
return json.Marshal(m)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Status) UnmarshalJSON(in []byte) (err error) {
|
|
||||||
var m map[string]json.RawMessage
|
|
||||||
if err := json.Unmarshal(in, &m); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
tJSON, ok := m["type"]
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("field 'type' not found")
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(tJSON, &s.Type); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
key := string(s.Type)
|
|
||||||
jobJSON, ok := m[key]
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("field '%s', not found", key)
|
|
||||||
}
|
|
||||||
switch s.Type {
|
|
||||||
case TypeSnap:
|
|
||||||
var st SnapJobStatus
|
|
||||||
err = json.Unmarshal(jobJSON, &st)
|
|
||||||
s.JobSpecific = &st
|
|
||||||
|
|
||||||
case TypePull:
|
|
||||||
fallthrough
|
|
||||||
case TypePush:
|
|
||||||
var st ActiveSideStatus
|
|
||||||
err = json.Unmarshal(jobJSON, &st)
|
|
||||||
s.JobSpecific = &st
|
|
||||||
|
|
||||||
case TypeSource:
|
|
||||||
fallthrough
|
|
||||||
case TypeSink:
|
|
||||||
var st PassiveStatus
|
|
||||||
err = json.Unmarshal(jobJSON, &st)
|
|
||||||
s.JobSpecific = &st
|
|
||||||
|
|
||||||
case TypeInternal:
|
|
||||||
// internal jobs do not report specifics
|
|
||||||
default:
|
|
||||||
err = fmt.Errorf("unknown job type '%s'", key)
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
@@ -1,191 +0,0 @@
|
|||||||
package job
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
"github.com/zrepl/zrepl/daemon/logging"
|
|
||||||
"github.com/zrepl/zrepl/daemon/snapper"
|
|
||||||
"github.com/zrepl/zrepl/endpoint"
|
|
||||||
"github.com/zrepl/zrepl/rpc"
|
|
||||||
"github.com/zrepl/zrepl/transport"
|
|
||||||
"github.com/zrepl/zrepl/transport/fromconfig"
|
|
||||||
"github.com/zrepl/zrepl/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
type PassiveSide struct {
|
|
||||||
mode passiveMode
|
|
||||||
name endpoint.JobID
|
|
||||||
listen transport.AuthenticatedListenerFactory
|
|
||||||
}
|
|
||||||
|
|
||||||
type passiveMode interface {
|
|
||||||
Handler() rpc.Handler
|
|
||||||
RunPeriodic(ctx context.Context)
|
|
||||||
SnapperReport() *snapper.Report // may be nil
|
|
||||||
Type() Type
|
|
||||||
}
|
|
||||||
|
|
||||||
type modeSink struct {
|
|
||||||
receiverConfig endpoint.ReceiverConfig
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *modeSink) Type() Type { return TypeSink }
|
|
||||||
|
|
||||||
func (m *modeSink) Handler() rpc.Handler {
|
|
||||||
return endpoint.NewReceiver(m.receiverConfig)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *modeSink) RunPeriodic(_ context.Context) {}
|
|
||||||
func (m *modeSink) SnapperReport() *snapper.Report { return nil }
|
|
||||||
|
|
||||||
func modeSinkFromConfig(g *config.Global, in *config.SinkJob, jobID endpoint.JobID) (m *modeSink, err error) {
|
|
||||||
m = &modeSink{}
|
|
||||||
|
|
||||||
m.receiverConfig, err = buildReceiverConfig(in, jobID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return m, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type modeSource struct {
|
|
||||||
senderConfig *endpoint.SenderConfig
|
|
||||||
snapper *snapper.PeriodicOrManual
|
|
||||||
}
|
|
||||||
|
|
||||||
func modeSourceFromConfig(g *config.Global, in *config.SourceJob, jobID endpoint.JobID) (m *modeSource, err error) {
|
|
||||||
// FIXME exact dedup of modePush
|
|
||||||
m = &modeSource{}
|
|
||||||
|
|
||||||
m.senderConfig, err = buildSenderConfig(in, jobID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "send options")
|
|
||||||
}
|
|
||||||
|
|
||||||
if m.snapper, err = snapper.FromConfig(g, m.senderConfig.FSF, in.Snapshotting); err != nil {
|
|
||||||
return nil, errors.Wrap(err, "cannot build snapper")
|
|
||||||
}
|
|
||||||
|
|
||||||
return m, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *modeSource) Type() Type { return TypeSource }
|
|
||||||
|
|
||||||
func (m *modeSource) Handler() rpc.Handler {
|
|
||||||
return endpoint.NewSender(*m.senderConfig)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *modeSource) RunPeriodic(ctx context.Context) {
|
|
||||||
m.snapper.Run(ctx, nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *modeSource) SnapperReport() *snapper.Report {
|
|
||||||
return m.snapper.Report()
|
|
||||||
}
|
|
||||||
|
|
||||||
func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, configJob interface{}) (s *PassiveSide, err error) {
|
|
||||||
|
|
||||||
s = &PassiveSide{}
|
|
||||||
|
|
||||||
s.name, err = endpoint.MakeJobID(in.Name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "invalid job name")
|
|
||||||
}
|
|
||||||
|
|
||||||
switch v := configJob.(type) {
|
|
||||||
case *config.SinkJob:
|
|
||||||
s.mode, err = modeSinkFromConfig(g, v, s.name) // shadow
|
|
||||||
case *config.SourceJob:
|
|
||||||
s.mode, err = modeSourceFromConfig(g, v, s.name) // shadow
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, err // no wrapping necessary
|
|
||||||
}
|
|
||||||
|
|
||||||
if s.listen, err = fromconfig.ListenerFactoryFromConfig(g, in.Serve); err != nil {
|
|
||||||
return nil, errors.Wrap(err, "cannot build listener factory")
|
|
||||||
}
|
|
||||||
|
|
||||||
return s, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *PassiveSide) Name() string { return j.name.String() }
|
|
||||||
|
|
||||||
type PassiveStatus struct {
|
|
||||||
Snapper *snapper.Report
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *PassiveSide) Status() *Status {
|
|
||||||
st := &PassiveStatus{
|
|
||||||
Snapper: s.mode.SnapperReport(),
|
|
||||||
}
|
|
||||||
return &Status{Type: s.mode.Type(), JobSpecific: st}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *PassiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
|
|
||||||
sink, ok := j.mode.(*modeSink)
|
|
||||||
if !ok {
|
|
||||||
_ = j.mode.(*modeSource) // make sure we didn't introduce a new job type
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
return sink.receiverConfig.RootWithoutClientComponent.Copy(), true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *PassiveSide) SenderConfig() *endpoint.SenderConfig {
|
|
||||||
source, ok := j.mode.(*modeSource)
|
|
||||||
if !ok {
|
|
||||||
_ = j.mode.(*modeSink) // make sure we didn't introduce a new job type
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return source.senderConfig
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*PassiveSide) RegisterMetrics(registerer prometheus.Registerer) {}
|
|
||||||
|
|
||||||
func (j *PassiveSide) Run(ctx context.Context) {
|
|
||||||
ctx, endTask := trace.WithTaskAndSpan(ctx, "passive-side-job", j.Name())
|
|
||||||
defer endTask()
|
|
||||||
log := GetLogger(ctx)
|
|
||||||
defer log.Info("job exiting")
|
|
||||||
{
|
|
||||||
ctx, endTask := trace.WithTask(ctx, "periodic") // shadowing
|
|
||||||
defer endTask()
|
|
||||||
ctx, cancel := context.WithCancel(ctx)
|
|
||||||
defer cancel()
|
|
||||||
go j.mode.RunPeriodic(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
handler := j.mode.Handler()
|
|
||||||
if handler == nil {
|
|
||||||
panic(fmt.Sprintf("implementation error: j.mode.Handler() returned nil: %#v", j))
|
|
||||||
}
|
|
||||||
|
|
||||||
ctxInterceptor := func(handlerCtx context.Context, info rpc.HandlerContextInterceptorData, handler func(ctx context.Context)) {
|
|
||||||
// the handlerCtx is clean => need to inherit logging and tracing config from job context
|
|
||||||
handlerCtx = logging.WithInherit(handlerCtx, ctx)
|
|
||||||
handlerCtx = trace.WithInherit(handlerCtx, ctx)
|
|
||||||
|
|
||||||
handlerCtx, endTask := trace.WithTaskAndSpan(handlerCtx, "handler", fmt.Sprintf("job=%q client=%q method=%q", j.Name(), info.ClientIdentity(), info.FullMethod()))
|
|
||||||
defer endTask()
|
|
||||||
handler(handlerCtx)
|
|
||||||
}
|
|
||||||
|
|
||||||
rpcLoggers := rpc.GetLoggersOrPanic(ctx) // WithSubsystemLoggers above
|
|
||||||
server := rpc.NewServer(handler, rpcLoggers, ctxInterceptor)
|
|
||||||
|
|
||||||
listener, err := j.listen()
|
|
||||||
if err != nil {
|
|
||||||
log.WithError(err).Error("cannot listen")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
server.Serve(ctx, listener)
|
|
||||||
}
|
|
||||||
@@ -1,35 +0,0 @@
|
|||||||
package reset
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
)
|
|
||||||
|
|
||||||
type contextKey int
|
|
||||||
|
|
||||||
const contextKeyReset contextKey = iota
|
|
||||||
|
|
||||||
func Wait(ctx context.Context) <-chan struct{} {
|
|
||||||
wc, ok := ctx.Value(contextKeyReset).(chan struct{})
|
|
||||||
if !ok {
|
|
||||||
wc = make(chan struct{})
|
|
||||||
}
|
|
||||||
return wc
|
|
||||||
}
|
|
||||||
|
|
||||||
type Func func() error
|
|
||||||
|
|
||||||
var AlreadyReset = errors.New("already reset")
|
|
||||||
|
|
||||||
func Context(ctx context.Context) (context.Context, Func) {
|
|
||||||
wc := make(chan struct{})
|
|
||||||
wuf := func() error {
|
|
||||||
select {
|
|
||||||
case wc <- struct{}{}:
|
|
||||||
return nil
|
|
||||||
default:
|
|
||||||
return AlreadyReset
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return context.WithValue(ctx, contextKeyReset, wc), wuf
|
|
||||||
}
|
|
||||||
@@ -1,191 +0,0 @@
|
|||||||
package job
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
|
||||||
"github.com/zrepl/zrepl/util/nodefault"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
"github.com/zrepl/zrepl/daemon/filters"
|
|
||||||
"github.com/zrepl/zrepl/daemon/job/wakeup"
|
|
||||||
"github.com/zrepl/zrepl/daemon/pruner"
|
|
||||||
"github.com/zrepl/zrepl/daemon/snapper"
|
|
||||||
"github.com/zrepl/zrepl/endpoint"
|
|
||||||
"github.com/zrepl/zrepl/replication/logic/pdu"
|
|
||||||
"github.com/zrepl/zrepl/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
type SnapJob struct {
|
|
||||||
name endpoint.JobID
|
|
||||||
fsfilter zfs.DatasetFilter
|
|
||||||
snapper *snapper.PeriodicOrManual
|
|
||||||
|
|
||||||
prunerFactory *pruner.LocalPrunerFactory
|
|
||||||
|
|
||||||
promPruneSecs *prometheus.HistogramVec // labels: prune_side
|
|
||||||
|
|
||||||
prunerMtx sync.Mutex
|
|
||||||
pruner *pruner.Pruner
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *SnapJob) Name() string { return j.name.String() }
|
|
||||||
|
|
||||||
func (j *SnapJob) Type() Type { return TypeSnap }
|
|
||||||
|
|
||||||
func snapJobFromConfig(g *config.Global, in *config.SnapJob) (j *SnapJob, err error) {
|
|
||||||
j = &SnapJob{}
|
|
||||||
fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "cannot build filesystem filter")
|
|
||||||
}
|
|
||||||
j.fsfilter = fsf
|
|
||||||
|
|
||||||
if j.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
|
|
||||||
return nil, errors.Wrap(err, "cannot build snapper")
|
|
||||||
}
|
|
||||||
j.name, err = endpoint.MakeJobID(in.Name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "invalid job name")
|
|
||||||
}
|
|
||||||
j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
||||||
Namespace: "zrepl",
|
|
||||||
Subsystem: "pruning",
|
|
||||||
Name: "time",
|
|
||||||
Help: "seconds spent in pruner",
|
|
||||||
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
|
|
||||||
}, []string{"prune_side"})
|
|
||||||
j.prunerFactory, err = pruner.NewLocalPrunerFactory(in.Pruning, j.promPruneSecs)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "cannot build snapjob pruning rules")
|
|
||||||
}
|
|
||||||
return j, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *SnapJob) RegisterMetrics(registerer prometheus.Registerer) {
|
|
||||||
registerer.MustRegister(j.promPruneSecs)
|
|
||||||
}
|
|
||||||
|
|
||||||
type SnapJobStatus struct {
|
|
||||||
Pruning *pruner.Report
|
|
||||||
Snapshotting *snapper.Report // may be nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *SnapJob) Status() *Status {
|
|
||||||
s := &SnapJobStatus{}
|
|
||||||
t := j.Type()
|
|
||||||
j.prunerMtx.Lock()
|
|
||||||
if j.pruner != nil {
|
|
||||||
s.Pruning = j.pruner.Report()
|
|
||||||
}
|
|
||||||
j.prunerMtx.Unlock()
|
|
||||||
s.Snapshotting = j.snapper.Report()
|
|
||||||
return &Status{Type: t, JobSpecific: s}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *SnapJob) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *SnapJob) SenderConfig() *endpoint.SenderConfig { return nil }
|
|
||||||
|
|
||||||
func (j *SnapJob) Run(ctx context.Context) {
|
|
||||||
ctx, endTask := trace.WithTaskAndSpan(ctx, "snap-job", j.Name())
|
|
||||||
defer endTask()
|
|
||||||
log := GetLogger(ctx)
|
|
||||||
|
|
||||||
defer log.Info("job exiting")
|
|
||||||
|
|
||||||
periodicDone := make(chan struct{})
|
|
||||||
ctx, cancel := context.WithCancel(ctx)
|
|
||||||
defer cancel()
|
|
||||||
periodicCtx, endTask := trace.WithTask(ctx, "snapshotting")
|
|
||||||
defer endTask()
|
|
||||||
go j.snapper.Run(periodicCtx, periodicDone)
|
|
||||||
|
|
||||||
invocationCount := 0
|
|
||||||
outer:
|
|
||||||
for {
|
|
||||||
log.Info("wait for wakeups")
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
log.WithError(ctx.Err()).Info("context")
|
|
||||||
break outer
|
|
||||||
|
|
||||||
case <-wakeup.Wait(ctx):
|
|
||||||
case <-periodicDone:
|
|
||||||
}
|
|
||||||
invocationCount++
|
|
||||||
|
|
||||||
invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", invocationCount))
|
|
||||||
j.doPrune(invocationCtx)
|
|
||||||
endSpan()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Adaptor that implements pruner.History around a pruner.Target.
|
|
||||||
// The ReplicationCursor method is Get-op only and always returns
|
|
||||||
// the filesystem's most recent version's GUID.
|
|
||||||
//
|
|
||||||
// TODO:
|
|
||||||
// This is a work-around for the current package daemon/pruner
|
|
||||||
// and package pruning.Snapshot limitation: they require the
|
|
||||||
// `Replicated` getter method be present, but obviously,
|
|
||||||
// a local job like SnapJob can't deliver on that.
|
|
||||||
// But the pruner.Pruner gives up on an FS if no replication
|
|
||||||
// cursor is present, which is why this pruner returns the
|
|
||||||
// most recent filesystem version.
|
|
||||||
type alwaysUpToDateReplicationCursorHistory struct {
|
|
||||||
// the Target passed as Target to BuildLocalPruner
|
|
||||||
target pruner.Target
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ pruner.History = (*alwaysUpToDateReplicationCursorHistory)(nil)
|
|
||||||
|
|
||||||
func (h alwaysUpToDateReplicationCursorHistory) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
|
|
||||||
fsvReq := &pdu.ListFilesystemVersionsReq{
|
|
||||||
Filesystem: req.GetFilesystem(),
|
|
||||||
}
|
|
||||||
res, err := h.target.ListFilesystemVersions(ctx, fsvReq)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
fsvs := res.GetVersions()
|
|
||||||
if len(fsvs) <= 0 {
|
|
||||||
return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Notexist{Notexist: true}}, nil
|
|
||||||
}
|
|
||||||
// always return must recent version
|
|
||||||
sort.Slice(fsvs, func(i, j int) bool {
|
|
||||||
return fsvs[i].CreateTXG < fsvs[j].CreateTXG
|
|
||||||
})
|
|
||||||
mostRecent := fsvs[len(fsvs)-1]
|
|
||||||
return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Guid{Guid: mostRecent.GetGuid()}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h alwaysUpToDateReplicationCursorHistory) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error) {
|
|
||||||
return h.target.ListFilesystems(ctx, req)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (j *SnapJob) doPrune(ctx context.Context) {
|
|
||||||
ctx, endSpan := trace.WithSpan(ctx, "snap-job-do-prune")
|
|
||||||
defer endSpan()
|
|
||||||
log := GetLogger(ctx)
|
|
||||||
sender := endpoint.NewSender(endpoint.SenderConfig{
|
|
||||||
JobID: j.name,
|
|
||||||
FSF: j.fsfilter,
|
|
||||||
// FIXME encryption setting is irrelevant for SnapJob because the endpoint is only used as pruner.Target
|
|
||||||
Encrypt: &nodefault.Bool{B: true},
|
|
||||||
})
|
|
||||||
j.prunerMtx.Lock()
|
|
||||||
j.pruner = j.prunerFactory.BuildLocalPruner(ctx, sender, alwaysUpToDateReplicationCursorHistory{sender})
|
|
||||||
j.prunerMtx.Unlock()
|
|
||||||
log.Info("start pruning")
|
|
||||||
j.pruner.Prune()
|
|
||||||
log.Info("finished pruning")
|
|
||||||
}
|
|
||||||
@@ -1,35 +0,0 @@
|
|||||||
package wakeup
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
)
|
|
||||||
|
|
||||||
type contextKey int
|
|
||||||
|
|
||||||
const contextKeyWakeup contextKey = iota
|
|
||||||
|
|
||||||
func Wait(ctx context.Context) <-chan struct{} {
|
|
||||||
wc, ok := ctx.Value(contextKeyWakeup).(chan struct{})
|
|
||||||
if !ok {
|
|
||||||
wc = make(chan struct{})
|
|
||||||
}
|
|
||||||
return wc
|
|
||||||
}
|
|
||||||
|
|
||||||
type Func func() error
|
|
||||||
|
|
||||||
var AlreadyWokenUp = errors.New("already woken up")
|
|
||||||
|
|
||||||
func Context(ctx context.Context) (context.Context, Func) {
|
|
||||||
wc := make(chan struct{})
|
|
||||||
wuf := func() error {
|
|
||||||
select {
|
|
||||||
case wc <- struct{}{}:
|
|
||||||
return nil
|
|
||||||
default:
|
|
||||||
return AlreadyWokenUp
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return context.WithValue(ctx, contextKeyWakeup, wc), wuf
|
|
||||||
}
|
|
||||||
@@ -1,303 +0,0 @@
|
|||||||
package logging
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/tls"
|
|
||||||
"crypto/x509"
|
|
||||||
"log/syslog"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/mattn/go-isatty"
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/config"
|
|
||||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
|
||||||
"github.com/zrepl/zrepl/logger"
|
|
||||||
"github.com/zrepl/zrepl/tlsconf"
|
|
||||||
)
|
|
||||||
|
|
||||||
func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error) {
|
|
||||||
|
|
||||||
outlets := logger.NewOutlets()
|
|
||||||
|
|
||||||
if len(in) == 0 {
|
|
||||||
// Default config
|
|
||||||
out := WriterOutlet{&HumanFormatter{}, os.Stdout}
|
|
||||||
outlets.Add(out, logger.Warn)
|
|
||||||
return outlets, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var syslogOutlets, stdoutOutlets int
|
|
||||||
for lei, le := range in {
|
|
||||||
|
|
||||||
outlet, minLevel, err := ParseOutlet(le)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrapf(err, "cannot parse outlet #%d", lei)
|
|
||||||
}
|
|
||||||
var _ logger.Outlet = WriterOutlet{}
|
|
||||||
var _ logger.Outlet = &SyslogOutlet{}
|
|
||||||
switch outlet.(type) {
|
|
||||||
case *SyslogOutlet:
|
|
||||||
syslogOutlets++
|
|
||||||
case WriterOutlet:
|
|
||||||
stdoutOutlets++
|
|
||||||
}
|
|
||||||
|
|
||||||
outlets.Add(outlet, minLevel)
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
if syslogOutlets > 1 {
|
|
||||||
return nil, errors.Errorf("can only define one 'syslog' outlet")
|
|
||||||
}
|
|
||||||
if stdoutOutlets > 1 {
|
|
||||||
return nil, errors.Errorf("can only define one 'stdout' outlet")
|
|
||||||
}
|
|
||||||
|
|
||||||
return outlets, nil
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
type Subsystem string
|
|
||||||
|
|
||||||
const (
|
|
||||||
SubsysMeta Subsystem = "meta"
|
|
||||||
SubsysJob Subsystem = "job"
|
|
||||||
SubsysReplication Subsystem = "repl"
|
|
||||||
SubsysEndpoint Subsystem = "endpoint"
|
|
||||||
SubsysPruning Subsystem = "pruning"
|
|
||||||
SubsysSnapshot Subsystem = "snapshot"
|
|
||||||
SubsysHooks Subsystem = "hook"
|
|
||||||
SubsysTransport Subsystem = "transport"
|
|
||||||
SubsysTransportMux Subsystem = "transportmux"
|
|
||||||
SubsysRPC Subsystem = "rpc"
|
|
||||||
SubsysRPCControl Subsystem = "rpc.ctrl"
|
|
||||||
SubsysRPCData Subsystem = "rpc.data"
|
|
||||||
SubsysZFSCmd Subsystem = "zfs.cmd"
|
|
||||||
SubsysTraceData Subsystem = "trace.data"
|
|
||||||
SubsysPlatformtest Subsystem = "platformtest"
|
|
||||||
)
|
|
||||||
|
|
||||||
var AllSubsystems = []Subsystem{
|
|
||||||
SubsysMeta,
|
|
||||||
SubsysJob,
|
|
||||||
SubsysReplication,
|
|
||||||
SubsysEndpoint,
|
|
||||||
SubsysPruning,
|
|
||||||
SubsysSnapshot,
|
|
||||||
SubsysHooks,
|
|
||||||
SubsysTransport,
|
|
||||||
SubsysTransportMux,
|
|
||||||
SubsysRPC,
|
|
||||||
SubsysRPCControl,
|
|
||||||
SubsysRPCData,
|
|
||||||
SubsysZFSCmd,
|
|
||||||
SubsysTraceData,
|
|
||||||
SubsysPlatformtest,
|
|
||||||
}
|
|
||||||
|
|
||||||
type injectedField struct {
|
|
||||||
field string
|
|
||||||
value interface{}
|
|
||||||
parent *injectedField
|
|
||||||
}
|
|
||||||
|
|
||||||
func WithInjectedField(ctx context.Context, field string, value interface{}) context.Context {
|
|
||||||
var parent *injectedField
|
|
||||||
parentI := ctx.Value(contextKeyInjectedField)
|
|
||||||
if parentI != nil {
|
|
||||||
parent = parentI.(*injectedField)
|
|
||||||
}
|
|
||||||
// TODO sanity-check `field` now
|
|
||||||
this := &injectedField{field, value, parent}
|
|
||||||
return context.WithValue(ctx, contextKeyInjectedField, this)
|
|
||||||
}
|
|
||||||
|
|
||||||
func iterInjectedFields(ctx context.Context, cb func(field string, value interface{})) {
|
|
||||||
injI := ctx.Value(contextKeyInjectedField)
|
|
||||||
if injI == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
inj := injI.(*injectedField)
|
|
||||||
for ; inj != nil; inj = inj.parent {
|
|
||||||
cb(inj.field, inj.value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type SubsystemLoggers map[Subsystem]logger.Logger
|
|
||||||
|
|
||||||
func SubsystemLoggersWithUniversalLogger(l logger.Logger) SubsystemLoggers {
|
|
||||||
loggers := make(SubsystemLoggers)
|
|
||||||
for _, s := range AllSubsystems {
|
|
||||||
loggers[s] = l
|
|
||||||
}
|
|
||||||
return loggers
|
|
||||||
}
|
|
||||||
|
|
||||||
func WithLoggers(ctx context.Context, loggers SubsystemLoggers) context.Context {
|
|
||||||
return context.WithValue(ctx, contextKeyLoggers, loggers)
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetLoggers(ctx context.Context) SubsystemLoggers {
|
|
||||||
loggers, ok := ctx.Value(contextKeyLoggers).(SubsystemLoggers)
|
|
||||||
if !ok {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return loggers
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetLogger(ctx context.Context, subsys Subsystem) logger.Logger {
|
|
||||||
return getLoggerImpl(ctx, subsys, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
func getLoggerImpl(ctx context.Context, subsys Subsystem, panicIfEnded bool) logger.Logger {
|
|
||||||
loggers, ok := ctx.Value(contextKeyLoggers).(SubsystemLoggers)
|
|
||||||
if !ok || loggers == nil {
|
|
||||||
return logger.NewNullLogger()
|
|
||||||
}
|
|
||||||
l, ok := loggers[subsys]
|
|
||||||
if !ok {
|
|
||||||
return logger.NewNullLogger()
|
|
||||||
}
|
|
||||||
|
|
||||||
l = l.WithField(SubsysField, subsys)
|
|
||||||
|
|
||||||
l = l.WithField(SpanField, trace.GetSpanStackOrDefault(ctx, *trace.StackKindId, "NOSPAN"))
|
|
||||||
|
|
||||||
fields := make(logger.Fields)
|
|
||||||
iterInjectedFields(ctx, func(field string, value interface{}) {
|
|
||||||
fields[field] = value
|
|
||||||
})
|
|
||||||
l = l.WithFields(fields)
|
|
||||||
|
|
||||||
return l
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
|
|
||||||
var is string
|
|
||||||
switch j := i.(type) {
|
|
||||||
case string:
|
|
||||||
is = j
|
|
||||||
default:
|
|
||||||
return nil, errors.Errorf("invalid log format: wrong type: %T", i)
|
|
||||||
}
|
|
||||||
|
|
||||||
switch is {
|
|
||||||
case "human":
|
|
||||||
return &HumanFormatter{}, nil
|
|
||||||
case "logfmt":
|
|
||||||
return &LogfmtFormatter{}, nil
|
|
||||||
case "json":
|
|
||||||
return &JSONFormatter{}, nil
|
|
||||||
default:
|
|
||||||
return nil, errors.Errorf("invalid log format: '%s'", is)
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func ParseOutlet(in config.LoggingOutletEnum) (o logger.Outlet, level logger.Level, err error) {
|
|
||||||
|
|
||||||
parseCommon := func(common config.LoggingOutletCommon) (logger.Level, EntryFormatter, error) {
|
|
||||||
if common.Level == "" || common.Format == "" {
|
|
||||||
return 0, nil, errors.Errorf("must specify 'level' and 'format' field")
|
|
||||||
}
|
|
||||||
|
|
||||||
minLevel, err := logger.ParseLevel(common.Level)
|
|
||||||
if err != nil {
|
|
||||||
return 0, nil, errors.Wrap(err, "cannot parse 'level' field")
|
|
||||||
}
|
|
||||||
formatter, err := parseLogFormat(common.Format)
|
|
||||||
if err != nil {
|
|
||||||
return 0, nil, errors.Wrap(err, "cannot parse 'formatter' field")
|
|
||||||
}
|
|
||||||
return minLevel, formatter, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var f EntryFormatter
|
|
||||||
|
|
||||||
switch v := in.Ret.(type) {
|
|
||||||
case *config.StdoutLoggingOutlet:
|
|
||||||
level, f, err = parseCommon(v.LoggingOutletCommon)
|
|
||||||
if err != nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
o, err = parseStdoutOutlet(v, f)
|
|
||||||
case *config.TCPLoggingOutlet:
|
|
||||||
level, f, err = parseCommon(v.LoggingOutletCommon)
|
|
||||||
if err != nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
o, err = parseTCPOutlet(v, f)
|
|
||||||
case *config.SyslogLoggingOutlet:
|
|
||||||
level, f, err = parseCommon(v.LoggingOutletCommon)
|
|
||||||
if err != nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
o, err = parseSyslogOutlet(v, f)
|
|
||||||
default:
|
|
||||||
panic(v)
|
|
||||||
}
|
|
||||||
return o, level, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseStdoutOutlet(in *config.StdoutLoggingOutlet, formatter EntryFormatter) (WriterOutlet, error) {
|
|
||||||
flags := MetadataAll
|
|
||||||
writer := os.Stdout
|
|
||||||
if !isatty.IsTerminal(writer.Fd()) && !in.Time {
|
|
||||||
flags &= ^MetadataTime
|
|
||||||
}
|
|
||||||
if isatty.IsTerminal(writer.Fd()) && !in.Color {
|
|
||||||
flags &= ^MetadataColor
|
|
||||||
}
|
|
||||||
|
|
||||||
formatter.SetMetadataFlags(flags)
|
|
||||||
return WriterOutlet{
|
|
||||||
formatter,
|
|
||||||
os.Stdout,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseTCPOutlet(in *config.TCPLoggingOutlet, formatter EntryFormatter) (out *TCPOutlet, err error) {
|
|
||||||
var tlsConfig *tls.Config
|
|
||||||
if in.TLS != nil {
|
|
||||||
tlsConfig, err = func(m *config.TCPLoggingOutletTLS, host string) (*tls.Config, error) {
|
|
||||||
clientCert, err := tls.LoadX509KeyPair(m.Cert, m.Key)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "cannot load client cert")
|
|
||||||
}
|
|
||||||
|
|
||||||
var rootCAs *x509.CertPool
|
|
||||||
if m.CA == "" {
|
|
||||||
if rootCAs, err = x509.SystemCertPool(); err != nil {
|
|
||||||
return nil, errors.Wrap(err, "cannot open system cert pool")
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
rootCAs, err = tlsconf.ParseCAFile(m.CA)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "cannot parse CA cert")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if rootCAs == nil {
|
|
||||||
panic("invariant violated")
|
|
||||||
}
|
|
||||||
|
|
||||||
return tlsconf.ClientAuthClient(host, rootCAs, clientCert)
|
|
||||||
}(in.TLS, in.Address)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.New("cannot not parse TLS config in field 'tls'")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
formatter.SetMetadataFlags(MetadataAll)
|
|
||||||
return NewTCPOutlet(formatter, in.Net, in.Address, tlsConfig, in.RetryInterval), nil
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseSyslogOutlet(in *config.SyslogLoggingOutlet, formatter EntryFormatter) (out *SyslogOutlet, err error) {
|
|
||||||
out = &SyslogOutlet{}
|
|
||||||
out.Formatter = formatter
|
|
||||||
out.Formatter.SetMetadataFlags(MetadataNone)
|
|
||||||
out.Facility = syslog.Priority(*in.Facility)
|
|
||||||
out.RetryInterval = in.RetryInterval
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
package logging
|
|
||||||
|
|
||||||
import "context"
|
|
||||||
|
|
||||||
type contextKey int
|
|
||||||
|
|
||||||
const (
|
|
||||||
contextKeyLoggers contextKey = 1 + iota
|
|
||||||
contextKeyInjectedField
|
|
||||||
)
|
|
||||||
|
|
||||||
var contextKeys = []contextKey{
|
|
||||||
contextKeyLoggers,
|
|
||||||
contextKeyInjectedField,
|
|
||||||
}
|
|
||||||
|
|
||||||
func WithInherit(ctx, inheritFrom context.Context) context.Context {
|
|
||||||
for _, k := range contextKeys {
|
|
||||||
if v := inheritFrom.Value(k); v != nil {
|
|
||||||
ctx = context.WithValue(ctx, k, v) // no shadow
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ctx
|
|
||||||
}
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user