Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 InsanePrawn
|
893d686eef |
+121
-344
@@ -1,79 +1,74 @@
|
||||
version: 2.1
|
||||
version: 2.0
|
||||
workflows:
|
||||
version: 2
|
||||
build:
|
||||
jobs:
|
||||
- build-1.11
|
||||
- build-1.12
|
||||
- build-1.13
|
||||
- build-1.14
|
||||
- build-latest
|
||||
- test-build-in-docker
|
||||
jobs:
|
||||
|
||||
commands:
|
||||
setup-home-local-bin:
|
||||
steps:
|
||||
- run:
|
||||
shell: /bin/bash -euo pipefail
|
||||
command: |
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
line='export PATH="$HOME/.local/bin:$PATH"'
|
||||
if grep "$line" $BASH_ENV >/dev/null; then
|
||||
echo "$line" >> $BASH_ENV
|
||||
fi
|
||||
|
||||
invoke-lazy-sh:
|
||||
# build-latest serves as the template
|
||||
# we use YAML anchors & aliases to exchange the docker image (and hence Go version used for the build)
|
||||
build-latest: &build-latest
|
||||
description: Builds zrepl
|
||||
parameters:
|
||||
subcommand:
|
||||
image:
|
||||
description: "the docker image that the job should use"
|
||||
type: string
|
||||
docker:
|
||||
- image: circleci/golang:latest
|
||||
environment:
|
||||
# required by lazy.sh
|
||||
TERM: xterm
|
||||
working_directory: /go/src/github.com/zrepl/zrepl
|
||||
steps:
|
||||
- run:
|
||||
environment:
|
||||
TERM: xterm
|
||||
command: ./lazy.sh <<parameters.subcommand>>
|
||||
|
||||
apt-update-and-install-common-deps:
|
||||
steps:
|
||||
- run: sudo apt update && sudo apt install gawk make
|
||||
|
||||
restore-cache-gomod:
|
||||
steps:
|
||||
- restore_cache:
|
||||
key: go-mod-v4-{{ checksum "go.sum" }}
|
||||
save-cache-gomod:
|
||||
steps:
|
||||
- save_cache:
|
||||
key: go-mod-v4-{{ checksum "go.sum" }}
|
||||
paths:
|
||||
- "/go/pkg/mod"
|
||||
|
||||
install-godep:
|
||||
steps:
|
||||
- apt-update-and-install-common-deps
|
||||
- invoke-lazy-sh:
|
||||
subcommand: godep
|
||||
|
||||
install-docdep:
|
||||
steps:
|
||||
- apt-update-and-install-common-deps
|
||||
- run: sudo apt install python3 python3-pip libgirepository1.0-dev
|
||||
- invoke-lazy-sh:
|
||||
subcommand: docdep
|
||||
|
||||
download-and-install-minio-client:
|
||||
steps:
|
||||
- setup-home-local-bin
|
||||
- restore_cache:
|
||||
key: minio-client-v2
|
||||
- run:
|
||||
shell: /bin/bash -eo pipefail
|
||||
name: Setup environment variables
|
||||
command: |
|
||||
if which mc; then exit 0; fi
|
||||
sudo curl -sSL https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2020-08-20T00-23-01Z \
|
||||
-o "$HOME/.local/bin/mc"
|
||||
sudo chmod +x "$HOME/.local/bin/mc"
|
||||
- save_cache:
|
||||
key: minio-client-v2
|
||||
paths:
|
||||
- "$HOME/.local/bin/mc"
|
||||
# used by pip (for docs)
|
||||
echo 'export PATH="$HOME/.local/bin:$PATH"' >> $BASH_ENV
|
||||
# we use modules
|
||||
echo 'export GO111MODULE=on' >> $BASH_ENV
|
||||
|
||||
- restore_cache:
|
||||
keys:
|
||||
- source
|
||||
- protobuf
|
||||
|
||||
- checkout
|
||||
|
||||
- save_cache:
|
||||
key: source
|
||||
paths:
|
||||
- ".git"
|
||||
|
||||
# install deps
|
||||
- run: wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
|
||||
- run: echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807 protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
|
||||
- run: sudo unzip -d /usr protoc-3.6.1-linux-x86_64.zip
|
||||
- save_cache:
|
||||
key: protobuf
|
||||
paths:
|
||||
- "/usr/include/google/protobuf"
|
||||
|
||||
- run: sudo apt update && sudo apt install python3 python3-pip libgirepository1.0-dev gawk
|
||||
- run: ./lazy.sh devsetup
|
||||
|
||||
- run: make zrepl-bin
|
||||
- run: make vet
|
||||
- run: make lint
|
||||
- run: make release
|
||||
- run: make test-go
|
||||
# cannot run test-platform because circle-ci runs in linux containers
|
||||
|
||||
- store_artifacts:
|
||||
path: ./artifacts/release
|
||||
when: always
|
||||
|
||||
upload-minio:
|
||||
parameters:
|
||||
src:
|
||||
type: string
|
||||
dst:
|
||||
type: string
|
||||
steps:
|
||||
- run:
|
||||
shell: /bin/bash -eo pipefail
|
||||
when: always
|
||||
@@ -83,296 +78,78 @@ commands:
|
||||
exit 0
|
||||
fi
|
||||
set -u # from now on
|
||||
|
||||
|
||||
# Download and install minio
|
||||
curl -sSL https://dl.minio.io/client/mc/release/linux-amd64/mc -o ${GOPATH}/bin/mc
|
||||
chmod +x ${GOPATH}/bin/mc
|
||||
mc config host add --api s3v4 zrepl-minio https://minio.cschwarz.com ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
|
||||
|
||||
# keep in sync with set-github-minio-status
|
||||
jobprefix=zrepl-ci-artifacts/${CIRCLE_SHA1}-pipeline-<<pipeline.number>>/${CIRCLE_JOB}
|
||||
|
||||
|
||||
# Upload artifacts
|
||||
mkdir -p ./artifacts
|
||||
mc cp -r <<parameters.src>> "zrepl-minio/$jobprefix/<<parameters.dst>>"
|
||||
echo "$CIRCLE_BUILD_URL" > ./artifacts/release/cirlceci_build_url
|
||||
mc cp -r artifacts/release "zrepl-minio/zrepl-ci-artifacts/${CIRCLE_SHA1}/${CIRCLE_JOB}/"
|
||||
|
||||
set-github-minio-status:
|
||||
parameters:
|
||||
context:
|
||||
type: string
|
||||
description:
|
||||
type: string
|
||||
minio-dst:
|
||||
type: string
|
||||
steps:
|
||||
- run:
|
||||
shell: /bin/bash -eo pipefail
|
||||
command: |
|
||||
if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
|
||||
echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
|
||||
exit 0
|
||||
fi
|
||||
set -u # from now on
|
||||
|
||||
# keep in sync with with upload-minio command
|
||||
jobprefix=zrepl-ci-artifacts/${CIRCLE_SHA1}-pipeline-<<pipeline.number>>/${CIRCLE_JOB}
|
||||
# Push Artifact Link to GitHub
|
||||
REPO="zrepl/zrepl"
|
||||
COMMIT="${CIRCLE_SHA1}"
|
||||
JOB_NAME="${CIRCLE_JOB}"
|
||||
CONTEXT="<<parameters.context>>"
|
||||
DESCRIPTION="<<parameters.description>>"
|
||||
TARGETURL=https://minio.cschwarz.com/minio/"$jobprefix"/"<<parameters.minio-dst>>"
|
||||
curl "https://api.github.com/repos/$REPO/statuses/$COMMIT" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: token $GITHUB_COMMIT_STATUS_TOKEN" \
|
||||
-X POST \
|
||||
-d '{"context":"'"$CONTEXT"'", "state": "success", "description":"'"$DESCRIPTION"'", "target_url":"'"$TARGETURL"'"}'
|
||||
-d '{"context":"zrepl/publish-ci-artifacts", "state": "success", "description":"CI Build Artifacts for '"$JOB_NAME"'", "target_url":"https://minio.cschwarz.com/minio/zrepl-ci-artifacts/'"$COMMIT"'/"}'
|
||||
|
||||
- run:
|
||||
shell: /bin/bash -euo pipefail
|
||||
command: |
|
||||
# Trigger Debian Package Build
|
||||
curl -v -X POST https://api.github.com/repos/zrepl/debian-binary-packaging/dispatches \
|
||||
-H 'Accept: application/vnd.github.v3+json' \
|
||||
-H "Authorization: token $ZREPL_DEBIAN_BINARYPACKAGIN_TRIGGER_BUILD_GITHUB_TOKEN" \
|
||||
--data '{"event_type": "push", "client_payload": { "zrepl_main_repo_commit": "'"$CIRCLE_SHA1"'", "go_version": "'"${CIRCLE_JOB##build-}"'" }}'
|
||||
|
||||
trigger-pipeline:
|
||||
parameters:
|
||||
body_no_shell_subst:
|
||||
type: string
|
||||
steps:
|
||||
- run: |
|
||||
curl -X POST https://circleci.com/api/v2/project/github/zrepl/zrepl/pipeline \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Accept: application/json' \
|
||||
-H "Circle-Token: $ZREPL_BOT_CIRCLE_TOKEN" \
|
||||
--data '<<parameters.body_no_shell_subst>>'
|
||||
|
||||
parameters:
|
||||
do_ci:
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
do_release:
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
release_docker_baseimage_tag:
|
||||
type: string
|
||||
default: "1.15"
|
||||
|
||||
workflows:
|
||||
version: 2
|
||||
|
||||
ci:
|
||||
when: << pipeline.parameters.do_ci >>
|
||||
jobs:
|
||||
- quickcheck-docs
|
||||
- quickcheck-go: &quickcheck-go-smoketest
|
||||
name: quickcheck-go-amd64-linux-1.15
|
||||
goversion: &latest-go-release "1.15"
|
||||
goos: linux
|
||||
goarch: amd64
|
||||
- test-go-on-latest-go-release:
|
||||
goversion: *latest-go-release
|
||||
- quickcheck-go:
|
||||
requires:
|
||||
- quickcheck-go-amd64-linux-1.15 #quickcheck-go-smoketest.name
|
||||
matrix: &quickcheck-go-matrix
|
||||
alias: quickcheck-go-matrix
|
||||
parameters:
|
||||
goversion: [*latest-go-release, "1.11"]
|
||||
goos: ["linux", "freebsd"]
|
||||
goarch: ["amd64", "arm64"]
|
||||
exclude:
|
||||
# don't re-do quickcheck-go-smoketest
|
||||
- goversion: *latest-go-release
|
||||
goos: linux
|
||||
goarch: amd64
|
||||
# not supported by Go 1.11
|
||||
- goversion: "1.11"
|
||||
goos: freebsd
|
||||
goarch: arm64
|
||||
|
||||
release:
|
||||
when: << pipeline.parameters.do_release >>
|
||||
jobs:
|
||||
- release-build
|
||||
- release-deb:
|
||||
requires:
|
||||
- release-build
|
||||
- release-rpm:
|
||||
requires:
|
||||
- release-build
|
||||
- release-upload:
|
||||
requires:
|
||||
- release-build
|
||||
- release-deb
|
||||
- release-rpm
|
||||
|
||||
periodic:
|
||||
triggers:
|
||||
- schedule:
|
||||
cron: "00 17 * * *"
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
- stable
|
||||
- problame/circleci-build
|
||||
jobs:
|
||||
- periodic-full-pipeline-run
|
||||
|
||||
zrepl.github.io:
|
||||
jobs:
|
||||
- publish-zrepl-github-io:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- stable
|
||||
|
||||
jobs:
|
||||
quickcheck-docs:
|
||||
build-1.11:
|
||||
<<: *build-latest
|
||||
docker:
|
||||
- image: cimg/base:2020.08
|
||||
steps:
|
||||
- checkout
|
||||
- install-docdep
|
||||
- run: make docs
|
||||
- image: circleci/golang:1.11
|
||||
|
||||
- store_artifacts:
|
||||
path: artifacts
|
||||
- download-and-install-minio-client
|
||||
- upload-minio:
|
||||
src: artifacts
|
||||
dst: ""
|
||||
- set-github-minio-status:
|
||||
context: artifacts/${CIRCLE_JOB}
|
||||
description: artifacts of CI job ${CIRCLE_JOB}
|
||||
minio-dst: ""
|
||||
|
||||
quickcheck-go:
|
||||
parameters:
|
||||
goversion:
|
||||
type: string
|
||||
goos:
|
||||
type: string
|
||||
goarch:
|
||||
type: string
|
||||
build-1.12:
|
||||
<<: *build-latest
|
||||
docker:
|
||||
- image: circleci/golang:<<parameters.goversion>>
|
||||
- image: circleci/golang:1.12
|
||||
|
||||
build-1.13:
|
||||
<<: *build-latest
|
||||
docker:
|
||||
- image: circleci/golang:1.13
|
||||
|
||||
build-1.14:
|
||||
<<: *build-latest
|
||||
docker:
|
||||
- image: circleci/golang:1.14
|
||||
|
||||
# this job tries to mimic the build-in-docker instructions
|
||||
# given in docs/installation.rst
|
||||
#
|
||||
# However, CircleCI doesn't support volume mounts, so we have to copy
|
||||
# the source into the build-container by modifying the Dockerfile
|
||||
test-build-in-docker:
|
||||
description: Check that build-in-docker works
|
||||
docker:
|
||||
- image: circleci/golang:latest
|
||||
environment:
|
||||
GOOS: <<parameters.goos>>
|
||||
GOARCH: <<parameters.goarch>>
|
||||
|
||||
working_directory: /go/src/github.com/zrepl/zrepl
|
||||
steps:
|
||||
- checkout
|
||||
|
||||
- restore-cache-gomod
|
||||
- run: go mod download
|
||||
- run: cd build && go mod download
|
||||
- save-cache-gomod
|
||||
|
||||
- install-godep
|
||||
- run: make formatcheck
|
||||
- run: make generate-platform-test-list
|
||||
- run: make zrepl-bin test-platform-bin
|
||||
- run: make vet
|
||||
- run: make lint
|
||||
|
||||
- download-and-install-minio-client
|
||||
- run: rm -f artifacts/generate-platform-test-list
|
||||
- store_artifacts:
|
||||
path: artifacts
|
||||
- upload-minio:
|
||||
src: artifacts
|
||||
dst: ""
|
||||
- set-github-minio-status:
|
||||
context: artifacts/${CIRCLE_JOB}
|
||||
description: artifacts of CI job ${CIRCLE_JOB}
|
||||
minio-dst: ""
|
||||
|
||||
test-go-on-latest-go-release:
|
||||
parameters:
|
||||
goversion:
|
||||
type: string
|
||||
docker:
|
||||
- image: circleci/golang:<<parameters.goversion>>
|
||||
steps:
|
||||
- checkout
|
||||
- restore-cache-gomod
|
||||
- run: make test-go
|
||||
# don't save-cache-gomod here, test-go doesn't pull all the dependencies
|
||||
|
||||
release-build:
|
||||
machine: true
|
||||
steps:
|
||||
- checkout
|
||||
- run: make release-docker RELEASE_DOCKER_BASEIMAGE_TAG=<<pipeline.parameters.release_docker_baseimage_tag>>
|
||||
- persist_to_workspace:
|
||||
root: .
|
||||
paths: [.]
|
||||
release-deb:
|
||||
machine: true
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- run: make debs-docker
|
||||
- persist_to_workspace:
|
||||
root: .
|
||||
paths:
|
||||
- "artifacts/*.deb"
|
||||
|
||||
release-rpm:
|
||||
machine: true
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- run: make rpms-docker
|
||||
- persist_to_workspace:
|
||||
root: .
|
||||
paths:
|
||||
- "artifacts/*.rpm"
|
||||
|
||||
release-upload:
|
||||
docker:
|
||||
- image: cimg/base:2020.08
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- store_artifacts:
|
||||
path: artifacts
|
||||
- download-and-install-minio-client
|
||||
- upload-minio:
|
||||
src: artifacts
|
||||
dst: ""
|
||||
- set-github-minio-status:
|
||||
context: artifacts/release
|
||||
description: CI-generated release artifacts
|
||||
minio-dst: ""
|
||||
|
||||
periodic-full-pipeline-run:
|
||||
docker:
|
||||
- image: cimg/base:2020.08
|
||||
steps:
|
||||
- trigger-pipeline:
|
||||
body_no_shell_subst: '{"branch":"<<pipeline.git.branch>>", "parameters": { "do_ci": true, "do_release": true }}'
|
||||
|
||||
publish-zrepl-github-io:
|
||||
docker:
|
||||
- image: cimg/python:3.7
|
||||
steps:
|
||||
- checkout
|
||||
- invoke-lazy-sh:
|
||||
subcommand: docdep
|
||||
- run:
|
||||
command: |
|
||||
git config --global user.email "me@cschwarz.com"
|
||||
git config --global user.name "zrepl-github-io-ci"
|
||||
|
||||
# https://circleci.com/docs/2.0/add-ssh-key/#adding-multiple-keys-with-blank-hostnames
|
||||
- run: ssh-add -D
|
||||
# the default circleci ssh config only additional ssh keys for Host !github.com
|
||||
- run:
|
||||
command: |
|
||||
cat > ~/.ssh/config \<<EOF
|
||||
Host *
|
||||
IdentityFile /home/circleci/.ssh/id_rsa_458e62c517f6c480e40452126ce47421
|
||||
EOF
|
||||
- add_ssh_keys:
|
||||
fingerprints:
|
||||
# deploy key for zrepl.github.io
|
||||
- "45:8e:62:c5:17:f6:c4:80:e4:04:52:12:6c:e4:74:21"
|
||||
|
||||
- run: bash -x docs/publish.sh -c -a
|
||||
- checkout
|
||||
- setup_remote_docker
|
||||
- run:
|
||||
name: (hacky) circleci doesn't allow volume mounts, so copy src to container
|
||||
command: echo "ADD . /src" >> build.Dockerfile
|
||||
- run:
|
||||
name: (hacky) commit modified Dockerfile to avoid failing git clean check in Makefile
|
||||
command: git -c user.name='circleci' -c user.email='circleci@localhost' commit -m 'CIRCLECI modified Dockerfile with zrepl src' --author 'autoauthor <circleci@localhost>' -- build.Dockerfile
|
||||
- run:
|
||||
name: build the build image (build deps)
|
||||
command: docker build -t zrepl_build -f build.Dockerfile .
|
||||
- run:
|
||||
name: try compiling
|
||||
command: docker run -it zrepl_build make release
|
||||
@@ -1,10 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
COMMIT="$1"
|
||||
GO_VERSION="$2"
|
||||
|
||||
curl -v -X POST https://api.github.com/repos/zrepl/debian-binary-packaging/dispatches \
|
||||
-H 'Accept: application/vnd.github.v3+json' \
|
||||
-H "Authorization: token $GITHUB_ACCESS_TOKEN" \
|
||||
--data '{"event_type": "push", "client_payload": { "zrepl_main_repo_commit": "'"$COMMIT"'", "go_version": "'"$GO_VERSION"'" }}'
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
dist: xenial
|
||||
services:
|
||||
- docker
|
||||
|
||||
env: # for allow_failures: https://docs.travis-ci.com/user/customizing-the-build/
|
||||
matrix:
|
||||
include:
|
||||
|
||||
- language: go
|
||||
name: "Build in Docker (docs/installation.rst)"
|
||||
script:
|
||||
- sudo docker build -t zrepl_build -f build.Dockerfile .
|
||||
- |
|
||||
sudo docker run -it --rm \
|
||||
-v "${PWD}:/go/src/github.com/zrepl/zrepl" \
|
||||
--user "$(id -u):$(id -g)" \
|
||||
zrepl_build make vendordeps release
|
||||
|
||||
- &zrepl_build_template
|
||||
language: go
|
||||
go_import_path: github.com/zrepl/zrepl
|
||||
before_install:
|
||||
- wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
|
||||
- echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807 protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
|
||||
- sudo unzip -d /usr protoc-3.6.1-linux-x86_64.zip
|
||||
- ./lazy.sh godep
|
||||
- make vendordeps
|
||||
script:
|
||||
- make
|
||||
- make vet
|
||||
- make test
|
||||
- make lint
|
||||
- make artifacts/zrepl-freebsd-amd64
|
||||
- make artifacts/zrepl-linux-amd64
|
||||
- make artifacts/zrepl-darwin-amd64
|
||||
go:
|
||||
- "1.11"
|
||||
|
||||
- <<: *zrepl_build_template
|
||||
go:
|
||||
- "1.12"
|
||||
|
||||
- <<: *zrepl_build_template
|
||||
go:
|
||||
- "master"
|
||||
|
||||
- &zrepl_docs_template
|
||||
language: python
|
||||
python:
|
||||
- "3.4"
|
||||
install:
|
||||
- sudo apt-get install libgirepository1.0-dev
|
||||
- pip install -r docs/requirements.txt
|
||||
script:
|
||||
- make docs
|
||||
- <<: *zrepl_docs_template
|
||||
python:
|
||||
- "3.5"
|
||||
- <<: *zrepl_docs_template
|
||||
python:
|
||||
- "3.6"
|
||||
- <<: *zrepl_docs_template
|
||||
python:
|
||||
- "3.7"
|
||||
|
||||
|
||||
allow_failures:
|
||||
- <<: *zrepl_build_template
|
||||
go:
|
||||
- "master"
|
||||
@@ -28,8 +28,6 @@ GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
|
||||
GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
|
||||
GOLANGCI_LINT := golangci-lint
|
||||
GOCOVMERGE := gocovmerge
|
||||
RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.15
|
||||
RELEASE_DOCKER_BASEIMAGE ?= golang:$(RELEASE_DOCKER_BASEIMAGE_TAG)
|
||||
|
||||
ifneq ($(GOARM),)
|
||||
ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM)
|
||||
@@ -59,85 +57,6 @@ ifeq (SIGN, 1)
|
||||
endif
|
||||
@echo "ZREPL RELEASE ARTIFACTS AVAILABLE IN artifacts/release"
|
||||
|
||||
release-docker: $(ARTIFACTDIR)
|
||||
sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > artifacts/release-docker.Dockerfile
|
||||
docker build -t zrepl_release --pull -f artifacts/release-docker.Dockerfile .
|
||||
docker run --rm -i -v $(CURDIR):/src -u $$(id -u):$$(id -g) \
|
||||
zrepl_release \
|
||||
make release GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
|
||||
|
||||
debs-docker:
|
||||
$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=deb
|
||||
rpms-docker:
|
||||
$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=rpm
|
||||
_debs_or_rpms_docker: # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
|
||||
$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=amd64
|
||||
$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=arm64
|
||||
$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=arm GOARM=7
|
||||
$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=386
|
||||
|
||||
rpm: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
|
||||
$(eval _ZREPL_RPM_VERSION := $(subst -,.,$(_ZREPL_VERSION)))
|
||||
$(eval _ZREPL_RPM_TOPDIR_ABS := $(CURDIR)/$(ARTIFACTDIR)/rpmbuild)
|
||||
rm -rf "$(_ZREPL_RPM_TOPDIR_ABS)"
|
||||
mkdir "$(_ZREPL_RPM_TOPDIR_ABS)"
|
||||
mkdir -p "$(_ZREPL_RPM_TOPDIR_ABS)"/{SPECS,RPMS,BUILD,BUILDROOT}
|
||||
sed "s/^Version:.*/Version: $(_ZREPL_RPM_VERSION)/g" \
|
||||
packaging/rpm/zrepl.spec > $(_ZREPL_RPM_TOPDIR_ABS)/SPECS/zrepl.spec
|
||||
|
||||
# see /usr/lib/rpm/platform
|
||||
ifeq ($(GOARCH),amd64)
|
||||
$(eval _ZREPL_RPMBUILD_TARGET := x86_64)
|
||||
else ifeq ($(GOARCH), 386)
|
||||
$(eval _ZREPL_RPMBUILD_TARGET := i386)
|
||||
else ifeq ($(GOARCH), arm64)
|
||||
$(eval _ZREPL_RPMBUILD_TARGET := aarch64)
|
||||
else ifeq ($(GOARCH), arm)
|
||||
$(eval _ZREPL_RPMBUILD_TARGET := armv7hl)
|
||||
else
|
||||
$(eval _ZREPL_RPMBUILD_TARGET := $(GOARCH))
|
||||
endif
|
||||
rpmbuild \
|
||||
--build-in-place \
|
||||
--define "_sourcedir $(CURDIR)" \
|
||||
--define "_topdir $(_ZREPL_RPM_TOPDIR_ABS)" \
|
||||
--define "_zrepl_binary_filename zrepl-$(ZREPL_TARGET_TUPLE)" \
|
||||
--target $(_ZREPL_RPMBUILD_TARGET) \
|
||||
-bb "$(_ZREPL_RPM_TOPDIR_ABS)"/SPECS/zrepl.spec
|
||||
cp "$(_ZREPL_RPM_TOPDIR_ABS)"/RPMS/$(_ZREPL_RPMBUILD_TARGET)/zrepl-$(_ZREPL_RPM_VERSION)*.rpm $(ARTIFACTDIR)/
|
||||
|
||||
rpm-docker:
|
||||
docker build -t zrepl_rpm_pkg --pull -f packaging/rpm/Dockerfile .
|
||||
docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
|
||||
zrepl_rpm_pkg \
|
||||
make rpm GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
|
||||
|
||||
deb: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
|
||||
|
||||
cp packaging/deb/debian/changelog.template packaging/deb/debian/changelog
|
||||
sed -i 's/DATE_DASH_R_OUTPUT/$(shell date -R)/' packaging/deb/debian/changelog
|
||||
VERSION="$(subst -,.,$(_ZREPL_VERSION))"; \
|
||||
export VERSION="$${VERSION#v}"; \
|
||||
sed -i 's/VERSION/'"$$VERSION"'/' packaging/deb/debian/changelog
|
||||
|
||||
ifeq ($(GOARCH), arm)
|
||||
$(eval DEB_HOST_ARCH := armhf)
|
||||
else ifeq ($(GOARCH), 386)
|
||||
$(eval DEB_HOST_ARCH := i386)
|
||||
else
|
||||
$(eval DEB_HOST_ARCH := $(GOARCH))
|
||||
endif
|
||||
|
||||
export ZREPL_DPKG_ZREPL_BINARY_FILENAME=zrepl-$(ZREPL_TARGET_TUPLE); \
|
||||
dpkg-buildpackage -b --no-sign --host-arch $(DEB_HOST_ARCH)
|
||||
cp ../*.deb artifacts/
|
||||
|
||||
deb-docker:
|
||||
docker build -t zrepl_debian_pkg --pull -f packaging/deb/Dockerfile .
|
||||
docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
|
||||
zrepl_debian_pkg \
|
||||
make deb GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
|
||||
|
||||
# expects `release` target to have run before
|
||||
NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
|
||||
wrapup-and-checksum:
|
||||
@@ -228,41 +147,26 @@ zrepl-bin:
|
||||
generate-platform-test-list:
|
||||
$(GO_BUILD) -o $(ARTIFACTDIR)/generate-platform-test-list ./platformtest/tests/gen
|
||||
|
||||
COVER_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-cover-$(ZREPL_TARGET_TUPLE)
|
||||
cover-platform-bin:
|
||||
test-platform-bin:
|
||||
$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
|
||||
-c -o "$(COVER_PLATFORM_BIN_PATH)" \
|
||||
-c -o "$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" \
|
||||
-covermode=atomic -cover -coverpkg github.com/zrepl/zrepl/... \
|
||||
./platformtest/harness
|
||||
cover-platform:
|
||||
# do not track dependency on cover-platform-bin to allow build of binary outside of test VM
|
||||
export _TEST_PLATFORM_CMD="$(COVER_PLATFORM_BIN_PATH) \
|
||||
-test.coverprofile \"$(ARTIFACTDIR)/platformtest.cover\" \
|
||||
-test.v \
|
||||
__DEVEL--i-heard-you-like-tests"; \
|
||||
$(MAKE) _test-or-cover-platform-impl
|
||||
|
||||
TEST_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)
|
||||
test-platform-bin:
|
||||
$(GO_BUILD) -o "$(TEST_PLATFORM_BIN_PATH)" ./platformtest/harness
|
||||
test-platform:
|
||||
export _TEST_PLATFORM_CMD="\"$(TEST_PLATFORM_BIN_PATH)\""; \
|
||||
$(MAKE) _test-or-cover-platform-impl
|
||||
|
||||
ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
|
||||
ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
|
||||
ZREPL_PLATFORMTEST_MOUNTPOINT := /tmp/zreplplatformtest.pool
|
||||
ZREPL_PLATFORMTEST_ZFS_LOG := /tmp/zreplplatformtest.zfs.log
|
||||
# ZREPL_PLATFORMTEST_STOP_AND_KEEP := -failure.stop-and-keep-pool
|
||||
ZREPL_PLATFORMTEST_ARGS :=
|
||||
_test-or-cover-platform-impl: $(ARTIFACTDIR)
|
||||
ifndef _TEST_PLATFORM_CMD
|
||||
$(error _TEST_PLATFORM_CMD is undefined, caller 'cover-platform' or 'test-platform' should have defined it)
|
||||
endif
|
||||
ZREPL_PLATFORMTEST_ARGS :=
|
||||
test-platform: $(ARTIFACTDIR) # do not track dependency on test-platform-bin to allow build of platformtest outside of test VM
|
||||
rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
|
||||
rm -f "$(ARTIFACTDIR)/platformtest.cover"
|
||||
platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
|
||||
$(_TEST_PLATFORM_CMD) \
|
||||
"$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" \
|
||||
-test.coverprofile "$(ARTIFACTDIR)/platformtest.cover" \
|
||||
-test.v \
|
||||
__DEVEL--i-heard-you-like-tests \
|
||||
-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
|
||||
-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
|
||||
-mountpoint "$(ZREPL_PLATFORMTEST_MOUNTPOINT)" \
|
||||
@@ -274,31 +178,22 @@ cover-merge: $(ARTIFACTDIR)
|
||||
cover-html: cover-merge
|
||||
$(GO) tool cover -html "$(ARTIFACTDIR)/merged.cover" -o "$(ARTIFACTDIR)/merged.cover.html"
|
||||
|
||||
cover-full:
|
||||
test-full:
|
||||
test "$$(id -u)" = "0" || echo "MUST RUN AS ROOT" 1>&2
|
||||
$(MAKE) test-go COVER=1
|
||||
$(MAKE) cover-platform-bin
|
||||
$(MAKE) cover-platform
|
||||
$(MAKE) test-platform
|
||||
$(MAKE) cover-html
|
||||
|
||||
##################### DEV TARGETS #####################
|
||||
# not part of the build, must do that manually
|
||||
.PHONY: generate formatcheck format
|
||||
.PHONY: generate format
|
||||
|
||||
generate: generate-platform-test-list
|
||||
protoc -I=replication/logic/pdu --go_out=plugins=grpc:replication/logic/pdu replication/logic/pdu/pdu.proto
|
||||
$(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./...
|
||||
|
||||
GOIMPORTS := goimports -srcdir . -local 'github.com/zrepl/zrepl'
|
||||
FINDSRCFILES := find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go'
|
||||
|
||||
formatcheck:
|
||||
@# goimports doesn't have a knob to exit with non-zero status code if formatting is needed
|
||||
@# see https://go-review.googlesource.com/c/tools/+/237378
|
||||
@ affectedfiles=$$($(GOIMPORTS) -l $(shell $(FINDSRCFILES)) | tee /dev/stderr | wc -l); test "$$affectedfiles" = 0
|
||||
|
||||
format:
|
||||
@ $(GOIMPORTS) -w -d $(shell $(FINDSRCFILES))
|
||||
goimports -srcdir . -local 'github.com/zrepl/zrepl' -w $(shell find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go')
|
||||
|
||||
##################### NOARCH #####################
|
||||
.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
|
||||
@@ -322,14 +217,11 @@ $(ARTIFACTDIR)/_zrepl.zsh_completion:
|
||||
|
||||
$(ARTIFACTDIR)/go_env.txt:
|
||||
$(GO_ENV_VARS) $(GO) env > $@
|
||||
$(GO) version >> $@
|
||||
|
||||
docs: $(ARTIFACTDIR)/docs
|
||||
# https://www.sphinx-doc.org/en/master/man/sphinx-build.html
|
||||
make -C docs \
|
||||
html \
|
||||
BUILDDIR=../artifacts/docs \
|
||||
SPHINXOPTS="-W --keep-going -n"
|
||||
|
||||
docs-clean:
|
||||
make -C docs \
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
[](https://github.com/zrepl/zrepl/blob/master/LICENSE)
|
||||
[](https://golang.org/)
|
||||
[](https://zrepl.github.io)
|
||||
[](https://patreon.com/zrepl)
|
||||
[](https://www.patreon.com/zrepl)
|
||||
[](https://github.com/sponsors/problame)
|
||||
[](https://liberapay.com/zrepl/donate)
|
||||
[](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
|
||||
@@ -21,7 +21,7 @@ zrepl is a one-stop ZFS backup & replication solution.
|
||||
|
||||
## Feature Requests
|
||||
|
||||
1. Does your feature request require default values / some kind of configuration?
|
||||
1. Does you feature request require default values / some kind of configuration?
|
||||
If so, think of an expressive configuration example.
|
||||
2. Think of at least one use case that generalizes from your concrete application.
|
||||
3. Open an issue on GitHub with example conf & use case attached.
|
||||
@@ -30,12 +30,20 @@ zrepl is a one-stop ZFS backup & replication solution.
|
||||
The above does not apply if you already implemented everything.
|
||||
Check out the *Coding Workflow* section below for details.
|
||||
|
||||
## Building, Releasing, Downstream-Packaging
|
||||
## Package Maintainer Information
|
||||
|
||||
This section provides an overview of the zrepl build & release process.
|
||||
Check out `docs/installation/compile-from-source.rst` for build-from-source instructions.
|
||||
* Follow the steps in `docs/installation.rst -> Compiling from Source` and read the Makefile / shell scripts used in this process.
|
||||
* Make sure your distro is compatible with the paths in `docs/installation.rst`.
|
||||
* Ship a default config that adheres to your distro's `hier` and logging system.
|
||||
* Ship a service manager file and _please_ try to upstream it to this repository.
|
||||
* `dist/systemd` contains a Systemd unit template.
|
||||
* Ship other material provided in `./dist`, e.g. in `/usr/share/zrepl/`.
|
||||
* Use `make release ZREPL_VERSION='mydistro-1.2.3_1'`
|
||||
* Your distro's name and any versioning supplemental to zrepl's (e.g. package revision) should be in this string
|
||||
* Use `sudo make test-platform` **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
|
||||
* Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
|
||||
|
||||
### Overview
|
||||
## Developer Documentation
|
||||
|
||||
zrepl is written in [Go](https://golang.org) and uses [Go modules](https://github.com/golang/go/wiki/Modules) to manage dependencies.
|
||||
The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
|
||||
@@ -52,61 +60,60 @@ An HTML report can be generated using `make cover-html`.
|
||||
|
||||
**Code generation** is triggered by `make generate`. Generated code is committed to the source tree.
|
||||
|
||||
### Build & Release Process
|
||||
### Project Structure
|
||||
|
||||
**The `Makefile` is catering to the needs of developers & CI, not distro packagers**.
|
||||
It provides phony targets for
|
||||
* local development (building, running tests, etc)
|
||||
* building a release in Docker (used by the CI & release management)
|
||||
* building .deb and .rpm packages out of the release artifacts.
|
||||
```
|
||||
├── artifacts # build artifcats generate by make
|
||||
├── cli # wrapper around CLI package cobra
|
||||
├── client # all subcommands that are not `daemon`
|
||||
├── config # config data types (=> package yaml-config)
|
||||
│ └── samples
|
||||
├── daemon # the implementation of `zrepl daemon` subcommand
|
||||
│ ├── filters
|
||||
│ ├── hooks # snapshot hooks
|
||||
│ ├── job # job implementations
|
||||
│ ├── logging # logging outlets + formatters
|
||||
│ ├── nethelpers
|
||||
│ ├── prometheus
|
||||
│ ├── pruner # pruner implementation
|
||||
│ ├── snapper # snapshotter implementation
|
||||
├── dist # supplemental material for users & package maintainers
|
||||
├── docs # sphinx-based documentation
|
||||
│ ├── **/*.rst # documentation in reStructuredText
|
||||
│ ├── sphinxconf
|
||||
│ │ └── conf.py # sphinx config (see commit 445a280 why its not in docs/)
|
||||
│ ├── requirements.txt # pip3 requirements to build documentation
|
||||
│ ├── publish.sh # shell script for automated rendering & deploy to zrepl.github.io repo
|
||||
│ └── public_git # checkout of zrepl.github.io managed by above shell script
|
||||
├── endpoint # implementation of replication endpoints (=> package replication)
|
||||
├── logger # our own logger package
|
||||
├── platformtest # test suite for our zfs abstractions (error classification, etc)
|
||||
├── pruning # pruning rules (the logic, not the actual execution)
|
||||
│ └── retentiongrid
|
||||
├── replication
|
||||
│ ├── driver # the driver of the replication logic (status reporting, error handling)
|
||||
│ ├── logic # planning & executing replication steps via rpc
|
||||
| | └── pdu # the generated gRPC & protobuf code used in replication (and endpoints)
|
||||
│ └── report # the JSON-serializable report datastructures exposed to the client
|
||||
├── rpc # the hybrid gRPC + ./dataconn RPC client: connects to a remote replication.Endpoint
|
||||
│ ├── dataconn # Bulk data-transfer RPC protocol
|
||||
│ ├── grpcclientidentity # adaptor to inject package transport's 'client identity' concept into gRPC contexts
|
||||
│ ├── netadaptor # adaptor to convert a package transport's Connecter and Listener into net.* primitives
|
||||
│ ├── transportmux # TCP connecter and listener used to split control & data traffic
|
||||
│ └── versionhandshake # replication protocol version handshake perfomed on newly established connections
|
||||
├── tlsconf # abstraction for Go TLS server + client config
|
||||
├── transport # transport implementations
|
||||
│ ├── fromconfig
|
||||
│ ├── local
|
||||
│ ├── ssh
|
||||
│ ├── tcp
|
||||
│ └── tls
|
||||
├── util
|
||||
├── version # abstraction for versions (filled during build by Makefile)
|
||||
└── zfs # zfs(8) wrappers
|
||||
```
|
||||
|
||||
**Build tooling & dependencies** are documented as code in `lazy.sh`.
|
||||
Go dependencies are then fetched by the go command and pip dependencies are pinned through a `requirements.txt`.
|
||||
|
||||
**We use CircleCI for continuous integration**.
|
||||
There are two workflows:
|
||||
|
||||
* `ci` runs for every commit / branch / tag pushed to GitHub.
|
||||
It is supposed to run very fast (<5min and provides quick feedback to developers).
|
||||
It runs formatting checks, lints and tests on the most important OSes / architectures.
|
||||
Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
|
||||
|
||||
* `release` runs
|
||||
* on manual triggers through the CircleCI API (in order to produce a release)
|
||||
* periodically on `master`
|
||||
Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
|
||||
|
||||
**Releases** are issued via Git tags + GitHub Releases feature.
|
||||
The procedure to issue a release is as follows:
|
||||
* Issue the source release:
|
||||
* Git tag the release on the `master` branch.
|
||||
* Push the tag.
|
||||
* Run `./docs/publish.sh` to re-build & push zrepl.github.io.
|
||||
* Issue the official binary release:
|
||||
* Run the `release` pipeline (triggered via CircleCI API)
|
||||
* Download the artifacts to the release manager's machine.
|
||||
* Create a GitHub release, edit the changelog, upload all the release artifacts, including .rpm and .deb files.
|
||||
* Issue the GitHub release.
|
||||
* Add the .rpm and .deb files to the official zrepl repos, publish those.
|
||||
|
||||
**Official binary releases are not re-built when Go receives an update. If the Go update is critical to zrepl (e.g. a Go security update that affects zrepl), we'd issue a new source release**.
|
||||
The rationale for this is that whereas distros provide a mechanism for this (`$zrepl_source_release-$distro_package_revision`), GitHub Releases doesn't which means we'd need to update the existing GitHub release's assets, which nobody would notice (no RSS feed updates, etc.).
|
||||
Downstream packagers can read the changelog to determine whether they want to push that minor release into their distro or simply skip it.
|
||||
|
||||
### Additional Notes to Distro Package Maintainers
|
||||
|
||||
* Use `sudo make test-platform-bin && sudo make test-platform` **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
|
||||
* Ship a default config that adheres to your distro's `hier` and logging system.
|
||||
* Ship a service manager file and _please_ try to upstream it to this repository.
|
||||
* `dist/systemd` contains a Systemd unit template.
|
||||
* Ship other material provided in `./dist`, e.g. in `/usr/share/zrepl/`.
|
||||
* Have a look at the `Makefile`'s `ZREPL_VERSION` variable and how it passed to Go's `ldFlags`.
|
||||
This is how `zrepl version` knows what version number to show.
|
||||
Your build system should set the `ldFlags` flags appropriately and add a prefix or suffix that indicates that the given zrepl binary is a distro build, not an official one.
|
||||
* Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
|
||||
|
||||
|
||||
## Contributing Code
|
||||
### Coding Workflow
|
||||
|
||||
* Open an issue when starting to hack on a new feature
|
||||
* Commits should reference the issue they are related to
|
||||
@@ -116,6 +123,9 @@ Downstream packagers can read the changelog to determine whether they want to pu
|
||||
|
||||
Backward-incompatible changes must be documented in the git commit message and are listed in `docs/changelog.rst`.
|
||||
|
||||
* Config-breaking changes must contain a line `BREAK CONFIG` in the commit message
|
||||
* Other breaking changes must contain a line `BREAK` in the commit message
|
||||
|
||||
### Glossary & Naming Inconsistencies
|
||||
|
||||
In ZFS, *dataset* refers to the objects *filesystem*, *ZVOL* and *snapshot*. <br />
|
||||
@@ -132,3 +142,16 @@ variables and types are often named *dataset* when they in fact refer to a *file
|
||||
There will not be a big refactoring (an attempt was made, but it's destroying too much history without much gain).
|
||||
|
||||
However, new contributions & patches should fix naming without further notice in the commit message.
|
||||
|
||||
### RPC debugging
|
||||
|
||||
Optionally, there are various RPC-related environment variables, that if set to something != `""` will produce additional debug output on stderr:
|
||||
|
||||
https://github.com/zrepl/zrepl/blob/master/rpc/rpc_debug.go#L11
|
||||
|
||||
https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/dataconn_debug.go#L11
|
||||
|
||||
https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/stream/stream_debug.go#L11
|
||||
|
||||
https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/heartbeatconn/heartbeatconn_debug.go#L11
|
||||
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
FROM !SUBSTITUTED_BY_MAKEFILE
|
||||
FROM golang:latest
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
python3-pip \
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
|
||||
@@ -480,11 +480,6 @@ func (t *tui) renderReplicationReport(rep *report.Report, history *bytesProgress
|
||||
t.newline()
|
||||
}
|
||||
|
||||
if len(latest.Filesystems) == 0 {
|
||||
t.write("NOTE: no filesystems were considered for replication!")
|
||||
t.newline()
|
||||
}
|
||||
|
||||
var maxFSLen int
|
||||
for _, fs := range latest.Filesystems {
|
||||
if len(fs.Info.Name) > maxFSLen {
|
||||
|
||||
@@ -63,7 +63,7 @@ func doZabsList(ctx context.Context, sc *cli.Subcommand, args []string) error {
|
||||
defer line.Lock().Unlock()
|
||||
if zabsListFlags.Json {
|
||||
enc.SetIndent("", " ")
|
||||
if err := enc.Encode(a); err != nil {
|
||||
if err := enc.Encode(abstractions); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
+51
-28
@@ -12,6 +12,8 @@ import (
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/zrepl/yaml-config"
|
||||
|
||||
zfsprop "github.com/zrepl/zrepl/zfs/property"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
@@ -52,12 +54,11 @@ func (j JobEnum) Name() string {
|
||||
}
|
||||
|
||||
type ActiveJob struct {
|
||||
Type string `yaml:"type"`
|
||||
Name string `yaml:"name"`
|
||||
Connect ConnectEnum `yaml:"connect"`
|
||||
Pruning PruningSenderReceiver `yaml:"pruning"`
|
||||
Debug JobDebugSettings `yaml:"debug,optional"`
|
||||
Replication *Replication `yaml:"replication,optional,fromdefaults"`
|
||||
Type string `yaml:"type"`
|
||||
Name string `yaml:"name"`
|
||||
Connect ConnectEnum `yaml:"connect"`
|
||||
Pruning PruningSenderReceiver `yaml:"pruning"`
|
||||
Debug JobDebugSettings `yaml:"debug,optional"`
|
||||
}
|
||||
|
||||
type PassiveJob struct {
|
||||
@@ -77,24 +78,61 @@ type SnapJob struct {
|
||||
}
|
||||
|
||||
type SendOptions struct {
|
||||
Encrypted bool `yaml:"encrypted,optional,default=false"`
|
||||
StepHolds SendOptionsStepHolds `yaml:"step_holds,optional"`
|
||||
|
||||
Encrypted bool `yaml:"encrypted,optional"`
|
||||
Raw bool `yaml:"raw,optional"`
|
||||
SendProperties bool `yaml:"send_properties,optional"`
|
||||
BackupProperties bool `yaml:"backup_properties,optional"`
|
||||
LargeBlocks bool `yaml:"large_blocks,optional"`
|
||||
Compressed bool `yaml:"compressed,optional"`
|
||||
EmbeddedData bool `yaml:"embbeded_data,optional"`
|
||||
}
|
||||
|
||||
type SendOptionsStepHolds struct {
|
||||
DisableIncremental bool `yaml:"disable_incremental,optional"`
|
||||
}
|
||||
|
||||
var _ yaml.Defaulter = (*SendOptions)(nil)
|
||||
|
||||
func (l *SendOptions) SetDefault() {
|
||||
*l = SendOptions{
|
||||
Encrypted: false,
|
||||
Raw: false,
|
||||
SendProperties: false,
|
||||
BackupProperties: false,
|
||||
LargeBlocks: false,
|
||||
Compressed: false,
|
||||
EmbeddedData: false,
|
||||
}
|
||||
}
|
||||
|
||||
type RecvOptions struct {
|
||||
// Note: we cannot enforce encrypted recv as the ZFS cli doesn't provide a mechanism for it
|
||||
// Encrypted bool `yaml:"may_encrypted"`
|
||||
|
||||
// Future:
|
||||
// Reencrypt bool `yaml:"reencrypt"`
|
||||
|
||||
Properties *PropertyRecvOptions `yaml:"properties,fromdefaults"`
|
||||
}
|
||||
|
||||
type Replication struct {
|
||||
Protection *ReplicationOptionsProtection `yaml:"protection,optional,fromdefaults"`
|
||||
var _ yaml.Defaulter = (*RecvOptions)(nil)
|
||||
|
||||
func (l *RecvOptions) SetDefault() {
|
||||
*l = RecvOptions{Properties: &PropertyRecvOptions{}}
|
||||
}
|
||||
|
||||
type ReplicationOptionsProtection struct {
|
||||
Initial string `yaml:"initial,optional,default=guarantee_resumability"`
|
||||
Incremental string `yaml:"incremental,optional,default=guarantee_resumability"`
|
||||
type PropertyRecvOptions struct {
|
||||
Inherit []zfsprop.Property `yaml:"inherit,optional"`
|
||||
Override map[zfsprop.Property]string `yaml:"override,optional"`
|
||||
}
|
||||
|
||||
var _ yaml.Defaulter = (*PropertyRecvOptions)(nil)
|
||||
|
||||
func (l *PropertyRecvOptions) SetDefault() {
|
||||
//*l = PropertyRecvOptions{}
|
||||
//TODO: is below necessary?
|
||||
*l = PropertyRecvOptions{Inherit: make([]zfsprop.Property, 0), Override: make(map[zfsprop.Property]string)}
|
||||
}
|
||||
|
||||
type PushJob struct {
|
||||
@@ -104,9 +142,6 @@ type PushJob struct {
|
||||
Send *SendOptions `yaml:"send,fromdefaults,optional"`
|
||||
}
|
||||
|
||||
func (j *PushJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
|
||||
func (j *PushJob) GetSendOptions() *SendOptions { return j.Send }
|
||||
|
||||
type PullJob struct {
|
||||
ActiveJob `yaml:",inline"`
|
||||
RootFS string `yaml:"root_fs"`
|
||||
@@ -114,10 +149,6 @@ type PullJob struct {
|
||||
Recv *RecvOptions `yaml:"recv,fromdefaults,optional"`
|
||||
}
|
||||
|
||||
func (j *PullJob) GetRootFS() string { return j.RootFS }
|
||||
func (j *PullJob) GetAppendClientIdentity() bool { return false }
|
||||
func (j *PullJob) GetRecvOptions() *RecvOptions { return j.Recv }
|
||||
|
||||
type PositiveDurationOrManual struct {
|
||||
Interval time.Duration
|
||||
Manual bool
|
||||
@@ -155,10 +186,6 @@ type SinkJob struct {
|
||||
Recv *RecvOptions `yaml:"recv,optional,fromdefaults"`
|
||||
}
|
||||
|
||||
func (j *SinkJob) GetRootFS() string { return j.RootFS }
|
||||
func (j *SinkJob) GetAppendClientIdentity() bool { return true }
|
||||
func (j *SinkJob) GetRecvOptions() *RecvOptions { return j.Recv }
|
||||
|
||||
type SourceJob struct {
|
||||
PassiveJob `yaml:",inline"`
|
||||
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
|
||||
@@ -166,9 +193,6 @@ type SourceJob struct {
|
||||
Send *SendOptions `yaml:"send,optional,fromdefaults"`
|
||||
}
|
||||
|
||||
func (j *SourceJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
|
||||
func (j *SourceJob) GetSendOptions() *SendOptions { return j.Send }
|
||||
|
||||
type FilesystemsFilter map[string]bool
|
||||
|
||||
type SnapshottingEnum struct {
|
||||
@@ -324,7 +348,6 @@ type PruneKeepNotReplicated struct {
|
||||
type PruneKeepLastN struct {
|
||||
Type string `yaml:"type"`
|
||||
Count int `yaml:"count"`
|
||||
Regex string `yaml:"regex,optional"`
|
||||
}
|
||||
|
||||
type PruneKeepRegex struct { // FIXME rename to KeepRegex
|
||||
|
||||
@@ -1,90 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/zrepl/yaml-config"
|
||||
)
|
||||
|
||||
type A struct {
|
||||
B *B `yaml:"b,optional,fromdefaults"`
|
||||
A1 string `yaml:"a1,optional"`
|
||||
}
|
||||
|
||||
type B struct {
|
||||
C *C `yaml:"c,optional,fromdefaults"`
|
||||
D string `yaml:"d,default=ddd"`
|
||||
E string `yaml:"e,optional"`
|
||||
}
|
||||
|
||||
type C struct {
|
||||
Q string `yaml:"q,optional"`
|
||||
R string `yaml:"r,default=r"`
|
||||
}
|
||||
|
||||
func TestDepFromDefaults(t *testing.T) {
|
||||
|
||||
type testcase struct {
|
||||
name string
|
||||
yaml string
|
||||
expect *A
|
||||
}
|
||||
|
||||
tcs := []testcase{
|
||||
{
|
||||
name: "empty",
|
||||
yaml: `{}`,
|
||||
expect: &A{
|
||||
B: &B{
|
||||
C: &C{
|
||||
R: "r",
|
||||
},
|
||||
D: "ddd",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "a1 set",
|
||||
yaml: `{"a1":"blah"}`,
|
||||
expect: &A{
|
||||
A1: "blah",
|
||||
B: &B{
|
||||
C: &C{
|
||||
R: "r",
|
||||
},
|
||||
D: "ddd",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "D set",
|
||||
yaml: `
|
||||
b:
|
||||
d: 4d
|
||||
`,
|
||||
expect: &A{
|
||||
B: &B{
|
||||
D: "4d",
|
||||
C: &C{
|
||||
R: "r",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for tci := range tcs {
|
||||
t.Run(fmt.Sprintf("%d-%s", tci, tcs[tci].name), func(t *testing.T) {
|
||||
tc := tcs[tci]
|
||||
|
||||
var a A
|
||||
err := yaml.UnmarshalStrict([]byte(tc.yaml), &a)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, tc.expect, &a)
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestRecvOptions(t *testing.T) {
|
||||
tmpl := `
|
||||
jobs:
|
||||
- name: foo
|
||||
type: pull
|
||||
connect:
|
||||
type: local
|
||||
listener_name: foo
|
||||
client_identity: bar
|
||||
root_fs: "zreplplatformtest"
|
||||
%s
|
||||
interval: manual
|
||||
pruning:
|
||||
keep_sender:
|
||||
- type: last_n
|
||||
count: 10
|
||||
keep_receiver:
|
||||
- type: last_n
|
||||
count: 10
|
||||
|
||||
`
|
||||
|
||||
recv_properties_empty := `
|
||||
recv:
|
||||
properties:
|
||||
`
|
||||
|
||||
recv_inherit_empty := `
|
||||
recv:
|
||||
properties:
|
||||
inherit:
|
||||
`
|
||||
|
||||
recv_inherit := `
|
||||
recv:
|
||||
properties:
|
||||
inherit:
|
||||
- testprop
|
||||
`
|
||||
|
||||
recv_override_empty := `
|
||||
recv:
|
||||
properties:
|
||||
override:
|
||||
`
|
||||
|
||||
recv_override := `
|
||||
recv:
|
||||
properties:
|
||||
override:
|
||||
testprop2: "test123"
|
||||
`
|
||||
|
||||
recv_override_and_inherit := `
|
||||
recv:
|
||||
properties:
|
||||
inherit:
|
||||
- testprop
|
||||
override:
|
||||
testprop2: "test123"
|
||||
`
|
||||
|
||||
recv_empty := `
|
||||
recv: {}
|
||||
`
|
||||
|
||||
recv_not_specified := `
|
||||
`
|
||||
|
||||
fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
|
||||
var c *Config
|
||||
|
||||
t.Run("recv_inherit_empty", func(t *testing.T) {
|
||||
c, err := testConfig(t, fill(recv_inherit_empty))
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, c)
|
||||
})
|
||||
|
||||
t.Run("recv_inherit", func(t *testing.T) {
|
||||
c = testValidConfig(t, fill(recv_inherit))
|
||||
inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
|
||||
assert.NotEmpty(t, inherit)
|
||||
})
|
||||
|
||||
t.Run("recv_override_empty", func(t *testing.T) {
|
||||
c, err := testConfig(t, fill(recv_override_empty))
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, c)
|
||||
})
|
||||
|
||||
t.Run("recv_override", func(t *testing.T) {
|
||||
c = testValidConfig(t, fill(recv_override))
|
||||
override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
|
||||
assert.NotEmpty(t, override)
|
||||
})
|
||||
|
||||
t.Run("recv_override_and_inherit", func(t *testing.T) {
|
||||
c = testValidConfig(t, fill(recv_override_and_inherit))
|
||||
inherit := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Inherit
|
||||
override := c.Jobs[0].Ret.(*PullJob).Recv.Properties.Override
|
||||
assert.NotEmpty(t, inherit)
|
||||
assert.NotEmpty(t, override)
|
||||
})
|
||||
|
||||
t.Run("recv_properties_empty", func(t *testing.T) {
|
||||
c, err := testConfig(t, fill(recv_properties_empty))
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, c)
|
||||
})
|
||||
|
||||
t.Run("recv_empty", func(t *testing.T) {
|
||||
c, err := testConfig(t, fill(recv_empty))
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, c)
|
||||
})
|
||||
|
||||
t.Run("send_not_specified", func(t *testing.T) {
|
||||
c, err := testConfig(t, fill(recv_not_specified))
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, c)
|
||||
})
|
||||
|
||||
}
|
||||
@@ -38,7 +38,41 @@ jobs:
|
||||
encrypted: true
|
||||
`
|
||||
|
||||
encrypted_unspecified := `
|
||||
raw_true := `
|
||||
send:
|
||||
raw: true
|
||||
|
||||
`
|
||||
|
||||
raw_false := `
|
||||
send:
|
||||
raw: false
|
||||
|
||||
`
|
||||
|
||||
raw_and_encrypted := `
|
||||
send:
|
||||
encrypted: true
|
||||
raw: true
|
||||
`
|
||||
|
||||
properties_and_encrypted := `
|
||||
send:
|
||||
encrypted: true
|
||||
send_properties: true
|
||||
`
|
||||
|
||||
properties_true := `
|
||||
send:
|
||||
send_properties: true
|
||||
`
|
||||
|
||||
properties_false := `
|
||||
send:
|
||||
send_properties: false
|
||||
`
|
||||
|
||||
send_empty := `
|
||||
send: {}
|
||||
`
|
||||
|
||||
@@ -59,12 +93,52 @@ jobs:
|
||||
assert.Equal(t, true, encrypted)
|
||||
})
|
||||
|
||||
t.Run("encrypted_unspecified", func(t *testing.T) {
|
||||
c = testValidConfig(t, fill(encrypted_unspecified))
|
||||
t.Run("send_empty", func(t *testing.T) {
|
||||
c := testValidConfig(t, fill(send_empty))
|
||||
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
|
||||
assert.Equal(t, false, encrypted)
|
||||
})
|
||||
|
||||
t.Run("properties_and_encrypted", func(t *testing.T) {
|
||||
c := testValidConfig(t, fill(properties_and_encrypted))
|
||||
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
|
||||
properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
|
||||
assert.Equal(t, true, encrypted)
|
||||
assert.Equal(t, true, properties)
|
||||
})
|
||||
|
||||
t.Run("properties_false", func(t *testing.T) {
|
||||
c := testValidConfig(t, fill(properties_false))
|
||||
properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
|
||||
assert.Equal(t, false, properties)
|
||||
})
|
||||
|
||||
t.Run("properties_true", func(t *testing.T) {
|
||||
c := testValidConfig(t, fill(properties_true))
|
||||
properties := c.Jobs[0].Ret.(*PushJob).Send.SendProperties
|
||||
assert.Equal(t, true, properties)
|
||||
})
|
||||
|
||||
t.Run("raw_true", func(t *testing.T) {
|
||||
c := testValidConfig(t, fill(raw_true))
|
||||
raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
|
||||
assert.Equal(t, true, raw)
|
||||
})
|
||||
|
||||
t.Run("raw_false", func(t *testing.T) {
|
||||
c := testValidConfig(t, fill(raw_false))
|
||||
raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
|
||||
assert.Equal(t, false, raw)
|
||||
})
|
||||
|
||||
t.Run("raw_and_encrypted", func(t *testing.T) {
|
||||
c := testValidConfig(t, fill(raw_and_encrypted))
|
||||
raw := c.Jobs[0].Ret.(*PushJob).Send.Raw
|
||||
encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
|
||||
assert.Equal(t, true, raw)
|
||||
assert.Equal(t, true, encrypted)
|
||||
})
|
||||
|
||||
t.Run("send_not_specified", func(t *testing.T) {
|
||||
c, err := testConfig(t, fill(send_not_specified))
|
||||
assert.NoError(t, err)
|
||||
|
||||
@@ -37,7 +37,7 @@ func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (
|
||||
return err
|
||||
}
|
||||
|
||||
intervals, err := ParseRetentionIntervalSpec(in)
|
||||
intervals, err := parseRetentionGridIntervalsString(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -102,7 +102,7 @@ func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval,
|
||||
|
||||
}
|
||||
|
||||
func ParseRetentionIntervalSpec(s string) (intervals []RetentionInterval, err error) {
|
||||
func parseRetentionGridIntervalsString(s string) (intervals []RetentionInterval, err error) {
|
||||
|
||||
ges := strings.Split(s, "|")
|
||||
intervals = make([]RetentionInterval, 0, 7*len(ges))
|
||||
|
||||
@@ -52,15 +52,12 @@ jobs:
|
||||
}
|
||||
send:
|
||||
encrypted: true
|
||||
replication:
|
||||
protection:
|
||||
initial: guarantee_resumability
|
||||
# Downgrade protection to guarantee_incremental which uses zfs bookmarks instead of zfs holds.
|
||||
# Thus, when we yank out the backup drive during replication
|
||||
# - we might not be able to resume the interrupted replication step because the partially received `to` snapshot of a `from`->`to` step may be pruned any time
|
||||
# - but in exchange we get back the disk space allocated by `to` when we prune it
|
||||
# - and because we still have the bookmarks created by `guarantee_incremental`, we can still do incremental replication of `from`->`to2` in the future
|
||||
incremental: guarantee_incremental
|
||||
# disable incremental step holds so that
|
||||
# - we can yank out the backup drive during replication
|
||||
# - thereby sacrificing resumability
|
||||
# - in exchange for the replicating snapshot not sticking around until we reconnect the backup drive
|
||||
step_holds:
|
||||
disable_incremental: true
|
||||
snapshotting:
|
||||
type: manual
|
||||
pruning:
|
||||
|
||||
@@ -3,7 +3,6 @@ package daemon
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
@@ -13,7 +12,6 @@ import (
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/util/envconst"
|
||||
@@ -39,12 +37,6 @@ func Run(ctx context.Context, conf *config.Config) error {
|
||||
cancel()
|
||||
}()
|
||||
|
||||
// The math/rand package is used presently for generating trace IDs, we
|
||||
// seed it with the current time and pid so that the IDs are mostly
|
||||
// unique.
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
rand.Seed(int64(os.Getpid()))
|
||||
|
||||
outlets, err := logging.OutletsFromConfig(*conf.Global.Logging)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "cannot build logging from config")
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"text/template"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
|
||||
+39
-36
@@ -9,10 +9,10 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/common/log"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/daemon/job/reset"
|
||||
"github.com/zrepl/zrepl/daemon/job/wakeup"
|
||||
"github.com/zrepl/zrepl/daemon/pruner"
|
||||
@@ -35,10 +35,9 @@ type ActiveSide struct {
|
||||
|
||||
prunerFactory *pruner.PrunerFactory
|
||||
|
||||
promRepStateSecs *prometheus.HistogramVec // labels: state
|
||||
promPruneSecs *prometheus.HistogramVec // labels: prune_side
|
||||
promBytesReplicated *prometheus.CounterVec // labels: filesystem
|
||||
promReplicationErrors prometheus.Gauge
|
||||
promRepStateSecs *prometheus.HistogramVec // labels: state
|
||||
promPruneSecs *prometheus.HistogramVec // labels: prune_side
|
||||
promBytesReplicated *prometheus.CounterVec // labels: filesystem
|
||||
|
||||
tasksMtx sync.Mutex
|
||||
tasks activeSideTasks
|
||||
@@ -146,24 +145,30 @@ func (m *modePush) ResetConnectBackoff() {
|
||||
|
||||
func modePushFromConfig(g *config.Global, in *config.PushJob, jobID endpoint.JobID) (*modePush, error) {
|
||||
m := &modePush{}
|
||||
var err error
|
||||
|
||||
m.senderConfig, err = buildSenderConfig(in, jobID)
|
||||
fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "sender config")
|
||||
return nil, errors.Wrap(err, "cannot build filesystem filter")
|
||||
}
|
||||
|
||||
replicationConfig, err := logic.ReplicationConfigFromConfig(in.Replication)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "field `replication`")
|
||||
}
|
||||
m.senderConfig = &endpoint.SenderConfig{
|
||||
FSF: fsf,
|
||||
JobID: jobID,
|
||||
DisableIncrementalStepHolds: in.Send.StepHolds.DisableIncremental,
|
||||
|
||||
Encrypt: &zfs.NilBool{B: in.Send.Encrypted},
|
||||
SendRaw: in.Send.Raw,
|
||||
SendProperties: in.Send.SendProperties,
|
||||
SendBackupProperties: in.Send.BackupProperties,
|
||||
SendLargeBlocks: in.Send.LargeBlocks,
|
||||
SendCompressed: in.Send.Compressed,
|
||||
SendEmbeddedData: in.Send.EmbeddedData,
|
||||
}
|
||||
m.plannerPolicy = &logic.PlannerPolicy{
|
||||
EncryptedSend: logic.TriFromBool(in.Send.Encrypted),
|
||||
ReplicationConfig: *replicationConfig,
|
||||
EncryptedSend: logic.TriFromBool(in.Send.Encrypted),
|
||||
}
|
||||
|
||||
if m.snapper, err = snapper.FromConfig(g, m.senderConfig.FSF, in.Snapshotting); err != nil {
|
||||
if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build snapper")
|
||||
}
|
||||
|
||||
@@ -175,6 +180,7 @@ type modePull struct {
|
||||
receiver *endpoint.Receiver
|
||||
receiverConfig endpoint.ReceiverConfig
|
||||
sender *rpc.Client
|
||||
rootFS *zfs.DatasetPath
|
||||
plannerPolicy *logic.PlannerPolicy
|
||||
interval config.PositiveDurationOrManual
|
||||
}
|
||||
@@ -248,19 +254,29 @@ func modePullFromConfig(g *config.Global, in *config.PullJob, jobID endpoint.Job
|
||||
m = &modePull{}
|
||||
m.interval = in.Interval
|
||||
|
||||
replicationConfig, err := logic.ReplicationConfigFromConfig(in.Replication)
|
||||
m.rootFS, err = zfs.NewDatasetPath(in.RootFS)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "field `replication`")
|
||||
return nil, errors.New("RootFS is not a valid zfs filesystem path")
|
||||
}
|
||||
if m.rootFS.Length() <= 0 {
|
||||
return nil, errors.New("RootFS must not be empty") // duplicates error check of receiver
|
||||
}
|
||||
|
||||
m.plannerPolicy = &logic.PlannerPolicy{
|
||||
EncryptedSend: logic.DontCare,
|
||||
ReplicationConfig: *replicationConfig,
|
||||
EncryptedSend: logic.DontCare,
|
||||
}
|
||||
|
||||
m.receiverConfig, err = buildReceiverConfig(in, jobID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
m.receiverConfig = endpoint.ReceiverConfig{
|
||||
JobID: jobID,
|
||||
RootWithoutClientComponent: m.rootFS,
|
||||
AppendClientIdentity: false, // !
|
||||
UpdateLastReceivedHold: true,
|
||||
|
||||
InheritProperties: in.Recv.Properties.Inherit,
|
||||
OverrideProperties: in.Recv.Properties.Override,
|
||||
}
|
||||
if err := m.receiverConfig.Validate(); err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build receiver config")
|
||||
}
|
||||
|
||||
return m, nil
|
||||
@@ -301,14 +317,6 @@ func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}) (
|
||||
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
|
||||
}, []string{"filesystem"})
|
||||
|
||||
j.promReplicationErrors = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "replication",
|
||||
Name: "filesystem_errors",
|
||||
Help: "number of filesystems that failed replication in the latest replication attempt, or -1 if the job failed before enumerating the filesystems",
|
||||
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
|
||||
})
|
||||
|
||||
j.connecter, err = fromconfig.ConnecterFromConfig(g, in.Connect)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build client")
|
||||
@@ -333,7 +341,6 @@ func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) {
|
||||
registerer.MustRegister(j.promRepStateSecs)
|
||||
registerer.MustRegister(j.promPruneSecs)
|
||||
registerer.MustRegister(j.promBytesReplicated)
|
||||
registerer.MustRegister(j.promReplicationErrors)
|
||||
}
|
||||
|
||||
func (j *ActiveSide) Name() string { return j.name.String() }
|
||||
@@ -368,7 +375,7 @@ func (j *ActiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
|
||||
_ = j.mode.(*modePush) // make sure we didn't introduce a new job type
|
||||
return nil, false
|
||||
}
|
||||
return pull.receiverConfig.RootWithoutClientComponent.Copy(), true
|
||||
return pull.rootFS.Copy(), true
|
||||
}
|
||||
|
||||
func (j *ActiveSide) SenderConfig() *endpoint.SenderConfig {
|
||||
@@ -466,10 +473,6 @@ func (j *ActiveSide) do(ctx context.Context) {
|
||||
GetLogger(ctx).Info("start replication")
|
||||
repWait(true) // wait blocking
|
||||
repCancel() // always cancel to free up context resources
|
||||
|
||||
replicationReport := j.tasks.replicationReport()
|
||||
j.promReplicationErrors.Set(float64(replicationReport.GetFailedFilesystemsCountInLatestAttempt()))
|
||||
|
||||
endSpan()
|
||||
}
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/transport"
|
||||
)
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type SendingJobConfig interface {
|
||||
GetFilesystems() config.FilesystemsFilter
|
||||
GetSendOptions() *config.SendOptions // must not be nil
|
||||
}
|
||||
|
||||
func buildSenderConfig(in SendingJobConfig, jobID endpoint.JobID) (*endpoint.SenderConfig, error) {
|
||||
|
||||
fsf, err := filters.DatasetMapFilterFromConfig(in.GetFilesystems())
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build filesystem filter")
|
||||
}
|
||||
|
||||
return &endpoint.SenderConfig{
|
||||
FSF: fsf,
|
||||
Encrypt: &zfs.NilBool{B: in.GetSendOptions().Encrypted},
|
||||
JobID: jobID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type ReceivingJobConfig interface {
|
||||
GetRootFS() string
|
||||
GetAppendClientIdentity() bool
|
||||
GetRecvOptions() *config.RecvOptions
|
||||
}
|
||||
|
||||
func buildReceiverConfig(in ReceivingJobConfig, jobID endpoint.JobID) (rc endpoint.ReceiverConfig, err error) {
|
||||
rootFs, err := zfs.NewDatasetPath(in.GetRootFS())
|
||||
if err != nil {
|
||||
return rc, errors.New("root_fs is not a valid zfs filesystem path")
|
||||
}
|
||||
if rootFs.Length() <= 0 {
|
||||
return rc, errors.New("root_fs must not be empty") // duplicates error check of receiver
|
||||
}
|
||||
|
||||
rc = endpoint.ReceiverConfig{
|
||||
JobID: jobID,
|
||||
RootWithoutClientComponent: rootFs,
|
||||
AppendClientIdentity: in.GetAppendClientIdentity(),
|
||||
}
|
||||
if err := rc.Validate(); err != nil {
|
||||
return rc, errors.Wrap(err, "cannot build receiver config")
|
||||
}
|
||||
|
||||
return rc, nil
|
||||
}
|
||||
@@ -2,16 +2,12 @@ package job
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/kr/pretty"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/transport/tls"
|
||||
)
|
||||
|
||||
func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
|
||||
@@ -112,35 +108,3 @@ jobs:
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestSampleConfigsAreBuiltWithoutErrors(t *testing.T) {
|
||||
paths, err := filepath.Glob("../../config/samples/*")
|
||||
if err != nil {
|
||||
t.Errorf("glob failed: %+v", err)
|
||||
}
|
||||
|
||||
for _, p := range paths {
|
||||
|
||||
if path.Ext(p) != ".yml" {
|
||||
t.Logf("skipping file %s", p)
|
||||
continue
|
||||
}
|
||||
|
||||
t.Run(p, func(t *testing.T) {
|
||||
c, err := config.ParseConfig(p)
|
||||
if err != nil {
|
||||
t.Errorf("error parsing %s:\n%+v", p, err)
|
||||
}
|
||||
|
||||
t.Logf("file: %s", p)
|
||||
t.Log(pretty.Sprint(c))
|
||||
|
||||
tls.FakeCertificateLoading(t)
|
||||
jobs, err := JobsFromConfig(c)
|
||||
t.Logf("jobs: %#v", jobs)
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+32
-7
@@ -6,10 +6,10 @@ import (
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/daemon/logging"
|
||||
"github.com/zrepl/zrepl/daemon/snapper"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
@@ -48,9 +48,22 @@ func (m *modeSink) SnapperReport() *snapper.Report { return nil }
|
||||
func modeSinkFromConfig(g *config.Global, in *config.SinkJob, jobID endpoint.JobID) (m *modeSink, err error) {
|
||||
m = &modeSink{}
|
||||
|
||||
m.receiverConfig, err = buildReceiverConfig(in, jobID)
|
||||
rootDataset, err := zfs.NewDatasetPath(in.RootFS)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, errors.New("root dataset is not a valid zfs filesystem path")
|
||||
}
|
||||
|
||||
m.receiverConfig = endpoint.ReceiverConfig{
|
||||
JobID: jobID,
|
||||
RootWithoutClientComponent: rootDataset,
|
||||
AppendClientIdentity: true, // !
|
||||
UpdateLastReceivedHold: true,
|
||||
|
||||
InheritProperties: in.Recv.Properties.Inherit,
|
||||
OverrideProperties: in.Recv.Properties.Override,
|
||||
}
|
||||
if err := m.receiverConfig.Validate(); err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build receiver config")
|
||||
}
|
||||
|
||||
return m, nil
|
||||
@@ -64,13 +77,25 @@ type modeSource struct {
|
||||
func modeSourceFromConfig(g *config.Global, in *config.SourceJob, jobID endpoint.JobID) (m *modeSource, err error) {
|
||||
// FIXME exact dedup of modePush
|
||||
m = &modeSource{}
|
||||
|
||||
m.senderConfig, err = buildSenderConfig(in, jobID)
|
||||
fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "send options")
|
||||
return nil, errors.Wrap(err, "cannot build filesystem filter")
|
||||
}
|
||||
m.senderConfig = &endpoint.SenderConfig{
|
||||
FSF: fsf,
|
||||
JobID: jobID,
|
||||
DisableIncrementalStepHolds: in.Send.StepHolds.DisableIncremental,
|
||||
|
||||
Encrypt: &zfs.NilBool{B: in.Send.Encrypted},
|
||||
SendRaw: in.Send.Raw,
|
||||
SendProperties: in.Send.SendProperties,
|
||||
SendBackupProperties: in.Send.BackupProperties,
|
||||
SendLargeBlocks: in.Send.LargeBlocks,
|
||||
SendCompressed: in.Send.Compressed,
|
||||
SendEmbeddedData: in.Send.EmbeddedData,
|
||||
}
|
||||
|
||||
if m.snapper, err = snapper.FromConfig(g, m.senderConfig.FSF, in.Snapshotting); err != nil {
|
||||
if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build snapper")
|
||||
}
|
||||
|
||||
|
||||
@@ -4,11 +4,9 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
@@ -30,8 +28,7 @@ type SnapJob struct {
|
||||
|
||||
promPruneSecs *prometheus.HistogramVec // labels: prune_side
|
||||
|
||||
prunerMtx sync.Mutex
|
||||
pruner *pruner.Pruner
|
||||
pruner *pruner.Pruner
|
||||
}
|
||||
|
||||
func (j *SnapJob) Name() string { return j.name.String() }
|
||||
@@ -79,11 +76,9 @@ type SnapJobStatus struct {
|
||||
func (j *SnapJob) Status() *Status {
|
||||
s := &SnapJobStatus{}
|
||||
t := j.Type()
|
||||
j.prunerMtx.Lock()
|
||||
if j.pruner != nil {
|
||||
s.Pruning = j.pruner.Report()
|
||||
}
|
||||
j.prunerMtx.Unlock()
|
||||
s.Snapshotting = j.snapper.Report()
|
||||
return &Status{Type: t, JobSpecific: s}
|
||||
}
|
||||
@@ -180,10 +175,10 @@ func (j *SnapJob) doPrune(ctx context.Context) {
|
||||
FSF: j.fsfilter,
|
||||
// FIXME encryption setting is irrelevant for SnapJob because the endpoint is only used as pruner.Target
|
||||
Encrypt: &zfs.NilBool{B: true},
|
||||
// FIXME DisableIncrementalStepHolds setting is irrelevant for SnapJob because the endpoint is only used as pruner.Target
|
||||
DisableIncrementalStepHolds: false,
|
||||
})
|
||||
j.prunerMtx.Lock()
|
||||
j.pruner = j.prunerFactory.BuildLocalPruner(ctx, sender, alwaysUpToDateReplicationCursorHistory{sender})
|
||||
j.prunerMtx.Unlock()
|
||||
log.Info("start pruning")
|
||||
j.pruner.Prune()
|
||||
log.Info("finished pruning")
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
|
||||
"github.com/mattn/go-isatty"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
|
||||
@@ -93,15 +93,11 @@ package trace
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
runtimedebug "runtime/debug"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/kr/pretty"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/util/chainlock"
|
||||
)
|
||||
|
||||
@@ -124,19 +120,15 @@ func RegisterMetrics(r prometheus.Registerer) {
|
||||
}
|
||||
|
||||
type traceNode struct {
|
||||
// NOTE: members with prefix debug are only valid if the package variable debugEnabled is true.
|
||||
|
||||
id string
|
||||
annotation string
|
||||
parentTask *traceNode
|
||||
debugCreationStack string // debug: stack trace of when the traceNode was created
|
||||
id string
|
||||
annotation string
|
||||
parentTask *traceNode
|
||||
|
||||
mtx chainlock.L
|
||||
|
||||
activeChildTasks int32 // only for task nodes, insignificant for span nodes
|
||||
debugActiveChildTasks map[*traceNode]bool // debug: set of active child tasks
|
||||
parentSpan *traceNode
|
||||
activeChildSpan *traceNode // nil if task or span doesn't have an active child span
|
||||
activeChildTasks int32 // only for task nodes, insignificant for span nodes
|
||||
parentSpan *traceNode
|
||||
activeChildSpan *traceNode // nil if task or span doesn't have an active child span
|
||||
|
||||
startedAt time.Time
|
||||
endedAt time.Time
|
||||
@@ -145,14 +137,6 @@ type traceNode struct {
|
||||
func (s *traceNode) StartedAt() time.Time { return s.startedAt }
|
||||
func (s *traceNode) EndedAt() time.Time { return s.endedAt }
|
||||
|
||||
// caller must hold mtx
|
||||
func (s *traceNode) debugString() string {
|
||||
if !debugEnabled {
|
||||
return "<debugEnabled=false>"
|
||||
}
|
||||
return pretty.Sprint(s)
|
||||
}
|
||||
|
||||
// Returned from WithTask or WithSpan.
|
||||
// Must be called once the task or span ends.
|
||||
// See package-level docs for nesting rules.
|
||||
@@ -212,16 +196,8 @@ func WithTask(ctx context.Context, taskName string) (context.Context, DoneFunc)
|
||||
endedAt: time.Time{},
|
||||
}
|
||||
|
||||
if debugEnabled {
|
||||
this.debugCreationStack = string(runtimedebug.Stack())
|
||||
this.debugActiveChildTasks = map[*traceNode]bool{}
|
||||
}
|
||||
|
||||
if parentTask != nil {
|
||||
this.parentTask.activeChildTasks++
|
||||
if debugEnabled {
|
||||
this.parentTask.debugActiveChildTasks[this] = true
|
||||
}
|
||||
parentTask.mtx.Unlock()
|
||||
}
|
||||
|
||||
@@ -242,11 +218,7 @@ func WithTask(ctx context.Context, taskName string) (context.Context, DoneFunc)
|
||||
defer this.mtx.Lock().Unlock()
|
||||
|
||||
if this.activeChildTasks != 0 {
|
||||
if debugEnabled {
|
||||
// the debugString can be quite long and panic won't print it completely
|
||||
fmt.Fprintf(os.Stderr, "going to panic due to activeChildTasks:\n%s\n", this.debugString())
|
||||
}
|
||||
panic(errors.WithMessagef(ErrTaskStillHasActiveChildTasks, "end task: %v active child tasks\n", this.activeChildTasks))
|
||||
panic(errors.Wrapf(ErrTaskStillHasActiveChildTasks, "end task: %v active child tasks", this.activeChildTasks))
|
||||
}
|
||||
|
||||
// support idempotent task ends
|
||||
@@ -257,15 +229,8 @@ func WithTask(ctx context.Context, taskName string) (context.Context, DoneFunc)
|
||||
|
||||
if this.parentTask != nil {
|
||||
this.parentTask.activeChildTasks--
|
||||
if debugEnabled {
|
||||
delete(this.parentTask.debugActiveChildTasks, this)
|
||||
}
|
||||
if this.parentTask.activeChildTasks < 0 {
|
||||
if debugEnabled {
|
||||
// the debugString can be quite long and panic won't print it completely
|
||||
fmt.Fprintf(os.Stderr, "going to panic due to activeChildTasks < 0:\n%s\n", this.parentTask.debugString())
|
||||
}
|
||||
panic(fmt.Sprintf("impl error: parent task with negative activeChildTasks count: %v", this.parentTask.activeChildTasks))
|
||||
panic("impl error: parent task with negative activeChildTasks count")
|
||||
}
|
||||
}
|
||||
return false
|
||||
@@ -314,10 +279,6 @@ func WithSpan(ctx context.Context, annotation string) (context.Context, DoneFunc
|
||||
endedAt: time.Time{},
|
||||
}
|
||||
|
||||
if debugEnabled {
|
||||
this.debugCreationStack = string(runtimedebug.Stack())
|
||||
}
|
||||
|
||||
parentSpan.mtx.HoldWhile(func() {
|
||||
if parentSpan.activeChildSpan != nil {
|
||||
panic(ErrAlreadyActiveChildSpan)
|
||||
|
||||
@@ -3,11 +3,9 @@ package trace
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/zrepl/zrepl/util/envconst"
|
||||
)
|
||||
|
||||
var debugEnabled = envconst.Bool("ZREPL_TRACE_DEBUG_ENABLED", false)
|
||||
const debugEnabled = false
|
||||
|
||||
func debug(format string, args ...interface{}) {
|
||||
if !debugEnabled {
|
||||
|
||||
@@ -3,11 +3,20 @@ package trace
|
||||
import (
|
||||
"encoding/base64"
|
||||
"math/rand"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/zrepl/zrepl/util/envconst"
|
||||
)
|
||||
|
||||
var genIdPRNG = rand.New(rand.NewSource(1))
|
||||
|
||||
func init() {
|
||||
genIdPRNG.Seed(time.Now().UnixNano())
|
||||
genIdPRNG.Seed(int64(os.Getpid()))
|
||||
}
|
||||
|
||||
var genIdNumBytes = envconst.Int("ZREPL_TRACE_ID_NUM_BYTES", 3)
|
||||
|
||||
func init() {
|
||||
@@ -21,7 +30,7 @@ func genID() string {
|
||||
enc := base64.NewEncoder(base64.RawStdEncoding, &out)
|
||||
buf := make([]byte, genIdNumBytes)
|
||||
for i := 0; i < len(buf); {
|
||||
n, err := rand.Read(buf[i:])
|
||||
n, err := genIdPRNG.Read(buf[i:])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
@@ -8,10 +8,10 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/daemon/hooks"
|
||||
"github.com/zrepl/zrepl/daemon/logging"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
@@ -49,7 +49,7 @@ type args struct {
|
||||
ctx context.Context
|
||||
prefix string
|
||||
interval time.Duration
|
||||
fsf zfs.DatasetFilter
|
||||
fsf *filters.DatasetMapFilter
|
||||
snapshotsTaken chan<- struct{}
|
||||
hooks *hooks.List
|
||||
dryRun bool
|
||||
@@ -109,7 +109,7 @@ func getLogger(ctx context.Context) Logger {
|
||||
return logging.GetLogger(ctx, logging.SubsysSnapshot)
|
||||
}
|
||||
|
||||
func PeriodicFromConfig(g *config.Global, fsf zfs.DatasetFilter, in *config.SnapshottingPeriodic) (*Snapper, error) {
|
||||
func PeriodicFromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in *config.SnapshottingPeriodic) (*Snapper, error) {
|
||||
if in.Prefix == "" {
|
||||
return nil, errors.New("prefix must not be empty")
|
||||
}
|
||||
@@ -383,7 +383,7 @@ func wait(a args, u updater) state {
|
||||
}
|
||||
}
|
||||
|
||||
func listFSes(ctx context.Context, mf zfs.DatasetFilter) (fss []*zfs.DatasetPath, err error) {
|
||||
func listFSes(ctx context.Context, mf *filters.DatasetMapFilter) (fss []*zfs.DatasetPath, err error) {
|
||||
return zfs.ZFSListMapping(ctx, mf)
|
||||
}
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"fmt"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
)
|
||||
|
||||
// FIXME: properly abstract snapshotting:
|
||||
@@ -32,7 +32,7 @@ func (s *PeriodicOrManual) Report() *Report {
|
||||
return nil
|
||||
}
|
||||
|
||||
func FromConfig(g *config.Global, fsf zfs.DatasetFilter, in config.SnapshottingEnum) (*PeriodicOrManual, error) {
|
||||
func FromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in config.SnapshottingEnum) (*PeriodicOrManual, error) {
|
||||
switch v := in.Ret.(type) {
|
||||
case *config.SnapshottingPeriodic:
|
||||
snapper, err := PeriodicFromConfig(g, fsf, v)
|
||||
|
||||
+15
-175
@@ -69,139 +69,6 @@
|
||||
"title": "Panel Title",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"cacheTimeout": null,
|
||||
"colorBackground": true,
|
||||
"colorPostfix": false,
|
||||
"colorPrefix": false,
|
||||
"colorValue": false,
|
||||
"colors": [
|
||||
"#bf1b00",
|
||||
"#508642",
|
||||
"#bf1b00"
|
||||
],
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"description": "Number of filesystems that failed replications",
|
||||
"format": "none",
|
||||
"gauge": {
|
||||
"maxValue": 100,
|
||||
"minValue": 0,
|
||||
"show": false,
|
||||
"thresholdLabels": false,
|
||||
"thresholdMarkers": true
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 3,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 10
|
||||
},
|
||||
"id": 50,
|
||||
"interval": null,
|
||||
"links": [],
|
||||
"mappingType": 1,
|
||||
"mappingTypes": [
|
||||
{
|
||||
"name": "value to text",
|
||||
"value": 1
|
||||
},
|
||||
{
|
||||
"name": "range to text",
|
||||
"value": 2
|
||||
}
|
||||
],
|
||||
"maxDataPoints": 100,
|
||||
"nullPointMode": "connected",
|
||||
"nullText": null,
|
||||
"postfix": "",
|
||||
"postfixFontSize": "50%",
|
||||
"prefix": "",
|
||||
"prefixFontSize": "50%",
|
||||
"rangeMaps": [
|
||||
{
|
||||
"from": "",
|
||||
"text": "",
|
||||
"to": ""
|
||||
}
|
||||
],
|
||||
"repeat": "zrepl_job_name",
|
||||
"repeatDirection": "h",
|
||||
"scopedVars": {
|
||||
"zrepl_job_name": {
|
||||
"selected": false,
|
||||
"text": "desktop_to_homesrv",
|
||||
"value": "desktop_to_homesrv"
|
||||
}
|
||||
},
|
||||
"sparkline": {
|
||||
"fillColor": "rgba(31, 118, 189, 0.18)",
|
||||
"full": true,
|
||||
"lineColor": "rgb(31, 120, 193)",
|
||||
"show": true
|
||||
},
|
||||
"tableColumn": "__name__",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "zrepl_replication_filesystem_errors{job=\"$prom_job_name\",zrepl_job=\"$zrepl_job_name\"}",
|
||||
"format": "time_series",
|
||||
"groupBy": [
|
||||
{
|
||||
"params": [
|
||||
"$__interval"
|
||||
],
|
||||
"type": "time"
|
||||
},
|
||||
{
|
||||
"params": [
|
||||
"null"
|
||||
],
|
||||
"type": "fill"
|
||||
}
|
||||
],
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "",
|
||||
"orderByTime": "ASC",
|
||||
"policy": "default",
|
||||
"refId": "A",
|
||||
"resultFormat": "time_series",
|
||||
"select": [
|
||||
[
|
||||
{
|
||||
"params": [
|
||||
"value"
|
||||
],
|
||||
"type": "field"
|
||||
},
|
||||
{
|
||||
"params": [],
|
||||
"type": "mean"
|
||||
}
|
||||
]
|
||||
],
|
||||
"tags": []
|
||||
}
|
||||
],
|
||||
"thresholds": "0,1",
|
||||
"title": "Failed replications $zrepl_job_name",
|
||||
"transparent": false,
|
||||
"type": "singlestat",
|
||||
"valueFontSize": "80%",
|
||||
"valueMaps": [
|
||||
{
|
||||
"op": "=",
|
||||
"text": "All failed",
|
||||
"value": "-1"
|
||||
},
|
||||
{
|
||||
"op": "=",
|
||||
"text": "All OK",
|
||||
"value": "0"
|
||||
}
|
||||
],
|
||||
"valueName": "avg"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
@@ -220,7 +87,7 @@
|
||||
"h": 4,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 13
|
||||
"y": 10
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 48,
|
||||
@@ -314,7 +181,7 @@
|
||||
"h": 5,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 13
|
||||
"y": 10
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 44,
|
||||
@@ -406,7 +273,7 @@
|
||||
"h": 5,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 17
|
||||
"y": 14
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 42,
|
||||
@@ -506,7 +373,7 @@
|
||||
"h": 4,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 18
|
||||
"y": 15
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 22,
|
||||
@@ -598,7 +465,7 @@
|
||||
"h": 5,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 22
|
||||
"y": 19
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 33,
|
||||
@@ -706,7 +573,7 @@
|
||||
"h": 5,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 22
|
||||
"y": 19
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 23,
|
||||
@@ -798,7 +665,7 @@
|
||||
"h": 5,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 27
|
||||
"y": 24
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 41,
|
||||
@@ -891,7 +758,7 @@
|
||||
"h": 5,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 27
|
||||
"y": 24
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 47,
|
||||
@@ -920,7 +787,7 @@
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [{
|
||||
"expr": "zrepl_endpoint_abstractions_cache_entry_count",
|
||||
"expr": "zrepl_endpoint_send_abstractions_cache_entry_count",
|
||||
"format": "time_series",
|
||||
"intervalFactor": 1,
|
||||
"refId": "A"
|
||||
@@ -929,7 +796,7 @@
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "zfs abstractions cache entry count (should not be zero and not grow unboundedly)",
|
||||
"title": "send abstractions cache entry count (should not be zero and not grow unboundedly)",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
@@ -983,7 +850,7 @@
|
||||
"h": 5,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 32
|
||||
"y": 29
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 17,
|
||||
@@ -1076,7 +943,7 @@
|
||||
"h": 5,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 37
|
||||
"y": 34
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 19,
|
||||
@@ -1177,33 +1044,6 @@
|
||||
"tagsQuery": "",
|
||||
"type": "query",
|
||||
"useTags": false
|
||||
},
|
||||
{
|
||||
"allValue": null,
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": [
|
||||
"$__all"
|
||||
]
|
||||
},
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"definition": "label_values(zrepl_replication_filesystem_errors{job=\"$prom_job_name\"}, zrepl_job)",
|
||||
"hide": 2,
|
||||
"includeAll": true,
|
||||
"label": "Zrepl Job Name",
|
||||
"multi": true,
|
||||
"name": "zrepl_job_name",
|
||||
"options": [],
|
||||
"query": "label_values(zrepl_replication_filesystem_errors{job=\"$prom_job_name\"}, zrepl_job)",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 1,
|
||||
"tagValuesQuery": "",
|
||||
"tags": [],
|
||||
"tagsQuery": "",
|
||||
"type": "query",
|
||||
"useTags": false
|
||||
}]
|
||||
},
|
||||
"time": {
|
||||
@@ -1236,6 +1076,6 @@
|
||||
},
|
||||
"timezone": "",
|
||||
"title": "zrepl 0.3",
|
||||
"uid": "etQuvBnGz",
|
||||
"version": 7
|
||||
}
|
||||
"uid": "etJuvBmGz",
|
||||
"version": 6
|
||||
}
|
||||
+16
-48
@@ -27,33 +27,6 @@ We use the following annotations for classifying changes:
|
||||
* |bugfix| Change that fixes a bug, no regressions or incompatibilities expected.
|
||||
* |docs| Change to the documentation.
|
||||
|
||||
0.3.1
|
||||
-----
|
||||
|
||||
Mostly a bugfix release for :ref:`zrepl 0.3 <release-0.3>`.
|
||||
|
||||
* |feature| pruning: add optional ``regex`` field to ``last_n`` rule
|
||||
* |docs| pruning: ``grid`` : improve documentation and add an example
|
||||
* |bugfix| pruning: ``grid``: add all snapshots that do not match the regex to the rule's destroy list.
|
||||
This brings the implementation in line with the docs.
|
||||
* |bugfix| ``easyrsa`` script in docs
|
||||
* |bugfix| platformtest: fix skipping encryption-only tests on systems that don't support encryption
|
||||
* |bugfix| replication: report AttemptDone if no filesystems are replicated
|
||||
* |feature| status + replication: warning if replication succeeeded without any filesystem being replicated
|
||||
* |docs| update multi-job & multi-host setup section
|
||||
* RPM Packaging
|
||||
* CI infrastructure rework
|
||||
* Continuous deployment of that new `stable` branch to zrepl.github.io.
|
||||
|
||||
.. NOTE::
|
||||
| zrepl is a spare-time project primarily developed by `Christian Schwarz <https://cschwarz.com>`_.
|
||||
| You can support maintenance and feature development through one of the following services:
|
||||
| |Donate via Patreon| |Donate via GitHub Sponsors| |Donate via Liberapay| |Donate via PayPal|
|
||||
| Note that PayPal processing fees are relatively high for small donations.
|
||||
| For SEPA wire transfer and **commercial support**, please `contact Christian directly <https://cschwarz.com>`_.
|
||||
|
||||
.. _release-0.3:
|
||||
|
||||
0.3
|
||||
---
|
||||
|
||||
@@ -61,38 +34,27 @@ This is a big one! Headlining features:
|
||||
|
||||
* **Resumable Send & Recv Support**
|
||||
No knobs required, automatically used where supported.
|
||||
* **Hold-Protected Send & Recv**
|
||||
Automatic ZFS holds to ensure that we can always use resumable send&recv for a replication step.
|
||||
* **Encrypted Send & Recv Support** for OpenZFS native encryption,
|
||||
:ref:`configurable <job-send-options>` at the job level, i.e., for all filesystems a job is responsible for.
|
||||
* **Replication Guarantees**
|
||||
Automatic use of ZFS holds and bookmarks to protect a replicated filesystem from losing synchronization between sender and receiver.
|
||||
By default, zrepl guarantees that incremental replication will always be possible and interrupted steps will always be resumable.
|
||||
* **Receive-side hold on last received dataset**
|
||||
The counterpart to the replication cursor bookmark on the send-side.
|
||||
Ensures that incremental replication will always be possible between a sender and receiver.
|
||||
|
||||
.. TIP::
|
||||
|
||||
We highly recommend studying the updated :ref:`overview section of the configuration chapter <overview-how-replication-works>` to understand how replication works.
|
||||
|
||||
.. TIP::
|
||||
|
||||
Go 1.15 changed the default TLS validation policy to **require Subject Alternative Names (SAN) in certificates**.
|
||||
The openssl commands we provided in the quick-start guides up to and including the zrepl 0.3 docs seem not to work properly.
|
||||
If you encounter certificate validation errors regarding SAN and wish to continue to use your old certificates, start the zrepl daemon with env var ``GODEBUG=x509ignoreCN=0``.
|
||||
Alternatively, generate new certificates with SANs (see :ref:`both options int the TLS transport docs <transport-tcp+tlsclientauth-certgen>` ).
|
||||
|
||||
Quick-start guides:
|
||||
|
||||
* We have added :ref:`another quick-start guide for a typical workstation use case for zrepl <quickstart-backup-to-external-disk>`.
|
||||
Check it out to learn how you can use zrepl to back up your workstation's OpenZFS natively-encrypted root filesystem to an external disk.
|
||||
We highly recommend studying the :ref:`overview section of the configuration chapter <overview-how-replication-works>` to understand how replication works.
|
||||
|
||||
Additional changelog:
|
||||
|
||||
* |break| Go 1.15 TLS changes mentioned above.
|
||||
* |break| |break_config| **more restrictive job names than in prior zrepl versions**
|
||||
Starting with this version, job names are going to be embedded into ZFS holds and bookmark names (see :ref:`this section for details <zrepl-zfs-abstractions>`).
|
||||
Starting with this version, job names are going to be embedded into ZFS holds and bookmark names (see :ref:`here<replication-cursor-and-last-received-hold>` and :ref:`here<step-holds-and-bookmarks>`).
|
||||
Therefore you might need to adjust your job names.
|
||||
**Note that jobs** cannot be renamed easily **once you start using zrepl 0.3.**
|
||||
**Note that jobs** :issue:`cannot be renamed easily` **once you start using zrepl 0.3.**
|
||||
* |break| |mig| replication cursor representation changed
|
||||
|
||||
* zrepl now manages the :ref:`replication cursor bookmark <zrepl-zfs-abstractions>` per job-filesystem tuple instead of a single replication cursor per filesystem.
|
||||
* zrepl now manages the :ref:`replication cursor bookmark <replication-cursor-and-last-received-hold>` per job-filesystem tuple instead of a single replication cursor per filesystem.
|
||||
In the future, this will permit multiple sending jobs to send from the same filesystems.
|
||||
* ZFS does not allow bookmark renaming, thus we cannot migrate the old replication cursors.
|
||||
* zrepl 0.3 will automatically create cursors in the new format for new replications, and warn if it still finds ones in the old format.
|
||||
@@ -100,7 +62,6 @@ Additional changelog:
|
||||
The migration will ensure that only those old-format cursors are destroyed that have been superseeded by new-format cursors.
|
||||
|
||||
* |feature| New option ``listen_freebind`` (tcp, tls, prometheus listener)
|
||||
* |feature| :issue:`341` Prometheus metric for failing replications + corresponding Grafana panel
|
||||
* |feature| :issue:`265` transport/tcp: support for CIDR masks in client IP whitelist
|
||||
* |feature| documented subcommand to generate ``bash`` and ``zsh`` completions
|
||||
* |feature| :issue:`307` ``chrome://trace`` -compatible activity tracing of zrepl daemon activity
|
||||
@@ -116,6 +77,13 @@ Additional changelog:
|
||||
* **[MAINTAINER NOTICE]** New platform tests in this version, please make sure you run them for your distro!
|
||||
* **[MAINTAINER NOTICE]** Please add the shell completions to the zrepl packages.
|
||||
|
||||
.. NOTE::
|
||||
| zrepl is a spare-time project primarily developed by `Christian Schwarz <https://cschwarz.com>`_.
|
||||
| You can support maintenance and feature development through one of the following services:
|
||||
| |Donate via Patreon| |Donate via GitHub Sponsors| |Donate via Liberapay| |Donate via PayPal|
|
||||
| Note that PayPal processing fees are relatively high for small donations.
|
||||
| For SEPA wire transfer and **commercial support**, please `contact Christian directly <https://cschwarz.com>`_.
|
||||
|
||||
0.2.1
|
||||
-----
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@ Configuration
|
||||
configuration/transports
|
||||
configuration/filter_syntax
|
||||
configuration/sendrecvoptions
|
||||
configuration/replication
|
||||
configuration/snapshotting
|
||||
configuration/prune
|
||||
configuration/logging
|
||||
|
||||
@@ -59,8 +59,8 @@ Note that snapshot-creation denoted by "(snap)" is orthogonal to whether a job i
|
||||
| Pull mode | ``pull`` | ``source`` | * Central backup-server for many nodes |
|
||||
| | | (snap) | * Remote server to NAS behind NAT |
|
||||
+-----------------------+--------------+----------------------------------+------------------------------------------------------------------------------------+
|
||||
| Local replication | | ``push`` + ``sink`` in one config | * Backup to :ref:`locally attached disk <quickstart-backup-to-external-disk>` |
|
||||
| | | with :ref:`local transport <transport-local>` | * Backup FreeBSD boot pool |
|
||||
| Local replication | | ``push`` + ``sink`` in one config | * Backup FreeBSD boot pool |
|
||||
| | | with :ref:`local transport <transport-local>` | |
|
||||
+-----------------------+--------------+----------------------------------+------------------------------------------------------------------------------------+
|
||||
| Snap & prune-only | ``snap`` | N/A | * | Snapshots & pruning but no replication |
|
||||
| | (snap) | | | required |
|
||||
@@ -80,19 +80,16 @@ The active side (:ref:`push <job-push>` and :ref:`pull <job-pull>` job) executes
|
||||
.. TIP::
|
||||
The progress of the active side can be watched live using the ``zrepl status`` subcommand.
|
||||
|
||||
.. _overview-passive-side--client-identity:
|
||||
|
||||
How the Passive Side Works
|
||||
--------------------------
|
||||
|
||||
The passive side (:ref:`sink <job-sink>` and :ref:`source <job-source>`) waits for connections from the corresponding active side,
|
||||
using the transport listener type specified in the ``serve`` field of the job configuration.
|
||||
When a client connects, the transport listener performS listener-specific access control (cert validation, IP ACLs, etc)
|
||||
and determines the *client identity*.
|
||||
The passive side job then uses this client identity as follows:
|
||||
Each transport listener provides a client's identity to the passive side job.
|
||||
It uses the client identity for access control:
|
||||
|
||||
* The ``sink`` job maps requests from different client identities to their respective sub-filesystem tree ``root_fs/${client_identity}``.
|
||||
* The ``source`` might, in the future, embed the client identity in :ref:`zrepl's ZFS abstraction names <zrepl-zfs-abstractions>` in order to support multi-host replication.
|
||||
* The ``source`` job has a whitelist of client identities that are allowed pull access.
|
||||
|
||||
.. TIP::
|
||||
The implementation of the ``sink`` job requires that the connecting client identities be a valid ZFS filesystem name components.
|
||||
@@ -135,10 +132,23 @@ The following high-level steps take place during replication and can be monitore
|
||||
|
||||
The idea behind the execution order of replication steps is that if the sender snapshots all filesystems simultaneously at fixed intervals, the receiver will have all filesystems snapshotted at time ``T1`` before the first snapshot at ``T2 = T1 + $interval`` is replicated.
|
||||
|
||||
Placeholder Filesystems
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
.. _replication-placeholder-property:
|
||||
|
||||
**Placeholder filesystems** on the receiving side are regular ZFS filesystems with the placeholder property ``zrepl:placeholder=on``.
|
||||
Placeholders allow the receiving side to mirror the sender's ZFS dataset hierarchy without replicating every filesystem at every intermediary dataset path component.
|
||||
Consider the following example: ``S/H/J`` shall be replicated to ``R/sink/job/S/H/J``, but neither ``S/H`` nor ``S`` shall be replicated.
|
||||
ZFS requires the existence of ``R/sink/job/S`` and ``R/sink/job/S/H`` in order to receive into ``R/sink/job/S/H/J``.
|
||||
Thus, zrepl creates the parent filesystems as placeholders on the receiving side.
|
||||
If at some point ``S/H`` and ``S`` shall be replicated, the receiving side invalidates the placeholder flag automatically.
|
||||
The ``zrepl test placeholder`` command can be used to check whether a filesystem is a placeholder.
|
||||
|
||||
ZFS Background Knowledge
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
This section gives some background knowledge about ZFS features that zrepl uses to provide guarantees for a replication filesystem.
|
||||
Specifically, zrepl guarantees by default that **incremental replication is always possible and that started replication steps can always be resumed if they are interrupted.**
|
||||
|
||||
This section gives some background knowledge about ZFS features that zrepl uses to guarantee that
|
||||
**incremental replication is always possible and that started replication steps can always be resumed if they are interrupted.**
|
||||
|
||||
**ZFS Send Modes & Bookmarks**
|
||||
ZFS supports full sends (``zfs send fs@to``) and incremental sends (``zfs send -i @from fs@to``).
|
||||
@@ -156,56 +166,41 @@ An incremental send can only be resumed if ``@to`` still exists *and* either ``@
|
||||
|
||||
**ZFS Holds**
|
||||
ZFS holds prevent a snapshot from being deleted through ``zfs destroy``, letting the destroy fail with a ``datset is busy`` error.
|
||||
Holds are created and referred to by a *tag*. They can be thought of as a named, persistent lock on the snapshot.
|
||||
|
||||
|
||||
.. _zrepl-zfs-abstractions:
|
||||
|
||||
ZFS Abstractions Managed By zrepl
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
With the background knowledge from the previous paragraph, we now summarize the different on-disk ZFS objects that zrepl manages to provide its functionality.
|
||||
|
||||
.. _replication-placeholder-property:
|
||||
|
||||
**Placeholder filesystems** on the receiving side are regular ZFS filesystems with the ZFS property ``zrepl:placeholder=on``.
|
||||
Placeholders allow the receiving side to mirror the sender's ZFS dataset hierarchy without replicating every filesystem at every intermediary dataset path component.
|
||||
Consider the following example: ``S/H/J`` shall be replicated to ``R/sink/job/S/H/J``, but neither ``S/H`` nor ``S`` shall be replicated.
|
||||
ZFS requires the existence of ``R/sink/job/S`` and ``R/sink/job/S/H`` in order to receive into ``R/sink/job/S/H/J``.
|
||||
Thus, zrepl creates the parent filesystems as placeholders on the receiving side.
|
||||
If at some point ``S/H`` and ``S`` shall be replicated, the receiving side invalidates the placeholder flag automatically.
|
||||
The ``zrepl test placeholder`` command can be used to check whether a filesystem is a placeholder.
|
||||
Holds are created and referred to by a user-defined *tag*. They can be thought of as a named, persistent lock on the snapshot.
|
||||
|
||||
.. _replication-cursor-and-last-received-hold:
|
||||
|
||||
The **replication cursor** bookmark and **last-received-hold** are managed by zrepl to ensure that future replications can always be done incrementally.
|
||||
Guaranteeing That Incremental Sends Are Always Possible
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
**Replication cursor** bookmark and **last-received-hold** are managed by zrepl to ensure that future replications can always be done incrementally.
|
||||
The replication cursor is a send-side bookmark of the most recent successfully replicated snapshot,
|
||||
and the last-received-hold is a hold of that snapshot on the receiving side.
|
||||
Both are moved atomically after the receiving side has confirmed that a replication step is complete.
|
||||
Both are moved aomically after the receiving side has confirmed that a replication step is complete.
|
||||
|
||||
The replication cursor has the format ``#zrepl_CUSOR_G_<GUID>_J_<JOBNAME>``.
|
||||
The last-received-hold tag has the format ``zrepl_last_received_J_<JOBNAME>``.
|
||||
Encoding the job name in the names ensures that multiple sending jobs can replicate the same filesystem to different receivers without interference.
|
||||
|
||||
.. _tentative-replication-cursor-bookmarks:
|
||||
|
||||
**Tentative replication cursor bookmarks** are short-lived bookmarks that protect the atomic moving-forward of the replication cursor and last-received-hold (see :issue:`this issue <340>`).
|
||||
They are only necessary if step holds are not used as per the :ref:`replication.protection <replication-option-protection>` setting.
|
||||
The tentative replication cursor has the format ``#zrepl_CUSORTENTATIVE_G_<GUID>_J_<JOBNAME>``.
|
||||
The ``zrepl zfs-abstraction list`` command provides a listing of all bookmarks and holds managed by zrepl.
|
||||
|
||||
.. _step-holds:
|
||||
.. _step-holds-and-bookmarks:
|
||||
|
||||
Guaranteeing That Sends Are Always Resumable
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
**Step holds** are zfs holds managed by zrepl to ensure that a replication step can always be resumed if it is interrupted, e.g., due to network outage.
|
||||
zrepl creates step holds before it attempts a replication step and releases them after the receiver confirms that the replication step is complete.
|
||||
For an initial replication ``full @initial_snap``, zrepl puts a zfs hold on ``@initial_snap``.
|
||||
For an incremental send ``@from -> @to``, zrepl puts a zfs hold on both ``@from`` and ``@to``.
|
||||
Note that ``@from`` is not strictly necessary for resumability -- a bookmark on the sending side would be sufficient --, but size-estimation in currently used OpenZFS versions only works if ``@from`` is a snapshot.
|
||||
|
||||
The hold tag has the format ``zrepl_STEP_J_<JOBNAME>``.
|
||||
A job only ever has one active send per filesystem.
|
||||
Thus, there are never more than two step holds for a given pair of ``(job,filesystem)``.
|
||||
|
||||
**Step bookmarks** are zrepl's equivalent for holds on bookmarks (ZFS does not support putting holds on bookmarks).
|
||||
They are intended for a situation where a replication step uses a bookmark ``#bm`` as incremental ``from`` where ``#bm`` is not managed by zrepl.
|
||||
They are intended for a situation where a replication step uses a bookmark ``#bm`` as incremental ``from`` that is not managed by zrepl.
|
||||
To ensure resumability, zrepl copies ``#bm`` to step bookmark ``#zrepl_STEP_G_<GUID>_J_<JOBNAME>``.
|
||||
If the replication is interrupted and ``#bm`` is deleted by the user, the step bookmark remains as an incremental source for the resumable send.
|
||||
Note that zrepl does not yet support creating step bookmarks because the `corresponding ZFS feature for copying bookmarks <https://github.com/openzfs/zfs/pull/9571>`_ is not yet widely available .
|
||||
@@ -232,53 +227,39 @@ Limitations
|
||||
Multiple Jobs & More than 2 Machines
|
||||
------------------------------------
|
||||
|
||||
The quick-start guides focus on simple setups with a single sender and a single receiver.
|
||||
This section documents considerations for more complex setups.
|
||||
|
||||
.. ATTENTION::
|
||||
|
||||
Before you continue, make sure you have a working understanding of :ref:`how zrepl works <overview-how-replication-works>`
|
||||
and :ref:`what zrepl does to ensure <zrepl-zfs-abstractions>` that replication between sender and receiver is always
|
||||
possible without conflicts.
|
||||
This will help you understand why certain kinds of multi-machine setups do not (yet) work.
|
||||
When using multiple jobs across single or multiple machines, the following rules are critical to avoid race conditions & data loss:
|
||||
|
||||
.. NOTE::
|
||||
1. The sets of ZFS filesystems matched by the ``filesystems`` filter fields must be disjoint across all jobs configured on a machine.
|
||||
2. The ZFS filesystem subtrees of jobs with ``root_fs`` must be disjoint.
|
||||
3. Across all zrepl instances on all machines in the replication domain, there must be a 1:1 correspondence between active and passive jobs.
|
||||
|
||||
If you can't find your desired configuration, have questions or would like to see improvements to multi-job setups, please `open an issue on GitHub <https://github.com/zrepl/zrepl/issues/new>`_.
|
||||
Explanations & exceptions to above rules are detailed below.
|
||||
|
||||
Multiple Jobs on one Machine
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
As a general rule, multiple jobs configured on one machine **must operate on disjoint sets of filesystems**.
|
||||
Otherwise, concurrently running jobs might interfere when operating on the same filesystem.
|
||||
If you would like to see improvements to multi-job setups, please `open an issue on GitHub <https://github.com/zrepl/zrepl/issues/new>`_.
|
||||
|
||||
On your setup, ensure that
|
||||
No Overlapping
|
||||
^^^^^^^^^^^^^^
|
||||
|
||||
* all ``filesystems`` filter specifications are disjoint
|
||||
* no ``root_fs`` is a prefix or equal to another ``root_fs``
|
||||
* no ``filesystems`` filter matches any ``root_fs``
|
||||
Jobs run independently of each other.
|
||||
If two jobs match the same filesystem with their ``filesystems`` filter, they will operate on that filesystem independently and potentially in parallel.
|
||||
For example, if job A prunes snapshots that job B is planning to replicate, the replication will fail because B assumed the snapshot to still be present.
|
||||
However, the next replication attempt will re-examine the situation from scratch and should work.
|
||||
|
||||
**Exceptions to the rule**:
|
||||
N push jobs to 1 sink
|
||||
^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
* A ``snap`` and ``push`` job on the same machine can match the same ``filesystems``.
|
||||
To avoid interference, only one of the jobs should be pruning snapshots on the sender, the other one should keep all snapshots.
|
||||
Since the jobs won't coordinate, errors in the log are to be expected, but :ref:`zrepl's ZFS abstractions <zrepl-zfs-abstractions>` ensure that ``push`` and ``sink`` can always replicate incrementally.
|
||||
This scenario is detailed in one of the :ref:`quick-start guides <quickstart-backup-to-external-disk>`.
|
||||
The :ref:`sink job <job-sink>` namespaces by client identity.
|
||||
It is thus safe to push to one sink job with different client identities.
|
||||
If the push jobs have the same client identity, the filesystems matched by the push jobs must be disjoint to avoid races.
|
||||
|
||||
N pull jobs from 1 source
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
More Than 2 Machines
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
Multiple pull jobs pulling from the same source have potential for race conditions during pruning:
|
||||
each pull job prunes the source side independently, causing replication-prune and prune-prune races.
|
||||
|
||||
This section might be relevant to users who wish to *fan-in* (N machines replicate to 1) or *fan-out* (replicate 1 machine to N machines).
|
||||
|
||||
**Working setups**:
|
||||
|
||||
* N ``push`` identities, 1 ``sink`` (as long as the different push jobs have a different :ref:`client identity <overview-passive-side--client-identity>`)
|
||||
|
||||
* ``sink`` constrains each client to a disjoint sub-tree of the sink-side dataset hierarchy ``${root_fs}/${client_identity}``.
|
||||
Therefore, the different clients cannot interfere.
|
||||
|
||||
|
||||
**Setups that do not work**:
|
||||
|
||||
* N ``pull`` identities, 1 ``source`` job. Tracking :issue:`380`.
|
||||
There is currently no way for a pull job to filter which snapshots it should attempt to replicate.
|
||||
Thus, it is not possible to just manually assert that the prune rules of all pull jobs are disjoint to avoid replication-prune and prune-prune races.
|
||||
|
||||
|
||||
@@ -46,8 +46,6 @@ Example Configuration:
|
||||
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
|
||||
regex: "^zrepl_.*"
|
||||
# manually created snapshots will be kept forever on receiver
|
||||
- type: regex
|
||||
regex: "^manual_.*"
|
||||
|
||||
.. DANGER::
|
||||
You might have **existing snapshots** of filesystems affected by pruning which you want to keep, i.e. not be destroyed by zrepl.
|
||||
@@ -84,82 +82,34 @@ Policy ``grid``
|
||||
- type: grid
|
||||
regex: "^zrepl_.*"
|
||||
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
|
||||
│ │ │
|
||||
└─ 1 repetition of a one-hour interval with keep=all
|
||||
│ │
|
||||
└─ 24 repetitions of a one-hour interval with keep=1
|
||||
│
|
||||
└─ 6 repetitions of a 30-day interval with keep=1
|
||||
│ │
|
||||
└─ one hour interval
|
||||
│
|
||||
└─ 24 adjacent one-hour intervals
|
||||
...
|
||||
|
||||
The retention grid can be thought of as a time-based sieve that thins out snapshots as they get older.
|
||||
The retention grid can be thought of as a time-based sieve:
|
||||
The ``grid`` field specifies a list of adjacent time intervals:
|
||||
the left edge of the leftmost (first) interval is the ``creation`` date of the youngest snapshot.
|
||||
All intervals to its right describe time intervals further in the past.
|
||||
|
||||
Each interval carries a maximum number of snapshots to keep.
|
||||
It is specified via ``(keep=N)``, where ``N`` is either ``all`` (all snapshots are kept) or a positive integer.
|
||||
The default value is **keep=1**.
|
||||
|
||||
The ``grid`` field specifies a list of adjacent time intervals.
|
||||
Each interval is a bucket with a maximum capacity of ``keep`` snapshots.
|
||||
The following procedure happens during pruning:
|
||||
|
||||
#. The list of snapshots is filtered by the regular expression in ``regex``.
|
||||
Only snapshots names that match the regex are considered for this rule, all others will be pruned unless another rule keeps them.
|
||||
#. The snapshots that match ``regex`` are placed onto a time axis according to their ``creation`` date.
|
||||
The youngest snapshot is on the left, the oldest on the right.
|
||||
#. The first buckets are placed "under" that axis so that the ``grid`` spec's first bucket's left edge aligns with youngest snapshot.
|
||||
#. All subsequent buckets are placed adjacent to their predecessor bucket.
|
||||
#. Now each snapshot on the axis either falls into one bucket or it is older than our rightmost bucket.
|
||||
Buckets are left-inclusive and right-exclusive which means that a snapshot on the edge of bucket will always 'fall into the right one'.
|
||||
#. Snapshots older than the rightmost bucket **not kept** by this gridspec.
|
||||
#. For each bucket, we only keep the ``keep`` oldest snapshots.
|
||||
Only snapshots names that match the regex are considered for this rule, all others are not affected.
|
||||
#. The filtered list of snapshots is sorted by ``creation``
|
||||
#. The left edge of the first interval is aligned to the ``creation`` date of the youngest snapshot
|
||||
#. A list of buckets is created, one for each interval
|
||||
#. The list of snapshots is split up into the buckets.
|
||||
#. For each bucket
|
||||
|
||||
The syntax to describe the bucket list is as follows:
|
||||
|
||||
::
|
||||
|
||||
Repeat x Duration (keep=all)
|
||||
|
||||
* The **duration** specifies the length of the interval.
|
||||
* The **keep** count specifies the number of snapshots that fit into the bucket.
|
||||
It can be either a positive integer or ``all`` (all snapshots are kept).
|
||||
* The **repeat** count repeats the bucket definition for the specified number of times.
|
||||
|
||||
**Example**:
|
||||
|
||||
::
|
||||
|
||||
This grid spec produces the following list of adjacent buckets. For the sake of simplicity,
|
||||
we subject all snapshots to the grid pruning policy by settings `regex: .*`.
|
||||
|
||||
`
|
||||
grid: 1x1h(keep=all) | 2x2h | 1x3h
|
||||
regex: .*
|
||||
`
|
||||
|
||||
0h 1h 2h 3h 4h 5h 6h 7h 8h 9h
|
||||
| | | | | | | | | |
|
||||
|-Bucket1-|-----Bucket 2------|------Bucket 3-----|-----------Bucket 4----------|
|
||||
| keep=all| keep=1 | keep=1 | keep=1 |
|
||||
|
||||
|
||||
|
||||
Let us consider the following set of snapshots @a-zA-C:
|
||||
|
||||
|
||||
| a b c d e f g h i j k l m n o p q r s t u v w x y z A B C D |
|
||||
|
||||
The `grid` algorithm maps them to their respective buckets:
|
||||
|
||||
Bucket 1: a, b, c
|
||||
Bucket 2: d,e,f,g,h,i,j
|
||||
Bucket 3: k,l,m,n,o,p
|
||||
Bucket 4: q,r, q,r,s,t,u,v,w,x,y,z
|
||||
None: A,B,C,D
|
||||
|
||||
It then applies the per-bucket pruning logic described above which resulting in the
|
||||
following list of remaining snapshots.
|
||||
|
||||
| a b c j p z |
|
||||
|
||||
Note that it only makes sense to grow (not shorten) the interval duration for buckets
|
||||
further in the past since each bucket acts like a low-pass filter for incoming snapshots
|
||||
and adding a less-low-pass-filter after a low-pass one has no effect.
|
||||
#. the contained snapshot list is sorted by creation.
|
||||
#. snapshots from the list, oldest first, are destroyed until the specified ``keep`` count is reached.
|
||||
#. all remaining snapshots on the list are kept.
|
||||
|
||||
|
||||
.. _prune-keep-last-n:
|
||||
@@ -175,11 +125,9 @@ Policy ``last_n``
|
||||
keep_receiver:
|
||||
- type: last_n
|
||||
count: 10
|
||||
regex: ^zrepl_.*$ # optional
|
||||
...
|
||||
|
||||
``last_n`` filters the snapshot list by ``regex``, then keeps the last ``count`` snapshots in that list (last = youngest = most recent creation date)
|
||||
All snapshots that don't match ``regex`` or exceed ``count`` in the filtered list are destroyed unless matched by other rules.
|
||||
``last_n`` keeps the last ``count`` snapshots (last = youngest = most recent creation date).
|
||||
|
||||
.. _prune-keep-regex:
|
||||
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
.. include:: ../global.rst.inc
|
||||
|
||||
|
||||
Replication Options
|
||||
===================
|
||||
|
||||
|
||||
::
|
||||
|
||||
jobs:
|
||||
- type: push
|
||||
filesystems: ...
|
||||
replication:
|
||||
protection:
|
||||
initial: guarantee_resumability # guarantee_{resumability,incremental,nothing}
|
||||
incremental: guarantee_resumability # guarantee_{resumability,incremental,nothing}
|
||||
...
|
||||
|
||||
.. _replication-option-protection:
|
||||
|
||||
``protection`` option
|
||||
--------------------------
|
||||
|
||||
The ``protection`` variable controls the degree to which a replicated filesystem is protected from getting out of sync through a zrepl pruner or external tools that destroy snapshots.
|
||||
zrepl can guarantee :ref:`resumability <step-holds>` or just :ref:`incremental replication <replication-cursor-and-last-received-hold>`.
|
||||
|
||||
``guarantee_resumability`` is the **default** value and guarantees that a replication step is always resumable and that incremental replication will always be possible.
|
||||
The implementation uses replication cursors, last-received-hold and step holds.
|
||||
|
||||
``guarantee_incremental`` only guarantees that incremental replication will always be possible.
|
||||
If a step ``from -> to`` is interrupted and its `to` snapshot is destroyed, zrepl will remove the half-received ``to``'s resume state and start a new step ``from -> to2``.
|
||||
The implementation uses replication cursors, tentative replication cursors and last-received-hold.
|
||||
|
||||
``guarantee_nothing`` does not make any guarantees with regards to keeping sending and receiving side in sync.
|
||||
No bookmarks or holds are created to protect sender and receiver from diverging.
|
||||
|
||||
**Tradeoffs**
|
||||
|
||||
Using ``guarantee_incremental`` instead of ``guarantee_resumability`` obviously removes the resumability guarantee.
|
||||
This means that replication progress is no longer monotonic which might lead to a replication setup that never makes progress if mid-step interruptions are too frequent (e.g. frequent network outages).
|
||||
However, the advantage and :issue:`reason for existence <288>` of the ``incremental`` mode is that it allows the pruner to delete snapshots of interrupted replication steps
|
||||
which is useful if replication happens so rarely (or fails so frequently) that the amount of disk space exclusively referenced by the step's snapshots becomes intolerable.
|
||||
|
||||
.. NOTE::
|
||||
|
||||
When changing this flag, obsoleted zrepl-managed bookmarks and holds will be destroyed on the next replication step that is attempted for each filesystem.
|
||||
|
||||
@@ -16,6 +16,8 @@ Send Options
|
||||
filesystems: ...
|
||||
send:
|
||||
encrypted: true
|
||||
step_holds:
|
||||
disable_incremental: false
|
||||
...
|
||||
|
||||
:ref:`Source<job-source>` and :ref:`push<job-push>` jobs have an optional ``send`` configuration section.
|
||||
@@ -34,6 +36,24 @@ Filesystems matched by ``filesystems`` that are not encrypted are not sent and w
|
||||
|
||||
If ``encryption=false``, zrepl expects that filesystems matching ``filesystems`` are not encrypted or have loaded encryption keys.
|
||||
|
||||
.. _job-send-option-step-holds-disable-incremental:
|
||||
|
||||
``step_holds.disable_incremental`` option
|
||||
-----------------------------------------
|
||||
|
||||
The ``step_holds.disable_incremental`` variable controls whether the creation of :ref:`step holds <step-holds-and-bookmarks>` should be disabled for incremental replication.
|
||||
The default value is ``false``.
|
||||
|
||||
Disabling step holds has the disadvantage that steps :ref:`might not be resumable <step-holds-and-bookmarks>` if interrupted.
|
||||
Non-resumability means that replication progress is no longer monotonic which might result in a replication setup that never makes progress if mid-step interruptions are too frequent (e.g. frequent network outages).
|
||||
|
||||
However, the advantage and :issue:`reason for existence <288>` of this flag is that it allows the pruner to delete snapshots of interrupted replication steps
|
||||
which is useful if replication happens so rarely (or fails so frequently) that the amount of disk space exclusively referenced by the step's snapshots becomes intolerable.
|
||||
|
||||
.. NOTE::
|
||||
|
||||
When setting this flag to ``true``, existing step holds for the job will be destroyed on the next replication attempt.
|
||||
|
||||
.. _job-recv-options:
|
||||
|
||||
Recv Options
|
||||
|
||||
@@ -131,7 +131,7 @@ The following environment variables are set:
|
||||
* ``ZREPL_SNAPNAME``: the zrepl-generated snapshot name (e.g. ``zrepl_20380119_031407_000``)
|
||||
* ``ZREPL_DRYRUN``: set to ``"true"`` if a dry run is in progress so scripts can print, but not run, their commands
|
||||
|
||||
An empty template hook can be found in :sampleconf:`/hooks/template.sh`.
|
||||
An empty template hook can be found in :sampleconf:`hooks/template.sh`.
|
||||
|
||||
.. _job-hook-type-postgres-checkpoint:
|
||||
|
||||
|
||||
@@ -91,8 +91,7 @@ The ``tls`` transport uses TCP + TLS with client authentication using client cer
|
||||
The client identity is the common name (CN) presented in the client certificate.
|
||||
|
||||
It is recommended to set up a dedicated CA infrastructure for this transport, e.g. using OpenVPN's `EasyRSA <https://github.com/OpenVPN/easy-rsa>`_.
|
||||
For a simple 2-machine setup, mutual TLS might also be sufficient.
|
||||
We provide :ref:`copy-pastable instructions to generate the certificates below <transport-tcp+tlsclientauth-certgen>`.
|
||||
For a simple 2-machine setup, see the :ref:`instructions below<transport-tcp+tlsclientauth-2machineopenssl>`.
|
||||
|
||||
The implementation uses `Go's TLS library <https://golang.org/pkg/crypto/tls/>`_.
|
||||
Since Go binaries are statically linked, you or your distribution need to recompile zrepl when vulnerabilities in that library are disclosed.
|
||||
@@ -104,16 +103,6 @@ If intermediate CAs are used, the **full chain** must be present in either in th
|
||||
Regardless, the client's certificate must be first in the ``cert`` file, with each following certificate directly certifying the one preceding it (see `TLS's specification <https://tools.ietf.org/html/rfc5246#section-7.4.2>`_).
|
||||
This is the common default when using a CA management tool.
|
||||
|
||||
.. NOTE::
|
||||
|
||||
As of Go 1.15 (zrepl 0.3.0 and newer), the Go TLS / x509 library **requrires Subject Alternative Names**
|
||||
be present in certificates. You might need to re-generate your certificates using one of the :ref:`two alternatives
|
||||
provided below<transport-tcp+tlsclientauth-certgen>`.
|
||||
|
||||
Note further that zrepl continues to use the CommonName field to assign client identities.
|
||||
Hence, we recommend to keep the Subject Alternative Name and the CommonName in sync.
|
||||
|
||||
|
||||
Serve
|
||||
~~~~~
|
||||
|
||||
@@ -158,25 +147,45 @@ The ``server_cn`` specifies the expected common name (CN) of the server's certif
|
||||
It overrides the hostname specified in ``address``.
|
||||
The connection fails if either do not match.
|
||||
|
||||
.. _transport-tcp+tlsclientauth-certgen:
|
||||
|
||||
.. _transport-tcp+tlsclientauth-2machineopenssl:
|
||||
|
||||
Mutual-TLS between Two Machines
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
Self-Signed Certificates
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Tools like `EasyRSA <https://github.com/OpenVPN/easy-rsa>`_ make it easy to manage CA infrastructure for multiple clients, e.g. a central zrepl backup server (in sink mode).
|
||||
However, for a two-machine setup, self-signed certificates distributed using an out-of-band mechanism will also work just fine:
|
||||
|
||||
Suppose you have a push-mode setup, with `backups.example.com` running the :ref:`sink job <job-sink>`, and `prod.example.com` running the :ref:`push job <job-push>`.
|
||||
Run the following OpenSSL commands on each host, substituting HOSTNAME in both filenames and the interactive input prompt by OpenSSL:
|
||||
|
||||
.. code-block:: bash
|
||||
:emphasize-lines: 1-5,24
|
||||
|
||||
(name=HOSTNAME; openssl req -x509 -sha256 -nodes \
|
||||
-newkey rsa:4096 \
|
||||
-days 365 \
|
||||
-keyout $name.key \
|
||||
-out $name.crt -addext "subjectAltName = DNS:$name" -subj "/CN=$name")
|
||||
openssl req -x509 -sha256 -nodes \
|
||||
-newkey rsa:4096 \
|
||||
-days 365 \
|
||||
-keyout HOSTNAME.key \
|
||||
-out HOSTNAME.crt
|
||||
|
||||
#Generating a 4096 bit RSA private key
|
||||
#................++++
|
||||
#.++++
|
||||
#writing new private key to 'backups.key'
|
||||
#-----
|
||||
#You are about to be asked to enter information that will be incorporated
|
||||
#into your certificate request.
|
||||
#What you are about to enter is what is called a Distinguished Name or a DN.
|
||||
#There are quite a few fields but you can leave some blank
|
||||
#For some fields there will be a default value,
|
||||
#If you enter '.', the field will be left blank.
|
||||
#-----
|
||||
#Country Name (2 letter code) [XX]:
|
||||
#State or Province Name (full name) []:
|
||||
#Locality Name (eg, city) [Default City]:
|
||||
#Organization Name (eg, company) [Default Company Ltd]:
|
||||
#Organizational Unit Name (eg, section) []:
|
||||
#Common Name (eg, your name or your server's hostname) []:HOSTNAME
|
||||
#Email Address []:
|
||||
|
||||
Now copy each machine's ``HOSTNAME.crt`` to the other machine's ``/etc/zrepl/HOSTNAME.crt``, for example using `scp`.
|
||||
The serve & connect configuration will thus look like the following:
|
||||
@@ -207,36 +216,6 @@ The serve & connect configuration will thus look like the following:
|
||||
...
|
||||
|
||||
|
||||
Certificate Authority using EasyRSA
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
For more than two machines, it might make sense to set up a CA infrastructure.
|
||||
Tools like `EasyRSA <https://github.com/OpenVPN/easy-rsa>`_ make this very easy:
|
||||
|
||||
::
|
||||
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
HOSTS=(backupserver prod1 prod2 prod3)
|
||||
|
||||
curl -L https://github.com/OpenVPN/easy-rsa/releases/download/v3.0.7/EasyRSA-3.0.7.tgz > EasyRSA-3.0.7.tgz
|
||||
echo "157d2e8c115c3ad070c1b2641a4c9191e06a32a8e50971847a718251eeb510a8 EasyRSA-3.0.7.tgz" | sha256sum -c
|
||||
rm -rf EasyRSA-3.0.7
|
||||
tar -xf EasyRSA-3.0.7.tgz
|
||||
cd EasyRSA-3.0.7
|
||||
./easyrsa
|
||||
./easyrsa init-pki
|
||||
./easyrsa build-ca nopass
|
||||
|
||||
for host in "${HOSTS[@]}"; do
|
||||
./easyrsa build-serverClient-full $host nopass
|
||||
echo cert for host $host available at pki/issued/$host.crt
|
||||
echo key for host $host available at pki/private/$host.key
|
||||
done
|
||||
echo ca cert available at pki/ca.crt
|
||||
|
||||
|
||||
.. _transport-ssh+stdinserver:
|
||||
|
||||
``ssh+stdinserver`` Transport
|
||||
|
||||
@@ -58,11 +58,13 @@ for latest_patch in latest_by_major_minor:
|
||||
cmdline.append("--whitelist-tags")
|
||||
cmdline.append(f"^{re.escape(latest_patch.orig)}$")
|
||||
|
||||
# we want flexibility to update docs for the latest stable release
|
||||
# => we have a branch for that, called `stable` which we move manually
|
||||
# TODO: in the future, have f"stable-{latest_by_major_minor[-1]}"
|
||||
default_version = "stable"
|
||||
cmdline.extend(["--whitelist-branches", default_version])
|
||||
# we want to render the latest non-rc version as the default page
|
||||
# (latest_by_major_minor is already sorted)
|
||||
default_version = latest_by_major_minor[-1]
|
||||
for tag in reversed(latest_by_major_minor):
|
||||
if tag.rc == 0:
|
||||
default_version = tag
|
||||
break
|
||||
|
||||
cmdline.extend(["--root-ref", f"{default_version}"])
|
||||
cmdline.extend(["--banner-main-ref", f"{default_version}"])
|
||||
@@ -71,4 +73,4 @@ cmdline.extend(["--sort", "semver"])
|
||||
|
||||
cmdline.extend(["--whitelist-branches", "master"])
|
||||
|
||||
print(" ".join(cmdline))
|
||||
print(" ".join(cmdline))
|
||||
+1
-1
@@ -13,7 +13,7 @@
|
||||
:target: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96
|
||||
.. |Donate via Liberapay| image:: https://img.shields.io/liberapay/patrons/zrepl.svg?logo=liberapay
|
||||
:target: https://liberapay.com/zrepl/donate
|
||||
.. |Donate via Patreon| image:: https://img.shields.io/badge/dynamic/json?color=yellow&label=Patreon&query=data.attributes.patron_count&suffix=%20patrons&url=https%3A%2F%2Fwww.patreon.com%2Fapi%2Fcampaigns%2F3095079
|
||||
.. |Donate via Patreon| image:: https://img.shields.io/endpoint.svg?url=https%3A%2F%2Fshieldsio-patreon.herokuapp.com%2Fzrepl%2Fpledges&style=flat&color=yellow
|
||||
:target: https://www.patreon.com/zrepl
|
||||
.. |Donate via GitHub Sponsors| image:: https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&style=flat&color=yellow
|
||||
:target: https://github.com/sponsors/problame
|
||||
|
||||
@@ -14,7 +14,6 @@ Installation
|
||||
installation/user-privileges
|
||||
installation/packages
|
||||
installation/apt-repos
|
||||
installation/rpm-repos
|
||||
installation/compile-from-source
|
||||
installation/freebsd-jail-with-iocage
|
||||
installation/what-next
|
||||
|
||||
@@ -7,7 +7,7 @@ Debian / Ubuntu APT repositories
|
||||
We maintain APT repositories for Debian, Ubuntu and derivatives.
|
||||
The fingerprint of the signing key is ``E101 418F D3D6 FBCB 9D65 A62D 7086 99FC 5F2E BF16``.
|
||||
It is available at `<https://zrepl.cschwarz.com/apt/apt-key.asc>`_ .
|
||||
Please open an issue in on GitHub if you encounter any issues with the repository.
|
||||
Please open an issue `in the packaging repository <https://github.com/zrepl/debian-binary-packaging>`_ if you encounter any issues with the repository.
|
||||
|
||||
The following snippet configure the repository for your Debian or Ubuntu release:
|
||||
|
||||
@@ -24,6 +24,6 @@ The following snippet configure the repository for your Debian or Ubuntu release
|
||||
|
||||
.. NOTE::
|
||||
|
||||
Until zrepl reaches 1.0, the repositories will be updated to the latest zrepl release immediately.
|
||||
Until zrepl reaches 1.0, all APT repositories will be updated to the latest zrepl release immediately.
|
||||
This includes breaking changes between zrepl versions.
|
||||
Use ``apt-mark hold zrepl`` to prevent upgrades of zrepl.
|
||||
|
||||
@@ -9,35 +9,37 @@ Producing a release requires **Go 1.11** or newer and **Python 3** + **pip3** +
|
||||
A tutorial to install Go is available over at `golang.org <https://golang.org/doc/install>`_.
|
||||
Python and pip3 should probably be installed via your distro's package manager.
|
||||
|
||||
::
|
||||
cd to/your/zrepl/checkout
|
||||
python3 -m venv3
|
||||
source venv3/bin/activate
|
||||
./lazy.sh devsetup
|
||||
make release
|
||||
# build artifacts are available in ./artifacts/release
|
||||
|
||||
The Python venv is used for the documentation build dependencies.
|
||||
If you just want to build the zrepl binary, leave it out and use `./lazy.sh godep` instead.
|
||||
|
||||
Alternatively, you can use the Docker build process:
|
||||
it is used to produce the official zrepl `binary releases`_
|
||||
and serves as a reference for build dependencies and procedure:
|
||||
|
||||
::
|
||||
|
||||
cd to/your/zrepl/checkout
|
||||
# make sure your user has access to the docker socket
|
||||
make release-docker
|
||||
# if you want .deb or .rpm packages, invoke the follwoing
|
||||
# targets _after_ you invoked release-docker
|
||||
make deb-docker
|
||||
make rpm-docker
|
||||
# build artifacts are available in ./artifacts/release
|
||||
# packages are available in ./artifacts
|
||||
git clone https://github.com/zrepl/zrepl.git && \
|
||||
cd zrepl && \
|
||||
sudo docker build -t zrepl_build -f build.Dockerfile . && \
|
||||
sudo docker run -it --rm \
|
||||
-v "${PWD}:/src" \
|
||||
--user "$(id -u):$(id -g)" \
|
||||
zrepl_build make release
|
||||
|
||||
Alternatively, you can install build dependencies on your local system and then build in your ``$GOPATH``:
|
||||
|
||||
::
|
||||
|
||||
mkdir -p "${GOPATH}/src/github.com/zrepl/zrepl"
|
||||
git clone https://github.com/zrepl/zrepl.git "${GOPATH}/src/github.com/zrepl/zrepl"
|
||||
cd "${GOPATH}/src/github.com/zrepl/zrepl"
|
||||
python3 -m venv3
|
||||
source venv3/bin/activate
|
||||
./lazy.sh devsetup
|
||||
make release
|
||||
|
||||
The Python venv is used for the documentation build dependencies.
|
||||
If you just want to build the zrepl binary, leave it out and use `./lazy.sh godep` instead.
|
||||
Either way, all build results are located in the ``artifacts/`` directory.
|
||||
|
||||
.. NOTE::
|
||||
|
||||
It is your job to install the built binary in the zrepl users's ``$PATH``, e.g. ``/usr/local/bin/zrepl``.
|
||||
It is your job to install the appropriate binary in the zrepl users's ``$PATH``, e.g. ``/usr/local/bin/zrepl``.
|
||||
Otherwise, the examples in the :ref:`quick-start guides <quickstart-toc>` may need to be adjusted.
|
||||
|
||||
@@ -30,12 +30,15 @@ The following list may be incomplete, feel free to submit a PR with an update:
|
||||
* - Arch Linux
|
||||
- ``yay install zrepl``
|
||||
- Available on `AUR <https://aur.archlinux.org/packages/zrepl>`_
|
||||
* - Fedora, CentOS, RHEL, OpenSUSE
|
||||
* - Fedora
|
||||
- ``dnf install zrepl``
|
||||
- :ref:`RPM repository config <installation-rpm-repos>`
|
||||
- Available on `COPR <https://copr.fedorainfracloud.org/coprs/poettlerric/zrepl/>`_
|
||||
* - CentOS/RHEL
|
||||
- ``yum install zrepl``
|
||||
- Available on `COPR <https://copr.fedorainfracloud.org/coprs/poettlerric/zrepl/>`_
|
||||
* - Debian + Ubuntu
|
||||
- ``apt install zrepl``
|
||||
- :ref:`APT repository config <installation-apt-repos>`
|
||||
- APT repository config :ref:`see below <installation-apt-repos>`
|
||||
* - OmniOS
|
||||
- ``pkg install zrepl``
|
||||
- Available since `r151030 <https://pkg.omniosce.org/r151030/extra/en/search.shtml?token=zrepl&action=Search>`_
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
.. _installation-rpm-repos:
|
||||
|
||||
RPM repositories
|
||||
~~~~~~~~~~~~~~~~
|
||||
|
||||
We provide a single RPM repository for all RPM-based Linux distros.
|
||||
The zrepl binary in the repo is the same as the one published to GitHub.
|
||||
Since Go binaries are statically linked, the RPM should work about everywhere.
|
||||
|
||||
The fingerprint of the signing key is ``F6F6 E8EA 6F2F 1462 2878 B5DE 50E3 4417 826E 2CE6``.
|
||||
It is available at `<https://zrepl.cschwarz.com/rpm/rpm-key.asc>`_ .
|
||||
Please open an issue on GitHub if you encounter any issues with the repository.
|
||||
|
||||
Copy-paste the following snippet into your shell to set up the zrepl repository.
|
||||
Then ``dnf install zrepl`` and make sure to confirm that the signing key matches the one shown above.
|
||||
|
||||
::
|
||||
|
||||
cat > /etc/yum.repos.d/zrepl.repo <<EOF
|
||||
[zrepl]
|
||||
name = zrepl
|
||||
baseurl = https://zrepl.cschwarz.com/rpm/repo
|
||||
gpgkey = https://zrepl.cschwarz.com/rpm/rpm-key.asc
|
||||
EOF
|
||||
|
||||
.. NOTE::
|
||||
|
||||
Until zrepl reaches 1.0, the repository will be updated to the latest zrepl release immediately.
|
||||
This includes breaking changes between zrepl versions.
|
||||
If that bothers you, use the `dnf versionlock plugin <https://dnf-plugins-core.readthedocs.io/en/latest/versionlock.html>`_ to pin the version of zrepl on your system.
|
||||
+7
-32
@@ -1,30 +1,14 @@
|
||||
#!/bin/bash
|
||||
set -eo pipefail
|
||||
|
||||
|
||||
NON_INTERACTIVE=false
|
||||
DO_CLONE=false
|
||||
while getopts "ca" arg; do
|
||||
case "$arg" in
|
||||
"a")
|
||||
NON_INTERACTIVE=true
|
||||
;;
|
||||
"c")
|
||||
DO_CLONE=true
|
||||
;;
|
||||
*)
|
||||
echo invalid option
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
GHPAGESREPO="git@github.com:zrepl/zrepl.github.io.git"
|
||||
SCRIPTDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
|
||||
PUBLICDIR="${SCRIPTDIR}/public_git"
|
||||
|
||||
checkout_repo_msg() {
|
||||
echo "clone ${GHPAGESREPO} to ${PUBLICDIR}:"
|
||||
echo " git clone ${GHPAGESREPO} ${PUBLICDIR}"
|
||||
git clone "${GHPAGESREPO}" "${PUBLICDIR}"
|
||||
}
|
||||
|
||||
exit_msg() {
|
||||
@@ -41,19 +25,11 @@ cd "$SCRIPTDIR"
|
||||
|
||||
if [ ! -d "$PUBLICDIR" ]; then
|
||||
checkout_repo_msg
|
||||
if $DO_CLONE; then
|
||||
git clone "${GHPAGESREPO}" "${PUBLICDIR}"
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if $NON_INTERACTIVE; then
|
||||
echo "non-interactive mode"
|
||||
else
|
||||
echo -n "PRESS ENTER to confirm you commited and pushed docs changes and tags to the zrepl repo"
|
||||
read -r
|
||||
fi
|
||||
echo -n "PRESS ENTER to confirm you commited and pushed docs changes and tags to the zrepl repo"
|
||||
read
|
||||
|
||||
pushd "$PUBLICDIR"
|
||||
|
||||
@@ -76,10 +52,9 @@ popd
|
||||
|
||||
echo "building site"
|
||||
|
||||
flags="$(python3 gen-sphinx-versioning-flags.py)"
|
||||
set -e
|
||||
sphinx-versioning build \
|
||||
$flags \
|
||||
$(python3 gen-sphinx-versioning-flags.py) \
|
||||
docs ./public_git \
|
||||
-- -c sphinxconf # older conf.py throw errors because they used
|
||||
# version = subprocess.show_output(["git", "describe"])
|
||||
@@ -91,7 +66,7 @@ git status --porcelain
|
||||
if [[ "$(git status --porcelain)" != "" ]]; then
|
||||
CURRENT_COMMIT="${CURRENT_COMMIT}(dirty)"
|
||||
fi
|
||||
COMMIT_MSG="sphinx-versioning render from publish.sh - $(date -u) - ${CURRENT_COMMIT}"
|
||||
COMMIT_MSG="sphinx-versioning render from publish.sh - `date -u` - ${CURRENT_COMMIT}"
|
||||
|
||||
pushd "$PUBLICDIR"
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ A few additional requirements:
|
||||
* We want to be able to put off the backups for more than three weeks, i.e., longer than the lifetime of the automatically created snapshots on our workstation.
|
||||
**zrepl should use bookmarks and holds to achieve this goal**.
|
||||
* When we yank out the drive during replication and go on a long vacation, we do *not* want the partially replicated snapshot to stick around as it would hold on to too much disk space over time.
|
||||
Therefore, we want zrepl to deviate from its :ref:`default behavior <replication-option-protection>` and sacrifice resumability, but nonetheless retain the ability to do incremental replication once we return from our vacation.
|
||||
Therefore, we want zrepl to deviate from its :ref:`default step-hold behavior <step-holds-and-bookmarks>` and sacrifice resumability, but nonetheless retain the ability to do incremental replication once we return from our vacation.
|
||||
**zrepl should provide an easy config knob to disable step holds for incremental replication**.
|
||||
|
||||
The following config snippet implements the setup described above.
|
||||
|
||||
@@ -51,18 +51,21 @@ To get things going quickly, we skip setting up a CA and generate two self-signe
|
||||
For convenience, we generate the key pairs on our local machine and distribute them using ssh:
|
||||
|
||||
.. code-block:: bash
|
||||
:emphasize-lines: 6,13
|
||||
|
||||
(name=backups; openssl req -x509 -sha256 -nodes \
|
||||
-newkey rsa:4096 \
|
||||
-days 365 \
|
||||
-keyout $name.key \
|
||||
-out $name.crt -addext "subjectAltName = DNS:$name" -subj "/CN=$name")
|
||||
openssl req -x509 -sha256 -nodes \
|
||||
-newkey rsa:4096 \
|
||||
-days 365 \
|
||||
-keyout backups.key \
|
||||
-out backups.crt
|
||||
# ... and use "backups" as Common Name (CN)
|
||||
|
||||
(name=prod; openssl req -x509 -sha256 -nodes \
|
||||
-newkey rsa:4096 \
|
||||
-days 365 \
|
||||
-keyout $name.key \
|
||||
-out $name.crt -addext "subjectAltName = DNS:$name" -subj "/CN=$name")
|
||||
openssl req -x509 -sha256 -nodes \
|
||||
-newkey rsa:4096 \
|
||||
-days 365 \
|
||||
-keyout prod.key \
|
||||
-out prod.crt
|
||||
# ... and use "prod" as Common Name (CN)
|
||||
|
||||
ssh root@backups "mkdir /etc/zrepl"
|
||||
scp backups.key backups.crt prod.crt root@backups:/etc/zrepl
|
||||
@@ -91,4 +94,4 @@ We define a corresponding **sink job** named ``sink`` in ``/etc/zrepl/zrepl.yml`
|
||||
Go Back To Quickstart Guide
|
||||
---------------------------
|
||||
|
||||
:ref:`Click here <quickstart-apply-config>` to go back to the quickstart guide.
|
||||
:ref:`Click here <quickstart-apply-config>` to go back to the quickstart guide.
|
||||
+1
-6
@@ -32,11 +32,6 @@ We would like to thank the following people and organizations for supporting zre
|
||||
|
||||
<div class="fa fa-code" style="width: 1em;"></div>
|
||||
|
||||
* |supporter-gold| Cyberiada GmbH
|
||||
* |supporter-std| `Gordon Schulz <https://github.com/azmodude>`_
|
||||
* |supporter-std| `@jwittlincohen <https://github.com/jwittlincohen>`_
|
||||
* |supporter-std| `Michael D. Schmitt <https://waterbendingscroll.dancingdragons.org>`_
|
||||
* |supporter-std| `Hans Schulz <https://github.com/schulzh>`_
|
||||
* |supporter-std| Henning Kessler
|
||||
* |supporter-std| `John Ramsden <https://github.com/johnramsden>`_
|
||||
* |supporter-std| `DrLuke <https://github.com/drluke>`_
|
||||
@@ -48,7 +43,7 @@ We would like to thank the following people and organizations for supporting zre
|
||||
* |supporter-gold| `MNX.io <https://mnx.io>`_
|
||||
* |supporter-std| `Marshall Clyburn <https://github.com/mdclyburn>`_
|
||||
* |supporter-code| `Ross Williams <https://github.com/overhacked>`_
|
||||
* |supporter-gold| Mike T.
|
||||
* |supporter-std| Mike T.
|
||||
* |supporter-code| `Justin Scholz <https://github.com/JMoVS>`_
|
||||
* |supporter-code| `InsanePrawn <https://github.com/InsanePrawn>`_
|
||||
* |supporter-code| `Ben Woods <https://www.freshports.org/sysutils/zrepl/>`_
|
||||
|
||||
+169
-101
@@ -10,7 +10,6 @@ import (
|
||||
|
||||
"github.com/kr/pretty"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
|
||||
"github.com/zrepl/zrepl/replication/logic/pdu"
|
||||
@@ -19,12 +18,21 @@ import (
|
||||
"github.com/zrepl/zrepl/util/envconst"
|
||||
"github.com/zrepl/zrepl/util/semaphore"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
zfsprop "github.com/zrepl/zrepl/zfs/property"
|
||||
)
|
||||
|
||||
type SenderConfig struct {
|
||||
FSF zfs.DatasetFilter
|
||||
Encrypt *zfs.NilBool
|
||||
JobID JobID
|
||||
FSF zfs.DatasetFilter
|
||||
DisableIncrementalStepHolds bool
|
||||
JobID JobID
|
||||
|
||||
Encrypt *zfs.NilBool
|
||||
SendRaw bool
|
||||
SendProperties bool
|
||||
SendBackupProperties bool
|
||||
SendLargeBlocks bool
|
||||
SendCompressed bool
|
||||
SendEmbeddedData bool
|
||||
}
|
||||
|
||||
func (c *SenderConfig) Validate() error {
|
||||
@@ -40,9 +48,17 @@ func (c *SenderConfig) Validate() error {
|
||||
|
||||
// Sender implements replication.ReplicationEndpoint for a sending side
|
||||
type Sender struct {
|
||||
FSFilter zfs.DatasetFilter
|
||||
encrypt *zfs.NilBool
|
||||
jobId JobID
|
||||
FSFilter zfs.DatasetFilter
|
||||
disableIncrementalStepHolds bool
|
||||
jobId JobID
|
||||
|
||||
encrypt *zfs.NilBool
|
||||
sendRaw bool
|
||||
sendProperties bool
|
||||
sendBackupProperties bool
|
||||
sendLargeBlocks bool
|
||||
sendCompressed bool
|
||||
sendEmbeddedData bool
|
||||
}
|
||||
|
||||
func NewSender(conf SenderConfig) *Sender {
|
||||
@@ -50,9 +66,17 @@ func NewSender(conf SenderConfig) *Sender {
|
||||
panic("invalid config" + err.Error())
|
||||
}
|
||||
return &Sender{
|
||||
FSFilter: conf.FSF,
|
||||
encrypt: conf.Encrypt,
|
||||
jobId: conf.JobID,
|
||||
FSFilter: conf.FSF,
|
||||
disableIncrementalStepHolds: conf.DisableIncrementalStepHolds,
|
||||
jobId: conf.JobID,
|
||||
|
||||
encrypt: conf.Encrypt,
|
||||
sendRaw: conf.SendRaw,
|
||||
sendProperties: conf.SendProperties,
|
||||
sendBackupProperties: conf.SendBackupProperties,
|
||||
sendLargeBlocks: conf.SendLargeBlocks,
|
||||
sendCompressed: conf.SendCompressed,
|
||||
sendEmbeddedData: conf.SendEmbeddedData,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,8 +193,15 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.Rea
|
||||
FS: r.Filesystem,
|
||||
From: uncheckedSendArgsFromPDU(r.GetFrom()), // validated by zfs.ZFSSendDry / zfs.ZFSSend
|
||||
To: uncheckedSendArgsFromPDU(r.GetTo()), // validated by zfs.ZFSSendDry / zfs.ZFSSend
|
||||
Encrypted: s.encrypt,
|
||||
ResumeToken: r.ResumeToken, // nil or not nil, depending on decoding success
|
||||
ResumeToken: r.ResumeToken, // nil or not nil, depending on decoding success
|
||||
|
||||
Encrypted: s.encrypt,
|
||||
Properties: s.sendProperties,
|
||||
BackupProperties: s.sendBackupProperties,
|
||||
Raw: s.sendRaw,
|
||||
LargeBlocks: s.sendLargeBlocks,
|
||||
Compressed: s.sendCompressed,
|
||||
EmbeddedData: s.sendEmbeddedData,
|
||||
}
|
||||
|
||||
sendArgs, err := sendArgsUnvalidated.Validate(ctx)
|
||||
@@ -178,6 +209,21 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.Rea
|
||||
return nil, nil, errors.Wrap(err, "validate send arguments")
|
||||
}
|
||||
|
||||
if s.sendRaw {
|
||||
encryptionSupported, err := zfs.EncryptionCLISupported(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Wrap(err, "cannot determine CLI native encryption support")
|
||||
}
|
||||
|
||||
if !encryptionSupported {
|
||||
// zfs-send(8) about `send -w`: "For unencrypted datasets, this flag will be equivalent to -Lec."
|
||||
sendArgs.Raw = false
|
||||
sendArgs.LargeBlocks = true // -L
|
||||
sendArgs.Compressed = true // -c
|
||||
sendArgs.EmbeddedData = true // -e
|
||||
}
|
||||
}
|
||||
|
||||
getLogger(ctx).Debug("acquire concurrent send semaphore")
|
||||
// TODO use try-acquire and fail with resource-exhaustion rpc status
|
||||
// => would require handling on the client-side
|
||||
@@ -209,27 +255,37 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.Rea
|
||||
return res, nil, nil
|
||||
}
|
||||
|
||||
// create holds or bookmarks of `From` and `To` to guarantee one of the following:
|
||||
// - that the replication step can always be resumed (`holds`),
|
||||
// - that the replication step can be interrupted and a future replication
|
||||
// step with same or different `To` but same `From` is still possible (`bookmarks`)
|
||||
// - nothing (`none`)
|
||||
//
|
||||
// ...
|
||||
//
|
||||
// ... actually create the abstractions
|
||||
replicationGuaranteeOptions, err := replicationGuaranteeOptionsFromPDU(r.GetReplicationConfig().Protection)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
// create a replication cursor for `From` (usually an idempotent no-op because SendCompleted already created it before)
|
||||
var fromReplicationCursor Abstraction
|
||||
if sendArgs.From != nil {
|
||||
// For all but the first replication, this should always be a no-op because SendCompleted already moved the cursor
|
||||
fromReplicationCursor, err = CreateReplicationCursor(ctx, sendArgs.FS, *sendArgs.FromVersion, s.jobId) // no shadow
|
||||
if err == zfs.ErrBookmarkCloningNotSupported {
|
||||
getLogger(ctx).Debug("not creating replication cursor from bookmark because ZFS does not support it")
|
||||
// fallthrough
|
||||
} else if err != nil {
|
||||
return nil, nil, errors.Wrap(err, "cannot set replication cursor to `from` version before starting send")
|
||||
}
|
||||
}
|
||||
replicationGuaranteeStrategy := replicationGuaranteeOptions.Strategy(sendArgs.From != nil)
|
||||
liveAbs, err := replicationGuaranteeStrategy.SenderPreSend(ctx, s.jobId, &sendArgs)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
|
||||
takeStepHolds := sendArgs.FromVersion == nil || !s.disableIncrementalStepHolds
|
||||
|
||||
var fromHold, toHold Abstraction
|
||||
// make sure `From` doesn't go away in order to make this step resumable
|
||||
if sendArgs.From != nil && takeStepHolds {
|
||||
fromHold, err = HoldStep(ctx, sendArgs.FS, *sendArgs.FromVersion, s.jobId) // no shadow
|
||||
if err == zfs.ErrBookmarkCloningNotSupported {
|
||||
getLogger(ctx).Debug("not creating step bookmark because ZFS does not support it")
|
||||
// fallthrough
|
||||
} else if err != nil {
|
||||
return nil, nil, errors.Wrapf(err, "cannot hold `from` version %q before starting send", *sendArgs.FromVersion)
|
||||
}
|
||||
}
|
||||
for _, a := range liveAbs {
|
||||
if a != nil {
|
||||
abstractionsCacheSingleton.Put(a)
|
||||
if takeStepHolds {
|
||||
// make sure `To` doesn't go away in order to make this step resumable
|
||||
toHold, err = HoldStep(ctx, sendArgs.FS, sendArgs.ToVersion, s.jobId)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Wrapf(err, "cannot hold `to` version %q before starting send", sendArgs.ToVersion)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -249,6 +305,7 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.Rea
|
||||
//
|
||||
// Note further that a resuming send, due to the idempotent nature of func CreateReplicationCursor and HoldStep,
|
||||
// will never lose its step holds because we just (idempotently re-)created them above, before attempting the cleanup.
|
||||
liveAbs := []Abstraction{fromHold, toHold, fromReplicationCursor}
|
||||
func() {
|
||||
ctx, endSpan := trace.WithSpan(ctx, "cleanup-stale-abstractions")
|
||||
defer endSpan()
|
||||
@@ -270,29 +327,22 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.Rea
|
||||
for _, staleVersion := range obsoleteAbs {
|
||||
for _, mustLiveVersion := range mustLiveVersions {
|
||||
isSendArg := zfs.FilesystemVersionEqualIdentity(mustLiveVersion, staleVersion.GetFilesystemVersion())
|
||||
stepHoldBasedGuaranteeStrategy := false
|
||||
k := replicationGuaranteeStrategy.Kind()
|
||||
switch k {
|
||||
case ReplicationGuaranteeKindResumability:
|
||||
stepHoldBasedGuaranteeStrategy = true
|
||||
case ReplicationGuaranteeKindIncremental:
|
||||
case ReplicationGuaranteeKindNone:
|
||||
default:
|
||||
panic(fmt.Sprintf("this is supposed to be an exhaustive match, got %v", k))
|
||||
}
|
||||
isSnapshot := mustLiveVersion.IsSnapshot()
|
||||
if isSendArg && (!isSnapshot || stepHoldBasedGuaranteeStrategy) {
|
||||
isStepHoldWeMightHaveCreatedWithCurrentValueOf_takeStepHolds :=
|
||||
takeStepHolds && staleVersion.GetType() == AbstractionStepHold
|
||||
if isSendArg && isStepHoldWeMightHaveCreatedWithCurrentValueOf_takeStepHolds {
|
||||
panic(fmt.Sprintf("impl error: %q would be destroyed because it is considered stale but it is part of of sendArgs=%s", mustLiveVersion.String(), pretty.Sprint(sendArgs)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
destroyTypes := AbstractionTypeSet{
|
||||
AbstractionStepHold: true,
|
||||
AbstractionTentativeReplicationCursorBookmark: true,
|
||||
}
|
||||
abstractionsCacheSingleton.TryBatchDestroy(ctx, s.jobId, sendArgs.FS, destroyTypes, keep, check)
|
||||
sendAbstractionsCacheSingleton.TryBatchDestroy(ctx, s.jobId, sendArgs.FS, keep, check)
|
||||
}()
|
||||
// now add the newly created abstractions to the cleaned-up cache
|
||||
for _, a := range liveAbs {
|
||||
if a != nil {
|
||||
sendAbstractionsCacheSingleton.Put(a)
|
||||
}
|
||||
}
|
||||
|
||||
sendStream, err := zfs.ZFSSend(ctx, sendArgs)
|
||||
if err != nil {
|
||||
@@ -326,32 +376,36 @@ func (p *Sender) SendCompleted(ctx context.Context, r *pdu.SendCompletedReq) (*p
|
||||
return nil, errors.Wrap(err, "validate `to` exists")
|
||||
}
|
||||
|
||||
replicationGuaranteeOptions, err := replicationGuaranteeOptionsFromPDU(orig.GetReplicationConfig().Protection)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
liveAbs, err := replicationGuaranteeOptions.Strategy(from != nil).SenderPostRecvConfirmed(ctx, p.jobId, fs, to)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, a := range liveAbs {
|
||||
if a != nil {
|
||||
abstractionsCacheSingleton.Put(a)
|
||||
log := func(ctx context.Context) Logger {
|
||||
log := getLogger(ctx).WithField("to_guid", to.Guid).
|
||||
WithField("fs", fs).
|
||||
WithField("to", to.RelName)
|
||||
if from != nil {
|
||||
log = log.WithField("from", from.RelName).WithField("from_guid", from.Guid)
|
||||
}
|
||||
return log
|
||||
}
|
||||
keep := func(a Abstraction) (keep bool) {
|
||||
keep = false
|
||||
for _, k := range liveAbs {
|
||||
keep = keep || AbstractionEquals(a, k)
|
||||
|
||||
toReplicationCursor, err := CreateReplicationCursor(ctx, fs, to, p.jobId)
|
||||
if err != nil {
|
||||
if err == zfs.ErrBookmarkCloningNotSupported {
|
||||
log(ctx).Debug("not setting replication cursor, bookmark cloning not supported")
|
||||
} else {
|
||||
msg := "cannot move replication cursor, keeping hold on `to` until successful"
|
||||
log(ctx).WithError(err).Error(msg)
|
||||
err = errors.Wrap(err, msg)
|
||||
// it is correct to not destroy from and to step holds if we can't move the cursor!
|
||||
return &pdu.SendCompletedRes{}, err
|
||||
}
|
||||
return keep
|
||||
} else {
|
||||
sendAbstractionsCacheSingleton.Put(toReplicationCursor)
|
||||
log(ctx).WithField("to_cursor", toReplicationCursor.String()).Info("successfully created `to` replication cursor")
|
||||
}
|
||||
destroyTypes := AbstractionTypeSet{
|
||||
AbstractionStepHold: true,
|
||||
AbstractionTentativeReplicationCursorBookmark: true,
|
||||
AbstractionReplicationCursorBookmarkV2: true,
|
||||
|
||||
keep := func(a Abstraction) bool {
|
||||
return AbstractionEquals(a, toReplicationCursor)
|
||||
}
|
||||
abstractionsCacheSingleton.TryBatchDestroy(ctx, p.jobId, fs, destroyTypes, keep, nil)
|
||||
sendAbstractionsCacheSingleton.TryBatchDestroy(ctx, p.jobId, fs, keep, nil)
|
||||
|
||||
return &pdu.SendCompletedRes{}, nil
|
||||
|
||||
@@ -422,19 +476,55 @@ type FSMap interface { // FIXME unused
|
||||
AsFilter() FSFilter
|
||||
}
|
||||
|
||||
// NOTE: when adding members to this struct, remember
|
||||
// to add them to `ReceiverConfig.copyIn()`
|
||||
type ReceiverConfig struct {
|
||||
JobID JobID
|
||||
|
||||
RootWithoutClientComponent *zfs.DatasetPath // TODO use
|
||||
AppendClientIdentity bool
|
||||
|
||||
UpdateLastReceivedHold bool
|
||||
|
||||
InheritProperties []zfsprop.Property
|
||||
OverrideProperties map[zfsprop.Property]string
|
||||
}
|
||||
|
||||
func (c *ReceiverConfig) copyIn() {
|
||||
c.RootWithoutClientComponent = c.RootWithoutClientComponent.Copy()
|
||||
|
||||
pInherit := make([]zfsprop.Property, len(c.InheritProperties))
|
||||
copy(pInherit, c.InheritProperties)
|
||||
c.InheritProperties = pInherit
|
||||
|
||||
pOverride := make(map[zfsprop.Property]string, len(c.OverrideProperties))
|
||||
for key, value := range c.OverrideProperties {
|
||||
pOverride[key] = value
|
||||
}
|
||||
c.OverrideProperties = pOverride
|
||||
}
|
||||
|
||||
func (c *ReceiverConfig) Validate() error {
|
||||
c.JobID.MustValidate()
|
||||
|
||||
if c.InheritProperties != nil {
|
||||
for _, prop := range c.InheritProperties {
|
||||
err := prop.Validate()
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "inherit property %q", prop)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if c.OverrideProperties != nil {
|
||||
for prop := range c.OverrideProperties {
|
||||
err := prop.Validate()
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "override property %q", prop)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if c.RootWithoutClientComponent.Length() <= 0 {
|
||||
return errors.New("RootWithoutClientComponent must not be an empty dataset path")
|
||||
}
|
||||
@@ -667,7 +757,7 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive io.
|
||||
f := zfs.NewDatasetPathForest()
|
||||
f.Add(lp)
|
||||
getLogger(ctx).Debug("begin tree-walk")
|
||||
f.WalkTopDown(func(v *zfs.DatasetPathVisit) (visitChildTree bool) {
|
||||
f.WalkTopDown(func(v zfs.DatasetPathVisit) (visitChildTree bool) {
|
||||
if v.Path.Equal(lp) {
|
||||
return false
|
||||
}
|
||||
@@ -695,7 +785,7 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive io.
|
||||
}
|
||||
l := getLogger(ctx).WithField("placeholder_fs", v.Path)
|
||||
l.Debug("create placeholder filesystem")
|
||||
err := zfs.ZFSCreatePlaceholderFilesystem(ctx, v.Path, v.Parent.Path)
|
||||
err := zfs.ZFSCreatePlaceholderFilesystem(ctx, v.Path)
|
||||
if err != nil {
|
||||
l.WithError(err).Error("cannot create placeholder filesystem")
|
||||
visitErr = err
|
||||
@@ -722,6 +812,10 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive io.
|
||||
return nil, errors.Wrap(err, "cannot get placeholder state")
|
||||
}
|
||||
log.WithField("placeholder_state", fmt.Sprintf("%#v", ph)).Debug("placeholder state")
|
||||
|
||||
recvOpts.InheritProperties = s.conf.InheritProperties
|
||||
recvOpts.OverrideProperties = s.conf.OverrideProperties
|
||||
|
||||
if ph.FSExists && ph.IsPlaceholder {
|
||||
recvOpts.RollbackAndForceRecv = true
|
||||
clearPlaceholderProperty = true
|
||||
@@ -851,38 +945,12 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive io.
|
||||
return nil, errors.Wrap(err, msg)
|
||||
}
|
||||
|
||||
replicationGuaranteeOptions, err := replicationGuaranteeOptionsFromPDU(req.GetReplicationConfig().Protection)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
replicationGuaranteeStrategy := replicationGuaranteeOptions.Strategy(ph.FSExists)
|
||||
liveAbs, err := replicationGuaranteeStrategy.ReceiverPostRecv(ctx, s.conf.JobID, lp.ToString(), toRecvd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, a := range liveAbs {
|
||||
if a != nil {
|
||||
abstractionsCacheSingleton.Put(a)
|
||||
if s.conf.UpdateLastReceivedHold {
|
||||
log.Debug("move last-received-hold")
|
||||
if err := MoveLastReceivedHold(ctx, lp.ToString(), toRecvd, s.conf.JobID); err != nil {
|
||||
return nil, errors.Wrap(err, "cannot move last-received-hold")
|
||||
}
|
||||
}
|
||||
keep := func(a Abstraction) (keep bool) {
|
||||
keep = false
|
||||
for _, k := range liveAbs {
|
||||
keep = keep || AbstractionEquals(a, k)
|
||||
}
|
||||
return keep
|
||||
}
|
||||
check := func(obsoleteAbs []Abstraction) {
|
||||
for _, abs := range obsoleteAbs {
|
||||
if zfs.FilesystemVersionEqualIdentity(abs.GetFilesystemVersion(), toRecvd) {
|
||||
panic(fmt.Sprintf("would destroy endpoint abstraction around the filesystem version we just received %s", abs))
|
||||
}
|
||||
}
|
||||
}
|
||||
destroyTypes := AbstractionTypeSet{
|
||||
AbstractionLastReceivedHold: true,
|
||||
}
|
||||
abstractionsCacheSingleton.TryBatchDestroy(ctx, s.conf.JobID, lp.ToString(), destroyTypes, keep, check)
|
||||
|
||||
return &pdu.ReceiveRes{}, nil
|
||||
}
|
||||
|
||||
@@ -1,214 +0,0 @@
|
||||
package endpoint
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/replication/logic/pdu"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type ReplicationGuaranteeOptions struct {
|
||||
Initial ReplicationGuaranteeKind
|
||||
Incremental ReplicationGuaranteeKind
|
||||
}
|
||||
|
||||
func replicationGuaranteeOptionsFromPDU(in *pdu.ReplicationConfigProtection) (o ReplicationGuaranteeOptions, _ error) {
|
||||
if in == nil {
|
||||
return o, errors.New("pdu.ReplicationConfigProtection must not be nil")
|
||||
}
|
||||
initial, err := replicationGuaranteeKindFromPDU(in.GetInitial())
|
||||
if err != nil {
|
||||
return o, errors.Wrap(err, "pdu.ReplicationConfigProtection: field Initial")
|
||||
}
|
||||
incremental, err := replicationGuaranteeKindFromPDU(in.GetIncremental())
|
||||
if err != nil {
|
||||
return o, errors.Wrap(err, "pdu.ReplicationConfigProtection: field Incremental")
|
||||
}
|
||||
o = ReplicationGuaranteeOptions{
|
||||
Initial: initial,
|
||||
Incremental: incremental,
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
func replicationGuaranteeKindFromPDU(in pdu.ReplicationGuaranteeKind) (k ReplicationGuaranteeKind, _ error) {
|
||||
switch in {
|
||||
case pdu.ReplicationGuaranteeKind_GuaranteeNothing:
|
||||
return ReplicationGuaranteeKindNone, nil
|
||||
case pdu.ReplicationGuaranteeKind_GuaranteeIncrementalReplication:
|
||||
return ReplicationGuaranteeKindIncremental, nil
|
||||
case pdu.ReplicationGuaranteeKind_GuaranteeResumability:
|
||||
return ReplicationGuaranteeKindResumability, nil
|
||||
|
||||
case pdu.ReplicationGuaranteeKind_GuaranteeInvalid:
|
||||
fallthrough
|
||||
default:
|
||||
return k, errors.Errorf("%q", in.String())
|
||||
}
|
||||
}
|
||||
|
||||
func (o ReplicationGuaranteeOptions) Strategy(incremental bool) ReplicationGuaranteeStrategy {
|
||||
g := o.Initial
|
||||
if incremental {
|
||||
g = o.Incremental
|
||||
}
|
||||
return ReplicationGuaranteeFromKind(g)
|
||||
}
|
||||
|
||||
//go:generate enumer -type=ReplicationGuaranteeKind -json -transform=snake -trimprefix=ReplicationGuaranteeKind
|
||||
type ReplicationGuaranteeKind int
|
||||
|
||||
const (
|
||||
ReplicationGuaranteeKindResumability ReplicationGuaranteeKind = 1 << iota
|
||||
ReplicationGuaranteeKindIncremental
|
||||
ReplicationGuaranteeKindNone
|
||||
)
|
||||
|
||||
type ReplicationGuaranteeStrategy interface {
|
||||
Kind() ReplicationGuaranteeKind
|
||||
SenderPreSend(ctx context.Context, jid JobID, sendArgs *zfs.ZFSSendArgsValidated) (keep []Abstraction, err error)
|
||||
ReceiverPostRecv(ctx context.Context, jid JobID, fs string, toRecvd zfs.FilesystemVersion) (keep []Abstraction, err error)
|
||||
SenderPostRecvConfirmed(ctx context.Context, jid JobID, fs string, to zfs.FilesystemVersion) (keep []Abstraction, err error)
|
||||
}
|
||||
|
||||
func ReplicationGuaranteeFromKind(k ReplicationGuaranteeKind) ReplicationGuaranteeStrategy {
|
||||
switch k {
|
||||
case ReplicationGuaranteeKindNone:
|
||||
return ReplicationGuaranteeNone{}
|
||||
case ReplicationGuaranteeKindIncremental:
|
||||
return ReplicationGuaranteeIncremental{}
|
||||
case ReplicationGuaranteeKindResumability:
|
||||
return ReplicationGuaranteeResumability{}
|
||||
default:
|
||||
panic(fmt.Sprintf("unreachable: %q %T", k, k))
|
||||
}
|
||||
}
|
||||
|
||||
type ReplicationGuaranteeNone struct{}
|
||||
|
||||
func (g ReplicationGuaranteeNone) Kind() ReplicationGuaranteeKind {
|
||||
return ReplicationGuaranteeKindNone
|
||||
}
|
||||
|
||||
func (g ReplicationGuaranteeNone) SenderPreSend(ctx context.Context, jid JobID, sendArgs *zfs.ZFSSendArgsValidated) (keep []Abstraction, err error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (g ReplicationGuaranteeNone) ReceiverPostRecv(ctx context.Context, jid JobID, fs string, toRecvd zfs.FilesystemVersion) (keep []Abstraction, err error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (g ReplicationGuaranteeNone) SenderPostRecvConfirmed(ctx context.Context, jid JobID, fs string, to zfs.FilesystemVersion) (keep []Abstraction, err error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
type ReplicationGuaranteeIncremental struct{}
|
||||
|
||||
func (g ReplicationGuaranteeIncremental) Kind() ReplicationGuaranteeKind {
|
||||
return ReplicationGuaranteeKindIncremental
|
||||
}
|
||||
|
||||
func (g ReplicationGuaranteeIncremental) SenderPreSend(ctx context.Context, jid JobID, sendArgs *zfs.ZFSSendArgsValidated) (keep []Abstraction, err error) {
|
||||
if sendArgs.FromVersion != nil {
|
||||
from, err := CreateTentativeReplicationCursor(ctx, sendArgs.FS, *sendArgs.FromVersion, jid)
|
||||
if err != nil {
|
||||
if err == zfs.ErrBookmarkCloningNotSupported {
|
||||
getLogger(ctx).WithField("replication_guarantee", g).
|
||||
WithField("bookmark", sendArgs.From.FullPath(sendArgs.FS)).
|
||||
Info("bookmark cloning is not supported, speculating that `from` will not be destroyed until step is done")
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
keep = append(keep, from)
|
||||
}
|
||||
to, err := CreateTentativeReplicationCursor(ctx, sendArgs.FS, sendArgs.ToVersion, jid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keep = append(keep, to)
|
||||
|
||||
return keep, nil
|
||||
}
|
||||
|
||||
func (g ReplicationGuaranteeIncremental) ReceiverPostRecv(ctx context.Context, jid JobID, fs string, toRecvd zfs.FilesystemVersion) (keep []Abstraction, err error) {
|
||||
return receiverPostRecvCommon(ctx, jid, fs, toRecvd)
|
||||
}
|
||||
|
||||
func (g ReplicationGuaranteeIncremental) SenderPostRecvConfirmed(ctx context.Context, jid JobID, fs string, to zfs.FilesystemVersion) (keep []Abstraction, err error) {
|
||||
return senderPostRecvConfirmedCommon(ctx, jid, fs, to)
|
||||
}
|
||||
|
||||
type ReplicationGuaranteeResumability struct{}
|
||||
|
||||
func (g ReplicationGuaranteeResumability) Kind() ReplicationGuaranteeKind {
|
||||
return ReplicationGuaranteeKindResumability
|
||||
}
|
||||
|
||||
func (g ReplicationGuaranteeResumability) SenderPreSend(ctx context.Context, jid JobID, sendArgs *zfs.ZFSSendArgsValidated) (keep []Abstraction, err error) {
|
||||
// try to hold the FromVersion
|
||||
if sendArgs.FromVersion != nil {
|
||||
if sendArgs.FromVersion.Type == zfs.Bookmark {
|
||||
getLogger(ctx).WithField("replication_guarantee", g).WithField("fromVersion", sendArgs.FromVersion.FullPath(sendArgs.FS)).
|
||||
Debug("cannot hold a bookmark, speculating that `from` will not be destroyed until step is done")
|
||||
} else {
|
||||
from, err := HoldStep(ctx, sendArgs.FS, *sendArgs.FromVersion, jid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keep = append(keep, from)
|
||||
}
|
||||
// fallthrough
|
||||
}
|
||||
|
||||
to, err := HoldStep(ctx, sendArgs.FS, sendArgs.ToVersion, jid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keep = append(keep, to)
|
||||
|
||||
return keep, nil
|
||||
}
|
||||
|
||||
func (g ReplicationGuaranteeResumability) ReceiverPostRecv(ctx context.Context, jid JobID, fs string, toRecvd zfs.FilesystemVersion) (keep []Abstraction, err error) {
|
||||
return receiverPostRecvCommon(ctx, jid, fs, toRecvd)
|
||||
}
|
||||
|
||||
func (g ReplicationGuaranteeResumability) SenderPostRecvConfirmed(ctx context.Context, jid JobID, fs string, to zfs.FilesystemVersion) (keep []Abstraction, err error) {
|
||||
return senderPostRecvConfirmedCommon(ctx, jid, fs, to)
|
||||
}
|
||||
|
||||
// helper function used by multiple strategies
|
||||
func senderPostRecvConfirmedCommon(ctx context.Context, jid JobID, fs string, to zfs.FilesystemVersion) (keep []Abstraction, err error) {
|
||||
|
||||
log := getLogger(ctx).WithField("toVersion", to.FullPath(fs))
|
||||
|
||||
toReplicationCursor, err := CreateReplicationCursor(ctx, fs, to, jid)
|
||||
if err != nil {
|
||||
if err == zfs.ErrBookmarkCloningNotSupported {
|
||||
log.Debug("not setting replication cursor, bookmark cloning not supported")
|
||||
} else {
|
||||
msg := "cannot move replication cursor, keeping hold on `to` until successful"
|
||||
log.WithError(err).Error(msg)
|
||||
err = errors.Wrap(err, msg)
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
log.WithField("to_cursor", toReplicationCursor.String()).Info("successfully created `to` replication cursor")
|
||||
}
|
||||
|
||||
return []Abstraction{toReplicationCursor}, nil
|
||||
}
|
||||
|
||||
// helper function used by multiple strategies
|
||||
func receiverPostRecvCommon(ctx context.Context, jid JobID, fs string, toRecvd zfs.FilesystemVersion) (keep []Abstraction, err error) {
|
||||
getLogger(ctx).Debug("create new last-received-hold")
|
||||
lrh, err := CreateLastReceivedHold(ctx, fs, toRecvd, jid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []Abstraction{lrh}, nil
|
||||
}
|
||||
@@ -3,5 +3,5 @@ package endpoint
|
||||
import "github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
func RegisterMetrics(r prometheus.Registerer) {
|
||||
r.MustRegister(abstractionsCacheMetrics.count)
|
||||
r.MustRegister(sendAbstractionsCacheMetrics.count)
|
||||
}
|
||||
|
||||
+42
-44
@@ -6,54 +6,53 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
"github.com/zrepl/zrepl/util/chainlock"
|
||||
)
|
||||
|
||||
var abstractionsCacheMetrics struct {
|
||||
var sendAbstractionsCacheMetrics struct {
|
||||
count prometheus.Gauge
|
||||
}
|
||||
|
||||
func init() {
|
||||
abstractionsCacheMetrics.count = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
sendAbstractionsCacheMetrics.count = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "endpoint",
|
||||
Name: "abstractions_cache_entry_count",
|
||||
Help: "number of abstractions tracked in the abstractionsCache data structure",
|
||||
Name: "send_abstractions_cache_entry_count",
|
||||
Help: "number of send abstractions tracked in the sendAbstractionsCache data structure",
|
||||
})
|
||||
}
|
||||
|
||||
var abstractionsCacheSingleton = newAbstractionsCache()
|
||||
var sendAbstractionsCacheSingleton = newSendAbstractionsCache()
|
||||
|
||||
func AbstractionsCacheInvalidate(fs string) {
|
||||
abstractionsCacheSingleton.InvalidateFSCache(fs)
|
||||
func SendAbstractionsCacheInvalidate(fs string) {
|
||||
sendAbstractionsCacheSingleton.InvalidateFSCache(fs)
|
||||
}
|
||||
|
||||
type abstractionsCacheDidLoadFSState int
|
||||
type sendAbstractionsCacheDidLoadFSState int
|
||||
|
||||
const (
|
||||
abstractionsCacheDidLoadFSStateNo abstractionsCacheDidLoadFSState = iota // 0-value has meaning
|
||||
abstractionsCacheDidLoadFSStateInProgress
|
||||
abstractionsCacheDidLoadFSStateDone
|
||||
sendAbstractionsCacheDidLoadFSStateNo sendAbstractionsCacheDidLoadFSState = iota // 0-value has meaning
|
||||
sendAbstractionsCacheDidLoadFSStateInProgress
|
||||
sendAbstractionsCacheDidLoadFSStateDone
|
||||
)
|
||||
|
||||
type abstractionsCache struct {
|
||||
type sendAbstractionsCache struct {
|
||||
mtx chainlock.L
|
||||
abstractions []Abstraction
|
||||
didLoadFS map[string]abstractionsCacheDidLoadFSState
|
||||
didLoadFS map[string]sendAbstractionsCacheDidLoadFSState
|
||||
didLoadFSChanged *sync.Cond
|
||||
}
|
||||
|
||||
func newAbstractionsCache() *abstractionsCache {
|
||||
c := &abstractionsCache{
|
||||
didLoadFS: make(map[string]abstractionsCacheDidLoadFSState),
|
||||
func newSendAbstractionsCache() *sendAbstractionsCache {
|
||||
c := &sendAbstractionsCache{
|
||||
didLoadFS: make(map[string]sendAbstractionsCacheDidLoadFSState),
|
||||
}
|
||||
c.didLoadFSChanged = c.mtx.NewCond()
|
||||
return c
|
||||
}
|
||||
|
||||
func (s *abstractionsCache) Put(a Abstraction) {
|
||||
func (s *sendAbstractionsCache) Put(a Abstraction) {
|
||||
defer s.mtx.Lock().Unlock()
|
||||
|
||||
var zeroJobId JobID
|
||||
@@ -64,10 +63,10 @@ func (s *abstractionsCache) Put(a Abstraction) {
|
||||
}
|
||||
|
||||
s.abstractions = append(s.abstractions, a)
|
||||
abstractionsCacheMetrics.count.Set(float64(len(s.abstractions)))
|
||||
sendAbstractionsCacheMetrics.count.Set(float64(len(s.abstractions)))
|
||||
}
|
||||
|
||||
func (s *abstractionsCache) InvalidateFSCache(fs string) {
|
||||
func (s *sendAbstractionsCache) InvalidateFSCache(fs string) {
|
||||
// FIXME: O(n)
|
||||
newAbs := make([]Abstraction, 0, len(s.abstractions))
|
||||
for _, a := range s.abstractions {
|
||||
@@ -76,9 +75,9 @@ func (s *abstractionsCache) InvalidateFSCache(fs string) {
|
||||
}
|
||||
}
|
||||
s.abstractions = newAbs
|
||||
abstractionsCacheMetrics.count.Set(float64(len(s.abstractions)))
|
||||
sendAbstractionsCacheMetrics.count.Set(float64(len(s.abstractions)))
|
||||
|
||||
s.didLoadFS[fs] = abstractionsCacheDidLoadFSStateNo
|
||||
s.didLoadFS[fs] = sendAbstractionsCacheDidLoadFSStateNo
|
||||
s.didLoadFSChanged.Broadcast()
|
||||
|
||||
}
|
||||
@@ -87,7 +86,7 @@ func (s *abstractionsCache) InvalidateFSCache(fs string) {
|
||||
// - only fetches on-disk abstractions once, but every time from the in-memory store
|
||||
//
|
||||
// That means that for precise results, all abstractions created by the endpoint must be .Put into this cache.
|
||||
func (s *abstractionsCache) GetAndDeleteByJobIDAndFS(ctx context.Context, jobID JobID, fs string, types AbstractionTypeSet, keep func(a Abstraction) bool) (ret []Abstraction) {
|
||||
func (s *sendAbstractionsCache) GetAndDeleteByJobIDAndFS(ctx context.Context, jobID JobID, fs string, keep func(a Abstraction) bool) (ret []Abstraction) {
|
||||
defer s.mtx.Lock().Unlock()
|
||||
defer trace.WithSpanFromStackUpdateCtx(&ctx)()
|
||||
var zeroJobId JobID
|
||||
@@ -98,50 +97,50 @@ func (s *abstractionsCache) GetAndDeleteByJobIDAndFS(ctx context.Context, jobID
|
||||
panic("must not pass zero-value fs")
|
||||
}
|
||||
|
||||
s.tryLoadOnDiskAbstractions(ctx, fs)
|
||||
s.tryLoadOnDiskSendAbstractions(ctx, fs)
|
||||
|
||||
// FIXME O(n)
|
||||
var remaining []Abstraction
|
||||
for _, a := range s.abstractions {
|
||||
aJobId := *a.GetJobID()
|
||||
aFS := a.GetFS()
|
||||
if aJobId == jobID && aFS == fs && types[a.GetType()] && !keep(a) {
|
||||
if aJobId == jobID && aFS == fs && !keep(a) {
|
||||
ret = append(ret, a)
|
||||
} else {
|
||||
remaining = append(remaining, a)
|
||||
}
|
||||
}
|
||||
s.abstractions = remaining
|
||||
abstractionsCacheMetrics.count.Set(float64(len(s.abstractions)))
|
||||
sendAbstractionsCacheMetrics.count.Set(float64(len(s.abstractions)))
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
// caller must hold s.mtx
|
||||
func (s *abstractionsCache) tryLoadOnDiskAbstractions(ctx context.Context, fs string) {
|
||||
for s.didLoadFS[fs] != abstractionsCacheDidLoadFSStateDone {
|
||||
if s.didLoadFS[fs] == abstractionsCacheDidLoadFSStateInProgress {
|
||||
func (s *sendAbstractionsCache) tryLoadOnDiskSendAbstractions(ctx context.Context, fs string) {
|
||||
for s.didLoadFS[fs] != sendAbstractionsCacheDidLoadFSStateDone {
|
||||
if s.didLoadFS[fs] == sendAbstractionsCacheDidLoadFSStateInProgress {
|
||||
s.didLoadFSChanged.Wait()
|
||||
continue
|
||||
}
|
||||
if s.didLoadFS[fs] != abstractionsCacheDidLoadFSStateNo {
|
||||
if s.didLoadFS[fs] != sendAbstractionsCacheDidLoadFSStateNo {
|
||||
panic(fmt.Sprintf("unreachable: %v", s.didLoadFS[fs]))
|
||||
}
|
||||
|
||||
s.didLoadFS[fs] = abstractionsCacheDidLoadFSStateInProgress
|
||||
s.didLoadFS[fs] = sendAbstractionsCacheDidLoadFSStateInProgress
|
||||
defer s.didLoadFSChanged.Broadcast()
|
||||
|
||||
var onDiskAbs []Abstraction
|
||||
var err error
|
||||
s.mtx.DropWhile(func() {
|
||||
onDiskAbs, err = s.tryLoadOnDiskAbstractionsImpl(ctx, fs) // no shadow
|
||||
onDiskAbs, err = s.tryLoadOnDiskSendAbstractionsImpl(ctx, fs) // no shadow
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
s.didLoadFS[fs] = abstractionsCacheDidLoadFSStateNo
|
||||
getLogger(ctx).WithField("fs", fs).WithError(err).Error("cannot list abstractions for filesystem")
|
||||
s.didLoadFS[fs] = sendAbstractionsCacheDidLoadFSStateNo
|
||||
getLogger(ctx).WithField("fs", fs).WithError(err).Error("cannot list send step abstractions for filesystem")
|
||||
} else {
|
||||
s.didLoadFS[fs] = abstractionsCacheDidLoadFSStateDone
|
||||
s.didLoadFS[fs] = sendAbstractionsCacheDidLoadFSStateDone
|
||||
s.abstractions = append(s.abstractions, onDiskAbs...)
|
||||
getLogger(ctx).WithField("fs", fs).WithField("abstractions", onDiskAbs).Debug("loaded step abstractions for filesystem")
|
||||
}
|
||||
@@ -150,7 +149,7 @@ func (s *abstractionsCache) tryLoadOnDiskAbstractions(ctx context.Context, fs st
|
||||
}
|
||||
|
||||
// caller should _not hold s.mtx
|
||||
func (s *abstractionsCache) tryLoadOnDiskAbstractionsImpl(ctx context.Context, fs string) ([]Abstraction, error) {
|
||||
func (s *sendAbstractionsCache) tryLoadOnDiskSendAbstractionsImpl(ctx context.Context, fs string) ([]Abstraction, error) {
|
||||
defer trace.WithSpanFromStackUpdateCtx(&ctx)()
|
||||
|
||||
q := ListZFSHoldsAndBookmarksQuery{
|
||||
@@ -159,10 +158,9 @@ func (s *abstractionsCache) tryLoadOnDiskAbstractionsImpl(ctx context.Context, f
|
||||
},
|
||||
JobID: nil,
|
||||
What: AbstractionTypeSet{
|
||||
AbstractionStepHold: true,
|
||||
AbstractionTentativeReplicationCursorBookmark: true,
|
||||
AbstractionReplicationCursorBookmarkV2: true,
|
||||
AbstractionLastReceivedHold: true,
|
||||
AbstractionStepHold: true,
|
||||
AbstractionStepBookmark: true,
|
||||
AbstractionReplicationCursorBookmarkV2: true,
|
||||
},
|
||||
Concurrency: 1,
|
||||
}
|
||||
@@ -177,12 +175,12 @@ func (s *abstractionsCache) tryLoadOnDiskAbstractionsImpl(ctx context.Context, f
|
||||
return abs, nil
|
||||
}
|
||||
|
||||
func (s *abstractionsCache) TryBatchDestroy(ctx context.Context, jobId JobID, fs string, types AbstractionTypeSet, keep func(a Abstraction) bool, check func(willDestroy []Abstraction)) {
|
||||
func (s *sendAbstractionsCache) TryBatchDestroy(ctx context.Context, jobId JobID, fs string, keep func(a Abstraction) bool, check func(willDestroy []Abstraction)) {
|
||||
// no s.mtx, we only use the public interface in this function
|
||||
|
||||
defer trace.WithSpanFromStackUpdateCtx(&ctx)()
|
||||
|
||||
obsoleteAbs := s.GetAndDeleteByJobIDAndFS(ctx, jobId, fs, types, keep)
|
||||
obsoleteAbs := s.GetAndDeleteByJobIDAndFS(ctx, jobId, fs, keep)
|
||||
|
||||
if check != nil {
|
||||
check(obsoleteAbs)
|
||||
@@ -195,11 +193,11 @@ func (s *abstractionsCache) TryBatchDestroy(ctx context.Context, jobId JobID, fs
|
||||
getLogger(ctx).
|
||||
WithField("abstraction", res.Abstraction).
|
||||
WithError(res.DestroyErr).
|
||||
Error("cannot destroy abstraction")
|
||||
Error("cannot destroy stale send step abstraction")
|
||||
} else {
|
||||
getLogger(ctx).
|
||||
WithField("abstraction", res.Abstraction).
|
||||
Info("destroyed abstraction")
|
||||
Info("destroyed stale send step abstraction")
|
||||
}
|
||||
}
|
||||
if hadErr {
|
||||
@@ -23,19 +23,19 @@ type AbstractionType string
|
||||
// There are a lot of exhaustive switches on AbstractionType in the code base.
|
||||
// When adding a new abstraction type, make sure to search and update them!
|
||||
const (
|
||||
AbstractionStepHold AbstractionType = "step-hold"
|
||||
AbstractionLastReceivedHold AbstractionType = "last-received-hold"
|
||||
AbstractionTentativeReplicationCursorBookmark AbstractionType = "tentative-replication-cursor-bookmark-v2"
|
||||
AbstractionReplicationCursorBookmarkV1 AbstractionType = "replication-cursor-bookmark-v1"
|
||||
AbstractionReplicationCursorBookmarkV2 AbstractionType = "replication-cursor-bookmark-v2"
|
||||
AbstractionStepBookmark AbstractionType = "step-bookmark"
|
||||
AbstractionStepHold AbstractionType = "step-hold"
|
||||
AbstractionLastReceivedHold AbstractionType = "last-received-hold"
|
||||
AbstractionReplicationCursorBookmarkV1 AbstractionType = "replication-cursor-bookmark-v1"
|
||||
AbstractionReplicationCursorBookmarkV2 AbstractionType = "replication-cursor-bookmark-v2"
|
||||
)
|
||||
|
||||
var AbstractionTypesAll = map[AbstractionType]bool{
|
||||
AbstractionStepHold: true,
|
||||
AbstractionLastReceivedHold: true,
|
||||
AbstractionTentativeReplicationCursorBookmark: true,
|
||||
AbstractionReplicationCursorBookmarkV1: true,
|
||||
AbstractionReplicationCursorBookmarkV2: true,
|
||||
AbstractionStepBookmark: true,
|
||||
AbstractionStepHold: true,
|
||||
AbstractionLastReceivedHold: true,
|
||||
AbstractionReplicationCursorBookmarkV1: true,
|
||||
AbstractionReplicationCursorBookmarkV2: true,
|
||||
}
|
||||
|
||||
// Implementation Note:
|
||||
@@ -80,12 +80,12 @@ func AbstractionEquals(a, b Abstraction) bool {
|
||||
|
||||
func (t AbstractionType) Validate() error {
|
||||
switch t {
|
||||
case AbstractionStepBookmark:
|
||||
return nil
|
||||
case AbstractionStepHold:
|
||||
return nil
|
||||
case AbstractionLastReceivedHold:
|
||||
return nil
|
||||
case AbstractionTentativeReplicationCursorBookmark:
|
||||
return nil
|
||||
case AbstractionReplicationCursorBookmarkV1:
|
||||
return nil
|
||||
case AbstractionReplicationCursorBookmarkV2:
|
||||
@@ -185,8 +185,8 @@ type BookmarkExtractor func(fs *zfs.DatasetPath, v zfs.FilesystemVersion) Abstra
|
||||
// returns nil if the abstraction type is not bookmark-based
|
||||
func (t AbstractionType) BookmarkExtractor() BookmarkExtractor {
|
||||
switch t {
|
||||
case AbstractionTentativeReplicationCursorBookmark:
|
||||
return TentativeReplicationCursorExtractor
|
||||
case AbstractionStepBookmark:
|
||||
return StepBookmarkExtractor
|
||||
case AbstractionReplicationCursorBookmarkV1:
|
||||
return ReplicationCursorV1Extractor
|
||||
case AbstractionReplicationCursorBookmarkV2:
|
||||
@@ -205,7 +205,7 @@ type HoldExtractor = func(fs *zfs.DatasetPath, v zfs.FilesystemVersion, tag stri
|
||||
// returns nil if the abstraction type is not hold-based
|
||||
func (t AbstractionType) HoldExtractor() HoldExtractor {
|
||||
switch t {
|
||||
case AbstractionTentativeReplicationCursorBookmark:
|
||||
case AbstractionStepBookmark:
|
||||
return nil
|
||||
case AbstractionReplicationCursorBookmarkV1:
|
||||
return nil
|
||||
@@ -220,23 +220,6 @@ func (t AbstractionType) HoldExtractor() HoldExtractor {
|
||||
}
|
||||
}
|
||||
|
||||
func (t AbstractionType) BookmarkNamer() func(fs string, guid uint64, jobId JobID) (string, error) {
|
||||
switch t {
|
||||
case AbstractionTentativeReplicationCursorBookmark:
|
||||
return TentativeReplicationCursorBookmarkName
|
||||
case AbstractionReplicationCursorBookmarkV1:
|
||||
panic("shouldn't be creating new ones")
|
||||
case AbstractionReplicationCursorBookmarkV2:
|
||||
return ReplicationCursorBookmarkName
|
||||
case AbstractionStepHold:
|
||||
return nil
|
||||
case AbstractionLastReceivedHold:
|
||||
return nil
|
||||
default:
|
||||
panic(fmt.Sprintf("unimpl: %q", t))
|
||||
}
|
||||
}
|
||||
|
||||
type ListZFSHoldsAndBookmarksQuery struct {
|
||||
FS ListZFSHoldsAndBookmarksQueryFilesystemFilter
|
||||
// What abstraction types should match (any contained in the set)
|
||||
@@ -714,9 +697,11 @@ func ListStale(ctx context.Context, q ListZFSHoldsAndBookmarksQuery) (*Staleness
|
||||
return nil, &ListStaleQueryError{errors.New("ListStale cannot have Until != nil set on query")}
|
||||
}
|
||||
|
||||
// if asking for step holds must also ask for replication cursor bookmarks (for firstNotStale)
|
||||
// if asking for step holds, must also as for step bookmarks (same kind of abstraction)
|
||||
// as well as replication cursor bookmarks (for firstNotStale)
|
||||
ifAnyThenAll := AbstractionTypeSet{
|
||||
AbstractionStepHold: true,
|
||||
AbstractionStepBookmark: true,
|
||||
AbstractionReplicationCursorBookmarkV2: true,
|
||||
}
|
||||
if q.What.ContainsAnyOf(ifAnyThenAll) && !q.What.ContainsAll(ifAnyThenAll) {
|
||||
@@ -745,7 +730,7 @@ type fsAjobAtype struct {
|
||||
}
|
||||
|
||||
// For step holds and bookmarks, only those older than the most recent replication cursor
|
||||
// of their (filesystem,job) are considered because younger ones cannot be stale by definition
|
||||
// of their (filesystem,job) is considered because younger ones cannot be stale by definition
|
||||
// (if we destroy them, we might actually lose the hold on the `To` for an ongoing incremental replication)
|
||||
//
|
||||
// For replication cursors and last-received-holds, only the most recent one is kept.
|
||||
@@ -787,6 +772,8 @@ func listStaleFiltering(abs []Abstraction, sinceBound *CreateTXGRangeBound) *Sta
|
||||
}
|
||||
|
||||
// stepFirstNotStaleCandidate.step
|
||||
case AbstractionStepBookmark:
|
||||
fallthrough
|
||||
case AbstractionStepHold:
|
||||
if c.step == nil || (*c.step).GetCreateTXG() < a.GetCreateTXG() {
|
||||
a := a
|
||||
@@ -810,7 +797,7 @@ func listStaleFiltering(abs []Abstraction, sinceBound *CreateTXGRangeBound) *Sta
|
||||
for k := range by {
|
||||
l := by[k]
|
||||
|
||||
if k.Type == AbstractionStepHold {
|
||||
if k.Type == AbstractionStepHold || k.Type == AbstractionStepBookmark {
|
||||
// all older than the most recent cursor are stale, others are always live
|
||||
|
||||
// if we don't have a replication cursor yet, use untilBound = nil
|
||||
|
||||
+139
-51
@@ -4,10 +4,12 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/util/errorarray"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
@@ -51,28 +53,6 @@ func ParseReplicationCursorBookmarkName(fullname string) (uint64, JobID, error)
|
||||
return guid, jobID, err
|
||||
}
|
||||
|
||||
const tentativeReplicationCursorBookmarkNamePrefix = "zrepl_CURSORTENTATIVE_"
|
||||
|
||||
// v must be validated by caller
|
||||
func TentativeReplicationCursorBookmarkName(fs string, guid uint64, id JobID) (string, error) {
|
||||
return tentativeReplicationCursorBookmarkNameImpl(fs, guid, id.String())
|
||||
}
|
||||
|
||||
func tentativeReplicationCursorBookmarkNameImpl(fs string, guid uint64, jobid string) (string, error) {
|
||||
return makeJobAndGuidBookmarkName(tentativeReplicationCursorBookmarkNamePrefix, fs, guid, jobid)
|
||||
}
|
||||
|
||||
// name is the full bookmark name, including dataset path
|
||||
//
|
||||
// err != nil always means that the bookmark is not a step bookmark
|
||||
func ParseTentativeReplicationCursorBookmarkName(fullname string) (guid uint64, jobID JobID, err error) {
|
||||
guid, jobID, err = parseJobAndGuidBookmarkName(fullname, tentativeReplicationCursorBookmarkNamePrefix)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "parse step bookmark name") // no shadow!
|
||||
}
|
||||
return guid, jobID, err
|
||||
}
|
||||
|
||||
// may return nil for both values, indicating there is no cursor
|
||||
func GetMostRecentReplicationCursorOfJob(ctx context.Context, fs string, jobID JobID) (*zfs.FilesystemVersion, error) {
|
||||
fsp, err := zfs.NewDatasetPath(fs)
|
||||
@@ -142,23 +122,23 @@ func GetReplicationCursors(ctx context.Context, dp *zfs.DatasetPath, jobID JobID
|
||||
//
|
||||
// returns ErrBookmarkCloningNotSupported if version is a bookmark and bookmarking bookmarks is not supported by ZFS
|
||||
func CreateReplicationCursor(ctx context.Context, fs string, target zfs.FilesystemVersion, jobID JobID) (a Abstraction, err error) {
|
||||
return createBookmarkAbstraction(ctx, AbstractionReplicationCursorBookmarkV2, fs, target, jobID)
|
||||
}
|
||||
|
||||
func CreateTentativeReplicationCursor(ctx context.Context, fs string, target zfs.FilesystemVersion, jobID JobID) (a Abstraction, err error) {
|
||||
return createBookmarkAbstraction(ctx, AbstractionTentativeReplicationCursorBookmark, fs, target, jobID)
|
||||
}
|
||||
|
||||
func createBookmarkAbstraction(ctx context.Context, abstractionType AbstractionType, fs string, target zfs.FilesystemVersion, jobID JobID) (a Abstraction, err error) {
|
||||
|
||||
bookmarkNamer := abstractionType.BookmarkNamer()
|
||||
if bookmarkNamer == nil {
|
||||
panic(abstractionType)
|
||||
bookmarkname, err := ReplicationCursorBookmarkName(fs, target.GetGuid(), jobID)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "determine replication cursor name")
|
||||
}
|
||||
|
||||
bookmarkname, err := bookmarkNamer(fs, target.GetGuid(), jobID)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "determine %s name", abstractionType)
|
||||
if target.IsBookmark() && target.GetName() == bookmarkname {
|
||||
return &bookmarkBasedAbstraction{
|
||||
Type: AbstractionReplicationCursorBookmarkV2,
|
||||
FS: fs,
|
||||
FilesystemVersion: target,
|
||||
JobID: jobID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
if !target.IsSnapshot() {
|
||||
return nil, zfs.ErrBookmarkCloningNotSupported
|
||||
}
|
||||
|
||||
// idempotently create bookmark (guid is encoded in it)
|
||||
@@ -172,13 +152,125 @@ func createBookmarkAbstraction(ctx context.Context, abstractionType AbstractionT
|
||||
}
|
||||
|
||||
return &bookmarkBasedAbstraction{
|
||||
Type: abstractionType,
|
||||
Type: AbstractionReplicationCursorBookmarkV2,
|
||||
FS: fs,
|
||||
FilesystemVersion: cursorBookmark,
|
||||
JobID: jobID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
const (
|
||||
ReplicationCursorBookmarkNamePrefix = "zrepl_last_received_J_"
|
||||
)
|
||||
|
||||
var lastReceivedHoldTagRE = regexp.MustCompile("^zrepl_last_received_J_(.+)$")
|
||||
|
||||
// err != nil always means that the bookmark is not a step bookmark
|
||||
func ParseLastReceivedHoldTag(tag string) (JobID, error) {
|
||||
match := lastReceivedHoldTagRE.FindStringSubmatch(tag)
|
||||
if match == nil {
|
||||
return JobID{}, errors.Errorf("parse last-received-hold tag: does not match regex %s", lastReceivedHoldTagRE.String())
|
||||
}
|
||||
jobId, err := MakeJobID(match[1])
|
||||
if err != nil {
|
||||
return JobID{}, errors.Wrap(err, "parse last-received-hold tag: invalid job id field")
|
||||
}
|
||||
return jobId, nil
|
||||
}
|
||||
|
||||
func LastReceivedHoldTag(jobID JobID) (string, error) {
|
||||
return lastReceivedHoldImpl(jobID.String())
|
||||
}
|
||||
|
||||
func lastReceivedHoldImpl(jobid string) (string, error) {
|
||||
tag := fmt.Sprintf("%s%s", ReplicationCursorBookmarkNamePrefix, jobid)
|
||||
if err := zfs.ValidHoldTag(tag); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return tag, nil
|
||||
}
|
||||
|
||||
func CreateLastReceivedHold(ctx context.Context, fs string, to zfs.FilesystemVersion, jobID JobID) (Abstraction, error) {
|
||||
|
||||
if !to.IsSnapshot() {
|
||||
return nil, errors.Errorf("last-received-hold: target must be a snapshot: %s", to.FullPath(fs))
|
||||
}
|
||||
|
||||
tag, err := LastReceivedHoldTag(jobID)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "last-received-hold: hold tag")
|
||||
}
|
||||
|
||||
// we never want to be without a hold
|
||||
// => hold new one before releasing old hold
|
||||
|
||||
err = zfs.ZFSHold(ctx, fs, to, tag)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "last-received-hold: hold newly received")
|
||||
}
|
||||
|
||||
return &holdBasedAbstraction{
|
||||
Type: AbstractionLastReceivedHold,
|
||||
FS: fs,
|
||||
FilesystemVersion: to,
|
||||
JobID: jobID,
|
||||
Tag: tag,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func MoveLastReceivedHold(ctx context.Context, fs string, to zfs.FilesystemVersion, jobID JobID) error {
|
||||
|
||||
_, err := CreateLastReceivedHold(ctx, fs, to, jobID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
q := ListZFSHoldsAndBookmarksQuery{
|
||||
What: AbstractionTypeSet{
|
||||
AbstractionLastReceivedHold: true,
|
||||
},
|
||||
FS: ListZFSHoldsAndBookmarksQueryFilesystemFilter{
|
||||
FS: &fs,
|
||||
},
|
||||
JobID: &jobID,
|
||||
CreateTXG: CreateTXGRange{
|
||||
Since: nil,
|
||||
Until: &CreateTXGRangeBound{
|
||||
CreateTXG: to.GetCreateTXG(),
|
||||
Inclusive: &zfs.NilBool{B: false},
|
||||
},
|
||||
},
|
||||
Concurrency: 1,
|
||||
}
|
||||
abs, absErrs, err := ListAbstractions(ctx, q)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "last-received-hold: list")
|
||||
}
|
||||
if len(absErrs) > 0 {
|
||||
return errors.Wrap(ListAbstractionsErrors(absErrs), "last-received-hold: list")
|
||||
}
|
||||
|
||||
getLogger(ctx).WithField("last-received-holds", fmt.Sprintf("%s", abs)).Debug("releasing last-received-holds")
|
||||
|
||||
var errs []error
|
||||
for res := range BatchDestroy(ctx, abs) {
|
||||
log := getLogger(ctx).
|
||||
WithField("last-received-hold", res.Abstraction)
|
||||
if res.DestroyErr != nil {
|
||||
errs = append(errs, res.DestroyErr)
|
||||
log.WithError(err).
|
||||
Error("cannot release last-received-hold")
|
||||
} else {
|
||||
log.Info("released last-received-hold")
|
||||
}
|
||||
}
|
||||
if len(errs) == 0 {
|
||||
return nil
|
||||
} else {
|
||||
return errorarray.Wrap(errs, "last-received-hold: release")
|
||||
}
|
||||
}
|
||||
|
||||
func ReplicationCursorV2Extractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion) (_ Abstraction) {
|
||||
if v.Type != zfs.Bookmark {
|
||||
panic("impl error")
|
||||
@@ -216,28 +308,24 @@ func ReplicationCursorV1Extractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion)
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ BookmarkExtractor = TentativeReplicationCursorExtractor
|
||||
var _ HoldExtractor = LastReceivedHoldExtractor
|
||||
|
||||
func TentativeReplicationCursorExtractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion) (_ Abstraction) {
|
||||
if v.Type != zfs.Bookmark {
|
||||
func LastReceivedHoldExtractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion, holdTag string) Abstraction {
|
||||
var err error
|
||||
|
||||
if v.Type != zfs.Snapshot {
|
||||
panic("impl error")
|
||||
}
|
||||
|
||||
fullname := v.ToAbsPath(fs)
|
||||
|
||||
guid, jobid, err := ParseTentativeReplicationCursorBookmarkName(fullname)
|
||||
if guid != v.Guid {
|
||||
// TODO log this possibly tinkered-with bookmark
|
||||
return nil
|
||||
}
|
||||
jobID, err := ParseLastReceivedHoldTag(holdTag)
|
||||
if err == nil {
|
||||
bm := &bookmarkBasedAbstraction{
|
||||
Type: AbstractionTentativeReplicationCursorBookmark,
|
||||
return &holdBasedAbstraction{
|
||||
Type: AbstractionLastReceivedHold,
|
||||
FS: fs.ToString(),
|
||||
FilesystemVersion: v,
|
||||
JobID: jobid,
|
||||
Tag: holdTag,
|
||||
JobID: jobID,
|
||||
}
|
||||
return bm
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,92 +0,0 @@
|
||||
package endpoint
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
const (
|
||||
LastReceivedHoldTagNamePrefix = "zrepl_last_received_J_"
|
||||
)
|
||||
|
||||
var lastReceivedHoldTagRE = regexp.MustCompile("^zrepl_last_received_J_(.+)$")
|
||||
|
||||
var _ HoldExtractor = LastReceivedHoldExtractor
|
||||
|
||||
func LastReceivedHoldExtractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion, holdTag string) Abstraction {
|
||||
var err error
|
||||
|
||||
if v.Type != zfs.Snapshot {
|
||||
panic("impl error")
|
||||
}
|
||||
|
||||
jobID, err := ParseLastReceivedHoldTag(holdTag)
|
||||
if err == nil {
|
||||
return &holdBasedAbstraction{
|
||||
Type: AbstractionLastReceivedHold,
|
||||
FS: fs.ToString(),
|
||||
FilesystemVersion: v,
|
||||
Tag: holdTag,
|
||||
JobID: jobID,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// err != nil always means that the bookmark is not a step bookmark
|
||||
func ParseLastReceivedHoldTag(tag string) (JobID, error) {
|
||||
match := lastReceivedHoldTagRE.FindStringSubmatch(tag)
|
||||
if match == nil {
|
||||
return JobID{}, errors.Errorf("parse last-received-hold tag: does not match regex %s", lastReceivedHoldTagRE.String())
|
||||
}
|
||||
jobId, err := MakeJobID(match[1])
|
||||
if err != nil {
|
||||
return JobID{}, errors.Wrap(err, "parse last-received-hold tag: invalid job id field")
|
||||
}
|
||||
return jobId, nil
|
||||
}
|
||||
|
||||
func LastReceivedHoldTag(jobID JobID) (string, error) {
|
||||
return lastReceivedHoldImpl(jobID.String())
|
||||
}
|
||||
|
||||
func lastReceivedHoldImpl(jobid string) (string, error) {
|
||||
tag := fmt.Sprintf("%s%s", LastReceivedHoldTagNamePrefix, jobid)
|
||||
if err := zfs.ValidHoldTag(tag); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return tag, nil
|
||||
}
|
||||
|
||||
func CreateLastReceivedHold(ctx context.Context, fs string, to zfs.FilesystemVersion, jobID JobID) (Abstraction, error) {
|
||||
|
||||
if !to.IsSnapshot() {
|
||||
return nil, errors.Errorf("last-received-hold: target must be a snapshot: %s", to.FullPath(fs))
|
||||
}
|
||||
|
||||
tag, err := LastReceivedHoldTag(jobID)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "last-received-hold: hold tag")
|
||||
}
|
||||
|
||||
// we never want to be without a hold
|
||||
// => hold new one before releasing old hold
|
||||
|
||||
err = zfs.ZFSHold(ctx, fs, to, tag)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "last-received-hold: hold newly received")
|
||||
}
|
||||
|
||||
return &holdBasedAbstraction{
|
||||
Type: AbstractionLastReceivedHold,
|
||||
FS: fs,
|
||||
FilesystemVersion: to,
|
||||
JobID: jobID,
|
||||
Tag: tag,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package endpoint
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
var stepHoldTagRE = regexp.MustCompile("^zrepl_STEP_J_(.+)")
|
||||
|
||||
func StepHoldTag(jobid JobID) (string, error) {
|
||||
return stepHoldTagImpl(jobid.String())
|
||||
}
|
||||
|
||||
func stepHoldTagImpl(jobid string) (string, error) {
|
||||
t := fmt.Sprintf("zrepl_STEP_J_%s", jobid)
|
||||
if err := zfs.ValidHoldTag(t); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// err != nil always means that the bookmark is not a step bookmark
|
||||
func ParseStepHoldTag(tag string) (JobID, error) {
|
||||
match := stepHoldTagRE.FindStringSubmatch(tag)
|
||||
if match == nil {
|
||||
return JobID{}, fmt.Errorf("parse hold tag: match regex %q", stepHoldTagRE)
|
||||
}
|
||||
jobID, err := MakeJobID(match[1])
|
||||
if err != nil {
|
||||
return JobID{}, errors.Wrap(err, "parse hold tag: invalid job id field")
|
||||
}
|
||||
return jobID, nil
|
||||
}
|
||||
|
||||
const stepBookmarkNamePrefix = "zrepl_STEP"
|
||||
|
||||
// v must be validated by caller
|
||||
func StepBookmarkName(fs string, guid uint64, id JobID) (string, error) {
|
||||
return stepBookmarkNameImpl(fs, guid, id.String())
|
||||
}
|
||||
|
||||
func stepBookmarkNameImpl(fs string, guid uint64, jobid string) (string, error) {
|
||||
return makeJobAndGuidBookmarkName(stepBookmarkNamePrefix, fs, guid, jobid)
|
||||
}
|
||||
|
||||
// name is the full bookmark name, including dataset path
|
||||
//
|
||||
// err != nil always means that the bookmark is not a step bookmark
|
||||
func ParseStepBookmarkName(fullname string) (guid uint64, jobID JobID, err error) {
|
||||
guid, jobID, err = parseJobAndGuidBookmarkName(fullname, stepBookmarkNamePrefix)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "parse step bookmark name") // no shadow!
|
||||
}
|
||||
return guid, jobID, err
|
||||
}
|
||||
|
||||
// idempotently hold / step-bookmark `version`
|
||||
//
|
||||
// returns ErrBookmarkCloningNotSupported if version is a bookmark and bookmarking bookmarks is not supported by ZFS
|
||||
func HoldStep(ctx context.Context, fs string, v zfs.FilesystemVersion, jobID JobID) (Abstraction, error) {
|
||||
if v.IsSnapshot() {
|
||||
|
||||
tag, err := StepHoldTag(jobID)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "step hold tag")
|
||||
}
|
||||
|
||||
if err := zfs.ZFSHold(ctx, fs, v, tag); err != nil {
|
||||
return nil, errors.Wrap(err, "step hold: zfs")
|
||||
}
|
||||
|
||||
return &holdBasedAbstraction{
|
||||
Type: AbstractionStepHold,
|
||||
FS: fs,
|
||||
Tag: tag,
|
||||
JobID: jobID,
|
||||
FilesystemVersion: v,
|
||||
}, nil
|
||||
}
|
||||
|
||||
if !v.IsBookmark() {
|
||||
panic(fmt.Sprintf("version must bei either snapshot or bookmark, got %#v", v))
|
||||
}
|
||||
|
||||
bmname, err := StepBookmarkName(fs, v.Guid, jobID)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "create step bookmark: determine bookmark name")
|
||||
}
|
||||
// idempotently create bookmark
|
||||
stepBookmark, err := zfs.ZFSBookmark(ctx, fs, v, bmname)
|
||||
if err != nil {
|
||||
if err == zfs.ErrBookmarkCloningNotSupported {
|
||||
// TODO we could actually try to find a local snapshot that has the requested GUID
|
||||
// however, the replication algorithm prefers snapshots anyways, so this quest
|
||||
// is most likely not going to be successful. Also, there's the possibility that
|
||||
// the caller might want to filter what snapshots are eligibile, and this would
|
||||
// complicate things even further.
|
||||
return nil, err // TODO go1.13 use wrapping
|
||||
}
|
||||
return nil, errors.Wrap(err, "create step bookmark: zfs")
|
||||
}
|
||||
return &bookmarkBasedAbstraction{
|
||||
Type: AbstractionStepBookmark,
|
||||
FS: fs,
|
||||
FilesystemVersion: stepBookmark,
|
||||
JobID: jobID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
var _ BookmarkExtractor = StepBookmarkExtractor
|
||||
|
||||
func StepBookmarkExtractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion) (_ Abstraction) {
|
||||
if v.Type != zfs.Bookmark {
|
||||
panic("impl error")
|
||||
}
|
||||
|
||||
fullname := v.ToAbsPath(fs)
|
||||
|
||||
guid, jobid, err := ParseStepBookmarkName(fullname)
|
||||
if guid != v.Guid {
|
||||
// TODO log this possibly tinkered-with bookmark
|
||||
return nil
|
||||
}
|
||||
if err == nil {
|
||||
bm := &bookmarkBasedAbstraction{
|
||||
Type: AbstractionStepBookmark,
|
||||
FS: fs.ToString(),
|
||||
FilesystemVersion: v,
|
||||
JobID: jobid,
|
||||
}
|
||||
return bm
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ HoldExtractor = StepHoldExtractor
|
||||
|
||||
func StepHoldExtractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion, holdTag string) Abstraction {
|
||||
if v.Type != zfs.Snapshot {
|
||||
panic("impl error")
|
||||
}
|
||||
|
||||
jobID, err := ParseStepHoldTag(holdTag)
|
||||
if err == nil {
|
||||
return &holdBasedAbstraction{
|
||||
Type: AbstractionStepHold,
|
||||
FS: fs.ToString(),
|
||||
Tag: holdTag,
|
||||
FilesystemVersion: v,
|
||||
JobID: jobID,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
package endpoint
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
var stepHoldTagRE = regexp.MustCompile("^zrepl_STEP_J_(.+)")
|
||||
|
||||
func StepHoldTag(jobid JobID) (string, error) {
|
||||
return stepHoldTagImpl(jobid.String())
|
||||
}
|
||||
|
||||
func stepHoldTagImpl(jobid string) (string, error) {
|
||||
t := fmt.Sprintf("zrepl_STEP_J_%s", jobid)
|
||||
if err := zfs.ValidHoldTag(t); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// err != nil always means that the bookmark is not a step bookmark
|
||||
func ParseStepHoldTag(tag string) (JobID, error) {
|
||||
match := stepHoldTagRE.FindStringSubmatch(tag)
|
||||
if match == nil {
|
||||
return JobID{}, fmt.Errorf("parse hold tag: match regex %q", stepHoldTagRE)
|
||||
}
|
||||
jobID, err := MakeJobID(match[1])
|
||||
if err != nil {
|
||||
return JobID{}, errors.Wrap(err, "parse hold tag: invalid job id field")
|
||||
}
|
||||
return jobID, nil
|
||||
}
|
||||
|
||||
// idempotently hold `version`
|
||||
func HoldStep(ctx context.Context, fs string, v zfs.FilesystemVersion, jobID JobID) (Abstraction, error) {
|
||||
if !v.IsSnapshot() {
|
||||
panic(fmt.Sprintf("version must be a snapshot got %#v", v))
|
||||
}
|
||||
|
||||
tag, err := StepHoldTag(jobID)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "step hold tag")
|
||||
}
|
||||
|
||||
if err := zfs.ZFSHold(ctx, fs, v, tag); err != nil {
|
||||
return nil, errors.Wrap(err, "step hold: zfs")
|
||||
}
|
||||
|
||||
return &holdBasedAbstraction{
|
||||
Type: AbstractionStepHold,
|
||||
FS: fs,
|
||||
Tag: tag,
|
||||
JobID: jobID,
|
||||
FilesystemVersion: v,
|
||||
}, nil
|
||||
|
||||
}
|
||||
|
||||
var _ HoldExtractor = StepHoldExtractor
|
||||
|
||||
func StepHoldExtractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion, holdTag string) Abstraction {
|
||||
if v.Type != zfs.Snapshot {
|
||||
panic("impl error")
|
||||
}
|
||||
|
||||
jobID, err := ParseStepHoldTag(holdTag)
|
||||
if err == nil {
|
||||
return &holdBasedAbstraction{
|
||||
Type: AbstractionStepHold,
|
||||
FS: fs.ToString(),
|
||||
Tag: holdTag,
|
||||
FilesystemVersion: v,
|
||||
JobID: jobID,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+2
-2
@@ -24,9 +24,9 @@ func MakeJobID(s string) (JobID, error) {
|
||||
return JobID{}, errors.Wrap(err, "must be usable as a dataset path component")
|
||||
}
|
||||
|
||||
if _, err := tentativeReplicationCursorBookmarkNameImpl("pool/ds", 0xface601d, s); err != nil {
|
||||
if _, err := stepBookmarkNameImpl("pool/ds", 0xface601d, s); err != nil {
|
||||
// note that this might still fail due to total maximum name length, but we can't enforce that
|
||||
return JobID{}, errors.Wrap(err, "must be usable for a tentative replication cursor bookmark")
|
||||
return JobID{}, errors.Wrap(err, "must be usable for a step bookmark")
|
||||
}
|
||||
|
||||
if _, err := stepHoldTagImpl(s); err != nil {
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
// Code generated by "enumer -type=ReplicationGuaranteeKind -json -transform=snake -trimprefix=ReplicationGuaranteeKind"; DO NOT EDIT.
|
||||
|
||||
//
|
||||
package endpoint
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
const (
|
||||
_ReplicationGuaranteeKindName_0 = "resumabilityincremental"
|
||||
_ReplicationGuaranteeKindName_1 = "none"
|
||||
)
|
||||
|
||||
var (
|
||||
_ReplicationGuaranteeKindIndex_0 = [...]uint8{0, 12, 23}
|
||||
_ReplicationGuaranteeKindIndex_1 = [...]uint8{0, 4}
|
||||
)
|
||||
|
||||
func (i ReplicationGuaranteeKind) String() string {
|
||||
switch {
|
||||
case 1 <= i && i <= 2:
|
||||
i -= 1
|
||||
return _ReplicationGuaranteeKindName_0[_ReplicationGuaranteeKindIndex_0[i]:_ReplicationGuaranteeKindIndex_0[i+1]]
|
||||
case i == 4:
|
||||
return _ReplicationGuaranteeKindName_1
|
||||
default:
|
||||
return fmt.Sprintf("ReplicationGuaranteeKind(%d)", i)
|
||||
}
|
||||
}
|
||||
|
||||
var _ReplicationGuaranteeKindValues = []ReplicationGuaranteeKind{1, 2, 4}
|
||||
|
||||
var _ReplicationGuaranteeKindNameToValueMap = map[string]ReplicationGuaranteeKind{
|
||||
_ReplicationGuaranteeKindName_0[0:12]: 1,
|
||||
_ReplicationGuaranteeKindName_0[12:23]: 2,
|
||||
_ReplicationGuaranteeKindName_1[0:4]: 4,
|
||||
}
|
||||
|
||||
// ReplicationGuaranteeKindString retrieves an enum value from the enum constants string name.
|
||||
// Throws an error if the param is not part of the enum.
|
||||
func ReplicationGuaranteeKindString(s string) (ReplicationGuaranteeKind, error) {
|
||||
if val, ok := _ReplicationGuaranteeKindNameToValueMap[s]; ok {
|
||||
return val, nil
|
||||
}
|
||||
return 0, fmt.Errorf("%s does not belong to ReplicationGuaranteeKind values", s)
|
||||
}
|
||||
|
||||
// ReplicationGuaranteeKindValues returns all values of the enum
|
||||
func ReplicationGuaranteeKindValues() []ReplicationGuaranteeKind {
|
||||
return _ReplicationGuaranteeKindValues
|
||||
}
|
||||
|
||||
// IsAReplicationGuaranteeKind returns "true" if the value is listed in the enum definition. "false" otherwise
|
||||
func (i ReplicationGuaranteeKind) IsAReplicationGuaranteeKind() bool {
|
||||
for _, v := range _ReplicationGuaranteeKindValues {
|
||||
if i == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// MarshalJSON implements the json.Marshaler interface for ReplicationGuaranteeKind
|
||||
func (i ReplicationGuaranteeKind) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(i.String())
|
||||
}
|
||||
|
||||
// UnmarshalJSON implements the json.Unmarshaler interface for ReplicationGuaranteeKind
|
||||
func (i *ReplicationGuaranteeKind) UnmarshalJSON(data []byte) error {
|
||||
var s string
|
||||
if err := json.Unmarshal(data, &s); err != nil {
|
||||
return fmt.Errorf("ReplicationGuaranteeKind should be a string, got %s", data)
|
||||
}
|
||||
|
||||
var err error
|
||||
*i, err = ReplicationGuaranteeKindString(s)
|
||||
return err
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
|
||||
@@ -14,32 +14,29 @@ step() {
|
||||
echo "${bold}$1${normal}"
|
||||
}
|
||||
|
||||
if ! type go >/dev/null; then
|
||||
step "go binary not installed or not in \$PATH" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$GOPATH" ]; then
|
||||
step "Make sure you have your GOPATH configured correctly" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CHECKOUTPATH="${GOPATH}/src/github.com/zrepl/zrepl"
|
||||
|
||||
godep() {
|
||||
step "install build dependencies (versions pinned in build/go.mod and build/tools.go)"
|
||||
|
||||
if ! type go >/dev/null; then
|
||||
step "go binary not installed or not in \$PATH" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$GOPATH" ]; then
|
||||
step "Your GOPATH is not configured correctly" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! (echo "$PATH" | grep "${GOPATH}/bin" > /dev/null); then
|
||||
step "GOPATH/bin is not in your PATH (it should be towards the start of it)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
pushd "$(dirname "${BASH_SOURCE[0]}")"/build
|
||||
set -x
|
||||
export GO111MODULE=on # otherwise, a checkout of this repo in GOPATH will disable modules on Go 1.12 and earlier
|
||||
source <(go env)
|
||||
export GOOS="$GOHOSTOS"
|
||||
export GOARCH="$GOHOSTARCH"
|
||||
# TODO GOARM=$GOHOSTARM?
|
||||
cat tools.go | grep _ | awk -F'"' '{print $2}' | tee | xargs -tI '{}' go install '{}'
|
||||
go build -v -mod=readonly -o "$GOPATH/bin/stringer" golang.org/x/tools/cmd/stringer
|
||||
go build -v -mod=readonly -o "$GOPATH/bin/protoc-gen-go" github.com/golang/protobuf/protoc-gen-go
|
||||
go build -v -mod=readonly -o "$GOPATH/bin/enumer" github.com/alvaroloes/enumer
|
||||
go build -v -mod=readonly -o "$GOPATH/bin/goimports" golang.org/x/tools/cmd/goimports
|
||||
go build -v -mod=readonly -o "$GOPATH/bin/golangci-lint" github.com/golangci/golangci-lint/cmd/golangci-lint
|
||||
go build -v -mod=readonly -o "$GOPATH/bin/gocovmerge" github.com/wadey/gocovmerge
|
||||
set +x
|
||||
popd
|
||||
if ! type stringer || ! type protoc-gen-go || ! type enumer || ! type goimports || ! type golangci-lint || ! type gocovmerge; then
|
||||
@@ -54,9 +51,8 @@ docdep() {
|
||||
exit 1
|
||||
fi
|
||||
step "Installing doc build dependencies"
|
||||
# shellcheck disable=SC2155
|
||||
local reqpath="$(dirname "${BASH_SOURCE[0]}")/docs/requirements.txt"
|
||||
if [ -n "$ZREPL_LAZY_DOCS_REQPATH" ]; then
|
||||
local reqpath="${CHECKOUTPATH}/docs/requirements.txt"
|
||||
if [ ! -z "$ZREPL_LAZY_DOCS_REQPATH" ]; then
|
||||
reqpath="$ZREPL_LAZY_DOCS_REQPATH"
|
||||
fi
|
||||
pip3 install -r "$reqpath"
|
||||
@@ -67,15 +63,9 @@ release() {
|
||||
make release
|
||||
}
|
||||
|
||||
# shellcheck disable=SC2198
|
||||
if [ -z "$@" ]; then
|
||||
step "No command specified, exiting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for cmd in "$@"; do
|
||||
case "$cmd" in
|
||||
godep|docdep|release|docs)
|
||||
godep|docdep|release_bins|docs)
|
||||
eval $cmd
|
||||
continue
|
||||
;;
|
||||
|
||||
@@ -82,7 +82,7 @@ func (l Level) String() string {
|
||||
case Error:
|
||||
return "error"
|
||||
default:
|
||||
return fmt.Sprintf("unknown level %d", l)
|
||||
return string(l)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
FROM debian:latest
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
build-essential \
|
||||
devscripts \
|
||||
dh-exec
|
||||
|
||||
RUN mkdir -p /build/src && chmod -R 0777 /build
|
||||
|
||||
WORKDIR /build/src
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
zrepl (VERSION) unstable;
|
||||
|
||||
Check https://zrepl.github.io/changelog.html.
|
||||
|
||||
-- zrepl build process <me@cschwarz.com> DATE_DASH_R_OUTPUT
|
||||
@@ -1 +0,0 @@
|
||||
11
|
||||
@@ -1,8 +0,0 @@
|
||||
Source: zrepl
|
||||
Maintainer: Christian Schwarz <me@cschwarz.com>
|
||||
Build-Depends: dh-exec
|
||||
|
||||
Package: zrepl
|
||||
Architecture: arm64 amd64 armhf i386
|
||||
Description: ZFS replication
|
||||
zrepl is a one-stop solution for ZFS filesystem replication
|
||||
@@ -1,50 +0,0 @@
|
||||
#!/usr/bin/make -f
|
||||
|
||||
%:
|
||||
dh $@
|
||||
|
||||
override_dh_strip:
|
||||
|
||||
override_dh_auto_build:
|
||||
|
||||
override_dh_auto_clean:
|
||||
|
||||
rm -rf debian/renamedir
|
||||
|
||||
override_dh_auto_install:
|
||||
|
||||
# install the zrepl.service for
|
||||
# dh_install_systemd and dh_systemd_{start,enable}
|
||||
install dist/systemd/zrepl.service debian/zrepl.service
|
||||
sed -i 's#ExecStart=/usr/local/bin/zrepl #ExecStart=/usr/bin/zrepl #' debian/zrepl.service
|
||||
sed -i 's#ExecStartPre=/usr/local/bin/zrepl #ExecStartPre=/usr/bin/zrepl #' debian/zrepl.service
|
||||
|
||||
mkdir -p debian/renamedir
|
||||
|
||||
# install binary
|
||||
stat artifacts/$(ZREPL_DPKG_ZREPL_BINARY_FILENAME)
|
||||
cp --preserve=all artifacts/$(ZREPL_DPKG_ZREPL_BINARY_FILENAME) debian/renamedir/zrepl
|
||||
dh_install debian/renamedir/zrepl usr/bin
|
||||
|
||||
# install bash completion
|
||||
dh_install artifacts/bash_completion etc/bash_completion.d/zrepl
|
||||
dh_install artifacts/_zrepl.zsh_completion usr/share/zsh/vendor-completions
|
||||
|
||||
# install docs
|
||||
dh_install artifacts/docs/html usr/share/doc/zrepl/docs/
|
||||
|
||||
# install examples
|
||||
dh_install config/samples/* usr/share/doc/zrepl/examples
|
||||
|
||||
# install default config
|
||||
mkdir -p debian/tmp/etc/zrepl
|
||||
chmod 0700 debian/tmp/etc/zrepl
|
||||
sed 's#USR_SHARE_ZREPL#/usr/share/doc/zrepl#' packaging/systemd-default-zrepl.yml > debian/renamedir/zrepl.yml
|
||||
dh_install debian/renamedir/zrepl.yml etc/zrepl
|
||||
|
||||
# save git revision of this packaging repo
|
||||
echo $$(git rev-parse HEAD) > debian/packaging-repo-git-revision
|
||||
dh_install debian/packaging-repo-git-revision usr/share/doc/zrepl
|
||||
|
||||
override_dh_auto_test:
|
||||
# don't run tests at this point
|
||||
@@ -1,7 +0,0 @@
|
||||
FROM fedora:latest
|
||||
|
||||
RUN dnf install -y git make bash rpm-build 'dnf-command(builddep)'
|
||||
ADD packaging/rpm/zrepl.spec /tmp/zrepl.spec
|
||||
RUN dnf builddep -y /tmp/zrepl.spec
|
||||
RUN mkdir -p /build/src && chmod -R 0777 /build
|
||||
WORKDIR /build/src
|
||||
@@ -1,78 +0,0 @@
|
||||
# This Spec file packages pre-built artifacts in the artifacts directory
|
||||
# into an RPM. This means that rpmbuild won't actually build any new artifacts
|
||||
#
|
||||
# Read the Makefile first, then come back here.
|
||||
|
||||
# Global meta data
|
||||
Version: SUBSTITUTED_BY_MAKEFILE
|
||||
%global common_description %{expand:
|
||||
zrepl is a one-stop, integrated solution for ZFS replication.}
|
||||
|
||||
# use gzip to be compatible with centos7
|
||||
%define _source_payload w9.gzdio
|
||||
%define _binary_payload w9.gzdio
|
||||
|
||||
# don't strip pre-built binaries
|
||||
%define __strip /usr/bin/true
|
||||
|
||||
Name: zrepl
|
||||
Release: 1
|
||||
Summary: One-stop, integrated solution for ZFS replication
|
||||
License: MIT
|
||||
URL: https://zrepl.github.io/
|
||||
# Source: we use rpmbuild --build-in-tree => no source
|
||||
BuildRequires: systemd
|
||||
Requires(post): systemd
|
||||
Requires(preun): systemd
|
||||
Requires(postun): systemd
|
||||
|
||||
%description
|
||||
%{common_description}
|
||||
|
||||
%prep
|
||||
# we use rpmbuild --build-in-tree => no prep or setup
|
||||
|
||||
%build
|
||||
# we don't actually need to build anything here, that has already been done by the Makefile
|
||||
|
||||
# Correct the path in the systemd unit file
|
||||
sed s:/usr/local/bin/:%{_bindir}/:g dist/systemd/zrepl.service > artifacts/rpmbuild/zrepl.service
|
||||
|
||||
# Generate the default configuration file
|
||||
sed 's#USR_SHARE_ZREPL#%{_datadir}/doc/zrepl#' packaging/systemd-default-zrepl.yml > artifacts/rpmbuild/zrepl.yml
|
||||
|
||||
%install
|
||||
install -Dm 0755 artifacts/%{_zrepl_binary_filename} %{buildroot}%{_bindir}/zrepl
|
||||
install -Dm 0644 artifacts/rpmbuild/zrepl.service %{buildroot}%{_unitdir}/zrepl.service
|
||||
install -Dm 0644 artifacts/_zrepl.zsh_completion %{buildroot}%{_datadir}/zsh/site-functions/_zrepl
|
||||
install -Dm 0644 artifacts/bash_completion %{buildroot}%{_datadir}/bash-completion/completions/zrepl
|
||||
install -Dm 0644 artifacts/rpmbuild/zrepl.yml %{buildroot}%{_sysconfdir}/zrepl/zrepl.yml
|
||||
install -d %{buildroot}%{_datadir}/doc/zrepl
|
||||
cp -a artifacts/docs/html %{buildroot}%{_datadir}/doc/zrepl/html
|
||||
cp -a config/samples %{buildroot}%{_datadir}/doc/zrepl/examples
|
||||
|
||||
%post
|
||||
%systemd_post zrepl.service
|
||||
|
||||
|
||||
%preun
|
||||
%systemd_preun zrepl.service
|
||||
|
||||
|
||||
%postun
|
||||
%systemd_postun_with_restart zrepl.service
|
||||
|
||||
|
||||
%files
|
||||
%defattr(-,root,root)
|
||||
%license LICENSE
|
||||
%{_bindir}/zrepl
|
||||
%config %{_unitdir}/zrepl.service
|
||||
%dir %{_sysconfdir}/zrepl
|
||||
%config %{_sysconfdir}/zrepl/zrepl.yml
|
||||
%{_datadir}/zsh/site-functions/_zrepl
|
||||
%{_datadir}/bash-completion/completions/zrepl
|
||||
%{_datadir}/doc/zrepl
|
||||
|
||||
%changelog
|
||||
# TODO: auto-fill changelog from git? -> need same solution for debian
|
||||
@@ -1,13 +0,0 @@
|
||||
global:
|
||||
logging:
|
||||
# use syslog instead of stdout because it makes journald happy
|
||||
- type: syslog
|
||||
format: human
|
||||
level: warn
|
||||
|
||||
jobs:
|
||||
# - name: foo
|
||||
# type: bar
|
||||
|
||||
# see USR_SHARE_ZREPL/examples
|
||||
# or https://zrepl.github.io/configuration/overview.html
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
|
||||
@@ -16,21 +16,15 @@ var Cases = []Case{BatchDestroy,
|
||||
ListFilesystemsNoFilter,
|
||||
ReceiveForceIntoEncryptedErr,
|
||||
ReceiveForceRollbackWorksUnencrypted,
|
||||
ReplicationFailingInitialParentProhibitsChildReplication,
|
||||
ReplicationIncrementalCleansUpStaleAbstractionsWithCacheOnSecondReplication,
|
||||
ReplicationIncrementalCleansUpStaleAbstractionsWithoutCacheOnSecondReplication,
|
||||
ReplicationIncrementalDestroysStepHoldsIffIncrementalStepHoldsAreDisabledButStepHoldsExist,
|
||||
ReplicationIncrementalIsPossibleIfCommonSnapshotIsDestroyed,
|
||||
ReplicationIsResumableFullSend__both_GuaranteeResumability,
|
||||
ReplicationIsResumableFullSend__initial_GuaranteeIncrementalReplication_incremental_GuaranteeIncrementalReplication,
|
||||
ReplicationIsResumableFullSend__initial_GuaranteeResumability_incremental_GuaranteeIncrementalReplication,
|
||||
ReplicationReceiverErrorWhileStillSending,
|
||||
ReplicationStepCompletedLostBehavior__GuaranteeIncrementalReplication,
|
||||
ReplicationStepCompletedLostBehavior__GuaranteeResumability,
|
||||
ReplicationIsResumableFullSend__DisableIncrementalStepHolds_False,
|
||||
ReplicationIsResumableFullSend__DisableIncrementalStepHolds_True,
|
||||
ResumableRecvAndTokenHandling,
|
||||
ResumeTokenParsing,
|
||||
SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden__EncryptionSupported_false,
|
||||
SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden__EncryptionSupported_true,
|
||||
SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden,
|
||||
SendArgsValidationResumeTokenDifferentFilesystemForbidden,
|
||||
SendArgsValidationResumeTokenEncryptionMismatchForbidden,
|
||||
UndestroyableSnapshotParsing,
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"path"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zrepl/zrepl/platformtest"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"fmt"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/zrepl/zrepl/platformtest"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
@@ -4,20 +4,11 @@ import (
|
||||
"fmt"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zrepl/zrepl/platformtest"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
func ReceiveForceIntoEncryptedErr(ctx *platformtest.Context) {
|
||||
|
||||
supported, err := zfs.EncryptionCLISupported(ctx)
|
||||
require.NoError(ctx, err, "encryption feature test failed")
|
||||
if !supported {
|
||||
ctx.SkipNow()
|
||||
return
|
||||
}
|
||||
|
||||
platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
|
||||
DESTROYROOT
|
||||
CREATEROOT
|
||||
@@ -41,7 +32,6 @@ func ReceiveForceIntoEncryptedErr(ctx *platformtest.Context) {
|
||||
|
||||
sendStream, err := zfs.ZFSSend(ctx, sendArgs)
|
||||
require.NoError(ctx, err)
|
||||
defer sendStream.Close()
|
||||
|
||||
recvOpts := zfs.RecvOptions{
|
||||
RollbackAndForceRecv: true,
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zrepl/zrepl/platformtest"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
@@ -4,13 +4,11 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
|
||||
"github.com/kr/pretty"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/platformtest"
|
||||
@@ -29,13 +27,11 @@ import (
|
||||
// of a new sender and receiver instance and one blocking invocation
|
||||
// of the replication engine without encryption
|
||||
type replicationInvocation struct {
|
||||
sjid, rjid endpoint.JobID
|
||||
sfs string
|
||||
sfilter *filters.DatasetMapFilter
|
||||
rfsRoot string
|
||||
interceptSender func(e *endpoint.Sender) logic.Sender
|
||||
interceptReceiver func(e *endpoint.Receiver) logic.Receiver
|
||||
guarantee pdu.ReplicationConfigProtection
|
||||
sjid, rjid endpoint.JobID
|
||||
sfs string
|
||||
rfsRoot string
|
||||
interceptSender func(e *endpoint.Sender) logic.Sender
|
||||
disableIncrementalStepHolds bool
|
||||
}
|
||||
|
||||
func (i replicationInvocation) Do(ctx *platformtest.Context) *report.Report {
|
||||
@@ -43,33 +39,24 @@ func (i replicationInvocation) Do(ctx *platformtest.Context) *report.Report {
|
||||
if i.interceptSender == nil {
|
||||
i.interceptSender = func(e *endpoint.Sender) logic.Sender { return e }
|
||||
}
|
||||
if i.interceptReceiver == nil {
|
||||
i.interceptReceiver = func(e *endpoint.Receiver) logic.Receiver { return e }
|
||||
}
|
||||
|
||||
if i.sfs != "" && i.sfilter != nil || i.sfs == "" && i.sfilter == nil {
|
||||
panic("either sfs or sfilter must be set")
|
||||
}
|
||||
if i.sfilter == nil {
|
||||
i.sfilter = filters.NewDatasetMapFilter(1, true)
|
||||
err := i.sfilter.Add(i.sfs, "ok")
|
||||
require.NoError(ctx, err)
|
||||
}
|
||||
sfilter := filters.NewDatasetMapFilter(1, true)
|
||||
err := sfilter.Add(i.sfs, "ok")
|
||||
require.NoError(ctx, err)
|
||||
sender := i.interceptSender(endpoint.NewSender(endpoint.SenderConfig{
|
||||
FSF: i.sfilter.AsFilter(),
|
||||
Encrypt: &zfs.NilBool{B: false},
|
||||
JobID: i.sjid,
|
||||
FSF: sfilter.AsFilter(),
|
||||
Encrypt: &zfs.NilBool{B: false},
|
||||
DisableIncrementalStepHolds: i.disableIncrementalStepHolds,
|
||||
JobID: i.sjid,
|
||||
}))
|
||||
receiver := i.interceptReceiver(endpoint.NewReceiver(endpoint.ReceiverConfig{
|
||||
receiver := endpoint.NewReceiver(endpoint.ReceiverConfig{
|
||||
JobID: i.rjid,
|
||||
AppendClientIdentity: false,
|
||||
RootWithoutClientComponent: mustDatasetPath(i.rfsRoot),
|
||||
}))
|
||||
UpdateLastReceivedHold: true,
|
||||
})
|
||||
plannerPolicy := logic.PlannerPolicy{
|
||||
EncryptedSend: logic.TriFromBool(false),
|
||||
ReplicationConfig: pdu.ReplicationConfig{
|
||||
Protection: &i.guarantee,
|
||||
},
|
||||
}
|
||||
|
||||
report, wait := replication.Do(
|
||||
@@ -102,11 +89,11 @@ func ReplicationIncrementalIsPossibleIfCommonSnapshotIsDestroyed(ctx *platformte
|
||||
snap1 := fsversion(ctx, sfs, "@1")
|
||||
|
||||
rep := replicationInvocation{
|
||||
sjid: sjid,
|
||||
rjid: rjid,
|
||||
sfs: sfs,
|
||||
rfsRoot: rfsRoot,
|
||||
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
|
||||
sjid: sjid,
|
||||
rjid: rjid,
|
||||
sfs: sfs,
|
||||
rfsRoot: rfsRoot,
|
||||
disableIncrementalStepHolds: false,
|
||||
}
|
||||
rfs := rep.ReceiveSideFilesystem()
|
||||
|
||||
@@ -166,11 +153,11 @@ func implReplicationIncrementalCleansUpStaleAbstractions(ctx *platformtest.Conte
|
||||
rfsRoot := ctx.RootDataset + "/receiver"
|
||||
|
||||
rep := replicationInvocation{
|
||||
sjid: sjid,
|
||||
rjid: rjid,
|
||||
sfs: sfs,
|
||||
rfsRoot: rfsRoot,
|
||||
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
|
||||
sjid: sjid,
|
||||
rjid: rjid,
|
||||
sfs: sfs,
|
||||
rfsRoot: rfsRoot,
|
||||
disableIncrementalStepHolds: false,
|
||||
}
|
||||
rfs := rep.ReceiveSideFilesystem()
|
||||
|
||||
@@ -220,7 +207,7 @@ func implReplicationIncrementalCleansUpStaleAbstractions(ctx *platformtest.Conte
|
||||
snap5 := fsversion(ctx, sfs, "@5")
|
||||
|
||||
if invalidateCacheBeforeSecondReplication {
|
||||
endpoint.AbstractionsCacheInvalidate(sfs)
|
||||
endpoint.SendAbstractionsCacheInvalidate(sfs)
|
||||
}
|
||||
|
||||
// do another replication
|
||||
@@ -340,59 +327,15 @@ func (s *PartialSender) Send(ctx context.Context, r *pdu.SendReq) (r1 *pdu.SendR
|
||||
return r1, r2, r3
|
||||
}
|
||||
|
||||
func ReplicationIsResumableFullSend__both_GuaranteeResumability(ctx *platformtest.Context) {
|
||||
|
||||
setup := replicationIsResumableFullSendSetup{
|
||||
protection: pdu.ReplicationConfigProtection{
|
||||
Initial: pdu.ReplicationGuaranteeKind_GuaranteeResumability,
|
||||
Incremental: pdu.ReplicationGuaranteeKind_GuaranteeResumability,
|
||||
},
|
||||
expectDatasetIsBusyErrorWhenDestroySnapshotWhilePartiallyReplicated: true,
|
||||
expectAllThreeSnapshotsToThreeBePresentAfterLoop: true,
|
||||
expectNoSnapshotsOnReceiverAfterLoop: false,
|
||||
}
|
||||
|
||||
implReplicationIsResumableFullSend(ctx, setup)
|
||||
func ReplicationIsResumableFullSend__DisableIncrementalStepHolds_False(ctx *platformtest.Context) {
|
||||
implReplicationIsResumableFullSend(ctx, false)
|
||||
}
|
||||
|
||||
func ReplicationIsResumableFullSend__initial_GuaranteeResumability_incremental_GuaranteeIncrementalReplication(ctx *platformtest.Context) {
|
||||
|
||||
setup := replicationIsResumableFullSendSetup{
|
||||
protection: pdu.ReplicationConfigProtection{
|
||||
Initial: pdu.ReplicationGuaranteeKind_GuaranteeResumability,
|
||||
Incremental: pdu.ReplicationGuaranteeKind_GuaranteeIncrementalReplication,
|
||||
},
|
||||
expectDatasetIsBusyErrorWhenDestroySnapshotWhilePartiallyReplicated: true,
|
||||
expectAllThreeSnapshotsToThreeBePresentAfterLoop: true,
|
||||
expectNoSnapshotsOnReceiverAfterLoop: false,
|
||||
}
|
||||
|
||||
implReplicationIsResumableFullSend(ctx, setup)
|
||||
func ReplicationIsResumableFullSend__DisableIncrementalStepHolds_True(ctx *platformtest.Context) {
|
||||
implReplicationIsResumableFullSend(ctx, true)
|
||||
}
|
||||
|
||||
func ReplicationIsResumableFullSend__initial_GuaranteeIncrementalReplication_incremental_GuaranteeIncrementalReplication(ctx *platformtest.Context) {
|
||||
|
||||
setup := replicationIsResumableFullSendSetup{
|
||||
protection: pdu.ReplicationConfigProtection{
|
||||
Initial: pdu.ReplicationGuaranteeKind_GuaranteeIncrementalReplication,
|
||||
Incremental: pdu.ReplicationGuaranteeKind_GuaranteeIncrementalReplication,
|
||||
},
|
||||
expectDatasetIsBusyErrorWhenDestroySnapshotWhilePartiallyReplicated: false,
|
||||
expectAllThreeSnapshotsToThreeBePresentAfterLoop: false,
|
||||
expectNoSnapshotsOnReceiverAfterLoop: true,
|
||||
}
|
||||
|
||||
implReplicationIsResumableFullSend(ctx, setup)
|
||||
}
|
||||
|
||||
type replicationIsResumableFullSendSetup struct {
|
||||
protection pdu.ReplicationConfigProtection
|
||||
expectDatasetIsBusyErrorWhenDestroySnapshotWhilePartiallyReplicated bool
|
||||
expectAllThreeSnapshotsToThreeBePresentAfterLoop bool
|
||||
expectNoSnapshotsOnReceiverAfterLoop bool
|
||||
}
|
||||
|
||||
func implReplicationIsResumableFullSend(ctx *platformtest.Context, setup replicationIsResumableFullSendSetup) {
|
||||
func implReplicationIsResumableFullSend(ctx *platformtest.Context, disableIncrementalStepHolds bool) {
|
||||
|
||||
platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
|
||||
CREATEROOT
|
||||
@@ -423,9 +366,8 @@ func implReplicationIsResumableFullSend(ctx *platformtest.Context, setup replica
|
||||
interceptSender: func(e *endpoint.Sender) logic.Sender {
|
||||
return &PartialSender{Sender: e, failAfterByteCount: 1 << 20}
|
||||
},
|
||||
guarantee: setup.protection,
|
||||
disableIncrementalStepHolds: disableIncrementalStepHolds,
|
||||
}
|
||||
|
||||
rfs := rep.ReceiveSideFilesystem()
|
||||
|
||||
for i := 2; i < 10; i++ {
|
||||
@@ -439,11 +381,8 @@ func implReplicationIsResumableFullSend(ctx *platformtest.Context, setup replica
|
||||
// and we wrote dummy data 1<<22 bytes, thus at least
|
||||
// for the first 4 times this should not be possible
|
||||
// due to step holds
|
||||
if setup.expectDatasetIsBusyErrorWhenDestroySnapshotWhilePartiallyReplicated {
|
||||
ctx.Logf("i=%v", i)
|
||||
require.Error(ctx, err)
|
||||
require.Contains(ctx, err.Error(), "dataset is busy")
|
||||
}
|
||||
require.Error(ctx, err)
|
||||
require.Contains(ctx, err.Error(), "dataset is busy")
|
||||
}
|
||||
|
||||
// and create some additional snapshots that could
|
||||
@@ -462,19 +401,11 @@ func implReplicationIsResumableFullSend(ctx *platformtest.Context, setup replica
|
||||
}
|
||||
}
|
||||
|
||||
if setup.expectAllThreeSnapshotsToThreeBePresentAfterLoop {
|
||||
// make sure all the filesystem versions we created
|
||||
// were replicated by the replication loop
|
||||
_ = fsversion(ctx, rfs, "@1")
|
||||
_ = fsversion(ctx, rfs, "@2")
|
||||
_ = fsversion(ctx, rfs, "@3")
|
||||
}
|
||||
|
||||
if setup.expectNoSnapshotsOnReceiverAfterLoop {
|
||||
versions, err := zfs.ZFSListFilesystemVersions(ctx, mustDatasetPath(rfs), zfs.ListFilesystemVersionsOptions{})
|
||||
require.NoError(ctx, err)
|
||||
require.Empty(ctx, versions)
|
||||
}
|
||||
// make sure all the filesystem versions we created
|
||||
// were replicated by the replication loop
|
||||
_ = fsversion(ctx, rfs, "@1")
|
||||
_ = fsversion(ctx, rfs, "@2")
|
||||
_ = fsversion(ctx, rfs, "@3")
|
||||
|
||||
}
|
||||
|
||||
@@ -497,11 +428,11 @@ func ReplicationIncrementalDestroysStepHoldsIffIncrementalStepHoldsAreDisabledBu
|
||||
{
|
||||
mustSnapshot(ctx, sfs+"@1")
|
||||
rep := replicationInvocation{
|
||||
sjid: sjid,
|
||||
rjid: rjid,
|
||||
sfs: sfs,
|
||||
rfsRoot: rfsRoot,
|
||||
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
|
||||
sjid: sjid,
|
||||
rjid: rjid,
|
||||
sfs: sfs,
|
||||
rfsRoot: rfsRoot,
|
||||
disableIncrementalStepHolds: false,
|
||||
}
|
||||
rfs := rep.ReceiveSideFilesystem()
|
||||
report := rep.Do(ctx)
|
||||
@@ -524,11 +455,11 @@ func ReplicationIncrementalDestroysStepHoldsIffIncrementalStepHoldsAreDisabledBu
|
||||
// to effect a step-holds situation
|
||||
{
|
||||
rep := replicationInvocation{
|
||||
sjid: sjid,
|
||||
rjid: rjid,
|
||||
sfs: sfs,
|
||||
rfsRoot: rfsRoot,
|
||||
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability), // !
|
||||
sjid: sjid,
|
||||
rjid: rjid,
|
||||
sfs: sfs,
|
||||
rfsRoot: rfsRoot,
|
||||
disableIncrementalStepHolds: false, // !
|
||||
interceptSender: func(e *endpoint.Sender) logic.Sender {
|
||||
return &PartialSender{Sender: e, failAfterByteCount: 1 << 20}
|
||||
},
|
||||
@@ -564,17 +495,17 @@ func ReplicationIncrementalDestroysStepHoldsIffIncrementalStepHoldsAreDisabledBu
|
||||
// end of test setup
|
||||
//
|
||||
|
||||
// retry replication with incremental step holds disabled (set to bookmarks-only in this case)
|
||||
// retry replication with incremental step holds disabled
|
||||
// - replication should not fail due to holds-related stuff
|
||||
// - replication should fail intermittently due to partial sender being fully read
|
||||
// - the partial sender is 1/4th the length of the stream, thus expect
|
||||
// successful replication after 5 more attempts
|
||||
rep := replicationInvocation{
|
||||
sjid: sjid,
|
||||
rjid: rjid,
|
||||
sfs: sfs,
|
||||
rfsRoot: rfsRoot,
|
||||
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeIncrementalReplication), // !
|
||||
sjid: sjid,
|
||||
rjid: rjid,
|
||||
sfs: sfs,
|
||||
rfsRoot: rfsRoot,
|
||||
disableIncrementalStepHolds: true, // !
|
||||
interceptSender: func(e *endpoint.Sender) logic.Sender {
|
||||
return &PartialSender{Sender: e, failAfterByteCount: 1 << 20}
|
||||
},
|
||||
@@ -620,298 +551,3 @@ func ReplicationIncrementalDestroysStepHoldsIffIncrementalStepHoldsAreDisabledBu
|
||||
require.Len(ctx, abs, 1)
|
||||
require.True(ctx, zfs.FilesystemVersionEqualIdentity(abs[0].GetFilesystemVersion(), snap2sfs))
|
||||
}
|
||||
|
||||
func ReplicationStepCompletedLostBehavior__GuaranteeResumability(ctx *platformtest.Context) {
|
||||
scenario := replicationStepCompletedLostBehavior_impl(ctx, pdu.ReplicationGuaranteeKind_GuaranteeResumability)
|
||||
|
||||
require.Error(ctx, scenario.deleteSfs1Err, "protected by holds")
|
||||
require.Contains(ctx, scenario.deleteSfs1Err.Error(), "dataset is busy")
|
||||
|
||||
require.Error(ctx, scenario.deleteSfs2Err, "protected by holds")
|
||||
require.Contains(ctx, scenario.deleteSfs2Err.Error(), "dataset is busy")
|
||||
|
||||
require.Nil(ctx, scenario.finalReport.Error())
|
||||
_ = fsversion(ctx, scenario.rfs, "@3") // @3 ade it to the other side
|
||||
}
|
||||
|
||||
func ReplicationStepCompletedLostBehavior__GuaranteeIncrementalReplication(ctx *platformtest.Context) {
|
||||
scenario := replicationStepCompletedLostBehavior_impl(ctx, pdu.ReplicationGuaranteeKind_GuaranteeIncrementalReplication)
|
||||
|
||||
require.NoError(ctx, scenario.deleteSfs1Err, "not protected by holds")
|
||||
require.NoError(ctx, scenario.deleteSfs2Err, "not protected by holds")
|
||||
|
||||
// step bookmarks should protect against loss of StepCompleted message
|
||||
require.Nil(ctx, scenario.finalReport.Error())
|
||||
_ = fsversion(ctx, scenario.rfs, "@3") // @3 ade it to the other side
|
||||
}
|
||||
|
||||
type FailSendCompletedSender struct {
|
||||
*endpoint.Sender
|
||||
}
|
||||
|
||||
var _ logic.Sender = (*FailSendCompletedSender)(nil)
|
||||
|
||||
func (p *FailSendCompletedSender) SendCompleted(ctx context.Context, r *pdu.SendCompletedReq) (*pdu.SendCompletedRes, error) {
|
||||
return nil, fmt.Errorf("[mock] SendCompleted not delivered to actual endpoint")
|
||||
}
|
||||
|
||||
type replicationStepCompletedLost_scenario struct {
|
||||
rfs string
|
||||
deleteSfs1Err, deleteSfs2Err error
|
||||
finalReport *report.FilesystemReport
|
||||
}
|
||||
|
||||
func replicationStepCompletedLostBehavior_impl(ctx *platformtest.Context, guaranteeKind pdu.ReplicationGuaranteeKind) *replicationStepCompletedLost_scenario {
|
||||
|
||||
platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
|
||||
CREATEROOT
|
||||
+ "sender"
|
||||
+ "receiver"
|
||||
R zfs create -p "${ROOTDS}/receiver/${ROOTDS}"
|
||||
`)
|
||||
|
||||
sjid := endpoint.MustMakeJobID("sender-job")
|
||||
rjid := endpoint.MustMakeJobID("receiver-job")
|
||||
|
||||
sfs := ctx.RootDataset + "/sender"
|
||||
rfsRoot := ctx.RootDataset + "/receiver"
|
||||
|
||||
// fully replicate snapshots @1
|
||||
{
|
||||
mustSnapshot(ctx, sfs+"@1")
|
||||
rep := replicationInvocation{
|
||||
sjid: sjid,
|
||||
rjid: rjid,
|
||||
sfs: sfs,
|
||||
rfsRoot: rfsRoot,
|
||||
guarantee: *pdu.ReplicationConfigProtectionWithKind(guaranteeKind),
|
||||
}
|
||||
rfs := rep.ReceiveSideFilesystem()
|
||||
report := rep.Do(ctx)
|
||||
ctx.Logf("\n%s", pretty.Sprint(report))
|
||||
// assert this worked (not the main subject of the test)
|
||||
_ = fsversion(ctx, rfs, "@1")
|
||||
}
|
||||
|
||||
// create a second snapshot @2
|
||||
mustSnapshot(ctx, sfs+"@2")
|
||||
|
||||
// fake loss of stepcompleted message
|
||||
rep := replicationInvocation{
|
||||
sjid: sjid,
|
||||
rjid: rjid,
|
||||
sfs: sfs,
|
||||
rfsRoot: rfsRoot,
|
||||
guarantee: *pdu.ReplicationConfigProtectionWithKind(guaranteeKind),
|
||||
interceptSender: func(e *endpoint.Sender) logic.Sender {
|
||||
return &FailSendCompletedSender{e}
|
||||
},
|
||||
}
|
||||
rfs := rep.ReceiveSideFilesystem()
|
||||
report := rep.Do(ctx)
|
||||
ctx.Logf("\n%s", pretty.Sprint(report))
|
||||
|
||||
// assert the replication worked
|
||||
_ = fsversion(ctx, rfs, "@2")
|
||||
// and that we hold it using a last-received-hold
|
||||
abs, absErrs, err := endpoint.ListAbstractions(ctx, endpoint.ListZFSHoldsAndBookmarksQuery{
|
||||
FS: endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
|
||||
FS: &rfs,
|
||||
},
|
||||
Concurrency: 1,
|
||||
JobID: &rjid,
|
||||
What: endpoint.AbstractionTypeSet{endpoint.AbstractionLastReceivedHold: true},
|
||||
})
|
||||
require.NoError(ctx, err)
|
||||
require.Empty(ctx, absErrs)
|
||||
require.Len(ctx, abs, 1)
|
||||
require.True(ctx, zfs.FilesystemVersionEqualIdentity(abs[0].GetFilesystemVersion(), fsversion(ctx, rfs, "@2")))
|
||||
|
||||
// now try to delete @2 on the sender, this should work because don't have step holds on it
|
||||
deleteSfs2Err := zfs.ZFSDestroy(ctx, sfs+"@2")
|
||||
// defer check to caller
|
||||
|
||||
// and create a new snapshot on the sender
|
||||
mustSnapshot(ctx, sfs+"@3")
|
||||
|
||||
// now we have: sender @1, @3
|
||||
// recver @1, @2
|
||||
|
||||
// delete @1 on both sides to demonstrate that, if we didn't have bookmarks, we would be out of sync
|
||||
deleteSfs1Err := zfs.ZFSDestroy(ctx, sfs+"@1")
|
||||
// defer check to caller
|
||||
err = zfs.ZFSDestroy(ctx, rfs+"@1")
|
||||
require.NoError(ctx, err)
|
||||
|
||||
// attempt replication and return the filesystem report report
|
||||
{
|
||||
rep := replicationInvocation{
|
||||
sjid: sjid,
|
||||
rjid: rjid,
|
||||
sfs: sfs,
|
||||
rfsRoot: rfsRoot,
|
||||
guarantee: *pdu.ReplicationConfigProtectionWithKind(guaranteeKind),
|
||||
}
|
||||
report := rep.Do(ctx)
|
||||
ctx.Logf("expecting failure:\n%s", pretty.Sprint(report))
|
||||
require.Len(ctx, report.Attempts, 1)
|
||||
require.Len(ctx, report.Attempts[0].Filesystems, 1)
|
||||
return &replicationStepCompletedLost_scenario{
|
||||
rfs: rfs,
|
||||
deleteSfs1Err: deleteSfs1Err,
|
||||
deleteSfs2Err: deleteSfs2Err,
|
||||
finalReport: report.Attempts[0].Filesystems[0],
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
type ErroringReceiver struct {
|
||||
recvErr error
|
||||
*endpoint.Receiver
|
||||
}
|
||||
|
||||
func (r *ErroringReceiver) Receive(ctx context.Context, req *pdu.ReceiveReq, stream io.ReadCloser) (*pdu.ReceiveRes, error) {
|
||||
return nil, r.recvErr
|
||||
}
|
||||
|
||||
type NeverEndingSender struct {
|
||||
*endpoint.Sender
|
||||
}
|
||||
|
||||
func (s *NeverEndingSender) Send(ctx context.Context, req *pdu.SendReq) (r *pdu.SendRes, stream io.ReadCloser, _ error) {
|
||||
stream = nil
|
||||
r = &pdu.SendRes{
|
||||
UsedResumeToken: false,
|
||||
ExpectedSize: 1 << 30,
|
||||
}
|
||||
if req.DryRun {
|
||||
return r, stream, nil
|
||||
}
|
||||
dz, err := os.Open("/dev/zero")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return r, dz, nil
|
||||
}
|
||||
|
||||
func ReplicationReceiverErrorWhileStillSending(ctx *platformtest.Context) {
|
||||
|
||||
platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
|
||||
CREATEROOT
|
||||
+ "sender"
|
||||
+ "sender@1"
|
||||
+ "receiver"
|
||||
R zfs create -p "${ROOTDS}/receiver/${ROOTDS}"
|
||||
`)
|
||||
|
||||
sjid := endpoint.MustMakeJobID("sender-job")
|
||||
rjid := endpoint.MustMakeJobID("receiver-job")
|
||||
|
||||
sfs := ctx.RootDataset + "/sender"
|
||||
rfsRoot := ctx.RootDataset + "/receiver"
|
||||
|
||||
mockRecvErr := fmt.Errorf("YiezahK3thie8ahKiel5sah2uugei2ize1yi8feivuu7musoat")
|
||||
|
||||
rep := replicationInvocation{
|
||||
sjid: sjid,
|
||||
rjid: rjid,
|
||||
sfs: sfs,
|
||||
rfsRoot: rfsRoot,
|
||||
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeNothing),
|
||||
interceptReceiver: func(r *endpoint.Receiver) logic.Receiver {
|
||||
return &ErroringReceiver{recvErr: mockRecvErr, Receiver: r}
|
||||
},
|
||||
interceptSender: func(s *endpoint.Sender) logic.Sender {
|
||||
return &NeverEndingSender{s}
|
||||
},
|
||||
}
|
||||
|
||||
// first replication
|
||||
report := rep.Do(ctx)
|
||||
ctx.Logf("\n%s", pretty.Sprint(report))
|
||||
|
||||
require.Len(ctx, report.Attempts, 1)
|
||||
attempt := report.Attempts[0]
|
||||
require.Nil(ctx, attempt.PlanError)
|
||||
require.Len(ctx, attempt.Filesystems, 1)
|
||||
afs := attempt.Filesystems[0]
|
||||
require.Nil(ctx, afs.PlanError)
|
||||
require.Len(ctx, afs.Steps, 1)
|
||||
require.Nil(ctx, afs.PlanError)
|
||||
require.NotNil(ctx, afs.StepError)
|
||||
require.Contains(ctx, afs.StepError.Err, mockRecvErr.Error())
|
||||
}
|
||||
|
||||
func ReplicationFailingInitialParentProhibitsChildReplication(ctx *platformtest.Context) {
|
||||
|
||||
platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
|
||||
CREATEROOT
|
||||
+ "sender"
|
||||
+ "sender/a"
|
||||
+ "sender/a/child"
|
||||
+ "sender/aa"
|
||||
+ "receiver"
|
||||
R zfs create -p "${ROOTDS}/receiver/${ROOTDS}"
|
||||
R zfs snapshot -r ${ROOTDS}/sender@initial
|
||||
`)
|
||||
|
||||
sjid := endpoint.MustMakeJobID("sender-job")
|
||||
rjid := endpoint.MustMakeJobID("receiver-job")
|
||||
|
||||
fsA := ctx.RootDataset + "/sender/a"
|
||||
fsAChild := ctx.RootDataset + "/sender/a/child"
|
||||
fsAA := ctx.RootDataset + "/sender/aa"
|
||||
|
||||
sfilter := filters.NewDatasetMapFilter(3, true)
|
||||
mustAddToSFilter := func(fs string) {
|
||||
err := sfilter.Add(fs, "ok")
|
||||
require.NoError(ctx, err)
|
||||
}
|
||||
mustAddToSFilter(fsA)
|
||||
mustAddToSFilter(fsAChild)
|
||||
mustAddToSFilter(fsAA)
|
||||
rfsRoot := ctx.RootDataset + "/receiver"
|
||||
|
||||
mockRecvErr := fmt.Errorf("yifae4ohPhaquaes0hohghiep9oufie4roo7quoWooluaj2ee8")
|
||||
|
||||
rep := replicationInvocation{
|
||||
sjid: sjid,
|
||||
rjid: rjid,
|
||||
sfilter: sfilter,
|
||||
rfsRoot: rfsRoot,
|
||||
guarantee: *pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeNothing),
|
||||
interceptReceiver: func(r *endpoint.Receiver) logic.Receiver {
|
||||
return &ErroringReceiver{recvErr: mockRecvErr, Receiver: r}
|
||||
},
|
||||
}
|
||||
|
||||
r := rep.Do(ctx)
|
||||
ctx.Logf("\n%s", pretty.Sprint(r))
|
||||
|
||||
require.Len(ctx, r.Attempts, 1)
|
||||
attempt := r.Attempts[0]
|
||||
require.Nil(ctx, attempt.PlanError)
|
||||
require.Len(ctx, attempt.Filesystems, 3)
|
||||
|
||||
fsByName := make(map[string]*report.FilesystemReport, len(attempt.Filesystems))
|
||||
for _, fs := range attempt.Filesystems {
|
||||
fsByName[fs.Info.Name] = fs
|
||||
}
|
||||
|
||||
require.Contains(ctx, fsByName, fsA)
|
||||
require.Contains(ctx, fsByName, fsAA)
|
||||
require.Contains(ctx, fsByName, fsAA)
|
||||
|
||||
checkFS := func(fs string, expectErrMsg string) {
|
||||
rep := fsByName[fs]
|
||||
require.Len(ctx, rep.Steps, 1)
|
||||
require.Nil(ctx, rep.PlanError)
|
||||
require.NotNil(ctx, rep.StepError)
|
||||
require.Contains(ctx, rep.StepError.Err, expectErrMsg)
|
||||
}
|
||||
|
||||
checkFS(fsA, mockRecvErr.Error())
|
||||
checkFS(fsAChild, "parent(s) failed during initial replication")
|
||||
checkFS(fsAA, mockRecvErr.Error()) // fsAA is not treated as a child of fsA
|
||||
}
|
||||
|
||||
@@ -22,7 +22,6 @@ func CreateReplicationCursor(ctx *platformtest.Context) {
|
||||
R zfs bookmark "${ROOTDS}/foo bar@2 with space" "${ROOTDS}/foo bar#2 with space"
|
||||
+ "foo bar@3 with space"
|
||||
R zfs bookmark "${ROOTDS}/foo bar@3 with space" "${ROOTDS}/foo bar#3 with space"
|
||||
- "foo bar@3 with space"
|
||||
`)
|
||||
|
||||
jobid := endpoint.MustMakeJobID("zreplplatformtest")
|
||||
@@ -43,7 +42,6 @@ func CreateReplicationCursor(ctx *platformtest.Context) {
|
||||
|
||||
snap := fsversion(ctx, fs, "@1 with space")
|
||||
book := fsversion(ctx, fs, "#1 with space")
|
||||
book3 := fsversion(ctx, fs, "#3 with space")
|
||||
|
||||
// create first cursor
|
||||
cursorOfSnap, err := endpoint.CreateReplicationCursor(ctx, fs, snap, jobid)
|
||||
@@ -51,11 +49,8 @@ func CreateReplicationCursor(ctx *platformtest.Context) {
|
||||
// check CreateReplicationCursor is idempotent (for snapshot target)
|
||||
cursorOfSnapIdemp, err := endpoint.CreateReplicationCursor(ctx, fs, snap, jobid)
|
||||
checkCreateCursor(err, cursorOfSnap, snap)
|
||||
// check CreateReplicationCursor is idempotent (for bookmark target of snapshot)
|
||||
cursorOfBook, err := endpoint.CreateReplicationCursor(ctx, fs, book, jobid)
|
||||
checkCreateCursor(err, cursorOfBook, snap)
|
||||
// ... for target = non-cursor bookmark
|
||||
_, err = endpoint.CreateReplicationCursor(ctx, fs, book3, jobid)
|
||||
_, err = endpoint.CreateReplicationCursor(ctx, fs, book, jobid)
|
||||
assert.Equal(ctx, zfs.ErrBookmarkCloningNotSupported, err)
|
||||
// ... for target = replication cursor bookmark to be created
|
||||
cursorOfCursor, err := endpoint.CreateReplicationCursor(ctx, fs, cursorOfSnapIdemp.GetFilesystemVersion(), jobid)
|
||||
|
||||
@@ -9,22 +9,11 @@ import (
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
func SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden__EncryptionSupported_true(ctx *platformtest.Context) {
|
||||
sendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden_impl(ctx, true)
|
||||
}
|
||||
|
||||
func SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden__EncryptionSupported_false(ctx *platformtest.Context) {
|
||||
sendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden_impl(ctx, false)
|
||||
}
|
||||
|
||||
func sendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden_impl(ctx *platformtest.Context, testForEncryptionSupported bool) {
|
||||
func SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden(ctx *platformtest.Context) {
|
||||
|
||||
supported, err := zfs.EncryptionCLISupported(ctx)
|
||||
check(err)
|
||||
if supported != testForEncryptionSupported {
|
||||
ctx.SkipNow()
|
||||
}
|
||||
noEncryptionCLISupport := !supported
|
||||
expectNotSupportedErr := !supported
|
||||
|
||||
platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
|
||||
DESTROYROOT
|
||||
@@ -55,11 +44,9 @@ func sendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden_impl(ctx *plat
|
||||
// fallthrough
|
||||
}
|
||||
|
||||
if noEncryptionCLISupport {
|
||||
if expectNotSupportedErr {
|
||||
require.Error(ctx, err)
|
||||
saverr, ok := err.(*zfs.ZFSSendArgsValidationError)
|
||||
require.True(ctx, ok, "%T", err)
|
||||
require.Equal(ctx, zfs.ZFSSendArgsEncryptedSendRequestedButFSUnencrypted, saverr.What)
|
||||
require.Equal(ctx, zfs.ErrEncryptedSendNotSupported, err)
|
||||
return
|
||||
}
|
||||
require.Error(ctx, err)
|
||||
@@ -151,7 +138,13 @@ func SendArgsValidationResumeTokenEncryptionMismatchForbidden(ctx *platformtest.
|
||||
}
|
||||
|
||||
func SendArgsValidationResumeTokenDifferentFilesystemForbidden(ctx *platformtest.Context) {
|
||||
supported, err := zfs.ResumeSendSupported(ctx)
|
||||
|
||||
supported, err := zfs.EncryptionCLISupported(ctx)
|
||||
check(err)
|
||||
if !supported {
|
||||
ctx.SkipNow()
|
||||
}
|
||||
supported, err = zfs.ResumeSendSupported(ctx)
|
||||
check(err)
|
||||
if !supported {
|
||||
ctx.SkipNow()
|
||||
|
||||
+42
-54
@@ -3,6 +3,7 @@ package pruning
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
@@ -29,88 +30,75 @@ func NewKeepGrid(in *config.PruneGrid) (p *KeepGrid, err error) {
|
||||
return nil, errors.Wrap(err, "Regex is invalid")
|
||||
}
|
||||
|
||||
return newKeepGrid(re, in.Grid)
|
||||
}
|
||||
|
||||
func MustNewKeepGrid(regex, gridspec string) *KeepGrid {
|
||||
|
||||
ris, err := config.ParseRetentionIntervalSpec(gridspec)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
re := regexp.MustCompile(regex)
|
||||
|
||||
grid, err := newKeepGrid(re, ris)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return grid
|
||||
}
|
||||
|
||||
func newKeepGrid(re *regexp.Regexp, configIntervals []config.RetentionInterval) (*KeepGrid, error) {
|
||||
if re == nil {
|
||||
panic("re must not be nil")
|
||||
}
|
||||
|
||||
if len(configIntervals) == 0 {
|
||||
return nil, errors.New("retention grid must specify at least one interval")
|
||||
}
|
||||
|
||||
intervals := make([]retentiongrid.Interval, len(configIntervals))
|
||||
for i := range configIntervals {
|
||||
intervals[i] = &configIntervals[i]
|
||||
}
|
||||
|
||||
// Assert intervals are of increasing length (not necessarily required, but indicates config mistake)
|
||||
lastDuration := time.Duration(0)
|
||||
for i := range intervals {
|
||||
for i := range in.Grid {
|
||||
|
||||
if intervals[i].Length() < lastDuration {
|
||||
if in.Grid[i].Length() < lastDuration {
|
||||
// If all intervals before were keep=all, this is ok
|
||||
allPrevKeepCountAll := true
|
||||
for j := i - 1; allPrevKeepCountAll && j >= 0; j-- {
|
||||
allPrevKeepCountAll = intervals[j].KeepCount() == config.RetentionGridKeepCountAll
|
||||
allPrevKeepCountAll = in.Grid[j].KeepCount() == config.RetentionGridKeepCountAll
|
||||
}
|
||||
if allPrevKeepCountAll {
|
||||
goto isMonotonicIncrease
|
||||
}
|
||||
return nil, errors.New("retention grid interval length must be monotonically increasing")
|
||||
err = errors.New("retention grid interval length must be monotonically increasing")
|
||||
return
|
||||
}
|
||||
isMonotonicIncrease:
|
||||
lastDuration = intervals[i].Length()
|
||||
lastDuration = in.Grid[i].Length()
|
||||
|
||||
}
|
||||
|
||||
retentionIntervals := make([]retentiongrid.Interval, len(in.Grid))
|
||||
for i := range in.Grid {
|
||||
retentionIntervals[i] = &in.Grid[i]
|
||||
}
|
||||
|
||||
return &KeepGrid{
|
||||
retentionGrid: retentiongrid.NewGrid(intervals),
|
||||
re: re,
|
||||
retentiongrid.NewGrid(retentionIntervals),
|
||||
re,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type retentionGridAdaptor struct {
|
||||
Snapshot
|
||||
}
|
||||
|
||||
func (a retentionGridAdaptor) LessThan(b retentiongrid.Entry) bool {
|
||||
return a.Date().Before(b.Date())
|
||||
}
|
||||
|
||||
// Prune filters snapshots with the retention grid.
|
||||
func (p *KeepGrid) KeepRule(snaps []Snapshot) (destroyList []Snapshot) {
|
||||
|
||||
matching, notMatching := partitionSnapList(snaps, func(snapshot Snapshot) bool {
|
||||
snaps = filterSnapList(snaps, func(snapshot Snapshot) bool {
|
||||
return p.re.MatchString(snapshot.Name())
|
||||
})
|
||||
|
||||
// snaps that don't match the regex are not kept by this rule
|
||||
destroyList = append(destroyList, notMatching...)
|
||||
|
||||
if len(matching) == 0 {
|
||||
return destroyList
|
||||
if len(snaps) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Build adaptors for retention grid
|
||||
adaptors := make([]retentiongrid.Entry, 0)
|
||||
for i := range snaps {
|
||||
adaptors = append(adaptors, retentionGridAdaptor{snaps[i]})
|
||||
}
|
||||
|
||||
// determine 'now' edge
|
||||
sort.SliceStable(adaptors, func(i, j int) bool {
|
||||
return adaptors[i].LessThan(adaptors[j])
|
||||
})
|
||||
now := adaptors[len(adaptors)-1].Date()
|
||||
|
||||
// Evaluate retention grid
|
||||
entrySlice := make([]retentiongrid.Entry, 0)
|
||||
for i := range matching {
|
||||
entrySlice = append(entrySlice, matching[i])
|
||||
}
|
||||
_, gridDestroyList := p.retentionGrid.FitEntries(entrySlice)
|
||||
_, removea := p.retentionGrid.FitEntries(now, adaptors)
|
||||
|
||||
// Revert adaptors
|
||||
for i := range gridDestroyList {
|
||||
destroyList = append(destroyList, gridDestroyList[i].(Snapshot))
|
||||
destroyList = make([]Snapshot, len(removea))
|
||||
for i := range removea {
|
||||
destroyList[i] = removea[i].(retentionGridAdaptor).Snapshot
|
||||
}
|
||||
return destroyList
|
||||
}
|
||||
|
||||
@@ -10,17 +10,6 @@ func filterSnapList(snaps []Snapshot, predicate func(Snapshot) bool) []Snapshot
|
||||
return r
|
||||
}
|
||||
|
||||
func partitionSnapList(snaps []Snapshot, predicate func(Snapshot) bool) (sTrue, sFalse []Snapshot) {
|
||||
for i := range snaps {
|
||||
if predicate(snaps[i]) {
|
||||
sTrue = append(sTrue, snaps[i])
|
||||
} else {
|
||||
sFalse = append(sFalse, snaps[i])
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func shallowCopySnapList(snaps []Snapshot) []Snapshot {
|
||||
c := make([]Snapshot, len(snaps))
|
||||
copy(c, snaps)
|
||||
|
||||
+8
-37
@@ -1,35 +1,20 @@
|
||||
package pruning
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type KeepLastN struct {
|
||||
n int
|
||||
re *regexp.Regexp
|
||||
n int
|
||||
}
|
||||
|
||||
func MustKeepLastN(n int, regex string) *KeepLastN {
|
||||
k, err := NewKeepLastN(n, regex)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
func NewKeepLastN(n int, regex string) (*KeepLastN, error) {
|
||||
func NewKeepLastN(n int) (*KeepLastN, error) {
|
||||
if n <= 0 {
|
||||
return nil, errors.Errorf("must specify positive number as 'keep last count', got %d", n)
|
||||
}
|
||||
re, err := regexp.Compile(regex)
|
||||
if err != nil {
|
||||
return nil, errors.Errorf("invalid regex %q: %s", regex, err)
|
||||
}
|
||||
return &KeepLastN{n, re}, nil
|
||||
return &KeepLastN{n}, nil
|
||||
}
|
||||
|
||||
func (k KeepLastN) KeepRule(snaps []Snapshot) (destroyList []Snapshot) {
|
||||
@@ -38,25 +23,11 @@ func (k KeepLastN) KeepRule(snaps []Snapshot) (destroyList []Snapshot) {
|
||||
return []Snapshot{}
|
||||
}
|
||||
|
||||
matching, notMatching := partitionSnapList(snaps, func(snapshot Snapshot) bool {
|
||||
return k.re.MatchString(snapshot.Name())
|
||||
})
|
||||
// snaps that don't match the regex are not kept by this rule
|
||||
destroyList = append(destroyList, notMatching...)
|
||||
res := shallowCopySnapList(snaps)
|
||||
|
||||
if len(matching) == 0 {
|
||||
return destroyList
|
||||
}
|
||||
|
||||
sort.Slice(matching, func(i, j int) bool {
|
||||
// by date (youngest first)
|
||||
id, jd := matching[i].Date(), matching[j].Date()
|
||||
if !id.Equal(jd) {
|
||||
return id.After(jd)
|
||||
}
|
||||
// then lexicographically descending (e.g. b, a)
|
||||
return strings.Compare(matching[i].Name(), matching[j].Name()) == 1
|
||||
sort.Slice(res, func(i, j int) bool {
|
||||
return res[i].Date().After(res[j].Date())
|
||||
})
|
||||
destroyList = append(destroyList, matching[k.n:]...)
|
||||
return destroyList
|
||||
|
||||
return res[k.n:]
|
||||
}
|
||||
|
||||
+11
-41
@@ -5,7 +5,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestKeepLastN(t *testing.T) {
|
||||
@@ -29,7 +28,7 @@ func TestKeepLastN(t *testing.T) {
|
||||
"keep2": {
|
||||
inputs: inputs["s1"],
|
||||
rules: []KeepRule{
|
||||
MustKeepLastN(2, ""),
|
||||
KeepLastN{2},
|
||||
},
|
||||
expDestroy: map[string]bool{
|
||||
"1": true, "2": true, "3": true,
|
||||
@@ -38,66 +37,37 @@ func TestKeepLastN(t *testing.T) {
|
||||
"keep1OfTwoWithSameTime": { // Keep one of two with same time
|
||||
inputs: inputs["s1"],
|
||||
rules: []KeepRule{
|
||||
MustKeepLastN(1, ""),
|
||||
KeepLastN{1},
|
||||
},
|
||||
expDestroyAlternatives: []map[string]bool{
|
||||
{"1": true, "2": true, "3": true, "4": true},
|
||||
{"1": true, "2": true, "3": true, "5": true},
|
||||
},
|
||||
expDestroy: map[string]bool{"1": true, "2": true, "3": true, "4": true},
|
||||
},
|
||||
"keepMany": {
|
||||
inputs: inputs["s1"],
|
||||
rules: []KeepRule{
|
||||
MustKeepLastN(100, ""),
|
||||
KeepLastN{100},
|
||||
},
|
||||
expDestroy: map[string]bool{},
|
||||
},
|
||||
"empty_input": {
|
||||
"empty": {
|
||||
inputs: inputs["s2"],
|
||||
rules: []KeepRule{
|
||||
MustKeepLastN(100, ""),
|
||||
KeepLastN{100},
|
||||
},
|
||||
expDestroy: map[string]bool{},
|
||||
},
|
||||
"empty_regex": {
|
||||
inputs: inputs["s1"],
|
||||
rules: []KeepRule{
|
||||
MustKeepLastN(4, ""),
|
||||
},
|
||||
expDestroy: map[string]bool{
|
||||
"1": true,
|
||||
},
|
||||
},
|
||||
"multiple_regexes": {
|
||||
inputs: []Snapshot{
|
||||
stubSnap{"a1", false, o(10)},
|
||||
stubSnap{"b1", false, o(11)},
|
||||
stubSnap{"a2", false, o(20)},
|
||||
stubSnap{"b2", false, o(21)},
|
||||
stubSnap{"a3", false, o(30)},
|
||||
stubSnap{"b3", false, o(31)},
|
||||
},
|
||||
rules: []KeepRule{
|
||||
MustKeepLastN(2, "^a"),
|
||||
MustKeepLastN(2, "^b"),
|
||||
},
|
||||
expDestroy: map[string]bool{
|
||||
"a1": true,
|
||||
"b1": true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
testTable(tcs, t)
|
||||
|
||||
t.Run("mustBePositive", func(t *testing.T) {
|
||||
var err error
|
||||
_, err = NewKeepLastN(0, "foo")
|
||||
_, err = NewKeepLastN(0)
|
||||
assert.Error(t, err)
|
||||
_, err = NewKeepLastN(-5, "foo")
|
||||
_, err = NewKeepLastN(-5)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("emptyRegexAllowed", func(t *testing.T) {
|
||||
_, err := NewKeepLastN(23, "")
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -60,7 +60,7 @@ func RuleFromConfig(in config.PruningEnum) (KeepRule, error) {
|
||||
case *config.PruneKeepNotReplicated:
|
||||
return NewKeepNotReplicated(), nil
|
||||
case *config.PruneKeepLastN:
|
||||
return NewKeepLastN(v.Count, v.Regex)
|
||||
return NewKeepLastN(v.Count)
|
||||
case *config.PruneKeepRegex:
|
||||
return NewKeepRegex(v.Regex, v.Negate)
|
||||
case *config.PruneGrid:
|
||||
|
||||
+35
-38
@@ -3,9 +3,6 @@ package pruning
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type stubSnap struct {
|
||||
@@ -21,9 +18,10 @@ func (s stubSnap) Replicated() bool { return s.replicated }
|
||||
func (s stubSnap) Date() time.Time { return s.date }
|
||||
|
||||
type testCase struct {
|
||||
inputs []Snapshot
|
||||
rules []KeepRule
|
||||
expDestroy map[string]bool
|
||||
inputs []Snapshot
|
||||
rules []KeepRule
|
||||
expDestroy, effDestroy map[string]bool
|
||||
expDestroyAlternatives []map[string]bool
|
||||
}
|
||||
|
||||
type snapshotList []Snapshot
|
||||
@@ -46,20 +44,41 @@ func (l snapshotList) NameList() []string {
|
||||
}
|
||||
|
||||
func testTable(tcs map[string]testCase, t *testing.T) {
|
||||
mapEqual := func(a, b map[string]bool) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for k, v := range a {
|
||||
if w, ok := b[k]; !ok || v != w {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
for name := range tcs {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
tc := tcs[name]
|
||||
destroyList := PruneSnapshots(tc.inputs, tc.rules)
|
||||
destroySet := make(map[string]bool, len(destroyList))
|
||||
for _, s := range destroyList {
|
||||
destroySet[s.Name()] = true
|
||||
remove := PruneSnapshots(tc.inputs, tc.rules)
|
||||
tc.effDestroy = make(map[string]bool)
|
||||
for _, s := range remove {
|
||||
tc.effDestroy[s.Name()] = true
|
||||
}
|
||||
t.Logf("destroySet:\n%#v", destroySet)
|
||||
t.Logf("expected:\n%#v", tc.expDestroy)
|
||||
|
||||
require.Equal(t, len(tc.expDestroy), len(destroySet))
|
||||
for name := range destroySet {
|
||||
assert.True(t, tc.expDestroy[name], "%q", name)
|
||||
if tc.expDestroyAlternatives == nil {
|
||||
if tc.expDestroy == nil {
|
||||
panic("must specify either expDestroyAlternatives or expDestroy")
|
||||
}
|
||||
tc.expDestroyAlternatives = []map[string]bool{tc.expDestroy}
|
||||
}
|
||||
var okAlt map[string]bool = nil
|
||||
for _, alt := range tc.expDestroyAlternatives {
|
||||
t.Logf("testing possible result: %v", alt)
|
||||
if mapEqual(alt, tc.effDestroy) {
|
||||
okAlt = alt
|
||||
}
|
||||
}
|
||||
if okAlt == nil {
|
||||
t.Errorf("no alternatives matched result: %v", tc.effDestroy)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -75,10 +94,6 @@ func TestPruneSnapshots(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
reltime := func(secs int64) time.Time {
|
||||
return time.Unix(secs, 0)
|
||||
}
|
||||
|
||||
tcs := map[string]testCase{
|
||||
"simple": {
|
||||
inputs: inputs["s1"],
|
||||
@@ -125,24 +140,6 @@ func TestPruneSnapshots(t *testing.T) {
|
||||
},
|
||||
expDestroy: map[string]bool{},
|
||||
},
|
||||
"multiple_grids_with_disjoint_regexes": {
|
||||
inputs: []Snapshot{
|
||||
stubSnap{"p1_a", false, reltime(4)},
|
||||
stubSnap{"p2_a", false, reltime(5)},
|
||||
stubSnap{"p1_b", false, reltime(14)},
|
||||
stubSnap{"p2_b", false, reltime(15)},
|
||||
stubSnap{"p1_c", false, reltime(29)},
|
||||
stubSnap{"p2_c", false, reltime(30)},
|
||||
},
|
||||
rules: []KeepRule{
|
||||
MustNewKeepGrid("^p1_", `1x10s | 1x10s`),
|
||||
MustNewKeepGrid("^p2_", `1x10s | 1x10s`),
|
||||
},
|
||||
expDestroy: map[string]bool{
|
||||
"p1_a": true,
|
||||
"p2_a": true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
testTable(tcs, t)
|
||||
|
||||
@@ -16,127 +16,85 @@ type Grid struct {
|
||||
intervals []Interval
|
||||
}
|
||||
|
||||
//A point inside the grid, i.e. a thing the grid can decide to remove
|
||||
type Entry interface {
|
||||
Date() time.Time
|
||||
LessThan(b Entry) bool
|
||||
}
|
||||
|
||||
func dateInInterval(date, startDateInterval time.Time, i Interval) bool {
|
||||
return date.After(startDateInterval) && date.Before(startDateInterval.Add(i.Length()))
|
||||
}
|
||||
|
||||
func NewGrid(l []Interval) *Grid {
|
||||
if len(l) == 0 {
|
||||
panic("must specify at least one interval")
|
||||
}
|
||||
// TODO Maybe check for ascending interval lengths here, although the algorithm
|
||||
// itself doesn't care about that.
|
||||
return &Grid{l}
|
||||
}
|
||||
|
||||
func (g Grid) FitEntries(entries []Entry) (keep, remove []Entry) {
|
||||
// Partition a list of RetentionGridEntries into the Grid,
|
||||
// relative to a given start date `now`.
|
||||
//
|
||||
// The `keepCount` oldest entries per `retentiongrid.Interval` are kept (`keep`),
|
||||
// the others are removed (`remove`).
|
||||
//
|
||||
// Entries that are younger than `now` are always kept.
|
||||
// Those that are older than the earliest beginning of an interval are removed.
|
||||
func (g Grid) FitEntries(now time.Time, entries []Entry) (keep, remove []Entry) {
|
||||
|
||||
if len(entries) == 0 {
|
||||
return
|
||||
type bucket struct {
|
||||
entries []Entry
|
||||
}
|
||||
|
||||
// determine 'now' based on youngest snapshot
|
||||
// => sort youngest-to-oldest
|
||||
sort.SliceStable(entries, func(i, j int) bool {
|
||||
return entries[i].Date().After(entries[j].Date())
|
||||
})
|
||||
now := entries[0].Date()
|
||||
|
||||
return g.fitEntriesWithNow(now, entries)
|
||||
}
|
||||
|
||||
type bucket struct {
|
||||
keepCount int
|
||||
youngerThan time.Time
|
||||
olderThanOrEq time.Time
|
||||
entries []Entry
|
||||
}
|
||||
|
||||
func makeBucketFromInterval(olderThanOrEq time.Time, i Interval) bucket {
|
||||
var b bucket
|
||||
kc := i.KeepCount()
|
||||
if kc == 0 {
|
||||
panic("keep count 0 is not allowed")
|
||||
}
|
||||
if (kc < 0) && kc != RetentionGridKeepCountAll {
|
||||
panic("negative keep counts are not allowed")
|
||||
}
|
||||
b.keepCount = kc
|
||||
b.olderThanOrEq = olderThanOrEq
|
||||
b.youngerThan = b.olderThanOrEq.Add(-i.Length())
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *bucket) Contains(e Entry) bool {
|
||||
d := e.Date()
|
||||
olderThan := d.Before(b.olderThanOrEq)
|
||||
eq := d.Equal(b.olderThanOrEq)
|
||||
youngerThan := d.After(b.youngerThan)
|
||||
return (olderThan || eq) && youngerThan
|
||||
}
|
||||
|
||||
func (b *bucket) AddIfContains(e Entry) (added bool) {
|
||||
added = b.Contains(e)
|
||||
if added {
|
||||
b.entries = append(b.entries, e)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (b *bucket) RemoveYoungerSnapsExceedingKeepCount() (removed []Entry) {
|
||||
|
||||
if b.keepCount == RetentionGridKeepCountAll {
|
||||
return nil
|
||||
}
|
||||
|
||||
removeCount := len(b.entries) - b.keepCount
|
||||
if removeCount <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// sort youngest-to-oldest
|
||||
sort.SliceStable(b.entries, func(i, j int) bool {
|
||||
return b.entries[i].Date().After(b.entries[j].Date())
|
||||
})
|
||||
|
||||
return b.entries[:removeCount]
|
||||
}
|
||||
|
||||
func (g Grid) fitEntriesWithNow(now time.Time, entries []Entry) (keep, remove []Entry) {
|
||||
|
||||
buckets := make([]bucket, len(g.intervals))
|
||||
|
||||
buckets[0] = makeBucketFromInterval(now, g.intervals[0])
|
||||
for i := 1; i < len(g.intervals); i++ {
|
||||
buckets[i] = makeBucketFromInterval(buckets[i-1].youngerThan, g.intervals[i])
|
||||
}
|
||||
|
||||
keep = make([]Entry, 0)
|
||||
remove = make([]Entry, 0)
|
||||
|
||||
assignEntriesToBuckets:
|
||||
for ei := 0; ei < len(entries); ei++ {
|
||||
e := entries[ei]
|
||||
// unconditionally keep entries that are in the future
|
||||
if now.Before(e.Date()) {
|
||||
keep = append(keep, e)
|
||||
continue assignEntriesToBuckets
|
||||
}
|
||||
// add to matching bucket, if any
|
||||
for bi := range buckets {
|
||||
if buckets[bi].AddIfContains(e) {
|
||||
continue assignEntriesToBuckets
|
||||
}
|
||||
}
|
||||
// unconditionally remove entries older than the oldest bucket
|
||||
remove = append(remove, e)
|
||||
oldestIntervalStart := now
|
||||
for i := range g.intervals {
|
||||
oldestIntervalStart = oldestIntervalStart.Add(-g.intervals[i].Length())
|
||||
}
|
||||
|
||||
// now apply the `KeepCount` per bucket
|
||||
for _, b := range buckets {
|
||||
destroy := b.RemoveYoungerSnapsExceedingKeepCount()
|
||||
remove = append(remove, destroy...)
|
||||
keep = append(keep, b.entries...)
|
||||
for ei := 0; ei < len(entries); ei++ {
|
||||
e := entries[ei]
|
||||
|
||||
date := e.Date()
|
||||
|
||||
if date == now || date.After(now) {
|
||||
keep = append(keep, e)
|
||||
continue
|
||||
} else if date.Before(oldestIntervalStart) {
|
||||
remove = append(remove, e)
|
||||
continue
|
||||
}
|
||||
|
||||
iStartTime := now
|
||||
for i := 0; i < len(g.intervals); i++ {
|
||||
iStartTime = iStartTime.Add(-g.intervals[i].Length())
|
||||
if date == iStartTime || dateInInterval(date, iStartTime, g.intervals[i]) {
|
||||
buckets[i].entries = append(buckets[i].entries, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for bi, b := range buckets {
|
||||
|
||||
interval := g.intervals[bi]
|
||||
|
||||
sort.SliceStable(b.entries, func(i, j int) bool {
|
||||
return b.entries[i].LessThan((b.entries[j]))
|
||||
})
|
||||
|
||||
i := 0
|
||||
for ; (interval.KeepCount() == RetentionGridKeepCountAll || i < interval.KeepCount()) && i < len(b.entries); i++ {
|
||||
keep = append(keep, b.entries[i])
|
||||
}
|
||||
for ; i < len(b.entries); i++ {
|
||||
remove = append(remove, b.entries[i])
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
@@ -10,18 +10,25 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
type testInterval struct {
|
||||
type retentionIntervalStub struct {
|
||||
length time.Duration
|
||||
keepCount int
|
||||
}
|
||||
|
||||
func (i *testInterval) Length() time.Duration { return i.length }
|
||||
func (i *testInterval) KeepCount() int { return i.keepCount }
|
||||
func (i *retentionIntervalStub) Length() time.Duration {
|
||||
return i.length
|
||||
}
|
||||
|
||||
func (i *retentionIntervalStub) KeepCount() int {
|
||||
return i.keepCount
|
||||
}
|
||||
|
||||
func gridFromString(gs string) (g *Grid) {
|
||||
sintervals := strings.Split(gs, "|")
|
||||
intervals := make([]Interval, len(sintervals))
|
||||
for idx, i := range sintervals {
|
||||
intervals := strings.Split(gs, "|")
|
||||
g = &Grid{
|
||||
intervals: make([]Interval, len(intervals)),
|
||||
}
|
||||
for idx, i := range intervals {
|
||||
comps := strings.SplitN(i, ",", 2)
|
||||
var durationStr, numSnapsStr string
|
||||
durationStr = comps[0]
|
||||
@@ -32,7 +39,7 @@ func gridFromString(gs string) (g *Grid) {
|
||||
}
|
||||
|
||||
var err error
|
||||
var interval testInterval
|
||||
var interval retentionIntervalStub
|
||||
|
||||
if interval.keepCount, err = strconv.Atoi(numSnapsStr); err != nil {
|
||||
panic(err)
|
||||
@@ -41,38 +48,44 @@ func gridFromString(gs string) (g *Grid) {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
intervals[idx] = &interval
|
||||
g.intervals[idx] = &interval
|
||||
}
|
||||
return NewGrid(intervals)
|
||||
return
|
||||
}
|
||||
|
||||
type testSnap struct {
|
||||
type dummySnap struct {
|
||||
Name string
|
||||
ShouldKeep bool
|
||||
date time.Time
|
||||
}
|
||||
|
||||
func (ds testSnap) Date() time.Time { return ds.date }
|
||||
func (ds dummySnap) Date() time.Time {
|
||||
return ds.date
|
||||
}
|
||||
|
||||
func (ds dummySnap) LessThan(b Entry) bool {
|
||||
return ds.date.Before(b.(dummySnap).date) // don't have a txg here
|
||||
}
|
||||
|
||||
func validateRetentionGridFitEntries(t *testing.T, now time.Time, input, keep, remove []Entry) {
|
||||
|
||||
snapDescr := func(d testSnap) string {
|
||||
snapDescr := func(d dummySnap) string {
|
||||
return fmt.Sprintf("%s@%s", d.Name, d.date.Sub(now))
|
||||
}
|
||||
|
||||
t.Logf("keep list:\n")
|
||||
for k := range keep {
|
||||
t.Logf("\t%s\n", snapDescr(keep[k].(testSnap)))
|
||||
t.Logf("\t%s\n", snapDescr(keep[k].(dummySnap)))
|
||||
}
|
||||
t.Logf("remove list:\n")
|
||||
for k := range remove {
|
||||
t.Logf("\t%s\n", snapDescr(remove[k].(testSnap)))
|
||||
t.Logf("\t%s\n", snapDescr(remove[k].(dummySnap)))
|
||||
}
|
||||
|
||||
t.Logf("\n\n")
|
||||
|
||||
for _, s := range input {
|
||||
d := s.(testSnap)
|
||||
d := s.(dummySnap)
|
||||
descr := snapDescr(d)
|
||||
t.Logf("testing %s\n", descr)
|
||||
if d.ShouldKeep {
|
||||
@@ -84,18 +97,21 @@ func validateRetentionGridFitEntries(t *testing.T, now time.Time, input, keep, r
|
||||
|
||||
t.Logf("resulting list:\n")
|
||||
for k := range keep {
|
||||
t.Logf("\t%s\n", snapDescr(keep[k].(testSnap)))
|
||||
t.Logf("\t%s\n", snapDescr(keep[k].(dummySnap)))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyInput(t *testing.T) {
|
||||
func TestRetentionGridFitEntriesEmptyInput(t *testing.T) {
|
||||
g := gridFromString("10m|10m|10m|1h")
|
||||
keep, remove := g.FitEntries([]Entry{})
|
||||
keep, remove := g.FitEntries(time.Now(), []Entry{})
|
||||
assert.Empty(t, keep)
|
||||
assert.Empty(t, remove)
|
||||
}
|
||||
|
||||
func TestIntervalBoundariesAndAlignment(t *testing.T) {
|
||||
func TestRetentionGridFitEntriesIntervalBoundariesAndAlignment(t *testing.T) {
|
||||
|
||||
// Intervals are (duration], i.e. 10min is in the first interval, not in the second
|
||||
|
||||
g := gridFromString("10m|10m|10m")
|
||||
|
||||
t.Logf("%#v\n", g)
|
||||
@@ -103,51 +119,20 @@ func TestIntervalBoundariesAndAlignment(t *testing.T) {
|
||||
now := time.Unix(0, 0)
|
||||
|
||||
snaps := []Entry{
|
||||
testSnap{"0", true, now.Add(1 * time.Minute)}, // before now => keep unconditionally
|
||||
testSnap{"1", true, now}, // 1st interval left edge => inclusive
|
||||
testSnap{"2", true, now.Add(-10 * time.Minute)}, // 2nd interval left edge => inclusive
|
||||
testSnap{"3", true, now.Add(-20 * time.Minute)}, // 3rd interval left edge => inclusuive
|
||||
testSnap{"4", false, now.Add(-30 * time.Minute)}, // 3rd interval right edge => excludive
|
||||
testSnap{"5", false, now.Add(-40 * time.Minute)}, // after last interval => remove unconditionally
|
||||
dummySnap{"0", true, now.Add(1 * time.Minute)}, // before now
|
||||
dummySnap{"1", true, now}, // before now
|
||||
dummySnap{"2", true, now.Add(-10 * time.Minute)}, // 1st interval
|
||||
dummySnap{"3", true, now.Add(-20 * time.Minute)}, // 2nd interval
|
||||
dummySnap{"4", true, now.Add(-30 * time.Minute)}, // 3rd interval
|
||||
dummySnap{"5", false, now.Add(-40 * time.Minute)}, // after last interval
|
||||
}
|
||||
|
||||
keep, remove := g.fitEntriesWithNow(now, snaps)
|
||||
keep, remove := g.FitEntries(now, snaps)
|
||||
validateRetentionGridFitEntries(t, now, snaps, keep, remove)
|
||||
|
||||
}
|
||||
|
||||
func TestKeepsOldestSnapsInABucket(t *testing.T) {
|
||||
g := gridFromString("1m,2")
|
||||
|
||||
relt := func(secs int64) time.Time { return time.Unix(secs, 0) }
|
||||
|
||||
snaps := []Entry{
|
||||
testSnap{"1", true, relt(1)},
|
||||
testSnap{"2", true, relt(2)},
|
||||
testSnap{"3", false, relt(3)},
|
||||
testSnap{"4", false, relt(4)},
|
||||
testSnap{"5", false, relt(5)},
|
||||
}
|
||||
|
||||
now := relt(6)
|
||||
keep, remove := g.FitEntries(snaps)
|
||||
validateRetentionGridFitEntries(t, now, snaps, keep, remove)
|
||||
}
|
||||
|
||||
func TestRespectsKeepCountAll(t *testing.T) {
|
||||
g := gridFromString("1m,-1|1m,1")
|
||||
relt := func(secs int64) time.Time { return time.Unix(secs, 0) }
|
||||
snaps := []Entry{
|
||||
testSnap{"a", true, relt(0)},
|
||||
testSnap{"b", true, relt(-1)},
|
||||
testSnap{"c", true, relt(-2)},
|
||||
testSnap{"d", false, relt(-60)},
|
||||
testSnap{"e", true, relt(-61)},
|
||||
}
|
||||
keep, remove := g.FitEntries(snaps)
|
||||
validateRetentionGridFitEntries(t, relt(61), snaps, keep, remove)
|
||||
}
|
||||
|
||||
func TestComplex(t *testing.T) {
|
||||
func TestRetentionGridFitEntries(t *testing.T) {
|
||||
|
||||
g := gridFromString("10m,-1|10m|10m,2|1h")
|
||||
|
||||
@@ -156,29 +141,20 @@ func TestComplex(t *testing.T) {
|
||||
now := time.Unix(0, 0)
|
||||
|
||||
snaps := []Entry{
|
||||
// pre-now must always be kept
|
||||
testSnap{"1", true, now.Add(3 * time.Minute)},
|
||||
// 1st interval allows unlimited entries
|
||||
testSnap{"b1", true, now.Add(-6 * time.Minute)},
|
||||
testSnap{"b3", true, now.Add(-8 * time.Minute)},
|
||||
testSnap{"b2", true, now.Add(-9 * time.Minute)},
|
||||
// 2nd interval allows 1 entry
|
||||
testSnap{"a", false, now.Add(-11 * time.Minute)},
|
||||
testSnap{"c", true, now.Add(-19 * time.Minute)},
|
||||
// 3rd interval allows 2 entries
|
||||
testSnap{"foo", true, now.Add(-25 * time.Minute)},
|
||||
testSnap{"bar", true, now.Add(-26 * time.Minute)},
|
||||
// this is at the left edge of the 4th interval
|
||||
testSnap{"border", false, now.Add(-30 * time.Minute)},
|
||||
// right in the 4th interval
|
||||
testSnap{"d", true, now.Add(-1*time.Hour - 15*time.Minute)},
|
||||
// on the right edge of 4th interval => not in it => delete
|
||||
testSnap{"q", false, now.Add(-1*time.Hour - 30*time.Minute)},
|
||||
// older then 4th interval => always delete
|
||||
testSnap{"e", false, now.Add(-1*time.Hour - 31*time.Minute)},
|
||||
testSnap{"f", false, now.Add(-2 * time.Hour)},
|
||||
dummySnap{"1", true, now.Add(3 * time.Minute)}, // pre-now must always be kept
|
||||
dummySnap{"b1", true, now.Add(-6 * time.Minute)}, // 1st interval allows unlimited entries
|
||||
dummySnap{"b3", true, now.Add(-8 * time.Minute)}, // 1st interval allows unlimited entries
|
||||
dummySnap{"b2", true, now.Add(-9 * time.Minute)}, // 1st interval allows unlimited entries
|
||||
dummySnap{"a", false, now.Add(-11 * time.Minute)},
|
||||
dummySnap{"c", true, now.Add(-19 * time.Minute)}, // 2nd interval allows 1 entry
|
||||
dummySnap{"foo", false, now.Add(-25 * time.Minute)},
|
||||
dummySnap{"bar", true, now.Add(-26 * time.Minute)}, // 3rd interval allows 2 entries
|
||||
dummySnap{"border", true, now.Add(-30 * time.Minute)},
|
||||
dummySnap{"d", true, now.Add(-1*time.Hour - 15*time.Minute)},
|
||||
dummySnap{"e", false, now.Add(-1*time.Hour - 31*time.Minute)}, // before earliest interval must always be deleted
|
||||
dummySnap{"f", false, now.Add(-2 * time.Hour)},
|
||||
}
|
||||
keep, remove := g.fitEntriesWithNow(now, snaps)
|
||||
keep, remove := g.FitEntries(now, snaps)
|
||||
|
||||
validateRetentionGridFitEntries(t, now, snaps, keep, remove)
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package driver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"sort"
|
||||
@@ -9,14 +10,10 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/kr/pretty"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
|
||||
"github.com/zrepl/zrepl/replication/report"
|
||||
"github.com/zrepl/zrepl/util/chainlock"
|
||||
"github.com/zrepl/zrepl/util/envconst"
|
||||
@@ -230,10 +227,7 @@ func Do(ctx context.Context, planner Planner) (ReportFunc, WaitFunc) {
|
||||
errRep := cur.errorReport()
|
||||
|
||||
if rep.State == report.AttemptDone {
|
||||
if len(rep.Filesystems) == 0 {
|
||||
log.Warn("no filesystems were considered for replication")
|
||||
}
|
||||
log.Debug("attempt completed")
|
||||
log.Debug("attempt completed successfully")
|
||||
break
|
||||
}
|
||||
|
||||
@@ -241,7 +235,7 @@ func Do(ctx context.Context, planner Planner) (ReportFunc, WaitFunc) {
|
||||
log.WithField("most_recent_err", mostRecentErr).WithField("most_recent_err_class", mostRecentErrClass).Debug("most recent error used for re-connect decision")
|
||||
if mostRecentErr == nil {
|
||||
// inconsistent reporting, let's bail out
|
||||
log.WithField("attempt_state", rep.State).Warn("attempt does not report done but error report does not report errors, aborting run")
|
||||
log.Warn("attempt does not report done but error report does not report errors, aborting run")
|
||||
break
|
||||
}
|
||||
log.WithError(mostRecentErr.Err).Error("most recent error in this attempt")
|
||||
@@ -315,9 +309,6 @@ func (a *attempt) doGlobalPlanning(ctx context.Context, prev *attempt) map[*fs]*
|
||||
return nil
|
||||
}
|
||||
|
||||
// a.fss != nil indicates that there was no planning error (see doc comment)
|
||||
a.fss = make([]*fs, 0)
|
||||
|
||||
for _, pfs := range pfss {
|
||||
fs := &fs{
|
||||
fs: pfs,
|
||||
@@ -383,28 +374,11 @@ func (a *attempt) doGlobalPlanning(ctx context.Context, prev *attempt) map[*fs]*
|
||||
// invariant: prevs contains an entry for each unambiguous correspondence
|
||||
|
||||
// build up parent-child relationship (FIXME (O(n^2), but who's going to have that many filesystems...))
|
||||
mustDatasetPathOrPlanFail := func(fs string) *zfs.DatasetPath {
|
||||
dp, err := zfs.NewDatasetPath(fs)
|
||||
if err != nil {
|
||||
now := time.Now()
|
||||
a.planErr = newTimedError(errors.Wrapf(err, "%q", fs), now)
|
||||
a.fss = nil
|
||||
a.finishedAt = now
|
||||
return nil
|
||||
}
|
||||
return dp
|
||||
}
|
||||
for _, f1 := range a.fss {
|
||||
fs1 := mustDatasetPathOrPlanFail(f1.fs.ReportInfo().Name)
|
||||
if fs1 == nil {
|
||||
return nil
|
||||
}
|
||||
fs1 := f1.fs.ReportInfo().Name
|
||||
for _, f2 := range a.fss {
|
||||
fs2 := mustDatasetPathOrPlanFail(f2.fs.ReportInfo().Name)
|
||||
if fs2 == nil {
|
||||
return nil
|
||||
}
|
||||
if fs1.HasPrefix(fs2) && !fs1.Equal(fs2) {
|
||||
fs2 := f2.fs.ReportInfo().Name
|
||||
if strings.HasPrefix(fs1, fs2) && fs1 != fs2 {
|
||||
f1.initialRepOrd.parents = append(f1.initialRepOrd.parents, f2)
|
||||
f2.initialRepOrd.children = append(f2.initialRepOrd.children, f1)
|
||||
}
|
||||
@@ -492,37 +466,57 @@ func (f *fs) do(ctx context.Context, pq *stepQueue, prev *fs) {
|
||||
// => don't set f.planning.done just yet
|
||||
f.debug("initial len(fs.planned.steps) = %d", len(f.planned.steps))
|
||||
|
||||
// for not-first attempts that succeeded in planning, only allow fs.planned.steps
|
||||
// up to and including the originally planned target snapshot
|
||||
// for not-first attempts, only allow fs.planned.steps
|
||||
// up to including the originally planned target snapshot
|
||||
if prev != nil && prev.planning.done && prev.planning.err == nil {
|
||||
f.debug("attempting to correlate plan with previous attempt to find out what is left to do")
|
||||
// find the highest of the previously uncompleted steps for which we can also find a step
|
||||
// in our current plan
|
||||
prevUncompleted := prev.planned.steps[prev.planned.step:]
|
||||
var target struct{ prev, cur int }
|
||||
target.prev = -1
|
||||
target.cur = -1
|
||||
out:
|
||||
for p := len(prevUncompleted) - 1; p >= 0; p-- {
|
||||
for q := len(f.planned.steps) - 1; q >= 0; q-- {
|
||||
if prevUncompleted[p].step.TargetEquals(f.planned.steps[q].step) {
|
||||
target.prev = p
|
||||
target.cur = q
|
||||
break out
|
||||
}
|
||||
}
|
||||
}
|
||||
if target.prev == -1 || target.cur == -1 {
|
||||
f.debug("no correlation possible between previous attempt and this attempt's plan")
|
||||
f.planning.err = newTimedError(fmt.Errorf("cannot correlate previously failed attempt to current plan"), time.Now())
|
||||
if len(prevUncompleted) == 0 {
|
||||
f.debug("prevUncompleted is empty")
|
||||
return
|
||||
}
|
||||
|
||||
f.planned.steps = f.planned.steps[0:target.cur]
|
||||
f.debug("found correlation, new steps are len(fs.planned.steps) = %d", len(f.planned.steps))
|
||||
} else {
|
||||
f.debug("previous attempt does not exist or did not finish planning, no correlation possible, taking this attempt's plan as is")
|
||||
if len(f.planned.steps) == 0 {
|
||||
f.debug("fs.planned.steps is empty")
|
||||
return
|
||||
}
|
||||
prevFailed := prevUncompleted[0]
|
||||
curFirst := f.planned.steps[0]
|
||||
// we assume that PlanFS retries prevFailed (using curFirst)
|
||||
if !prevFailed.step.TargetEquals(curFirst.step) {
|
||||
f.debug("Targets don't match")
|
||||
// Two options:
|
||||
// A: planning algorithm is broken
|
||||
// B: manual user intervention inbetween
|
||||
// Neither way will we make progress, so let's error out
|
||||
stepFmt := func(step *step) string {
|
||||
r := step.report()
|
||||
s := r.Info
|
||||
if r.IsIncremental() {
|
||||
return fmt.Sprintf("%s=>%s", s.From, s.To)
|
||||
} else {
|
||||
return fmt.Sprintf("full=>%s", s.To)
|
||||
}
|
||||
}
|
||||
msg := fmt.Sprintf("last attempt's uncompleted step %s does not correspond to this attempt's first planned step %s",
|
||||
stepFmt(prevFailed), stepFmt(curFirst))
|
||||
f.planned.stepErr = newTimedError(errors.New(msg), time.Now())
|
||||
return
|
||||
}
|
||||
// only allow until step targets diverge
|
||||
min := len(prevUncompleted)
|
||||
if min > len(f.planned.steps) {
|
||||
min = len(f.planned.steps)
|
||||
}
|
||||
diverge := 0
|
||||
for ; diverge < min; diverge++ {
|
||||
f.debug("diverge compare iteration %d", diverge)
|
||||
if !f.planned.steps[diverge].step.TargetEquals(prevUncompleted[diverge].step) {
|
||||
break
|
||||
}
|
||||
}
|
||||
f.debug("diverge is %d", diverge)
|
||||
f.planned.steps = f.planned.steps[0:diverge]
|
||||
}
|
||||
f.debug("post-prev-merge len(fs.planned.steps) = %d", len(f.planned.steps))
|
||||
|
||||
// now we are done planning (f.planned.steps won't change from now on)
|
||||
f.planning.done = true
|
||||
@@ -663,12 +657,10 @@ func (a *attempt) report() *report.AttemptReport {
|
||||
r.Filesystems[i] = a.fss[i].report()
|
||||
}
|
||||
|
||||
var state report.AttemptState
|
||||
if a.planErr == nil && a.fss == nil {
|
||||
state = report.AttemptPlanning
|
||||
} else if a.planErr != nil && a.fss == nil {
|
||||
state := report.AttemptPlanning
|
||||
if a.planErr != nil {
|
||||
state = report.AttemptPlanningError
|
||||
} else if a.planErr == nil && a.fss != nil {
|
||||
} else if a.fss != nil {
|
||||
if a.finishedAt.IsZero() {
|
||||
state = report.AttemptFanOutFSs
|
||||
} else {
|
||||
@@ -681,8 +673,6 @@ func (a *attempt) report() *report.AttemptReport {
|
||||
state = report.AttemptFanOutError
|
||||
}
|
||||
}
|
||||
} else {
|
||||
panic(fmt.Sprintf("attempt.planErr and attempt.fss must not both be != nil:\n%s\n%s", pretty.Sprint(a.planErr), pretty.Sprint(a.fss)))
|
||||
}
|
||||
r.State = state
|
||||
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
|
||||
"github.com/zrepl/zrepl/replication/report"
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
|
||||
"github.com/montanaflynn/stats"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
)
|
||||
|
||||
|
||||
@@ -77,48 +77,56 @@ func IncrementalPath(receiver, sender []*FilesystemVersion) (incPath []*Filesyst
|
||||
receiver = SortVersionListByCreateTXGThenBookmarkLTSnapshot(receiver)
|
||||
sender = SortVersionListByCreateTXGThenBookmarkLTSnapshot(sender)
|
||||
|
||||
var mrcaCandidate struct {
|
||||
found bool
|
||||
guid uint64
|
||||
r, s int
|
||||
}
|
||||
// Find most recent common ancestor by name, preferring snapshots over bookmarks
|
||||
|
||||
findCandidate:
|
||||
for r := len(receiver) - 1; r >= 0; r-- {
|
||||
for s := len(sender) - 1; s >= 0; s-- {
|
||||
if sender[s].GetGuid() == receiver[r].GetGuid() {
|
||||
mrcaCandidate.guid = sender[s].GetGuid()
|
||||
mrcaCandidate.s = s
|
||||
mrcaCandidate.r = r
|
||||
mrcaCandidate.found = true
|
||||
break findCandidate
|
||||
}
|
||||
mrcaRcv := len(receiver) - 1
|
||||
mrcaSnd := len(sender) - 1
|
||||
|
||||
for mrcaRcv >= 0 && mrcaSnd >= 0 {
|
||||
if receiver[mrcaRcv].Guid == sender[mrcaSnd].Guid {
|
||||
// Since we arrive from the end of the array, and because we defined bookmark < snapshot,
|
||||
// this condition will match snapshot first, which is what we want because it gives us
|
||||
// size estimation
|
||||
break
|
||||
}
|
||||
receiverCreation, err := receiver[mrcaRcv].CreationAsTime()
|
||||
if err != nil {
|
||||
panic(err) // FIXME move this to a sorting phase before
|
||||
}
|
||||
senderCreation, err := sender[mrcaSnd].CreationAsTime()
|
||||
if err != nil {
|
||||
panic(err) // FIXME move this to the sorting phase before
|
||||
}
|
||||
|
||||
if receiverCreation.Before(senderCreation) {
|
||||
mrcaSnd--
|
||||
} else {
|
||||
mrcaRcv--
|
||||
}
|
||||
}
|
||||
|
||||
// handle failure cases
|
||||
if !mrcaCandidate.found {
|
||||
if mrcaRcv == -1 || mrcaSnd == -1 {
|
||||
return nil, &ConflictNoCommonAncestor{
|
||||
SortedSenderVersions: sender,
|
||||
SortedReceiverVersions: receiver,
|
||||
}
|
||||
} else if mrcaCandidate.r != len(receiver)-1 {
|
||||
}
|
||||
|
||||
if mrcaRcv != len(receiver)-1 {
|
||||
return nil, &ConflictDiverged{
|
||||
SortedSenderVersions: sender,
|
||||
SortedReceiverVersions: receiver,
|
||||
CommonAncestor: sender[mrcaCandidate.s],
|
||||
SenderOnly: sender[mrcaCandidate.s+1:],
|
||||
ReceiverOnly: receiver[mrcaCandidate.r+1:],
|
||||
CommonAncestor: sender[mrcaSnd],
|
||||
SenderOnly: sender[mrcaSnd+1:],
|
||||
ReceiverOnly: receiver[mrcaRcv+1:],
|
||||
}
|
||||
}
|
||||
|
||||
// incPath is possible
|
||||
|
||||
// incPath must not contain bookmarks except initial one,
|
||||
incPath = make([]*FilesystemVersion, 0, len(sender))
|
||||
incPath = append(incPath, sender[mrcaCandidate.s])
|
||||
incPath = append(incPath, sender[mrcaSnd])
|
||||
// it's ok if incPath[0] is a bookmark, but not the subsequent ones in the incPath
|
||||
for i := mrcaCandidate.s + 1; i < len(sender); i++ {
|
||||
for i := mrcaSnd + 1; i < len(sender); i++ {
|
||||
if sender[i].Type == FilesystemVersion_Snapshot && incPath[len(incPath)-1].Guid != sender[i].Guid {
|
||||
incPath = append(incPath, sender[i])
|
||||
}
|
||||
|
||||
+88
-230
@@ -46,36 +46,7 @@ func (x Tri) String() string {
|
||||
return proto.EnumName(Tri_name, int32(x))
|
||||
}
|
||||
func (Tri) EnumDescriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{0}
|
||||
}
|
||||
|
||||
type ReplicationGuaranteeKind int32
|
||||
|
||||
const (
|
||||
ReplicationGuaranteeKind_GuaranteeInvalid ReplicationGuaranteeKind = 0
|
||||
ReplicationGuaranteeKind_GuaranteeResumability ReplicationGuaranteeKind = 1
|
||||
ReplicationGuaranteeKind_GuaranteeIncrementalReplication ReplicationGuaranteeKind = 2
|
||||
ReplicationGuaranteeKind_GuaranteeNothing ReplicationGuaranteeKind = 3
|
||||
)
|
||||
|
||||
var ReplicationGuaranteeKind_name = map[int32]string{
|
||||
0: "GuaranteeInvalid",
|
||||
1: "GuaranteeResumability",
|
||||
2: "GuaranteeIncrementalReplication",
|
||||
3: "GuaranteeNothing",
|
||||
}
|
||||
var ReplicationGuaranteeKind_value = map[string]int32{
|
||||
"GuaranteeInvalid": 0,
|
||||
"GuaranteeResumability": 1,
|
||||
"GuaranteeIncrementalReplication": 2,
|
||||
"GuaranteeNothing": 3,
|
||||
}
|
||||
|
||||
func (x ReplicationGuaranteeKind) String() string {
|
||||
return proto.EnumName(ReplicationGuaranteeKind_name, int32(x))
|
||||
}
|
||||
func (ReplicationGuaranteeKind) EnumDescriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{1}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{0}
|
||||
}
|
||||
|
||||
type FilesystemVersion_VersionType int32
|
||||
@@ -98,7 +69,7 @@ func (x FilesystemVersion_VersionType) String() string {
|
||||
return proto.EnumName(FilesystemVersion_VersionType_name, int32(x))
|
||||
}
|
||||
func (FilesystemVersion_VersionType) EnumDescriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{5, 0}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{5, 0}
|
||||
}
|
||||
|
||||
type ListFilesystemReq struct {
|
||||
@@ -111,7 +82,7 @@ func (m *ListFilesystemReq) Reset() { *m = ListFilesystemReq{} }
|
||||
func (m *ListFilesystemReq) String() string { return proto.CompactTextString(m) }
|
||||
func (*ListFilesystemReq) ProtoMessage() {}
|
||||
func (*ListFilesystemReq) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{0}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{0}
|
||||
}
|
||||
func (m *ListFilesystemReq) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_ListFilesystemReq.Unmarshal(m, b)
|
||||
@@ -142,7 +113,7 @@ func (m *ListFilesystemRes) Reset() { *m = ListFilesystemRes{} }
|
||||
func (m *ListFilesystemRes) String() string { return proto.CompactTextString(m) }
|
||||
func (*ListFilesystemRes) ProtoMessage() {}
|
||||
func (*ListFilesystemRes) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{1}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{1}
|
||||
}
|
||||
func (m *ListFilesystemRes) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_ListFilesystemRes.Unmarshal(m, b)
|
||||
@@ -183,7 +154,7 @@ func (m *Filesystem) Reset() { *m = Filesystem{} }
|
||||
func (m *Filesystem) String() string { return proto.CompactTextString(m) }
|
||||
func (*Filesystem) ProtoMessage() {}
|
||||
func (*Filesystem) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{2}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{2}
|
||||
}
|
||||
func (m *Filesystem) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_Filesystem.Unmarshal(m, b)
|
||||
@@ -242,7 +213,7 @@ func (m *ListFilesystemVersionsReq) Reset() { *m = ListFilesystemVersion
|
||||
func (m *ListFilesystemVersionsReq) String() string { return proto.CompactTextString(m) }
|
||||
func (*ListFilesystemVersionsReq) ProtoMessage() {}
|
||||
func (*ListFilesystemVersionsReq) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{3}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{3}
|
||||
}
|
||||
func (m *ListFilesystemVersionsReq) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_ListFilesystemVersionsReq.Unmarshal(m, b)
|
||||
@@ -280,7 +251,7 @@ func (m *ListFilesystemVersionsRes) Reset() { *m = ListFilesystemVersion
|
||||
func (m *ListFilesystemVersionsRes) String() string { return proto.CompactTextString(m) }
|
||||
func (*ListFilesystemVersionsRes) ProtoMessage() {}
|
||||
func (*ListFilesystemVersionsRes) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{4}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{4}
|
||||
}
|
||||
func (m *ListFilesystemVersionsRes) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_ListFilesystemVersionsRes.Unmarshal(m, b)
|
||||
@@ -322,7 +293,7 @@ func (m *FilesystemVersion) Reset() { *m = FilesystemVersion{} }
|
||||
func (m *FilesystemVersion) String() string { return proto.CompactTextString(m) }
|
||||
func (*FilesystemVersion) ProtoMessage() {}
|
||||
func (*FilesystemVersion) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{5}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{5}
|
||||
}
|
||||
func (m *FilesystemVersion) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_FilesystemVersion.Unmarshal(m, b)
|
||||
@@ -388,20 +359,19 @@ type SendReq struct {
|
||||
// SHOULD clear the resume token on their side and use From and To instead If
|
||||
// ResumeToken is not empty, the GUIDs of From and To MUST correspond to those
|
||||
// encoded in the ResumeToken. Otherwise, the Sender MUST return an error.
|
||||
ResumeToken string `protobuf:"bytes,4,opt,name=ResumeToken,proto3" json:"ResumeToken,omitempty"`
|
||||
Encrypted Tri `protobuf:"varint,5,opt,name=Encrypted,proto3,enum=Tri" json:"Encrypted,omitempty"`
|
||||
DryRun bool `protobuf:"varint,6,opt,name=DryRun,proto3" json:"DryRun,omitempty"`
|
||||
ReplicationConfig *ReplicationConfig `protobuf:"bytes,7,opt,name=ReplicationConfig,proto3" json:"ReplicationConfig,omitempty"`
|
||||
XXX_NoUnkeyedLiteral struct{} `json:"-"`
|
||||
XXX_unrecognized []byte `json:"-"`
|
||||
XXX_sizecache int32 `json:"-"`
|
||||
ResumeToken string `protobuf:"bytes,4,opt,name=ResumeToken,proto3" json:"ResumeToken,omitempty"`
|
||||
Encrypted Tri `protobuf:"varint,5,opt,name=Encrypted,proto3,enum=Tri" json:"Encrypted,omitempty"`
|
||||
DryRun bool `protobuf:"varint,6,opt,name=DryRun,proto3" json:"DryRun,omitempty"`
|
||||
XXX_NoUnkeyedLiteral struct{} `json:"-"`
|
||||
XXX_unrecognized []byte `json:"-"`
|
||||
XXX_sizecache int32 `json:"-"`
|
||||
}
|
||||
|
||||
func (m *SendReq) Reset() { *m = SendReq{} }
|
||||
func (m *SendReq) String() string { return proto.CompactTextString(m) }
|
||||
func (*SendReq) ProtoMessage() {}
|
||||
func (*SendReq) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{6}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{6}
|
||||
}
|
||||
func (m *SendReq) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_SendReq.Unmarshal(m, b)
|
||||
@@ -463,97 +433,6 @@ func (m *SendReq) GetDryRun() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (m *SendReq) GetReplicationConfig() *ReplicationConfig {
|
||||
if m != nil {
|
||||
return m.ReplicationConfig
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type ReplicationConfig struct {
|
||||
Protection *ReplicationConfigProtection `protobuf:"bytes,1,opt,name=protection,proto3" json:"protection,omitempty"`
|
||||
XXX_NoUnkeyedLiteral struct{} `json:"-"`
|
||||
XXX_unrecognized []byte `json:"-"`
|
||||
XXX_sizecache int32 `json:"-"`
|
||||
}
|
||||
|
||||
func (m *ReplicationConfig) Reset() { *m = ReplicationConfig{} }
|
||||
func (m *ReplicationConfig) String() string { return proto.CompactTextString(m) }
|
||||
func (*ReplicationConfig) ProtoMessage() {}
|
||||
func (*ReplicationConfig) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{7}
|
||||
}
|
||||
func (m *ReplicationConfig) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_ReplicationConfig.Unmarshal(m, b)
|
||||
}
|
||||
func (m *ReplicationConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
|
||||
return xxx_messageInfo_ReplicationConfig.Marshal(b, m, deterministic)
|
||||
}
|
||||
func (dst *ReplicationConfig) XXX_Merge(src proto.Message) {
|
||||
xxx_messageInfo_ReplicationConfig.Merge(dst, src)
|
||||
}
|
||||
func (m *ReplicationConfig) XXX_Size() int {
|
||||
return xxx_messageInfo_ReplicationConfig.Size(m)
|
||||
}
|
||||
func (m *ReplicationConfig) XXX_DiscardUnknown() {
|
||||
xxx_messageInfo_ReplicationConfig.DiscardUnknown(m)
|
||||
}
|
||||
|
||||
var xxx_messageInfo_ReplicationConfig proto.InternalMessageInfo
|
||||
|
||||
func (m *ReplicationConfig) GetProtection() *ReplicationConfigProtection {
|
||||
if m != nil {
|
||||
return m.Protection
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type ReplicationConfigProtection struct {
|
||||
Initial ReplicationGuaranteeKind `protobuf:"varint,1,opt,name=Initial,proto3,enum=ReplicationGuaranteeKind" json:"Initial,omitempty"`
|
||||
Incremental ReplicationGuaranteeKind `protobuf:"varint,2,opt,name=Incremental,proto3,enum=ReplicationGuaranteeKind" json:"Incremental,omitempty"`
|
||||
XXX_NoUnkeyedLiteral struct{} `json:"-"`
|
||||
XXX_unrecognized []byte `json:"-"`
|
||||
XXX_sizecache int32 `json:"-"`
|
||||
}
|
||||
|
||||
func (m *ReplicationConfigProtection) Reset() { *m = ReplicationConfigProtection{} }
|
||||
func (m *ReplicationConfigProtection) String() string { return proto.CompactTextString(m) }
|
||||
func (*ReplicationConfigProtection) ProtoMessage() {}
|
||||
func (*ReplicationConfigProtection) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{8}
|
||||
}
|
||||
func (m *ReplicationConfigProtection) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_ReplicationConfigProtection.Unmarshal(m, b)
|
||||
}
|
||||
func (m *ReplicationConfigProtection) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
|
||||
return xxx_messageInfo_ReplicationConfigProtection.Marshal(b, m, deterministic)
|
||||
}
|
||||
func (dst *ReplicationConfigProtection) XXX_Merge(src proto.Message) {
|
||||
xxx_messageInfo_ReplicationConfigProtection.Merge(dst, src)
|
||||
}
|
||||
func (m *ReplicationConfigProtection) XXX_Size() int {
|
||||
return xxx_messageInfo_ReplicationConfigProtection.Size(m)
|
||||
}
|
||||
func (m *ReplicationConfigProtection) XXX_DiscardUnknown() {
|
||||
xxx_messageInfo_ReplicationConfigProtection.DiscardUnknown(m)
|
||||
}
|
||||
|
||||
var xxx_messageInfo_ReplicationConfigProtection proto.InternalMessageInfo
|
||||
|
||||
func (m *ReplicationConfigProtection) GetInitial() ReplicationGuaranteeKind {
|
||||
if m != nil {
|
||||
return m.Initial
|
||||
}
|
||||
return ReplicationGuaranteeKind_GuaranteeInvalid
|
||||
}
|
||||
|
||||
func (m *ReplicationConfigProtection) GetIncremental() ReplicationGuaranteeKind {
|
||||
if m != nil {
|
||||
return m.Incremental
|
||||
}
|
||||
return ReplicationGuaranteeKind_GuaranteeInvalid
|
||||
}
|
||||
|
||||
type Property struct {
|
||||
Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
|
||||
Value string `protobuf:"bytes,2,opt,name=Value,proto3" json:"Value,omitempty"`
|
||||
@@ -566,7 +445,7 @@ func (m *Property) Reset() { *m = Property{} }
|
||||
func (m *Property) String() string { return proto.CompactTextString(m) }
|
||||
func (*Property) ProtoMessage() {}
|
||||
func (*Property) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{9}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{7}
|
||||
}
|
||||
func (m *Property) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_Property.Unmarshal(m, b)
|
||||
@@ -617,7 +496,7 @@ func (m *SendRes) Reset() { *m = SendRes{} }
|
||||
func (m *SendRes) String() string { return proto.CompactTextString(m) }
|
||||
func (*SendRes) ProtoMessage() {}
|
||||
func (*SendRes) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{10}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{8}
|
||||
}
|
||||
func (m *SendRes) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_SendRes.Unmarshal(m, b)
|
||||
@@ -669,7 +548,7 @@ func (m *SendCompletedReq) Reset() { *m = SendCompletedReq{} }
|
||||
func (m *SendCompletedReq) String() string { return proto.CompactTextString(m) }
|
||||
func (*SendCompletedReq) ProtoMessage() {}
|
||||
func (*SendCompletedReq) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{11}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{9}
|
||||
}
|
||||
func (m *SendCompletedReq) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_SendCompletedReq.Unmarshal(m, b)
|
||||
@@ -706,7 +585,7 @@ func (m *SendCompletedRes) Reset() { *m = SendCompletedRes{} }
|
||||
func (m *SendCompletedRes) String() string { return proto.CompactTextString(m) }
|
||||
func (*SendCompletedRes) ProtoMessage() {}
|
||||
func (*SendCompletedRes) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{12}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{10}
|
||||
}
|
||||
func (m *SendCompletedRes) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_SendCompletedRes.Unmarshal(m, b)
|
||||
@@ -731,18 +610,17 @@ type ReceiveReq struct {
|
||||
To *FilesystemVersion `protobuf:"bytes,2,opt,name=To,proto3" json:"To,omitempty"`
|
||||
// If true, the receiver should clear the resume token before performing the
|
||||
// zfs recv of the stream in the request
|
||||
ClearResumeToken bool `protobuf:"varint,3,opt,name=ClearResumeToken,proto3" json:"ClearResumeToken,omitempty"`
|
||||
ReplicationConfig *ReplicationConfig `protobuf:"bytes,4,opt,name=ReplicationConfig,proto3" json:"ReplicationConfig,omitempty"`
|
||||
XXX_NoUnkeyedLiteral struct{} `json:"-"`
|
||||
XXX_unrecognized []byte `json:"-"`
|
||||
XXX_sizecache int32 `json:"-"`
|
||||
ClearResumeToken bool `protobuf:"varint,3,opt,name=ClearResumeToken,proto3" json:"ClearResumeToken,omitempty"`
|
||||
XXX_NoUnkeyedLiteral struct{} `json:"-"`
|
||||
XXX_unrecognized []byte `json:"-"`
|
||||
XXX_sizecache int32 `json:"-"`
|
||||
}
|
||||
|
||||
func (m *ReceiveReq) Reset() { *m = ReceiveReq{} }
|
||||
func (m *ReceiveReq) String() string { return proto.CompactTextString(m) }
|
||||
func (*ReceiveReq) ProtoMessage() {}
|
||||
func (*ReceiveReq) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{13}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{11}
|
||||
}
|
||||
func (m *ReceiveReq) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_ReceiveReq.Unmarshal(m, b)
|
||||
@@ -783,13 +661,6 @@ func (m *ReceiveReq) GetClearResumeToken() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (m *ReceiveReq) GetReplicationConfig() *ReplicationConfig {
|
||||
if m != nil {
|
||||
return m.ReplicationConfig
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type ReceiveRes struct {
|
||||
XXX_NoUnkeyedLiteral struct{} `json:"-"`
|
||||
XXX_unrecognized []byte `json:"-"`
|
||||
@@ -800,7 +671,7 @@ func (m *ReceiveRes) Reset() { *m = ReceiveRes{} }
|
||||
func (m *ReceiveRes) String() string { return proto.CompactTextString(m) }
|
||||
func (*ReceiveRes) ProtoMessage() {}
|
||||
func (*ReceiveRes) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{14}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{12}
|
||||
}
|
||||
func (m *ReceiveRes) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_ReceiveRes.Unmarshal(m, b)
|
||||
@@ -833,7 +704,7 @@ func (m *DestroySnapshotsReq) Reset() { *m = DestroySnapshotsReq{} }
|
||||
func (m *DestroySnapshotsReq) String() string { return proto.CompactTextString(m) }
|
||||
func (*DestroySnapshotsReq) ProtoMessage() {}
|
||||
func (*DestroySnapshotsReq) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{15}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{13}
|
||||
}
|
||||
func (m *DestroySnapshotsReq) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_DestroySnapshotsReq.Unmarshal(m, b)
|
||||
@@ -879,7 +750,7 @@ func (m *DestroySnapshotRes) Reset() { *m = DestroySnapshotRes{} }
|
||||
func (m *DestroySnapshotRes) String() string { return proto.CompactTextString(m) }
|
||||
func (*DestroySnapshotRes) ProtoMessage() {}
|
||||
func (*DestroySnapshotRes) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{16}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{14}
|
||||
}
|
||||
func (m *DestroySnapshotRes) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_DestroySnapshotRes.Unmarshal(m, b)
|
||||
@@ -924,7 +795,7 @@ func (m *DestroySnapshotsRes) Reset() { *m = DestroySnapshotsRes{} }
|
||||
func (m *DestroySnapshotsRes) String() string { return proto.CompactTextString(m) }
|
||||
func (*DestroySnapshotsRes) ProtoMessage() {}
|
||||
func (*DestroySnapshotsRes) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{17}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{15}
|
||||
}
|
||||
func (m *DestroySnapshotsRes) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_DestroySnapshotsRes.Unmarshal(m, b)
|
||||
@@ -962,7 +833,7 @@ func (m *ReplicationCursorReq) Reset() { *m = ReplicationCursorReq{} }
|
||||
func (m *ReplicationCursorReq) String() string { return proto.CompactTextString(m) }
|
||||
func (*ReplicationCursorReq) ProtoMessage() {}
|
||||
func (*ReplicationCursorReq) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{18}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{16}
|
||||
}
|
||||
func (m *ReplicationCursorReq) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_ReplicationCursorReq.Unmarshal(m, b)
|
||||
@@ -1003,7 +874,7 @@ func (m *ReplicationCursorRes) Reset() { *m = ReplicationCursorRes{} }
|
||||
func (m *ReplicationCursorRes) String() string { return proto.CompactTextString(m) }
|
||||
func (*ReplicationCursorRes) ProtoMessage() {}
|
||||
func (*ReplicationCursorRes) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{19}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{17}
|
||||
}
|
||||
func (m *ReplicationCursorRes) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_ReplicationCursorRes.Unmarshal(m, b)
|
||||
@@ -1139,7 +1010,7 @@ func (m *PingReq) Reset() { *m = PingReq{} }
|
||||
func (m *PingReq) String() string { return proto.CompactTextString(m) }
|
||||
func (*PingReq) ProtoMessage() {}
|
||||
func (*PingReq) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{20}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{18}
|
||||
}
|
||||
func (m *PingReq) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_PingReq.Unmarshal(m, b)
|
||||
@@ -1178,7 +1049,7 @@ func (m *PingRes) Reset() { *m = PingRes{} }
|
||||
func (m *PingRes) String() string { return proto.CompactTextString(m) }
|
||||
func (*PingRes) ProtoMessage() {}
|
||||
func (*PingRes) Descriptor() ([]byte, []int) {
|
||||
return fileDescriptor_pdu_616c27178643eca4, []int{21}
|
||||
return fileDescriptor_pdu_483c6918b7b3d747, []int{19}
|
||||
}
|
||||
func (m *PingRes) XXX_Unmarshal(b []byte) error {
|
||||
return xxx_messageInfo_PingRes.Unmarshal(m, b)
|
||||
@@ -1213,8 +1084,6 @@ func init() {
|
||||
proto.RegisterType((*ListFilesystemVersionsRes)(nil), "ListFilesystemVersionsRes")
|
||||
proto.RegisterType((*FilesystemVersion)(nil), "FilesystemVersion")
|
||||
proto.RegisterType((*SendReq)(nil), "SendReq")
|
||||
proto.RegisterType((*ReplicationConfig)(nil), "ReplicationConfig")
|
||||
proto.RegisterType((*ReplicationConfigProtection)(nil), "ReplicationConfigProtection")
|
||||
proto.RegisterType((*Property)(nil), "Property")
|
||||
proto.RegisterType((*SendRes)(nil), "SendRes")
|
||||
proto.RegisterType((*SendCompletedReq)(nil), "SendCompletedReq")
|
||||
@@ -1229,7 +1098,6 @@ func init() {
|
||||
proto.RegisterType((*PingReq)(nil), "PingReq")
|
||||
proto.RegisterType((*PingRes)(nil), "PingRes")
|
||||
proto.RegisterEnum("Tri", Tri_name, Tri_value)
|
||||
proto.RegisterEnum("ReplicationGuaranteeKind", ReplicationGuaranteeKind_name, ReplicationGuaranteeKind_value)
|
||||
proto.RegisterEnum("FilesystemVersion_VersionType", FilesystemVersion_VersionType_name, FilesystemVersion_VersionType_value)
|
||||
}
|
||||
|
||||
@@ -1470,71 +1338,61 @@ var _Replication_serviceDesc = grpc.ServiceDesc{
|
||||
Metadata: "pdu.proto",
|
||||
}
|
||||
|
||||
func init() { proto.RegisterFile("pdu.proto", fileDescriptor_pdu_616c27178643eca4) }
|
||||
func init() { proto.RegisterFile("pdu.proto", fileDescriptor_pdu_483c6918b7b3d747) }
|
||||
|
||||
var fileDescriptor_pdu_616c27178643eca4 = []byte{
|
||||
// 995 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x56, 0xcf, 0x6e, 0xdb, 0xc6,
|
||||
0x13, 0x36, 0x25, 0xda, 0xa2, 0x46, 0xce, 0x2f, 0xf4, 0xd8, 0x09, 0x68, 0xfd, 0xd2, 0xd4, 0xd8,
|
||||
0x14, 0x85, 0x63, 0xa0, 0x44, 0xe1, 0xb4, 0x05, 0x8a, 0x14, 0x41, 0xeb, 0xbf, 0x31, 0xd2, 0xba,
|
||||
0xea, 0x5a, 0x0d, 0x8a, 0xdc, 0x18, 0x69, 0x2a, 0x2f, 0x4c, 0x71, 0xe9, 0x5d, 0x2a, 0x88, 0x7a,
|
||||
0xec, 0xa1, 0x87, 0x5e, 0x7a, 0xea, 0xeb, 0xf4, 0x29, 0xfa, 0x20, 0x7d, 0x84, 0x82, 0x6b, 0x92,
|
||||
0xa2, 0x44, 0xca, 0x70, 0x4f, 0xda, 0xf9, 0xe6, 0xdb, 0xd9, 0xd9, 0xd1, 0x37, 0xb3, 0x84, 0x76,
|
||||
0x3c, 0x9c, 0xf8, 0xb1, 0x92, 0x89, 0x64, 0x9b, 0xb0, 0xf1, 0xad, 0xd0, 0xc9, 0x89, 0x08, 0x49,
|
||||
0x4f, 0x75, 0x42, 0x63, 0x4e, 0xd7, 0xec, 0xa0, 0x0a, 0x6a, 0xfc, 0x04, 0x3a, 0x33, 0x40, 0x7b,
|
||||
0xd6, 0x4e, 0x73, 0xb7, 0xb3, 0xdf, 0xf1, 0x4b, 0xa4, 0xb2, 0x9f, 0xfd, 0x6e, 0x01, 0xcc, 0x6c,
|
||||
0x44, 0xb0, 0x7b, 0x41, 0x72, 0xe9, 0x59, 0x3b, 0xd6, 0x6e, 0x9b, 0x9b, 0x35, 0xee, 0x40, 0x87,
|
||||
0x93, 0x9e, 0x8c, 0xa9, 0x2f, 0xaf, 0x28, 0xf2, 0x1a, 0xc6, 0x55, 0x86, 0xf0, 0x23, 0xb8, 0x77,
|
||||
0xa6, 0x7b, 0x61, 0x30, 0xa0, 0x4b, 0x19, 0x0e, 0x49, 0x79, 0xcd, 0x1d, 0x6b, 0xd7, 0xe1, 0xf3,
|
||||
0x60, 0x1a, 0xe7, 0x4c, 0x1f, 0x47, 0x03, 0x35, 0x8d, 0x13, 0x1a, 0x7a, 0xb6, 0xe1, 0x94, 0x21,
|
||||
0xf6, 0x1c, 0xb6, 0xe7, 0x2f, 0xf4, 0x9a, 0x94, 0x16, 0x32, 0xd2, 0x9c, 0xae, 0xf1, 0x71, 0x39,
|
||||
0xd1, 0x2c, 0xc1, 0x12, 0xc2, 0x5e, 0x2d, 0xdf, 0xac, 0xd1, 0x07, 0x27, 0x37, 0xb3, 0x92, 0xa0,
|
||||
0x5f, 0x61, 0xf2, 0x82, 0xc3, 0xfe, 0xb6, 0x60, 0xa3, 0xe2, 0xc7, 0x7d, 0xb0, 0xfb, 0xd3, 0x98,
|
||||
0xcc, 0xe1, 0xff, 0xdb, 0x7f, 0x5c, 0x8d, 0xe0, 0x67, 0xbf, 0x29, 0x8b, 0x1b, 0x6e, 0x5a, 0xd1,
|
||||
0xf3, 0x60, 0x4c, 0x59, 0xd9, 0xcc, 0x3a, 0xc5, 0x4e, 0x27, 0x62, 0x68, 0xca, 0x64, 0x73, 0xb3,
|
||||
0xc6, 0x47, 0xd0, 0x3e, 0x54, 0x14, 0x24, 0xd4, 0xff, 0xe9, 0xd4, 0xd4, 0xc6, 0xe6, 0x33, 0x00,
|
||||
0xbb, 0xe0, 0x18, 0x43, 0xc8, 0xc8, 0x5b, 0x35, 0x91, 0x0a, 0x9b, 0x3d, 0x85, 0x4e, 0xe9, 0x58,
|
||||
0x5c, 0x07, 0xe7, 0x22, 0x0a, 0x62, 0x7d, 0x29, 0x13, 0x77, 0x25, 0xb5, 0x0e, 0xa4, 0xbc, 0x1a,
|
||||
0x07, 0xea, 0xca, 0xb5, 0xd8, 0x9f, 0x0d, 0x68, 0x5d, 0x50, 0x34, 0xbc, 0x43, 0x3d, 0xf1, 0x63,
|
||||
0xb0, 0x4f, 0x94, 0x1c, 0x9b, 0xc4, 0xeb, 0xcb, 0x65, 0xfc, 0xc8, 0xa0, 0xd1, 0x97, 0xe6, 0x2a,
|
||||
0xf5, 0xac, 0x46, 0x5f, 0x2e, 0x4a, 0xc8, 0xae, 0x4a, 0x88, 0x41, 0x7b, 0x26, 0x8d, 0x55, 0x53,
|
||||
0x5f, 0xdb, 0xef, 0x2b, 0xc1, 0x67, 0x30, 0x3e, 0x84, 0xb5, 0x23, 0x35, 0xe5, 0x93, 0xc8, 0x5b,
|
||||
0x33, 0xda, 0xc9, 0x2c, 0xfc, 0x1a, 0x36, 0x38, 0xc5, 0xa1, 0x18, 0x98, 0x7a, 0x1c, 0xca, 0xe8,
|
||||
0x67, 0x31, 0xf2, 0x5a, 0x59, 0x42, 0x15, 0x0f, 0xaf, 0x92, 0xd9, 0x0f, 0x35, 0x11, 0xf0, 0x2b,
|
||||
0x80, 0xb4, 0xf9, 0x68, 0x60, 0xaa, 0x6e, 0x99, 0x78, 0x8f, 0xaa, 0xf1, 0x7a, 0x05, 0x87, 0x97,
|
||||
0xf8, 0xec, 0x0f, 0x0b, 0xfe, 0x7f, 0x0b, 0x17, 0x9f, 0x41, 0xeb, 0x2c, 0x12, 0x89, 0x08, 0xc2,
|
||||
0x4c, 0x4e, 0xdb, 0xe5, 0xd0, 0xa7, 0x93, 0x40, 0x05, 0x51, 0x42, 0xf4, 0x4a, 0x44, 0x43, 0x9e,
|
||||
0x33, 0xf1, 0x39, 0x74, 0xce, 0xa2, 0x81, 0xa2, 0x31, 0x45, 0x49, 0x10, 0x9a, 0xbf, 0xe6, 0xd6,
|
||||
0x8d, 0x65, 0x36, 0xfb, 0x0c, 0x9c, 0x9e, 0x92, 0x31, 0xa9, 0x64, 0x5a, 0xa8, 0xd2, 0x2a, 0xa9,
|
||||
0x72, 0x0b, 0x56, 0x5f, 0x07, 0xe1, 0x24, 0x97, 0xea, 0x8d, 0xc1, 0x7e, 0xb5, 0x72, 0xc9, 0x68,
|
||||
0xdc, 0x85, 0xfb, 0x3f, 0x6a, 0x1a, 0x2e, 0x4e, 0x03, 0x87, 0x2f, 0xc2, 0xc8, 0x60, 0xfd, 0xf8,
|
||||
0x7d, 0x4c, 0x83, 0x84, 0x86, 0x17, 0xe2, 0x17, 0x32, 0xf2, 0x68, 0xf2, 0x39, 0x0c, 0x9f, 0x02,
|
||||
0x64, 0xf9, 0x08, 0xd2, 0x9e, 0x6d, 0xba, 0xb2, 0xed, 0xe7, 0x29, 0xf2, 0x92, 0x93, 0xbd, 0x00,
|
||||
0x37, 0xcd, 0xe1, 0x50, 0x8e, 0xe3, 0x90, 0x12, 0x32, 0xfa, 0xdd, 0x83, 0xce, 0xf7, 0x4a, 0x8c,
|
||||
0x44, 0x14, 0x84, 0x9c, 0xae, 0x33, 0x99, 0x3a, 0x7e, 0x26, 0x6f, 0x5e, 0x76, 0x32, 0xac, 0xec,
|
||||
0xd7, 0xec, 0x2f, 0x0b, 0x80, 0xd3, 0x80, 0xc4, 0x3b, 0xba, 0x4b, 0x3b, 0xdc, 0xc8, 0xbc, 0x71,
|
||||
0xab, 0xcc, 0xf7, 0xc0, 0x3d, 0x0c, 0x29, 0x50, 0xe5, 0x02, 0xdd, 0x8c, 0xc2, 0x0a, 0x5e, 0x2f,
|
||||
0x5a, 0xfb, 0xbf, 0x88, 0x76, 0xbd, 0x94, 0xbf, 0x66, 0x23, 0xd8, 0x3c, 0x22, 0x9d, 0x28, 0x39,
|
||||
0xcd, 0xbb, 0xff, 0x2e, 0x53, 0x13, 0x3f, 0x85, 0x76, 0xc1, 0xf7, 0x1a, 0x4b, 0x27, 0xe3, 0x8c,
|
||||
0xc4, 0xde, 0x00, 0x2e, 0x1c, 0x94, 0x0d, 0xd8, 0xdc, 0xcc, 0x5a, 0xa5, 0x76, 0xc0, 0xe6, 0x9c,
|
||||
0x54, 0x6c, 0xc7, 0x4a, 0x49, 0x95, 0x8b, 0xcd, 0x18, 0xec, 0xa8, 0xee, 0x12, 0xe9, 0x9b, 0xd6,
|
||||
0x4a, 0x4b, 0x17, 0x26, 0xf9, 0xf0, 0xde, 0xf4, 0xab, 0x29, 0xf0, 0x9c, 0xc3, 0xbe, 0x80, 0xad,
|
||||
0x72, 0xb5, 0x26, 0x4a, 0x4b, 0x75, 0x97, 0x17, 0xa4, 0x5f, 0xbb, 0x4f, 0xe3, 0x56, 0x36, 0xae,
|
||||
0xd3, 0x1d, 0xf6, 0xcb, 0x95, 0x62, 0x60, 0x3b, 0xe7, 0x32, 0xa1, 0xf7, 0x42, 0x27, 0x37, 0x5d,
|
||||
0xf0, 0x72, 0x85, 0x17, 0xc8, 0x81, 0x03, 0x6b, 0x37, 0xe9, 0xb0, 0x27, 0xd0, 0xea, 0x89, 0x68,
|
||||
0x94, 0x26, 0xe0, 0x41, 0xeb, 0x3b, 0xd2, 0x3a, 0x18, 0xe5, 0x8d, 0x97, 0x9b, 0xec, 0x83, 0x9c,
|
||||
0xa4, 0xd3, 0xd6, 0x3c, 0x1e, 0x5c, 0xca, 0xbc, 0x35, 0xd3, 0xf5, 0xde, 0x2e, 0x34, 0xfb, 0x4a,
|
||||
0xa4, 0xc3, 0xfc, 0x48, 0x46, 0xc9, 0x61, 0xa0, 0xc8, 0x5d, 0xc1, 0x36, 0xac, 0x9e, 0x04, 0xa1,
|
||||
0x26, 0xd7, 0x42, 0x07, 0xec, 0xbe, 0x9a, 0x90, 0xdb, 0xd8, 0xfb, 0xcd, 0x02, 0x6f, 0xd9, 0x38,
|
||||
0xc0, 0x2d, 0x70, 0x0b, 0xe0, 0x2c, 0x7a, 0x17, 0x84, 0x62, 0xe8, 0xae, 0xe0, 0x36, 0x3c, 0x28,
|
||||
0x50, 0xa3, 0xd0, 0xe0, 0xad, 0x08, 0x45, 0x32, 0x75, 0x2d, 0x7c, 0x02, 0x1f, 0x96, 0x36, 0x14,
|
||||
0xa3, 0xa4, 0x74, 0x80, 0xdb, 0x98, 0x8b, 0x7a, 0x2e, 0x93, 0x4b, 0x11, 0x8d, 0xdc, 0xe6, 0xfe,
|
||||
0x3f, 0x8d, 0x74, 0xe6, 0x17, 0x3c, 0xec, 0x82, 0x9d, 0xde, 0x10, 0x1d, 0x3f, 0xab, 0x46, 0x37,
|
||||
0x5f, 0x69, 0xfc, 0x12, 0xee, 0xcf, 0x3f, 0xdd, 0x1a, 0xd1, 0xaf, 0x7c, 0xef, 0x74, 0xab, 0x98,
|
||||
0xc6, 0x1e, 0x3c, 0xac, 0x7f, 0xf5, 0xb1, 0xeb, 0x2f, 0xfd, 0x96, 0xe8, 0x2e, 0xf7, 0x69, 0x7c,
|
||||
0x01, 0xee, 0xa2, 0x06, 0x71, 0xcb, 0xaf, 0xe9, 0xad, 0x6e, 0x1d, 0xaa, 0xf1, 0x9b, 0xf9, 0xc6,
|
||||
0x36, 0x2a, 0xc2, 0x07, 0x7e, 0x9d, 0x22, 0xbb, 0xb5, 0xb0, 0xc6, 0xcf, 0xe1, 0xde, 0xdc, 0xb8,
|
||||
0xc2, 0x0d, 0x7f, 0x71, 0xfc, 0x75, 0x2b, 0x90, 0x3e, 0x58, 0x7d, 0xd3, 0x8c, 0x87, 0x93, 0xb7,
|
||||
0x6b, 0xe6, 0x93, 0xf1, 0xd9, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x7e, 0xa4, 0xfb, 0xca, 0x3f,
|
||||
0x0a, 0x00, 0x00,
|
||||
var fileDescriptor_pdu_483c6918b7b3d747 = []byte{
|
||||
// 833 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x56, 0x5f, 0x6f, 0xe3, 0x44,
|
||||
0x10, 0xaf, 0x13, 0xa7, 0x75, 0x26, 0x3d, 0x2e, 0x9d, 0x96, 0x93, 0xb1, 0xe0, 0x54, 0x2d, 0x08,
|
||||
0xe5, 0x2a, 0x61, 0xa1, 0xf2, 0x47, 0x42, 0x48, 0x27, 0xd1, 0xb4, 0xbd, 0x3b, 0x01, 0x47, 0xb4,
|
||||
0x35, 0x27, 0x74, 0x6f, 0x26, 0x19, 0xb5, 0x56, 0x1d, 0xaf, 0xbb, 0xe3, 0xa0, 0x0b, 0xe2, 0x89,
|
||||
0x47, 0xbe, 0x1e, 0x7c, 0x10, 0x3e, 0x02, 0xf2, 0xc6, 0x4e, 0x9c, 0xd8, 0x41, 0x79, 0xca, 0xce,
|
||||
0x6f, 0x66, 0x77, 0x67, 0x7f, 0xf3, 0x9b, 0x71, 0xa0, 0x9b, 0x4e, 0x66, 0x7e, 0xaa, 0x55, 0xa6,
|
||||
0xc4, 0x31, 0x1c, 0xfd, 0x10, 0x71, 0x76, 0x1d, 0xc5, 0xc4, 0x73, 0xce, 0x68, 0x2a, 0xe9, 0x41,
|
||||
0x5c, 0xd4, 0x41, 0xc6, 0xcf, 0xa0, 0xb7, 0x02, 0xd8, 0xb5, 0x4e, 0xdb, 0x83, 0xde, 0x79, 0xcf,
|
||||
0xaf, 0x04, 0x55, 0xfd, 0xe2, 0x2f, 0x0b, 0x60, 0x65, 0x23, 0x82, 0x3d, 0x0a, 0xb3, 0x3b, 0xd7,
|
||||
0x3a, 0xb5, 0x06, 0x5d, 0x69, 0xd6, 0x78, 0x0a, 0x3d, 0x49, 0x3c, 0x9b, 0x52, 0xa0, 0xee, 0x29,
|
||||
0x71, 0x5b, 0xc6, 0x55, 0x85, 0xf0, 0x13, 0x78, 0xf4, 0x8a, 0x47, 0x71, 0x38, 0xa6, 0x3b, 0x15,
|
||||
0x4f, 0x48, 0xbb, 0xed, 0x53, 0x6b, 0xe0, 0xc8, 0x75, 0x30, 0x3f, 0xe7, 0x15, 0x5f, 0x25, 0x63,
|
||||
0x3d, 0x4f, 0x33, 0x9a, 0xb8, 0xb6, 0x89, 0xa9, 0x42, 0xe2, 0x5b, 0xf8, 0x60, 0xfd, 0x41, 0x6f,
|
||||
0x48, 0x73, 0xa4, 0x12, 0x96, 0xf4, 0x80, 0x4f, 0xab, 0x89, 0x16, 0x09, 0x56, 0x10, 0xf1, 0xfd,
|
||||
0xf6, 0xcd, 0x8c, 0x3e, 0x38, 0xa5, 0x59, 0x50, 0x82, 0x7e, 0x2d, 0x52, 0x2e, 0x63, 0xc4, 0x3f,
|
||||
0x16, 0x1c, 0xd5, 0xfc, 0x78, 0x0e, 0x76, 0x30, 0x4f, 0xc9, 0x5c, 0xfe, 0xde, 0xf9, 0xd3, 0xfa,
|
||||
0x09, 0x7e, 0xf1, 0x9b, 0x47, 0x49, 0x13, 0x9b, 0x33, 0xfa, 0x3a, 0x9c, 0x52, 0x41, 0x9b, 0x59,
|
||||
0xe7, 0xd8, 0x8b, 0x59, 0x34, 0x31, 0x34, 0xd9, 0xd2, 0xac, 0xf1, 0x43, 0xe8, 0x0e, 0x35, 0x85,
|
||||
0x19, 0x05, 0xbf, 0xbc, 0x30, 0xdc, 0xd8, 0x72, 0x05, 0xa0, 0x07, 0x8e, 0x31, 0x22, 0x95, 0xb8,
|
||||
0x1d, 0x73, 0xd2, 0xd2, 0x16, 0xcf, 0xa0, 0x57, 0xb9, 0x16, 0x0f, 0xc1, 0xb9, 0x49, 0xc2, 0x94,
|
||||
0xef, 0x54, 0xd6, 0xdf, 0xcb, 0xad, 0x0b, 0xa5, 0xee, 0xa7, 0xa1, 0xbe, 0xef, 0x5b, 0xe2, 0x6f,
|
||||
0x0b, 0x0e, 0x6e, 0x28, 0x99, 0xec, 0xc0, 0x27, 0x7e, 0x0a, 0xf6, 0xb5, 0x56, 0x53, 0x93, 0x78,
|
||||
0x33, 0x5d, 0xc6, 0x8f, 0x02, 0x5a, 0x81, 0x32, 0x4f, 0x69, 0x8e, 0x6a, 0x05, 0x6a, 0x53, 0x42,
|
||||
0x76, 0x5d, 0x42, 0x02, 0xba, 0x2b, 0x69, 0x74, 0x0c, 0xbf, 0xb6, 0x1f, 0xe8, 0x48, 0xae, 0x60,
|
||||
0x7c, 0x02, 0xfb, 0x97, 0x7a, 0x2e, 0x67, 0x89, 0xbb, 0x6f, 0xb4, 0x53, 0x58, 0xe2, 0x4b, 0x70,
|
||||
0x46, 0x5a, 0xa5, 0xa4, 0xb3, 0xf9, 0x92, 0x6e, 0xab, 0x42, 0xf7, 0x09, 0x74, 0xde, 0x84, 0xf1,
|
||||
0xac, 0xac, 0xc1, 0xc2, 0x10, 0x7f, 0x2e, 0xb9, 0x60, 0x1c, 0xc0, 0xe3, 0x9f, 0x99, 0x26, 0x9b,
|
||||
0x32, 0x77, 0xe4, 0x26, 0x8c, 0x02, 0x0e, 0xaf, 0xde, 0xa5, 0x34, 0xce, 0x68, 0x72, 0x13, 0xfd,
|
||||
0x4e, 0xe6, 0xdd, 0x6d, 0xb9, 0x86, 0xe1, 0x33, 0x80, 0x22, 0x9f, 0x88, 0xd8, 0xb5, 0x8d, 0xdc,
|
||||
0xba, 0x7e, 0x99, 0xa2, 0xac, 0x38, 0xc5, 0x73, 0xe8, 0xe7, 0x39, 0x0c, 0xd5, 0x34, 0x8d, 0x29,
|
||||
0x23, 0x53, 0x98, 0x33, 0xe8, 0xfd, 0xa4, 0xa3, 0xdb, 0x28, 0x09, 0x63, 0x49, 0x0f, 0x05, 0xff,
|
||||
0x8e, 0x5f, 0xd4, 0x4d, 0x56, 0x9d, 0x02, 0x6b, 0xfb, 0x59, 0xfc, 0x01, 0x20, 0x69, 0x4c, 0xd1,
|
||||
0x6f, 0xb4, 0x4b, 0x99, 0x17, 0xe5, 0x6b, 0xfd, 0x6f, 0xf9, 0xce, 0xa0, 0x3f, 0x8c, 0x29, 0xd4,
|
||||
0x55, 0x7e, 0x16, 0x2d, 0x5e, 0xc3, 0xc5, 0x61, 0xe5, 0x76, 0x16, 0xb7, 0x70, 0x7c, 0x49, 0x9c,
|
||||
0x69, 0x35, 0x2f, 0x35, 0xb9, 0x4b, 0x2f, 0xe3, 0xe7, 0xd0, 0x5d, 0xc6, 0xbb, 0xad, 0xad, 0xfd,
|
||||
0xba, 0x0a, 0x12, 0x6f, 0x01, 0x37, 0x2e, 0x2a, 0xda, 0xbe, 0x34, 0xcd, 0x2d, 0x5b, 0xda, 0xbe,
|
||||
0x8c, 0xc9, 0x95, 0x72, 0xa5, 0xb5, 0xd2, 0xa5, 0x52, 0x8c, 0x21, 0x2e, 0x9b, 0x1e, 0x91, 0x4f,
|
||||
0xda, 0x83, 0xfc, 0xe1, 0x71, 0x56, 0x8e, 0x94, 0x63, 0xbf, 0x9e, 0x82, 0x2c, 0x63, 0xc4, 0xd7,
|
||||
0x70, 0x22, 0x29, 0x8d, 0xa3, 0xb1, 0xe9, 0xda, 0xe1, 0x4c, 0xb3, 0xd2, 0xbb, 0xcc, 0xb5, 0xa0,
|
||||
0x71, 0x1f, 0xe3, 0x49, 0x31, 0x44, 0xf2, 0x1d, 0xf6, 0xcb, 0xbd, 0xe5, 0x18, 0x71, 0x5e, 0xab,
|
||||
0x8c, 0xde, 0x45, 0x9c, 0x2d, 0x24, 0xfc, 0x72, 0x4f, 0x2e, 0x91, 0x0b, 0x07, 0xf6, 0x17, 0xe9,
|
||||
0x88, 0x8f, 0xe1, 0x60, 0x14, 0x25, 0xb7, 0x79, 0x02, 0x2e, 0x1c, 0xfc, 0x48, 0xcc, 0xe1, 0x6d,
|
||||
0xd9, 0x35, 0xa5, 0x29, 0x3e, 0x2a, 0x83, 0x38, 0xef, 0xab, 0xab, 0xf1, 0x9d, 0x2a, 0xfb, 0x2a,
|
||||
0x5f, 0x9f, 0x0d, 0xa0, 0x1d, 0xe8, 0x28, 0x1f, 0x31, 0x97, 0x2a, 0xc9, 0x86, 0xa1, 0xa6, 0xfe,
|
||||
0x1e, 0x76, 0xa1, 0x73, 0x1d, 0xc6, 0x4c, 0x7d, 0x0b, 0x1d, 0xb0, 0x03, 0x3d, 0xa3, 0x7e, 0xeb,
|
||||
0xfc, 0xdf, 0x56, 0x3e, 0x00, 0x96, 0x8f, 0x40, 0x0f, 0xec, 0xfc, 0x60, 0x74, 0xfc, 0x22, 0x09,
|
||||
0xaf, 0x5c, 0x31, 0x7e, 0x03, 0x8f, 0xd7, 0xe7, 0x38, 0x23, 0xfa, 0xb5, 0x8f, 0x9f, 0x57, 0xc7,
|
||||
0x18, 0x47, 0xf0, 0xa4, 0xf9, 0x13, 0x80, 0x9e, 0xbf, 0xf5, 0xc3, 0xe2, 0x6d, 0xf7, 0x31, 0x3e,
|
||||
0x87, 0xfe, 0x66, 0xe9, 0xf1, 0xc4, 0x6f, 0x90, 0xb4, 0xd7, 0x84, 0x32, 0x7e, 0x07, 0x47, 0xb5,
|
||||
0xe2, 0xe1, 0xfb, 0x7e, 0x93, 0x10, 0xbc, 0x46, 0x98, 0xf1, 0x2b, 0x78, 0xb4, 0xd6, 0xe2, 0x78,
|
||||
0xe4, 0x6f, 0x8e, 0x0c, 0xaf, 0x06, 0xf1, 0x45, 0xe7, 0x6d, 0x3b, 0x9d, 0xcc, 0x7e, 0xdd, 0x37,
|
||||
0xff, 0x1f, 0xbe, 0xf8, 0x2f, 0x00, 0x00, 0xff, 0xff, 0x27, 0x95, 0xc1, 0x78, 0x4c, 0x08, 0x00,
|
||||
0x00,
|
||||
}
|
||||
|
||||
@@ -61,25 +61,6 @@ message SendReq {
|
||||
Tri Encrypted = 5;
|
||||
|
||||
bool DryRun = 6;
|
||||
|
||||
ReplicationConfig ReplicationConfig = 7;
|
||||
}
|
||||
|
||||
message ReplicationConfig {
|
||||
ReplicationConfigProtection protection = 1;
|
||||
}
|
||||
|
||||
|
||||
message ReplicationConfigProtection {
|
||||
ReplicationGuaranteeKind Initial = 1;
|
||||
ReplicationGuaranteeKind Incremental = 2;
|
||||
}
|
||||
|
||||
enum ReplicationGuaranteeKind {
|
||||
GuaranteeInvalid = 0;
|
||||
GuaranteeResumability = 1;
|
||||
GuaranteeIncrementalReplication = 2;
|
||||
GuaranteeNothing = 3;
|
||||
}
|
||||
|
||||
message Property {
|
||||
@@ -112,8 +93,6 @@ message ReceiveReq {
|
||||
// If true, the receiver should clear the resume token before performing the
|
||||
// zfs recv of the stream in the request
|
||||
bool ClearResumeToken = 3;
|
||||
|
||||
ReplicationConfig ReplicationConfig = 4;
|
||||
}
|
||||
|
||||
message ReceiveRes {}
|
||||
|
||||
@@ -83,10 +83,3 @@ func (v *FilesystemVersion) ZFSFilesystemVersion() (*zfs.FilesystemVersion, erro
|
||||
Creation: ct,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func ReplicationConfigProtectionWithKind(both ReplicationGuaranteeKind) *ReplicationConfigProtection {
|
||||
return &ReplicationConfigProtection{
|
||||
Initial: both,
|
||||
Incremental: both,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
@@ -53,6 +52,10 @@ type Receiver interface {
|
||||
Receive(ctx context.Context, req *pdu.ReceiveReq, receive io.ReadCloser) (*pdu.ReceiveRes, error)
|
||||
}
|
||||
|
||||
type PlannerPolicy struct {
|
||||
EncryptedSend tri // all sends must be encrypted (send -w, and encryption!=off)
|
||||
}
|
||||
|
||||
type Planner struct {
|
||||
sender Sender
|
||||
receiver Receiver
|
||||
@@ -551,25 +554,19 @@ func (s *Step) updateSizeEstimate(ctx context.Context) error {
|
||||
log.WithError(err).Error("dry run send request failed")
|
||||
return err
|
||||
}
|
||||
if sres == nil {
|
||||
err := fmt.Errorf("dry run send request returned nil send result")
|
||||
log.Error(err.Error())
|
||||
return err
|
||||
}
|
||||
s.expectedSize = sres.GetExpectedSize()
|
||||
s.expectedSize = sres.ExpectedSize
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Step) buildSendRequest(dryRun bool) (sr *pdu.SendReq) {
|
||||
fs := s.parent.Path
|
||||
sr = &pdu.SendReq{
|
||||
Filesystem: fs,
|
||||
From: s.from, // may be nil
|
||||
To: s.to,
|
||||
Encrypted: s.encrypt.ToPDU(),
|
||||
ResumeToken: s.resumeToken,
|
||||
DryRun: dryRun,
|
||||
ReplicationConfig: &s.parent.policy.ReplicationConfig,
|
||||
Filesystem: fs,
|
||||
From: s.from, // may be nil
|
||||
To: s.to,
|
||||
Encrypted: s.encrypt.ToPDU(),
|
||||
ResumeToken: s.resumeToken,
|
||||
DryRun: dryRun,
|
||||
}
|
||||
return sr
|
||||
}
|
||||
@@ -587,11 +584,6 @@ func (s *Step) doReplication(ctx context.Context) error {
|
||||
log.WithError(err).Error("send request failed")
|
||||
return err
|
||||
}
|
||||
if sres == nil {
|
||||
err := fmt.Errorf("send request returned nil send result")
|
||||
log.Error(err.Error())
|
||||
return err
|
||||
}
|
||||
if stream == nil {
|
||||
err := errors.New("send request did not return a stream, broken endpoint implementation")
|
||||
return err
|
||||
@@ -611,10 +603,9 @@ func (s *Step) doReplication(ctx context.Context) error {
|
||||
}()
|
||||
|
||||
rr := &pdu.ReceiveReq{
|
||||
Filesystem: fs,
|
||||
To: sr.GetTo(),
|
||||
ClearResumeToken: !sres.UsedResumeToken,
|
||||
ReplicationConfig: &s.parent.policy.ReplicationConfig,
|
||||
Filesystem: fs,
|
||||
To: sr.GetTo(),
|
||||
ClearResumeToken: !sres.UsedResumeToken,
|
||||
}
|
||||
log.Debug("initiate receive request")
|
||||
_, err = s.receiver.Receive(ctx, rr, byteCountingStream)
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
package logic
|
||||
|
||||
import (
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/replication/logic/pdu"
|
||||
)
|
||||
|
||||
type PlannerPolicy struct {
|
||||
EncryptedSend tri // all sends must be encrypted (send -w, and encryption!=off)
|
||||
ReplicationConfig pdu.ReplicationConfig
|
||||
}
|
||||
|
||||
func ReplicationConfigFromConfig(in *config.Replication) (*pdu.ReplicationConfig, error) {
|
||||
initial, err := pduReplicationGuaranteeKindFromConfig(in.Protection.Initial)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "field 'initial'")
|
||||
}
|
||||
incremental, err := pduReplicationGuaranteeKindFromConfig(in.Protection.Incremental)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "field 'incremental'")
|
||||
}
|
||||
return &pdu.ReplicationConfig{
|
||||
Protection: &pdu.ReplicationConfigProtection{
|
||||
Initial: initial,
|
||||
Incremental: incremental,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func pduReplicationGuaranteeKindFromConfig(in string) (k pdu.ReplicationGuaranteeKind, _ error) {
|
||||
switch in {
|
||||
case "guarantee_nothing":
|
||||
return pdu.ReplicationGuaranteeKind_GuaranteeNothing, nil
|
||||
case "guarantee_incremental":
|
||||
return pdu.ReplicationGuaranteeKind_GuaranteeIncrementalReplication, nil
|
||||
case "guarantee_resumability":
|
||||
return pdu.ReplicationGuaranteeKind_GuaranteeResumability, nil
|
||||
default:
|
||||
return k, errors.Errorf("%q is not in guarantee_{nothing,incremental,resumability}", in)
|
||||
}
|
||||
}
|
||||
@@ -162,30 +162,3 @@ func (f *FilesystemReport) NextStep() *StepReport {
|
||||
func (f *StepReport) IsIncremental() bool {
|
||||
return f.Info.From != ""
|
||||
}
|
||||
|
||||
// Returns, for the latest replication attempt,
|
||||
// 0 if there have not been any replication attempts,
|
||||
// -1 if the replication failed while enumerating file systems
|
||||
// N if N filesystems could not not be replicated successfully
|
||||
func (r *Report) GetFailedFilesystemsCountInLatestAttempt() int {
|
||||
|
||||
if len(r.Attempts) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
a := r.Attempts[len(r.Attempts)-1]
|
||||
switch a.State {
|
||||
case AttemptPlanningError:
|
||||
return -1
|
||||
case AttemptFanOutError:
|
||||
var count int
|
||||
for _, f := range a.Filesystems {
|
||||
if f.Error() != nil {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,7 +66,7 @@ type ProtocolError struct {
|
||||
}
|
||||
|
||||
func (e *ProtocolError) Error() string {
|
||||
return fmt.Sprintf("protocol error: %s", e.cause)
|
||||
return fmt.Sprintf("protocol error: %s", e)
|
||||
}
|
||||
|
||||
func (c *Client) recv(ctx context.Context, conn *stream.Conn, res proto.Message) error {
|
||||
|
||||
@@ -239,10 +239,6 @@ func (s *Server) serveConnRequest(ctx context.Context, endpoint string, c *strea
|
||||
|
||||
if sendStream != nil {
|
||||
err := c.SendStream(ctx, sendStream, ZFSStream)
|
||||
closeErr := sendStream.Close()
|
||||
if closeErr != nil {
|
||||
s.log.WithError(err).Error("cannot close send stream")
|
||||
}
|
||||
if err != nil {
|
||||
s.log.WithError(err).Error("cannot write send stream")
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user