Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Christian Schwarz
|
3de3e2b688 |
+88
-364
@@ -1,386 +1,110 @@
|
|||||||
version: 2.1
|
version: 2.0
|
||||||
orbs:
|
|
||||||
# NB: this is not the Go version, but the Orb version
|
|
||||||
# https://circleci.com/developer/orbs/orb/circleci/go#usage-go-modules-cache
|
|
||||||
go: circleci/go@1.11.0
|
|
||||||
|
|
||||||
commands:
|
|
||||||
setup-home-local-bin:
|
|
||||||
steps:
|
|
||||||
- run:
|
|
||||||
shell: /bin/bash -euo pipefail
|
|
||||||
command: |
|
|
||||||
mkdir -p "$HOME/.local/bin"
|
|
||||||
line='export PATH="$HOME/.local/bin:$PATH"'
|
|
||||||
if grep "$line" $BASH_ENV >/dev/null; then
|
|
||||||
echo "$line" >> $BASH_ENV
|
|
||||||
fi
|
|
||||||
|
|
||||||
# NOTE: UV version is defined in .uv-version file at repository root
|
|
||||||
install-docdep:
|
|
||||||
steps:
|
|
||||||
- run:
|
|
||||||
name: Read UV version from .uv-version
|
|
||||||
command: |
|
|
||||||
UV_VERSION=$(cat .uv-version)
|
|
||||||
echo "export UV_VERSION=$UV_VERSION" >> $BASH_ENV
|
|
||||||
# Python is managed by uv - it will automatically download the version
|
|
||||||
# specified in docs/.python-version when needed
|
|
||||||
- run:
|
|
||||||
name: Install uv
|
|
||||||
command: curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh
|
|
||||||
- run:
|
|
||||||
name: Add uv to PATH and set cache dir
|
|
||||||
command: |
|
|
||||||
echo 'export PATH="$HOME/.local/bin:$PATH"' >> $BASH_ENV
|
|
||||||
echo 'export UV_CACHE_DIR="$HOME/.cache/uv"' >> $BASH_ENV
|
|
||||||
- restore_cache:
|
|
||||||
name: Restore uv cache
|
|
||||||
keys:
|
|
||||||
- uv-cache-v1-${UV_VERSION}-{{ checksum "docs/uv.lock" }}
|
|
||||||
- uv-cache-v1-${UV_VERSION}-
|
|
||||||
|
|
||||||
save-uv-cache:
|
|
||||||
steps:
|
|
||||||
- run:
|
|
||||||
name: Prune uv cache for CI
|
|
||||||
command: uv cache prune --ci
|
|
||||||
- save_cache:
|
|
||||||
name: Save uv cache
|
|
||||||
key: uv-cache-v1-${UV_VERSION}-{{ checksum "docs/uv.lock" }}
|
|
||||||
paths:
|
|
||||||
- ~/.cache/uv
|
|
||||||
|
|
||||||
install-zfs-from-source:
|
|
||||||
parameters:
|
|
||||||
zfs_release:
|
|
||||||
type: string
|
|
||||||
steps:
|
|
||||||
- run:
|
|
||||||
name: Record kernel version for cache key
|
|
||||||
command: uname -r > /tmp/kernel-version
|
|
||||||
- restore_cache:
|
|
||||||
name: Restore ZFS native debs cache
|
|
||||||
keys:
|
|
||||||
- zfs-debs-v2-<<parameters.zfs_release>>-{{ checksum "/tmp/kernel-version" }}
|
|
||||||
- run:
|
|
||||||
# https://openzfs.github.io/openzfs-docs/Developer%20Resources/Building%20ZFS.html
|
|
||||||
name: Build ZFS <<parameters.zfs_release>> native debs (if not cached)
|
|
||||||
no_output_timeout: 20m
|
|
||||||
command: |
|
|
||||||
# CircleCI machine images have pyenv Python 3.13 shadowing the system
|
|
||||||
# Python 3.12. The apt python3-* packages (setuptools, cffi, etc.) only
|
|
||||||
# install for the system Python, and ZFS's dpkg-buildpackage needs them
|
|
||||||
# for --enable-pyzfs. Use the system Python so apt packages are visible.
|
|
||||||
export PYENV_VERSION=system
|
|
||||||
if [ -d /tmp/zfs-debs ]; then
|
|
||||||
echo "ZFS debs cache hit, skipping build"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
sudo apt-get update
|
|
||||||
sudo apt-get install -y \
|
|
||||||
alien autoconf automake build-essential debhelper-compat dh-autoreconf \
|
|
||||||
dh-dkms dh-python dkms fakeroot gawk git libaio-dev libattr1-dev \
|
|
||||||
libblkid-dev libcurl4-openssl-dev libelf-dev libffi-dev libpam0g-dev \
|
|
||||||
libssl-dev libtirpc-dev libtool libudev-dev linux-headers-$(uname -r) lsb-release \
|
|
||||||
parallel po-debconf python3 python3-all-dev python3-cffi python3-dev \
|
|
||||||
python3-packaging python3-setuptools python3-sphinx uuid-dev zlib1g-dev
|
|
||||||
mkdir -p /tmp/zfs-build
|
|
||||||
cd /tmp/zfs-build
|
|
||||||
git clone --depth 1 --branch <<parameters.zfs_release>> https://github.com/openzfs/zfs.git zfs-src
|
|
||||||
cd zfs-src
|
|
||||||
sh autogen.sh
|
|
||||||
./configure
|
|
||||||
make native-deb
|
|
||||||
mkdir -p /tmp/zfs-debs
|
|
||||||
find /tmp/zfs-build -name '*.deb' -exec mv -t /tmp/zfs-debs/ {} +
|
|
||||||
- save_cache:
|
|
||||||
name: Save ZFS native debs cache
|
|
||||||
key: zfs-debs-v2-<<parameters.zfs_release>>-{{ checksum "/tmp/kernel-version" }}
|
|
||||||
paths:
|
|
||||||
- /tmp/zfs-debs
|
|
||||||
- run:
|
|
||||||
name: Install ZFS <<parameters.zfs_release>>
|
|
||||||
command: |
|
|
||||||
ls /tmp/zfs-debs/
|
|
||||||
# Only install the debs we need. Exclude dracut (conflicts with
|
|
||||||
# initramfs-tools), dkms (we have prebuilt modules), and packages
|
|
||||||
# we don't need (pyzfs, test, doc).
|
|
||||||
sudo apt-get install -y $(find /tmp/zfs-debs -name '*.deb' \
|
|
||||||
! -name '*dracut*' \
|
|
||||||
! -name '*dkms*' \
|
|
||||||
! -name '*initramfs*' \
|
|
||||||
! -name '*pyzfs*' \
|
|
||||||
! -name '*doc*' \
|
|
||||||
! -name '*test*' \
|
|
||||||
-print)
|
|
||||||
sudo modprobe zfs
|
|
||||||
- run:
|
|
||||||
name: Verify ZFS version
|
|
||||||
command: |
|
|
||||||
sudo zfs version
|
|
||||||
sudo zpool version
|
|
||||||
|
|
||||||
docs-publish-sh:
|
|
||||||
parameters:
|
|
||||||
push:
|
|
||||||
type: boolean
|
|
||||||
steps:
|
|
||||||
- checkout
|
|
||||||
- run:
|
|
||||||
command: |
|
|
||||||
git config --global user.email "zreplbot@cschwarz.com"
|
|
||||||
git config --global user.name "zrepl-github-io-ci"
|
|
||||||
|
|
||||||
# Configure git to use the GitHub token for HTTPS authentication.
|
|
||||||
# The token is stored in the 'zrepl-github-io-deploy' context.
|
|
||||||
- when:
|
|
||||||
condition: << parameters.push >>
|
|
||||||
steps:
|
|
||||||
- run:
|
|
||||||
name: Configure git to use GitHub token for push
|
|
||||||
# GITHUB_PAGES_TOKEN is from the 'zrepl-github-io-deploy' context.
|
|
||||||
# CircleCI's secret masking automatically redacts context variables in logs.
|
|
||||||
command: |
|
|
||||||
# Unset CircleCI's SSH URL rewriting that checkout step configured
|
|
||||||
git config --global --unset-all url."ssh://git@github.com".insteadOf || true
|
|
||||||
# Set up credential helper with GitHub token
|
|
||||||
git config --global credential.helper store
|
|
||||||
echo "https://x-access-token:${GITHUB_PAGES_TOKEN}@github.com" > ~/.git-credentials
|
|
||||||
chmod 600 ~/.git-credentials
|
|
||||||
|
|
||||||
# caller must install-docdep
|
|
||||||
- when:
|
|
||||||
condition: << parameters.push >>
|
|
||||||
steps:
|
|
||||||
- run: bash -x docs/publish.sh -c -a -P
|
|
||||||
- when:
|
|
||||||
condition:
|
|
||||||
not: << parameters.push >>
|
|
||||||
steps:
|
|
||||||
- run: bash -x docs/publish.sh -c -a
|
|
||||||
|
|
||||||
parameters:
|
|
||||||
do_ci:
|
|
||||||
type: boolean
|
|
||||||
default: true
|
|
||||||
|
|
||||||
do_release:
|
|
||||||
type: boolean
|
|
||||||
default: false
|
|
||||||
|
|
||||||
workflows:
|
workflows:
|
||||||
version: 2
|
version: 2
|
||||||
|
build:
|
||||||
ci:
|
|
||||||
when: << pipeline.parameters.do_ci >>
|
|
||||||
jobs:
|
jobs:
|
||||||
- run-docs-publish-sh:
|
- build-1.11
|
||||||
name: quickcheck-docs
|
- build-1.12
|
||||||
push: false
|
- build-1.13
|
||||||
- quickcheck-go:
|
- build-latest
|
||||||
name: quickcheck-go-amd64-linux-1.25.7
|
|
||||||
goversion: &latest-go-release "1.25.7"
|
|
||||||
goos: linux
|
|
||||||
goarch: amd64
|
|
||||||
- test-go:
|
|
||||||
goversion: *latest-go-release
|
|
||||||
- quickcheck-go:
|
|
||||||
requires:
|
|
||||||
- quickcheck-go-amd64-linux-1.25.7 #quickcheck-go-smoketest.name
|
|
||||||
matrix:
|
|
||||||
alias: quickcheck-go-matrix
|
|
||||||
parameters:
|
|
||||||
goversion: [*latest-go-release, "1.24.13"]
|
|
||||||
goos: ["linux", "freebsd"]
|
|
||||||
goarch: ["amd64", "arm64"]
|
|
||||||
exclude:
|
|
||||||
# don't re-do quickcheck-go-smoketest
|
|
||||||
- goversion: *latest-go-release
|
|
||||||
goos: linux
|
|
||||||
goarch: amd64
|
|
||||||
- platformtest:
|
|
||||||
matrix:
|
|
||||||
parameters:
|
|
||||||
goversion: [*latest-go-release]
|
|
||||||
goos: ["linux"]
|
|
||||||
goarch: ["amd64"]
|
|
||||||
zfs_release: ["zfs-2.2.9", "zfs-2.3.5", "zfs-2.4.0"]
|
|
||||||
requires:
|
|
||||||
- test-go
|
|
||||||
- quickcheck-go-<< matrix.goarch >>-<< matrix.goos >>-<< matrix.goversion >>
|
|
||||||
|
|
||||||
release:
|
|
||||||
when: << pipeline.parameters.do_release >>
|
|
||||||
jobs:
|
|
||||||
- release-build
|
|
||||||
- release-deb:
|
|
||||||
requires:
|
|
||||||
- release-build
|
|
||||||
- release-rpm:
|
|
||||||
requires:
|
|
||||||
- release-build
|
|
||||||
- release-upload:
|
|
||||||
requires:
|
|
||||||
- release-build
|
|
||||||
- release-deb
|
|
||||||
- release-rpm
|
|
||||||
|
|
||||||
publish-zrepl.github.io:
|
|
||||||
jobs:
|
|
||||||
- run-docs-publish-sh:
|
|
||||||
name: publish-zrepl.github.io
|
|
||||||
push: true
|
|
||||||
context:
|
|
||||||
- zrepl-github-io-deploy
|
|
||||||
filters:
|
|
||||||
branches:
|
|
||||||
only:
|
|
||||||
- master
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
run-docs-publish-sh:
|
# build-latest serves as the template
|
||||||
|
# we use YAML anchors & aliases to exchange the docker image (and hence Go version used for the build)
|
||||||
|
build-latest: &build-latest
|
||||||
|
description: Builds zrepl
|
||||||
parameters:
|
parameters:
|
||||||
push:
|
image:
|
||||||
type: boolean
|
description: "the docker image that the job should use"
|
||||||
docker:
|
|
||||||
- image: cimg/base:current
|
|
||||||
steps:
|
|
||||||
- checkout
|
|
||||||
- install-docdep
|
|
||||||
- docs-publish-sh:
|
|
||||||
push: << parameters.push >>
|
|
||||||
- save-uv-cache
|
|
||||||
|
|
||||||
quickcheck-go:
|
|
||||||
parameters:
|
|
||||||
goversion:
|
|
||||||
type: string
|
|
||||||
goos:
|
|
||||||
type: string
|
|
||||||
goarch:
|
|
||||||
type: string
|
type: string
|
||||||
docker:
|
docker:
|
||||||
- image: &cimg_with_modern_go cimg/go:1.25
|
- image: circleci/golang:latest
|
||||||
environment:
|
environment:
|
||||||
GOOS: <<parameters.goos>>
|
# required by lazy.sh
|
||||||
GOARCH: <<parameters.goarch>>
|
TERM: xterm
|
||||||
GOTOOLCHAIN: "go<<parameters.goversion>>"
|
working_directory: /go/src/github.com/zrepl/zrepl
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
|
- run:
|
||||||
|
name: Setup environment variables
|
||||||
|
command: |
|
||||||
|
# used by pip (for docs)
|
||||||
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> $BASH_ENV
|
||||||
|
# we use modules
|
||||||
|
echo 'export GO111MODULE=on' >> $BASH_ENV
|
||||||
|
|
||||||
|
- restore_cache:
|
||||||
|
keys:
|
||||||
|
- source
|
||||||
|
- protobuf
|
||||||
|
|
||||||
- checkout
|
- checkout
|
||||||
|
|
||||||
- go/load-cache:
|
- save_cache:
|
||||||
key: quickcheck-<<parameters.goversion>>
|
key: source
|
||||||
- run: make build/install
|
paths:
|
||||||
- run: go mod download
|
- ".git"
|
||||||
- run: cd build && go mod download
|
|
||||||
- go/save-cache:
|
|
||||||
key: quickcheck-<<parameters.goversion>>
|
|
||||||
|
|
||||||
# ensure all code has been generated
|
# install deps
|
||||||
- run: make generate
|
- run: wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
|
||||||
- run: |
|
- run: echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807 protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
|
||||||
if output=$(git status --porcelain) && [ -z "$output" ]; then
|
- run: sudo unzip -d /usr protoc-3.6.1-linux-x86_64.zip
|
||||||
echo "Working directory clean"
|
- save_cache:
|
||||||
else
|
key: protobuf
|
||||||
echo "Uncommitted changes"
|
paths:
|
||||||
echo ""
|
- "/usr/include/google/protobuf"
|
||||||
echo "$output"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# other checks
|
- run: sudo apt update && sudo apt install python3 python3-pip libgirepository1.0-dev
|
||||||
- run: make zrepl-bin test-platform-bin
|
- run: ./lazy.sh devsetup
|
||||||
|
|
||||||
|
- run: make
|
||||||
- run: make vet
|
- run: make vet
|
||||||
|
- run: make test
|
||||||
- run: make lint
|
- run: make lint
|
||||||
|
- run: make release
|
||||||
|
|
||||||
|
|
||||||
- store_artifacts:
|
- store_artifacts:
|
||||||
path: artifacts
|
path: ./artifacts/release
|
||||||
- persist_to_workspace:
|
when: always
|
||||||
root: .
|
|
||||||
paths: [.]
|
|
||||||
|
|
||||||
platformtest:
|
- run:
|
||||||
parameters:
|
shell: /bin/bash -eo pipefail
|
||||||
goversion:
|
when: always
|
||||||
type: string
|
command: |
|
||||||
goos:
|
if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
|
||||||
type: string
|
echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
|
||||||
goarch:
|
exit 0
|
||||||
type: string
|
fi
|
||||||
zfs_release:
|
set -u # from now on
|
||||||
type: string
|
|
||||||
machine:
|
|
||||||
# pinned (not :current) to keep ZFS build cache valid across runs
|
|
||||||
image: ubuntu-2404:2025.09.1
|
|
||||||
resource_class: medium
|
|
||||||
environment:
|
|
||||||
GOOS: <<parameters.goos>>
|
|
||||||
GOARCH: <<parameters.goarch>>
|
|
||||||
steps:
|
|
||||||
- attach_workspace:
|
|
||||||
at: .
|
|
||||||
- install-zfs-from-source:
|
|
||||||
zfs_release: <<parameters.zfs_release>>
|
|
||||||
- run: sudo make test-platform GOOS="$GOOS" GOARCH="$GOARCH"
|
|
||||||
|
|
||||||
test-go:
|
# Download and install minio
|
||||||
parameters:
|
curl -sSL https://dl.minio.io/client/mc/release/linux-amd64/mc -o ${GOPATH}/bin/mc
|
||||||
goversion:
|
chmod +x ${GOPATH}/bin/mc
|
||||||
type: string
|
mc config host add --api s3v4 zrepl-minio https://minio.cschwarz.com ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
|
||||||
|
|
||||||
|
# Upload artifacts
|
||||||
|
echo "$CIRCLE_BUILD_URL" > ./artifacts/release/cirlceci_build_url
|
||||||
|
mc cp -r artifacts/release "zrepl-minio/zrepl-ci-artifacts/${CIRCLE_SHA1}/${CIRCLE_JOB}/"
|
||||||
|
|
||||||
|
# Push Artifact Link to GitHub
|
||||||
|
REPO="zrepl/zrepl"
|
||||||
|
COMMIT="${CIRCLE_SHA1}"
|
||||||
|
JOB_NAME="${CIRCLE_JOB}"
|
||||||
|
curl "https://api.github.com/repos/$REPO/statuses/$COMMIT?access_token=$GITHUB_COMMIT_STATUS_TOKEN" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-X POST \
|
||||||
|
-d '{"context":"zrepl/publish-ci-artifacts", "state": "success", "description":"CI Build Artifacts for '"$JOB_NAME"'", "target_url":"https://minio.cschwarz.com/minio/zrepl-ci-artifacts/'"$COMMIT"'/"}'
|
||||||
|
|
||||||
|
build-1.11:
|
||||||
|
<<: *build-latest
|
||||||
docker:
|
docker:
|
||||||
- image: *cimg_with_modern_go
|
- image: circleci/golang:1.11
|
||||||
environment:
|
|
||||||
GOTOOLCHAIN: "go<<parameters.goversion>>"
|
|
||||||
steps:
|
|
||||||
- checkout
|
|
||||||
- go/load-cache:
|
|
||||||
key: make-test-go
|
|
||||||
- run: make test-go
|
|
||||||
- go/save-cache:
|
|
||||||
key: make-test-go
|
|
||||||
|
|
||||||
release-build:
|
build-1.12:
|
||||||
machine:
|
<<: *build-latest
|
||||||
image: &release-vm-image "ubuntu-2404:current"
|
|
||||||
resource_class: large
|
|
||||||
steps:
|
|
||||||
- checkout
|
|
||||||
- run: make release-docker
|
|
||||||
- persist_to_workspace:
|
|
||||||
root: .
|
|
||||||
paths: [.]
|
|
||||||
release-deb:
|
|
||||||
machine:
|
|
||||||
image: *release-vm-image
|
|
||||||
steps:
|
|
||||||
- attach_workspace:
|
|
||||||
at: .
|
|
||||||
- run: make debs-docker
|
|
||||||
- persist_to_workspace:
|
|
||||||
root: .
|
|
||||||
paths:
|
|
||||||
- "artifacts/*.deb"
|
|
||||||
|
|
||||||
release-rpm:
|
|
||||||
machine:
|
|
||||||
image: *release-vm-image
|
|
||||||
steps:
|
|
||||||
- attach_workspace:
|
|
||||||
at: .
|
|
||||||
- run: make rpms-docker
|
|
||||||
- persist_to_workspace:
|
|
||||||
root: .
|
|
||||||
paths:
|
|
||||||
- "artifacts/*.rpm"
|
|
||||||
|
|
||||||
release-upload:
|
|
||||||
docker:
|
docker:
|
||||||
- image: cimg/base:2024.09
|
- image: circleci/golang:1.12
|
||||||
steps:
|
|
||||||
- attach_workspace:
|
build-1.13:
|
||||||
at: .
|
<<: *build-latest
|
||||||
- run: make wrapup-and-checksum
|
docker:
|
||||||
- store_artifacts:
|
- image: circleci/golang:1.13
|
||||||
path: artifacts/release
|
|
||||||
|
|||||||
@@ -1,92 +0,0 @@
|
|||||||
import argparse
|
|
||||||
from pathlib import Path
|
|
||||||
import re
|
|
||||||
|
|
||||||
import requests
|
|
||||||
|
|
||||||
import time
|
|
||||||
|
|
||||||
import os
|
|
||||||
import yaml
|
|
||||||
import argparse
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
|
|
||||||
circle_token = os.environ.get('CIRCLE_TOKEN')
|
|
||||||
if not circle_token:
|
|
||||||
cli_yml = Path.home() / ".circleci" / "cli.yml"
|
|
||||||
if cli_yml.exists():
|
|
||||||
with open(cli_yml) as f:
|
|
||||||
data = yaml.safe_load(f)
|
|
||||||
if data:
|
|
||||||
circle_token = data.get("token")
|
|
||||||
if not circle_token:
|
|
||||||
raise ValueError('CIRCLE_TOKEN not set and no token found in ~/.circleci/cli.yml')
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description='Download artifacts from CircleCI')
|
|
||||||
parser.add_argument('build_num', type=str, help='Build number')
|
|
||||||
parser.add_argument('dst', type=Path, help='Destination directory')
|
|
||||||
parser.add_argument('--prefix', type=str, default='', help='Filter for prefix')
|
|
||||||
parser.add_argument('--match', type=str, default='.*', help='Only include paths matching the given regex')
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
res = requests.get(
|
|
||||||
f"https://circleci.com/api/v1.1/project/github/zrepl/zrepl/{args.build_num}/artifacts",
|
|
||||||
headers={
|
|
||||||
"Circle-Token": circle_token,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
res.raise_for_status()
|
|
||||||
|
|
||||||
# https://circleci.com/docs/api/v1/index.html#artifacts-of-a-job
|
|
||||||
# [ {
|
|
||||||
# "path" : "raw-test-output/go-test-report.xml",
|
|
||||||
# "pretty_path" : "raw-test-output/go-test-report.xml",
|
|
||||||
# "node_index" : 0,
|
|
||||||
# "url" : "https://24-88881093-gh.circle-artifacts.com/0/raw-test-output/go-test-report.xml"
|
|
||||||
# }, {
|
|
||||||
# "path" : "raw-test-output/go-test.out",
|
|
||||||
# "pretty_path" : "raw-test-output/go-test.out",
|
|
||||||
# "node_index" : 0,
|
|
||||||
# "url" : "https://24-88881093-gh.circle-artifacts.com/0/raw-test-output/go-test.out"
|
|
||||||
# } ]
|
|
||||||
res = res.json()
|
|
||||||
|
|
||||||
for artifact in res:
|
|
||||||
if not artifact["pretty_path"].startswith(args.prefix):
|
|
||||||
continue
|
|
||||||
if not re.match(args.match, artifact["pretty_path"]):
|
|
||||||
continue
|
|
||||||
stripped = artifact["pretty_path"][len(args.prefix):]
|
|
||||||
print(f"Downloading {artifact['pretty_path']} to {args.dst / stripped}")
|
|
||||||
artifact_rel = Path(stripped)
|
|
||||||
artifact_dst = args.dst / artifact_rel
|
|
||||||
artifact_dst.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
res = requests.get(
|
|
||||||
artifact["url"],
|
|
||||||
headers={
|
|
||||||
"Circle-Token": circle_token,
|
|
||||||
},
|
|
||||||
stream=True,
|
|
||||||
)
|
|
||||||
res.raise_for_status()
|
|
||||||
|
|
||||||
total_size = int(res.headers.get("Content-Length", 0))
|
|
||||||
block_size = 128 * 1024
|
|
||||||
with open(artifact_dst, "wb") as f:
|
|
||||||
progress = 0
|
|
||||||
start_time = time.time()
|
|
||||||
for chunk in res.iter_content(chunk_size=block_size):
|
|
||||||
f.write(chunk)
|
|
||||||
progress += len(chunk)
|
|
||||||
percent = progress / total_size * 100
|
|
||||||
elapsed_time = time.time() - start_time
|
|
||||||
if elapsed_time >= 5:
|
|
||||||
print(f"Downloaded {progress}/{total_size} bytes ({percent:.2f}%)", end="\r")
|
|
||||||
start_time = time.time()
|
|
||||||
print(f"Downloaded {progress}/{total_size} bytes ({percent:.2f}%)")
|
|
||||||
print("Download complete!")
|
|
||||||
|
|
||||||
print("All files downloaded")
|
|
||||||
@@ -1,95 +0,0 @@
|
|||||||
---
|
|
||||||
description: Generate GitHub release notes from docs/changelog.rst and create a draft release with artifacts
|
|
||||||
argument-hint: <version-tag, e.g. v0.7.0>
|
|
||||||
model: sonnet
|
|
||||||
allowed-tools: Read, Write
|
|
||||||
---
|
|
||||||
|
|
||||||
# Task
|
|
||||||
|
|
||||||
Create a draft GitHub release for zrepl version $ARGUMENTS and upload release artifacts.
|
|
||||||
|
|
||||||
## Inputs
|
|
||||||
|
|
||||||
Here is the current changelog RST source:
|
|
||||||
|
|
||||||
!`cat docs/changelog.rst`
|
|
||||||
|
|
||||||
## Instructions
|
|
||||||
|
|
||||||
### Phase 1: Pre-flight Sanity Checks
|
|
||||||
|
|
||||||
Before creating the release, verify that the previous build steps have been completed.
|
|
||||||
Run these checks using bash commands:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
test -d artifacts/release && echo "✓ artifacts/release directory exists" || echo "✗ Missing artifacts/release directory - run: make download-circleci-release JOB_NUM=<num>"
|
|
||||||
test -f artifacts/release/sha512sum.txt && echo "✓ sha512sum.txt exists" || echo "✗ Missing sha512sum.txt - run: make download-circleci-release JOB_NUM=<num>"
|
|
||||||
test -f artifacts/release/sha512sum.txt.asc && echo "✓ sha512sum.txt.asc exists (signed)" || echo "✗ Missing signature - run: make verify-and-sign"
|
|
||||||
ls artifacts/release/zrepl-* >/dev/null 2>&1 && echo "✓ Release artifacts present" || echo "✗ No release artifacts found"
|
|
||||||
```
|
|
||||||
|
|
||||||
If ANY check fails (shows ✗), **STOP** and print the error messages. Do not proceed with release creation.
|
|
||||||
|
|
||||||
### Phase 2: Generate Release Notes
|
|
||||||
|
|
||||||
2. Extract the changelog section for version $ARGUMENTS from the RST source above.
|
|
||||||
3. Extract all GitHub usernames mentioned in that changelog section (look for @username patterns or other contributor attributions).
|
|
||||||
4. Write a short "Highlights" section (2-4 bullets) summarizing the most impactful user-facing changes in plain language. Do NOT include commit/issue links.
|
|
||||||
5. Write a "Contributors" thank you line for all GitHub users found in step 3. Format as: "Thanks to @user1, @user2, and @user3 for their contributions to this release!"
|
|
||||||
6. Write a "Breaking Changes" section. Convert RST formatting to plain Markdown text (no commit or issue links):
|
|
||||||
- `:commit:\`abc123\`` → just remove it or describe what it changed
|
|
||||||
- `:issue:\`123\`` → just remove it or describe the issue in plain text
|
|
||||||
- `:ref:\`display text <anchor>\`` → just use `display text` as plain text
|
|
||||||
- `:repomasterlink:\`path\`` → just mention the path without a link
|
|
||||||
- RST inline code ` ``code`` ` → markdown `` `code` ``
|
|
||||||
- RST links `` `text <url>`_ `` → just use `text` without the link
|
|
||||||
If there are no breaking changes, write "No breaking changes. vX.Y.Z-1 is interoperable with vX.Y.Z." (fill in actual versions).
|
|
||||||
7. Assemble the full release notes using the template below. **IMPORTANT**: Do NOT include a detailed changelog list. Only include what is specified in the template.
|
|
||||||
8. Write the result to `artifacts/release-notes.md`.
|
|
||||||
9. Validate all links in the release notes by checking HTTP status codes with curl:
|
|
||||||
- Extract all URLs from the markdown file
|
|
||||||
- Check each URL with `curl -sI -w "%{http_code}" -o /dev/null <url>`
|
|
||||||
- Report any broken links (non-200 status codes)
|
|
||||||
- If any links are broken, stop and notify the user before creating the release
|
|
||||||
|
|
||||||
### Phase 3: Create Draft Release and Upload Artifacts
|
|
||||||
|
|
||||||
10. Run: `gh release create $ARGUMENTS --title "$ARGUMENTS" --notes-file artifacts/release-notes.md --draft`
|
|
||||||
11. Upload all artifacts: `gh release upload $ARGUMENTS artifacts/release/*`
|
|
||||||
12. Verify upload succeeded: `gh release view $ARGUMENTS --json assets --jq '.assets | length'`
|
|
||||||
13. Print the release URL and confirm that artifacts were uploaded successfully.
|
|
||||||
|
|
||||||
## Template
|
|
||||||
|
|
||||||
```
|
|
||||||
The full changelog is [on the docs site](https://zrepl.github.io/changelog.html).
|
|
||||||
|
|
||||||
## Highlights
|
|
||||||
|
|
||||||
{2-4 bullet plain-language summary of the most impactful changes}
|
|
||||||
|
|
||||||
{Thank all GitHub users mentioned in the changelog, e.g., "Thanks to @user1, @user2, and @user3 for their contributions to this release!"}
|
|
||||||
|
|
||||||
## Breaking Changes
|
|
||||||
|
|
||||||
{breaking changes, or "No breaking changes."}
|
|
||||||
|
|
||||||
## New Users
|
|
||||||
|
|
||||||
We provide [quick-start guides](https://zrepl.github.io/quickstart.html) for different usage scenarios.
|
|
||||||
We also recommend studying the [overview section of the configuration chapter](https://zrepl.github.io/configuration/overview.html).
|
|
||||||
|
|
||||||
## Testing & Upgrading
|
|
||||||
|
|
||||||
* Read the [Changelog](https://zrepl.github.io/changelog.html)
|
|
||||||
* [Run the platform tests](https://zrepl.github.io/usage.html#platform-tests) on a test system.
|
|
||||||
* Download & deploy the `zrepl` binary / distro package.
|
|
||||||
|
|
||||||
## Donations
|
|
||||||
|
|
||||||
zrepl is a spare-time project primarily developed by [Christian Schwarz](https://cschwarz.com).
|
|
||||||
Express your support through a donation to keep maintenance and feature development going.
|
|
||||||
|
|
||||||
[](https://patreon.com/zrepl) [](https://github.com/sponsors/problame) [](https://liberapay.com/zrepl/donate) [](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
|
|
||||||
```
|
|
||||||
@@ -1,149 +0,0 @@
|
|||||||
---
|
|
||||||
description: Update supporters list from git history and changelog contributors
|
|
||||||
argument-hint: <time-range-or-release>
|
|
||||||
model: sonnet
|
|
||||||
---
|
|
||||||
|
|
||||||
# Task
|
|
||||||
|
|
||||||
Automatically update `docs/supporters.rst` by extracting GitHub contributors from:
|
|
||||||
1. Git commit history (code contributors)
|
|
||||||
2. `docs/changelog.rst` (all mentioned GitHub users)
|
|
||||||
|
|
||||||
## Arguments
|
|
||||||
|
|
||||||
Specify a time range or release range to filter contributors:
|
|
||||||
|
|
||||||
**Time range examples:**
|
|
||||||
- `--since="2024-01-01"` - Contributors since a specific date
|
|
||||||
- `--since="1 year ago"` - Contributors in the last year
|
|
||||||
- `--since="6 months ago"` - Contributors in the last 6 months
|
|
||||||
|
|
||||||
**Release range examples:**
|
|
||||||
- `v0.6.0..HEAD` - Contributors between v0.6.0 and current HEAD
|
|
||||||
- `v0.6.0..v0.7.0` - Contributors between two releases
|
|
||||||
- `--all` - All contributors (default if no argument provided)
|
|
||||||
|
|
||||||
## Instructions
|
|
||||||
|
|
||||||
### Step 1: Extract Contributors from Git History
|
|
||||||
|
|
||||||
Parse the arguments to determine the git log range:
|
|
||||||
- If `--since="..."` is provided, use: `git log --since="..." --format='%aN <%aE>' | sort -u`
|
|
||||||
- If a commit range like `v0.6.0..HEAD` is provided, use: `git log v0.6.0..HEAD --format='%aN <%aE>' | sort -u`
|
|
||||||
- If `--all` or no argument, use: `git log --format='%aN <%aE>' | sort -u`
|
|
||||||
|
|
||||||
Parse the output to identify:
|
|
||||||
- GitHub users (look for @users.noreply.github.com emails or known GitHub patterns)
|
|
||||||
- Real names of code contributors
|
|
||||||
- Filter out the main maintainer (Christian Schwarz / problame) as they're already credited
|
|
||||||
|
|
||||||
### Step 2: Extract GitHub Users from Changelog
|
|
||||||
|
|
||||||
Read `docs/changelog.rst` and extract all GitHub usernames mentioned:
|
|
||||||
- Pattern: `` `@username <https://github.com/username>`_ ``
|
|
||||||
- Pattern: `@username` in text
|
|
||||||
- Collect all unique usernames
|
|
||||||
|
|
||||||
### Step 3: Categorize Contributors
|
|
||||||
|
|
||||||
For each contributor, determine their tier:
|
|
||||||
|
|
||||||
**Code contributors** (`|supporter-code|`):
|
|
||||||
- Anyone who appears in git history as a commit author
|
|
||||||
- Prioritize those with substantial contributions (multiple commits)
|
|
||||||
|
|
||||||
**Other contributors** (keep existing `|supporter-gold|` and `|supporter-std|`):
|
|
||||||
- Keep all existing gold/std supporters as-is (monetary supporters)
|
|
||||||
- Only update code contributors
|
|
||||||
|
|
||||||
### Step 4: Read Current Supporters File
|
|
||||||
|
|
||||||
Read `docs/supporters.rst` and parse existing entries to:
|
|
||||||
- Preserve all `|supporter-gold|` entries (monetary supporters - don't touch these)
|
|
||||||
- Preserve all `|supporter-std|` entries (monetary supporters - don't touch these)
|
|
||||||
- Preserve ALL existing `|supporter-code|` entries in their current order
|
|
||||||
- Extract list of existing code contributor names/GitHub usernames to avoid duplicates
|
|
||||||
|
|
||||||
### Step 5: Identify Contributors to Promote/Add
|
|
||||||
|
|
||||||
**IMPORTANT: Find contributors in the range, and either ADD them (if new) or MOVE them up (if recurring).**
|
|
||||||
|
|
||||||
For each GitHub user found in changelog or git history for the specified range:
|
|
||||||
1. Check if they're already listed as `|supporter-code|` in the existing file
|
|
||||||
2. Categorize them:
|
|
||||||
- **NEW**: Not currently in the file → will be added
|
|
||||||
- **RECURRING**: Already in the file with new contributions in this range → will be moved to top
|
|
||||||
- **UNCHANGED**: Not in this range → stays in original position
|
|
||||||
3. For both NEW and RECURRING contributors:
|
|
||||||
- Get their latest contribution date from the range: `git log <range> --author="email" --format="%ai" -1`
|
|
||||||
- Format as: `* |supporter-code| `Name <https://github.com/username>`_`
|
|
||||||
- Use their real name if available from git history, otherwise use GitHub username
|
|
||||||
4. Sort NEW + RECURRING contributors together by their contribution date (newest first)
|
|
||||||
5. Keep list of UNCHANGED contributors in their original order
|
|
||||||
|
|
||||||
**The final order will be: [NEW + RECURRING from range, sorted newest first] + [UNCHANGED in original order]**
|
|
||||||
|
|
||||||
### Step 6: Generate Updated File
|
|
||||||
|
|
||||||
Create the updated supporters list:
|
|
||||||
- Keep all `|supporter-gold|` entries unchanged and in their exact positions
|
|
||||||
- Keep all `|supporter-std|` entries unchanged and in their exact positions
|
|
||||||
- For `|supporter-code|` entries, reorganize as follows:
|
|
||||||
1. **Top section**: Contributors from the analyzed range (NEW + RECURRING), sorted newest first
|
|
||||||
2. **Bottom section**: UNCHANGED contributors (not in this range), in their original order
|
|
||||||
- Update the spacer comment showing the range that was just processed:
|
|
||||||
```rst
|
|
||||||
..
|
|
||||||
↓ claude --permission-mode default /update-supporters <actual-range-used>
|
|
||||||
```
|
|
||||||
Replace `<actual-range-used>` with the actual argument passed (e.g., `v0.6.1..v0.7.0`)
|
|
||||||
This comment should appear immediately before the first code contributor entry
|
|
||||||
If there's an existing comment in this format, replace it; otherwise add it
|
|
||||||
- Maintain all other file structure and RST formatting, including the "Before /update-supporters" marker comment
|
|
||||||
|
|
||||||
### Step 7: Apply Changes
|
|
||||||
|
|
||||||
- Use Edit tool to update the `docs/supporters.rst` file
|
|
||||||
- Display summary of changes made:
|
|
||||||
- Time/release range analyzed
|
|
||||||
- Number of contributors found
|
|
||||||
- New code contributors added
|
|
||||||
- Final counts by tier
|
|
||||||
|
|
||||||
## Implementation Notes
|
|
||||||
|
|
||||||
- **DO NOT modify, remove, or re-sort** any `|supporter-gold|` or `|supporter-std|` entries
|
|
||||||
- **For code contributors**: ADD new ones and MOVE UP recurring ones (those with contributions in the analyzed range)
|
|
||||||
- When extracting from git history, focus on substantial contributors (filter out single-commit or trivial changes if needed)
|
|
||||||
- **CRITICAL:** Contributors from the analyzed range go to the top
|
|
||||||
- Use `git log <range> --author="<email>" --format="%ai" -1` to get latest contribution date in the range
|
|
||||||
- Both NEW and RECURRING contributors from this range are sorted together (newest first) and placed at the TOP
|
|
||||||
- UNCHANGED contributors (not in this range) stay in their original relative order below
|
|
||||||
- This creates a chronological history: most recent release's contributors appear first
|
|
||||||
- Add/update the spacer comment showing the processed range
|
|
||||||
- Handle edge cases:
|
|
||||||
- Contributors mentioned in changelog but not in git history (external contributors, reporters)
|
|
||||||
- Contributors in git history but not mentioned in changelog (decide whether to include)
|
|
||||||
|
|
||||||
## Example Output
|
|
||||||
|
|
||||||
```
|
|
||||||
Analyzing contributors for range: v0.6.0..v0.7.0
|
|
||||||
- Found 8 code contributors in git history
|
|
||||||
- Found 12 GitHub users mentioned in changelog.rst (v0.7.0 section)
|
|
||||||
- Current supporters.rst has 8 gold, 12 std, 10 code contributors
|
|
||||||
|
|
||||||
New contributors:
|
|
||||||
+ Jane Doe (@janedoe)
|
|
||||||
+ John Smith (@jsmith)
|
|
||||||
|
|
||||||
Promoted (recurring) contributors:
|
|
||||||
↑ Alice Developer (@alice) - new contributions in v0.6.0..v0.7.0
|
|
||||||
|
|
||||||
Updated docs/supporters.rst:
|
|
||||||
- Gold supporters: 8 (unchanged)
|
|
||||||
- Std supporters: 12 (unchanged)
|
|
||||||
- Code contributors: 10 → 12
|
|
||||||
- Promoted to top: 3 contributors (2 new + 1 recurring)
|
|
||||||
```
|
|
||||||
@@ -1,4 +1,3 @@
|
|||||||
github: problame
|
|
||||||
patreon: zrepl
|
patreon: zrepl
|
||||||
liberapay: zrepl
|
liberapay: zrepl
|
||||||
custom: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96
|
custom: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96
|
||||||
|
|||||||
@@ -1,9 +0,0 @@
|
|||||||
version: 2
|
|
||||||
updates:
|
|
||||||
# Docs use Python only for static site generation.
|
|
||||||
- package-ecosystem: "pip"
|
|
||||||
directory: /docs
|
|
||||||
schedule:
|
|
||||||
interval: "weekly"
|
|
||||||
ignore:
|
|
||||||
- dependency-name: "*"
|
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
# Build
|
# Build
|
||||||
artifacts/
|
artifacts/
|
||||||
build/install.tmp
|
|
||||||
|
|
||||||
# Golang
|
# Golang
|
||||||
vendor/
|
vendor/
|
||||||
|
|||||||
+8
-40
@@ -1,42 +1,10 @@
|
|||||||
version: "2"
|
|
||||||
linters:
|
linters:
|
||||||
enable:
|
enable:
|
||||||
- revive
|
- goimports
|
||||||
settings:
|
|
||||||
revive:
|
issues:
|
||||||
rules:
|
exclude-rules:
|
||||||
- name: time-equal
|
- path: _test\.go
|
||||||
exclusions:
|
linters:
|
||||||
generated: lax
|
- errcheck
|
||||||
presets:
|
|
||||||
- comments
|
|
||||||
- common-false-positives
|
|
||||||
- legacy
|
|
||||||
- std-error-handling
|
|
||||||
rules:
|
|
||||||
- linters:
|
|
||||||
- errcheck
|
|
||||||
path: _test\.go
|
|
||||||
- linters:
|
|
||||||
- staticcheck
|
|
||||||
text: 'SA9003:'
|
|
||||||
- linters:
|
|
||||||
- staticcheck
|
|
||||||
text: '(QF1001|QF1011|ST1012|QF1008|ST1005|ST1023|QF1003|ST1006|ST1001|QF1004):'
|
|
||||||
paths:
|
|
||||||
- third_party$
|
|
||||||
- builtin$
|
|
||||||
- examples$
|
|
||||||
formatters:
|
|
||||||
enable:
|
|
||||||
- goimports
|
|
||||||
settings:
|
|
||||||
goimports:
|
|
||||||
local-prefixes:
|
|
||||||
- github.com/zrepl/zrepl
|
|
||||||
exclusions:
|
|
||||||
generated: lax
|
|
||||||
paths:
|
|
||||||
- third_party$
|
|
||||||
- builtin$
|
|
||||||
- examples$
|
|
||||||
|
|||||||
+70
@@ -0,0 +1,70 @@
|
|||||||
|
dist: xenial
|
||||||
|
services:
|
||||||
|
- docker
|
||||||
|
|
||||||
|
env: # for allow_failures: https://docs.travis-ci.com/user/customizing-the-build/
|
||||||
|
matrix:
|
||||||
|
include:
|
||||||
|
|
||||||
|
- language: go
|
||||||
|
name: "Build in Docker (docs/installation.rst)"
|
||||||
|
script:
|
||||||
|
- sudo docker build -t zrepl_build -f build.Dockerfile .
|
||||||
|
- |
|
||||||
|
sudo docker run -it --rm \
|
||||||
|
-v "${PWD}:/go/src/github.com/zrepl/zrepl" \
|
||||||
|
--user "$(id -u):$(id -g)" \
|
||||||
|
zrepl_build make vendordeps release
|
||||||
|
|
||||||
|
- &zrepl_build_template
|
||||||
|
language: go
|
||||||
|
go_import_path: github.com/zrepl/zrepl
|
||||||
|
before_install:
|
||||||
|
- wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
|
||||||
|
- echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807 protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
|
||||||
|
- sudo unzip -d /usr protoc-3.6.1-linux-x86_64.zip
|
||||||
|
- ./lazy.sh godep
|
||||||
|
- make vendordeps
|
||||||
|
script:
|
||||||
|
- make
|
||||||
|
- make vet
|
||||||
|
- make test
|
||||||
|
- make lint
|
||||||
|
- make artifacts/zrepl-freebsd-amd64
|
||||||
|
- make artifacts/zrepl-linux-amd64
|
||||||
|
- make artifacts/zrepl-darwin-amd64
|
||||||
|
go:
|
||||||
|
- "1.11"
|
||||||
|
|
||||||
|
- <<: *zrepl_build_template
|
||||||
|
go:
|
||||||
|
- "1.12"
|
||||||
|
|
||||||
|
- <<: *zrepl_build_template
|
||||||
|
go:
|
||||||
|
- "master"
|
||||||
|
|
||||||
|
- &zrepl_docs_template
|
||||||
|
language: python
|
||||||
|
python:
|
||||||
|
- "3.4"
|
||||||
|
install:
|
||||||
|
- sudo apt-get install libgirepository1.0-dev
|
||||||
|
- pip install -r docs/requirements.txt
|
||||||
|
script:
|
||||||
|
- make docs
|
||||||
|
- <<: *zrepl_docs_template
|
||||||
|
python:
|
||||||
|
- "3.5"
|
||||||
|
- <<: *zrepl_docs_template
|
||||||
|
python:
|
||||||
|
- "3.6"
|
||||||
|
- <<: *zrepl_docs_template
|
||||||
|
python:
|
||||||
|
- "3.7"
|
||||||
|
|
||||||
|
|
||||||
|
allow_failures:
|
||||||
|
- <<: *zrepl_build_template
|
||||||
|
go:
|
||||||
|
- "master"
|
||||||
@@ -1 +0,0 @@
|
|||||||
0.9.30
|
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
.PHONY: generate build test vet cover release docs docs-clean clean format lint platformtest
|
.PHONY: generate build test vet cover release docs docs-clean clean format lint platformtest
|
||||||
.PHONY: release release-noarch
|
.DEFAULT_GOAL := build
|
||||||
.DEFAULT_GOAL := zrepl-bin
|
|
||||||
|
|
||||||
ARTIFACTDIR := artifacts
|
ARTIFACTDIR := artifacts
|
||||||
|
|
||||||
@@ -13,188 +12,97 @@ ifndef _ZREPL_VERSION
|
|||||||
$(error cannot infer variable ZREPL_VERSION using git and variable is not overriden by make invocation)
|
$(error cannot infer variable ZREPL_VERSION using git and variable is not overriden by make invocation)
|
||||||
endif
|
endif
|
||||||
endif
|
endif
|
||||||
|
|
||||||
ZREPL_PACKAGE_RELEASE := 1
|
|
||||||
|
|
||||||
GO := go
|
GO := go
|
||||||
GOOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOOS"')
|
GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
|
||||||
GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"')
|
|
||||||
GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"')
|
|
||||||
GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"')
|
|
||||||
GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
|
|
||||||
GO_ENV_VARS := CGO_ENABLED=0
|
|
||||||
GO_LDFLAGS := "-X github.com/zrepl/zrepl/internal/version.zreplVersion=$(_ZREPL_VERSION)"
|
|
||||||
GO_MOD_READONLY := -mod=readonly
|
GO_MOD_READONLY := -mod=readonly
|
||||||
GO_EXTRA_BUILDFLAGS :=
|
GO_BUILDFLAGS := $(GO_MOD_READONLY)
|
||||||
GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
|
GO_BUILD := GO111MODULE=on $(GO) build $(GO_BUILDFLAGS) -v -ldflags $(GO_LDFLAGS)
|
||||||
GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
|
|
||||||
GOCOVMERGE := gocovmerge
|
|
||||||
RELEASE_GOVERSION ?= go1.25.7
|
|
||||||
STRIPPED_GOVERSION := $(subst go,,$(RELEASE_GOVERSION))
|
|
||||||
RELEASE_DOCKER_BASEIMAGE ?= golang:$(STRIPPED_GOVERSION)
|
|
||||||
RELEASE_DOCKER_CACHEMOUNT :=
|
|
||||||
|
|
||||||
ifneq ($(GOARM),)
|
# keep in sync with vet target
|
||||||
ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM)
|
RELEASE_BINS := $(ARTIFACTDIR)/zrepl-freebsd-amd64
|
||||||
else
|
RELEASE_BINS += $(ARTIFACTDIR)/zrepl-linux-amd64
|
||||||
ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)
|
RELEASE_BINS += $(ARTIFACTDIR)/zrepl-linux-arm64
|
||||||
endif
|
RELEASE_BINS += $(ARTIFACTDIR)/zrepl-darwin-amd64
|
||||||
|
|
||||||
|
RELEASE_NOARCH := $(ARTIFACTDIR)/zrepl-noarch.tar
|
||||||
|
THIS_PLATFORM_RELEASE_BIN := $(shell bash -c 'source <($(GO) env) && echo "zrepl-$${GOOS}-$${GOARCH}"' )
|
||||||
|
|
||||||
ifneq ($(RELEASE_DOCKER_CACHEMOUNT),)
|
generate: #not part of the build, must do that manually
|
||||||
_RELEASE_DOCKER_CACHEMOUNT := -v $(RELEASE_DOCKER_CACHEMOUNT)/mod:/go/pkg/mod -v $(RELEASE_DOCKER_CACHEMOUNT)/xdg-cache:/.cache/go-build
|
protoc -I=replication/logic/pdu --go_out=plugins=grpc:replication/logic/pdu replication/logic/pdu/pdu.proto
|
||||||
.PHONY: release-docker-mkcachemount
|
$(GO) generate $(GO_BUILDFLAGS) -x ./...
|
||||||
release-docker-mkcachemount:
|
|
||||||
mkdir -p $(RELEASE_DOCKER_CACHEMOUNT)
|
|
||||||
mkdir -p $(RELEASE_DOCKER_CACHEMOUNT)/mod
|
|
||||||
mkdir -p $(RELEASE_DOCKER_CACHEMOUNT)/xdg-cache
|
|
||||||
else
|
|
||||||
_RELEASE_DOCKER_CACHEMOUNT :=
|
|
||||||
.PHONY: release-docker-mkcachemount
|
|
||||||
release-docker-mkcachemount:
|
|
||||||
# nothing to do
|
|
||||||
endif
|
|
||||||
|
|
||||||
##################### PRODUCING A RELEASE #############
|
format:
|
||||||
.PHONY: release wrapup-and-checksum check-git-clean verify-and-sign clean ensure-release-toolchain
|
goimports -srcdir . -local 'github.com/zrepl/zrepl' -w $(shell find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go')
|
||||||
|
|
||||||
ensure-release-toolchain:
|
lint:
|
||||||
# ensure the toolchain is actually the one we expect
|
golangci-lint run ./...
|
||||||
test $(RELEASE_GOVERSION) = "$$($(GO_ENV_VARS) $(GO) env GOVERSION)"
|
|
||||||
|
|
||||||
release: ensure-release-toolchain
|
build:
|
||||||
$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="vet"
|
$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl"
|
||||||
$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="lint"
|
|
||||||
$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="zrepl-bin"
|
|
||||||
$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="test-platform-bin"
|
|
||||||
$(MAKE) noarch
|
|
||||||
|
|
||||||
release-docker: $(ARTIFACTDIR) release-docker-mkcachemount
|
test:
|
||||||
sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build/build.Dockerfile > $(ARTIFACTDIR)/build.Dockerfile
|
$(GO) test $(GO_BUILDFLAGS) ./...
|
||||||
docker build -t zrepl_release --pull \
|
# TODO compile the tests for each supported platform
|
||||||
--build-arg BUILD_UID=$$(id -u) \
|
# but `go test -c ./...` is not supported
|
||||||
--build-arg BUILD_GID=$$(id -g) \
|
|
||||||
-f $(ARTIFACTDIR)/build.Dockerfile .
|
|
||||||
docker run --rm -i$$(test -t 0 && echo t) \
|
|
||||||
$(_RELEASE_DOCKER_CACHEMOUNT) \
|
|
||||||
-v $(CURDIR):/src \
|
|
||||||
zrepl_release \
|
|
||||||
make release \
|
|
||||||
GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
|
|
||||||
ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE) \
|
|
||||||
RELEASE_GOVERSION=$(RELEASE_GOVERSION)
|
|
||||||
|
|
||||||
debs-docker:
|
vet:
|
||||||
$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=deb
|
$(GO) vet $(GO_BUILDFLAGS) ./...
|
||||||
rpms-docker:
|
# for each supported platform to cover conditional compilation
|
||||||
$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=rpm
|
# (keep in sync with RELEASE_BINS)
|
||||||
_debs_or_rpms_docker: # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
|
GOOS=freebsd GOARCH=amd64 $(GO) vet $(GO_BUILDFLAGS) ./...
|
||||||
$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=amd64
|
GOOS=linux GOARCH=amd64 $(GO) vet $(GO_BUILDFLAGS) ./...
|
||||||
$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=arm64
|
GOOS=linux GOARCH=arm64 $(GO) vet $(GO_BUILDFLAGS) ./...
|
||||||
$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=arm GOARM=7
|
GOOS=darwin GOARCH=amd64 $(GO) vet $(GO_BUILDFLAGS) ./...
|
||||||
$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=386
|
|
||||||
|
|
||||||
rpm: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
|
ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
|
||||||
$(eval _ZREPL_RPM_VERSION := $(subst -,.,$(_ZREPL_VERSION)))
|
ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
|
||||||
$(eval _ZREPL_RPM_TOPDIR_ABS := $(CURDIR)/$(ARTIFACTDIR)/rpmbuild)
|
$(ARTIFACTDIR)/zrepl_platformtest:
|
||||||
rm -rf "$(_ZREPL_RPM_TOPDIR_ABS)"
|
$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl_platformtest" ./platformtest/harness
|
||||||
mkdir "$(_ZREPL_RPM_TOPDIR_ABS)"
|
platformtest: $(ARTIFACTDIR)/zrepl_platformtest
|
||||||
for d in BUILD BUILDROOT RPMS SOURCES SPECS SRPMS; do \
|
"$(ARTIFACTDIR)/zrepl_platformtest" -poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" -imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)"
|
||||||
mkdir -p "$(_ZREPL_RPM_TOPDIR_ABS)/$$d"; \
|
|
||||||
done
|
|
||||||
sed \
|
|
||||||
-e "s/^Version:.*/Version: $(_ZREPL_RPM_VERSION)/g" \
|
|
||||||
-e "s/^Release:.*/Release: $(ZREPL_PACKAGE_RELEASE)/g" \
|
|
||||||
packaging/rpm/zrepl.spec \
|
|
||||||
> $(_ZREPL_RPM_TOPDIR_ABS)/SPECS/zrepl.spec
|
|
||||||
|
|
||||||
# see /usr/lib/rpm/platform
|
$(ARTIFACTDIR):
|
||||||
ifeq ($(GOARCH),amd64)
|
mkdir -p "$@"
|
||||||
$(eval _ZREPL_RPMBUILD_TARGET := x86_64)
|
|
||||||
else ifeq ($(GOARCH), 386)
|
|
||||||
$(eval _ZREPL_RPMBUILD_TARGET := i386)
|
|
||||||
else ifeq ($(GOARCH), arm64)
|
|
||||||
$(eval _ZREPL_RPMBUILD_TARGET := aarch64)
|
|
||||||
else ifeq ($(GOARCH), arm)
|
|
||||||
$(eval _ZREPL_RPMBUILD_TARGET := armv7hl)
|
|
||||||
else
|
|
||||||
$(eval _ZREPL_RPMBUILD_TARGET := $(GOARCH))
|
|
||||||
endif
|
|
||||||
rpmbuild \
|
|
||||||
--build-in-place \
|
|
||||||
--define "_sourcedir $(CURDIR)" \
|
|
||||||
--define "_topdir $(_ZREPL_RPM_TOPDIR_ABS)" \
|
|
||||||
--define "_zrepl_binary_filename zrepl-$(ZREPL_TARGET_TUPLE)" \
|
|
||||||
--target $(_ZREPL_RPMBUILD_TARGET) \
|
|
||||||
-bb "$(_ZREPL_RPM_TOPDIR_ABS)"/SPECS/zrepl.spec
|
|
||||||
cp "$(_ZREPL_RPM_TOPDIR_ABS)"/RPMS/$(_ZREPL_RPMBUILD_TARGET)/zrepl-$(_ZREPL_RPM_VERSION)*.rpm $(ARTIFACTDIR)/
|
|
||||||
|
|
||||||
rpm-docker:
|
$(ARTIFACTDIR)/docs: $(ARTIFACTDIR)
|
||||||
docker build -t zrepl_rpm_pkg --pull -f packaging/rpm/Dockerfile .
|
mkdir -p "$@"
|
||||||
docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
|
|
||||||
zrepl_rpm_pkg \
|
|
||||||
make rpm \
|
|
||||||
GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
|
|
||||||
ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE)
|
|
||||||
|
|
||||||
|
$(ARTIFACTDIR)/bash_completion: $(RELEASE_BINS)
|
||||||
|
artifacts/$(THIS_PLATFORM_RELEASE_BIN) bashcomp "$@"
|
||||||
|
|
||||||
deb: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
|
.PHONY: $(ARTIFACTDIR)/go_version.txt
|
||||||
|
$(ARTIFACTDIR)/go_version.txt:
|
||||||
|
$(GO) version > $@
|
||||||
|
|
||||||
cp packaging/deb/debian/changelog.template packaging/deb/debian/changelog
|
docs: $(ARTIFACTDIR)/docs
|
||||||
sed -i 's/DATE_DASH_R_OUTPUT/$(shell date -R)/' packaging/deb/debian/changelog
|
make -C docs \
|
||||||
VERSION="$(subst -,.,$(_ZREPL_VERSION))-$(ZREPL_PACKAGE_RELEASE)"; \
|
html \
|
||||||
export VERSION="$${VERSION#v}"; \
|
BUILDDIR=../artifacts/docs \
|
||||||
sed -i 's/VERSION/'"$$VERSION"'/' packaging/deb/debian/changelog
|
|
||||||
|
|
||||||
ifeq ($(GOARCH), arm)
|
docs-clean:
|
||||||
$(eval DEB_HOST_ARCH := armhf)
|
make -C docs \
|
||||||
else ifeq ($(GOARCH), 386)
|
clean \
|
||||||
$(eval DEB_HOST_ARCH := i386)
|
BUILDDIR=../artifacts/docs
|
||||||
else
|
|
||||||
$(eval DEB_HOST_ARCH := $(GOARCH))
|
|
||||||
endif
|
|
||||||
|
|
||||||
export ZREPL_DPKG_ZREPL_BINARY_FILENAME=zrepl-$(ZREPL_TARGET_TUPLE); \
|
.PHONY: $(RELEASE_BINS)
|
||||||
dpkg-buildpackage -b --no-sign --host-arch $(DEB_HOST_ARCH)
|
# TODO: two wildcards possible
|
||||||
cp ../*.deb artifacts/
|
$(RELEASE_BINS): $(ARTIFACTDIR)/zrepl-%: generate $(ARTIFACTDIR) vet test lint
|
||||||
|
STEM=$*; GOOS="$${STEM%%-*}"; GOARCH="$${STEM##*-}"; export GOOS GOARCH; \
|
||||||
|
$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$$GOOS-$$GOARCH"
|
||||||
|
|
||||||
deb-docker:
|
$(RELEASE_NOARCH): docs $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_version.txt
|
||||||
docker build -t zrepl_debian_pkg --pull -f packaging/deb/Dockerfile .
|
|
||||||
# Use a small open file limit to make fakeroot work. If we don't
|
|
||||||
# specify it, docker daemon will use its file limit. I don't know
|
|
||||||
# what changed (Docker, its systemd service, its Go version). But I
|
|
||||||
# observed fakeroot iterating close(i) up to i > 1000000, which costs
|
|
||||||
# a good amount of CPU time and makes the build slow.
|
|
||||||
docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
|
|
||||||
--ulimit nofile=1024:1024 \
|
|
||||||
zrepl_debian_pkg \
|
|
||||||
make deb \
|
|
||||||
GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
|
|
||||||
ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE)
|
|
||||||
|
|
||||||
# expects `release`, `deb` & `rpm` targets to have run before
|
|
||||||
NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
|
|
||||||
wrapup-and-checksum:
|
|
||||||
rm -f $(NOARCH_TARBALL)
|
|
||||||
tar --mtime='1970-01-01' --sort=name \
|
tar --mtime='1970-01-01' --sort=name \
|
||||||
--transform 's/$(ARTIFACTDIR)/zrepl-$(_ZREPL_VERSION)-noarch/' \
|
--transform 's/$(ARTIFACTDIR)/zrepl-$(_ZREPL_VERSION)-noarch/' \
|
||||||
--transform 's#dist#zrepl-$(_ZREPL_VERSION)-noarch/dist#' \
|
-acf $@ \
|
||||||
--transform 's#internal/config/samples#zrepl-$(_ZREPL_VERSION)-noarch/config#' \
|
|
||||||
-acf $(NOARCH_TARBALL) \
|
|
||||||
$(ARTIFACTDIR)/docs/html \
|
$(ARTIFACTDIR)/docs/html \
|
||||||
$(ARTIFACTDIR)/bash_completion \
|
$(ARTIFACTDIR)/bash_completion \
|
||||||
$(ARTIFACTDIR)/_zrepl.zsh_completion \
|
$(ARTIFACTDIR)/go_version.txt
|
||||||
$(ARTIFACTDIR)/go_env.txt \
|
|
||||||
dist \
|
release: $(RELEASE_BINS) $(RELEASE_NOARCH)
|
||||||
internal/config/samples
|
|
||||||
rm -rf "$(ARTIFACTDIR)/release"
|
rm -rf "$(ARTIFACTDIR)/release"
|
||||||
mkdir -p "$(ARTIFACTDIR)/release"
|
mkdir -p "$(ARTIFACTDIR)/release"
|
||||||
cp -l $(ARTIFACTDIR)/zrepl* \
|
cp $^ "$(ARTIFACTDIR)/release"
|
||||||
$(ARTIFACTDIR)/platformtest-* \
|
|
||||||
"$(ARTIFACTDIR)/release"
|
|
||||||
cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
|
cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
|
||||||
|
|
||||||
check-git-clean:
|
|
||||||
@# note that we use ZREPL_VERSION and not _ZREPL_VERSION because we want to detect the override
|
@# note that we use ZREPL_VERSION and not _ZREPL_VERSION because we want to detect the override
|
||||||
@if git describe --always --dirty 2>/dev/null | grep dirty >/dev/null; then \
|
@if git describe --always --dirty 2>/dev/null | grep dirty >/dev/null; then \
|
||||||
echo '[INFO] either git reports checkout is dirty or git is not installed or this is not a git checkout'; \
|
echo '[INFO] either git reports checkout is dirty or git is not installed or this is not a git checkout'; \
|
||||||
@@ -207,186 +115,5 @@ check-git-clean:
|
|||||||
fi; \
|
fi; \
|
||||||
fi;
|
fi;
|
||||||
|
|
||||||
tag-release:
|
|
||||||
test -n "$(ZREPL_TAG_VERSION)" || exit 1
|
|
||||||
git tag -u '328A6627FA98061D!' -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)"
|
|
||||||
|
|
||||||
verify-and-sign:
|
|
||||||
cd "$(ARTIFACTDIR)/release" && sha512sum -c sha512sum.txt
|
|
||||||
gpg -u '328A6627FA98061D!' \
|
|
||||||
--armor \
|
|
||||||
--detach-sign $(ARTIFACTDIR)/release/sha512sum.txt
|
|
||||||
|
|
||||||
clean: docs-clean
|
clean: docs-clean
|
||||||
rm -rf "$(ARTIFACTDIR)"
|
rm -rf "$(ARTIFACTDIR)"
|
||||||
|
|
||||||
download-circleci-release:
|
|
||||||
rm -rf "$(ARTIFACTDIR)"
|
|
||||||
mkdir -p "$(ARTIFACTDIR)/release"
|
|
||||||
python3 .circleci/download_artifacts.py --prefix 'artifacts/release/' "$(JOB_NUM)" "$(ARTIFACTDIR)/release"
|
|
||||||
|
|
||||||
##################### MULTI-ARCH HELPERS #####################
|
|
||||||
|
|
||||||
_run_make_foreach_target_tuple:
|
|
||||||
if [ "$(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG)" = "" ]; then \
|
|
||||||
echo "RUN_MAKE_FOREACH_TARGET_TUPLE_ARG must be set"; \
|
|
||||||
exit 1; \
|
|
||||||
fi
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd GOARCH=amd64
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd GOARCH=386
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd GOARCH=arm GOARM=7
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd GOARCH=arm64
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux GOARCH=amd64
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux GOARCH=arm64
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux GOARCH=arm GOARM=7
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux GOARCH=386
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=darwin GOARCH=amd64
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=solaris GOARCH=amd64
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=illumos GOARCH=amd64
|
|
||||||
|
|
||||||
##################### REGULAR TARGETS #####################
|
|
||||||
.PHONY: lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin
|
|
||||||
|
|
||||||
lint: build/install
|
|
||||||
$(GO_ENV_VARS) build/install/golangci_lint/golangci-lint run ./...
|
|
||||||
|
|
||||||
vet:
|
|
||||||
$(GO_ENV_VARS) $(GO) vet $(GO_BUILDFLAGS) ./...
|
|
||||||
|
|
||||||
test-go: $(ARTIFACTDIR)
|
|
||||||
rm -f "$(ARTIFACTDIR)/gotest.cover"
|
|
||||||
ifeq ($(COVER),1)
|
|
||||||
$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
|
|
||||||
-coverpkg github.com/zrepl/zrepl/... \
|
|
||||||
-covermode atomic \
|
|
||||||
-coverprofile "$(ARTIFACTDIR)/gotest.cover" \
|
|
||||||
./...
|
|
||||||
else
|
|
||||||
$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
|
|
||||||
./...
|
|
||||||
endif
|
|
||||||
|
|
||||||
zrepl-bin:
|
|
||||||
$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(ZREPL_TARGET_TUPLE)"
|
|
||||||
|
|
||||||
COVER_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-cover-$(ZREPL_TARGET_TUPLE)
|
|
||||||
cover-platform-bin:
|
|
||||||
$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
|
|
||||||
-c -o "$(COVER_PLATFORM_BIN_PATH)" \
|
|
||||||
-covermode=atomic -cover -coverpkg github.com/zrepl/zrepl/... \
|
|
||||||
./platformtest/harness
|
|
||||||
cover-platform:
|
|
||||||
# do not track dependency on cover-platform-bin to allow build of binary outside of test VM
|
|
||||||
export _TEST_PLATFORM_CMD="$(COVER_PLATFORM_BIN_PATH) \
|
|
||||||
-test.coverprofile \"$(ARTIFACTDIR)/platformtest.cover\" \
|
|
||||||
-test.v \
|
|
||||||
__DEVEL--i-heard-you-like-tests"; \
|
|
||||||
$(MAKE) _test-or-cover-platform-impl
|
|
||||||
|
|
||||||
TEST_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)
|
|
||||||
test-platform-bin:
|
|
||||||
$(GO_BUILD) -o "$(TEST_PLATFORM_BIN_PATH)" ./internal/platformtest/harness
|
|
||||||
test-platform:
|
|
||||||
export _TEST_PLATFORM_CMD="\"$(TEST_PLATFORM_BIN_PATH)\""; \
|
|
||||||
$(MAKE) _test-or-cover-platform-impl
|
|
||||||
|
|
||||||
ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
|
|
||||||
ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
|
|
||||||
ZREPL_PLATFORMTEST_MOUNTPOINT := /tmp/zreplplatformtest.pool
|
|
||||||
ZREPL_PLATFORMTEST_ZFS_LOG := /tmp/zreplplatformtest.zfs.log
|
|
||||||
# ZREPL_PLATFORMTEST_STOP_AND_KEEP := -failure.stop-and-keep-pool
|
|
||||||
ZREPL_PLATFORMTEST_ARGS :=
|
|
||||||
_test-or-cover-platform-impl: $(ARTIFACTDIR)
|
|
||||||
ifndef _TEST_PLATFORM_CMD
|
|
||||||
$(error _TEST_PLATFORM_CMD is undefined, caller 'cover-platform' or 'test-platform' should have defined it)
|
|
||||||
endif
|
|
||||||
rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
|
|
||||||
rm -f "$(ARTIFACTDIR)/platformtest.cover"
|
|
||||||
internal/platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
|
|
||||||
$(_TEST_PLATFORM_CMD) \
|
|
||||||
-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
|
|
||||||
-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
|
|
||||||
-mountpoint "$(ZREPL_PLATFORMTEST_MOUNTPOINT)" \
|
|
||||||
$(ZREPL_PLATFORMTEST_STOP_AND_KEEP) \
|
|
||||||
$(ZREPL_PLATFORMTEST_ARGS)
|
|
||||||
|
|
||||||
cover-merge: $(ARTIFACTDIR)
|
|
||||||
$(GOCOVMERGE) $(ARTIFACTDIR)/platformtest.cover $(ARTIFACTDIR)/gotest.cover > $(ARTIFACTDIR)/merged.cover
|
|
||||||
cover-html: cover-merge
|
|
||||||
$(GO) tool cover -html "$(ARTIFACTDIR)/merged.cover" -o "$(ARTIFACTDIR)/merged.cover.html"
|
|
||||||
|
|
||||||
cover-full:
|
|
||||||
test "$$(id -u)" = "0" || echo "MUST RUN AS ROOT" 1>&2
|
|
||||||
$(MAKE) test-go COVER=1
|
|
||||||
$(MAKE) cover-platform-bin
|
|
||||||
$(MAKE) cover-platform
|
|
||||||
$(MAKE) cover-html
|
|
||||||
|
|
||||||
##################### DEV TARGETS #####################
|
|
||||||
# not part of the build, must do that manually
|
|
||||||
.PHONY: generate format
|
|
||||||
|
|
||||||
build/install:
|
|
||||||
rm -rf build/install.tmp
|
|
||||||
mkdir build/install.tmp
|
|
||||||
|
|
||||||
-echo "installing protoc"
|
|
||||||
mkdir build/install.tmp/protoc
|
|
||||||
bash -x build/get_protoc.bash build/install.tmp/protoc
|
|
||||||
|
|
||||||
-echo "installing golangci-lint"
|
|
||||||
mkdir -p build/install.tmp/golangci_lint
|
|
||||||
GOHOSTARCH=$(GOHOSTARCH) \
|
|
||||||
build/get_golangci_lint.bash build/install.tmp/golangci_lint
|
|
||||||
|
|
||||||
-echo "installing go tools"
|
|
||||||
build/go_install_tools.bash build/install.tmp/gobin
|
|
||||||
|
|
||||||
mv build/install.tmp build/install
|
|
||||||
|
|
||||||
|
|
||||||
generate: build/install
|
|
||||||
# TODO: would be nice to run with a pure path here
|
|
||||||
PATH="$(CURDIR)/build/install/gobin:$(CURDIR)/build/install/protoc/bin:$$PATH" && \
|
|
||||||
build/install/protoc/bin/protoc -I=internal/replication/logic/pdu --go_out=internal/replication/logic/pdu --go-grpc_out=internal/replication/logic/pdu internal/replication/logic/pdu/pdu.proto && \
|
|
||||||
build/install/protoc/bin/protoc -I=internal/rpc/grpcclientidentity/example --go_out=internal/rpc/grpcclientidentity/example/pdu --go-grpc_out=internal/rpc/grpcclientidentity/example/pdu internal/rpc/grpcclientidentity/example/grpcauth.proto && \
|
|
||||||
$(GO) generate $(GO_BUILDFLAGS) -x ./... && \
|
|
||||||
true
|
|
||||||
|
|
||||||
GOIMPORTS := goimports -srcdir . -local 'github.com/zrepl/zrepl'
|
|
||||||
|
|
||||||
format: build/install
|
|
||||||
@ build/install/gobin/goimports -w -local 'github.com/zrepl/zrepl' $$(find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go')
|
|
||||||
|
|
||||||
##################### NOARCH #####################
|
|
||||||
.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
|
|
||||||
|
|
||||||
|
|
||||||
$(ARTIFACTDIR):
|
|
||||||
mkdir -p "$@"
|
|
||||||
$(ARTIFACTDIR)/docs: $(ARTIFACTDIR)
|
|
||||||
mkdir -p "$@"
|
|
||||||
|
|
||||||
noarch: $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs
|
|
||||||
# pass
|
|
||||||
|
|
||||||
$(ARTIFACTDIR)/bash_completion:
|
|
||||||
$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
|
|
||||||
artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion bash "$@"
|
|
||||||
|
|
||||||
$(ARTIFACTDIR)/_zrepl.zsh_completion:
|
|
||||||
$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
|
|
||||||
artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion zsh "$@"
|
|
||||||
|
|
||||||
$(ARTIFACTDIR)/go_env.txt:
|
|
||||||
$(GO_ENV_VARS) $(GO) env > $@
|
|
||||||
$(GO) version >> $@
|
|
||||||
|
|
||||||
docs: $(ARTIFACTDIR)/docs
|
|
||||||
# https://www.sphinx-doc.org/en/master/man/sphinx-build.html
|
|
||||||
cd docs && uv sync --frozen
|
|
||||||
cd docs && uv run sphinx-build -W --keep-going -n . ../artifacts/docs/html
|
|
||||||
|
|
||||||
docs-clean:
|
|
||||||
rm -rf artifacts/docs
|
|
||||||
rm -rf docs/.venv
|
|
||||||
|
|||||||
@@ -1,12 +1,9 @@
|
|||||||
[](https://github.com/zrepl/zrepl/blob/master/LICENSE)
|
[](https://github.com/zrepl/zrepl/blob/master/LICENSE)
|
||||||
[](https://golang.org/)
|
[](https://golang.org/)
|
||||||
[](https://zrepl.github.io)
|
[](https://zrepl.github.io)
|
||||||
[](https://patreon.com/zrepl)
|
|
||||||
[](https://github.com/sponsors/problame)
|
|
||||||
[](https://liberapay.com/zrepl/donate)
|
|
||||||
[](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
|
[](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
|
||||||
|
[](https://liberapay.com/zrepl/donate)
|
||||||
[](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl)
|
[](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl)
|
||||||
[](https://matrix.to/#/#zrepl:matrix.org)
|
|
||||||
|
|
||||||
# zrepl
|
# zrepl
|
||||||
zrepl is a one-stop ZFS backup & replication solution.
|
zrepl is a one-stop ZFS backup & replication solution.
|
||||||
@@ -22,203 +19,118 @@ zrepl is a one-stop ZFS backup & replication solution.
|
|||||||
|
|
||||||
## Feature Requests
|
## Feature Requests
|
||||||
|
|
||||||
1. Does your feature request require default values / some kind of configuration?
|
1. Does you feature request require default values / some kind of configuration?
|
||||||
If so, think of an expressive configuration example.
|
If so, think of an expressive configuration example.
|
||||||
2. Think of at least one use case that generalizes from your concrete application.
|
2. Think of at least one use case that generalizes from your concrete application.
|
||||||
3. Open an issue on GitHub with example conf & use case attached.
|
3. Open an issue on GitHub with example conf & use case attached.
|
||||||
4. **Optional**: [Contact Christian Schwarz](https://cschwarz.com) for contract work.
|
|
||||||
|
|
||||||
The above does not apply if you already implemented everything.
|
The above does not apply if you already implemented everything.
|
||||||
Check out the *Coding Workflow* section below for details.
|
Check out the *Coding Workflow* section below for details.
|
||||||
|
|
||||||
## Development
|
## Package Maintainer Information
|
||||||
|
|
||||||
zrepl is written in [Go](https://golang.org) and uses [Go modules](https://github.com/golang/go/wiki/Modules) to manage dependencies.
|
* Follow the steps in `docs/installation.rst -> Compiling from Source` and read the Makefile / shell scripts used in this process.
|
||||||
The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
|
* Make sure your distro is compatible with the paths in `docs/installation.rst`.
|
||||||
|
|
||||||
### Building
|
|
||||||
|
|
||||||
#### Go Code
|
|
||||||
|
|
||||||
Dependencies:
|
|
||||||
|
|
||||||
* Go
|
|
||||||
* GNU Make
|
|
||||||
* Git
|
|
||||||
* wget (`make generate`)
|
|
||||||
* unzip (`make generate`)
|
|
||||||
|
|
||||||
Some Go code is **generated**, and generated code is committed to the source tree.
|
|
||||||
Therefore, building does not require having code generation tools set up.
|
|
||||||
When making changes that require code to be (re-)generated, run `make generate`.
|
|
||||||
I downloads and installs pinned versions of the code generation tools into `./build/install`.
|
|
||||||
There is a CI check that ensures Git state is clean, i.e., code generation has been done by a PR and is deterministic.
|
|
||||||
|
|
||||||
#### Docs
|
|
||||||
|
|
||||||
Install [uv](https://docs.astral.sh/uv/getting-started/installation/), then run `make docs`.
|
|
||||||
uv automatically manages Python and dependencies.
|
|
||||||
|
|
||||||
### Testing
|
|
||||||
|
|
||||||
The **test suite** is split into pure **Go tests** (`make test-go`) and **platform tests** that interact with ZFS and thus generally **require root privileges** (`sudo make test-platform`).
|
|
||||||
Platform tests run on their own pool with the name `zreplplatformtest`, which is created using the file vdev in `/tmp`.
|
|
||||||
|
|
||||||
For a full **code coverage** profile, run `make test-go COVER=1 && sudo make test-platform && make cover-merge`.
|
|
||||||
An HTML report can be generated using `make cover-html`.
|
|
||||||
|
|
||||||
### Circle CI
|
|
||||||
|
|
||||||
We use CircleCI for automated build & test pre- and post-merge.
|
|
||||||
|
|
||||||
There are two workflows:
|
|
||||||
|
|
||||||
* `ci` runs for every commit / branch / tag pushed to GitHub.
|
|
||||||
It is supposed to run very fast (<5min and provides quick feedback to developers).
|
|
||||||
It runs formatting checks, lints and tests on the most important OSes / architectures.
|
|
||||||
|
|
||||||
* `release` runs
|
|
||||||
* on manual triggers through the CircleCI API (in order to produce a release)
|
|
||||||
* periodically on `master`
|
|
||||||
|
|
||||||
Artifacts are stored in CircleCI.
|
|
||||||
|
|
||||||
### Releasing
|
|
||||||
|
|
||||||
All zrepl releases are git-tagged and then published as a GitHub Release.
|
|
||||||
There is a git tag for each zrepl release, usually `vMAJOR.MINOR.0`.
|
|
||||||
We don't move git tags once the release has been published.
|
|
||||||
|
|
||||||
The procedure to issue a release is as follows:
|
|
||||||
|
|
||||||
* Prepare the release (as a PR to `master`):
|
|
||||||
* Finalize `docs/changelog.rst` for the release.
|
|
||||||
* Update the supporters list based on contributors for this release:
|
|
||||||
```bash
|
|
||||||
claude --permission-mode default '/update-supporters v0.6.1..v0.7.0'
|
|
||||||
```
|
|
||||||
Replace version tags with the appropriate range for your release.
|
|
||||||
* Merge the PR. Docs are auto-published to zrepl.github.io on merge.
|
|
||||||
* Tag the release:
|
|
||||||
* Git tag the release on the `master` branch (e.g., `vMAJOR.MINOR.0`).
|
|
||||||
```
|
|
||||||
make tag-release ZREPL_TAG_VERSION=v0.7.0
|
|
||||||
```
|
|
||||||
* Push the tag.
|
|
||||||
* Build and publish:
|
|
||||||
* Run the `release` pipeline against the `master` branch (trigger via CircleCI UI).
|
|
||||||
This URL: https://app.circleci.com/pipelines/github/zrepl/zrepl?branch=master.
|
|
||||||
Example pipeline: https://app.circleci.com/pipelines/github/zrepl/zrepl/8547
|
|
||||||
* Download artifacts using this handy makefile target.
|
|
||||||
Note: `JOB_NUM` must be the **job number** from the `release-upload` job, **not the pipeline number**.
|
|
||||||
Find it via: pipeline → `release` workflow → `release-upload` job number.
|
|
||||||
Example URL: https://app.circleci.com/pipelines/github/zrepl/zrepl/8547/workflows/65feb2c9-15d7-46ab-a551-46d62a5769b0/jobs/66079/steps
|
|
||||||
|
|
||||||
```
|
|
||||||
make download-circleci-release JOB_NUM=66079
|
|
||||||
```
|
|
||||||
* Verify checksums and sign the checksums file:
|
|
||||||
```
|
|
||||||
make verify-and-sign
|
|
||||||
```
|
|
||||||
* Create GitHub draft release and upload artifacts:
|
|
||||||
```bash
|
|
||||||
claude --permission-mode default '/draft-release v0.7.0'
|
|
||||||
```
|
|
||||||
This command will verify that artifacts are ready, create the draft release, and upload all artifacts.
|
|
||||||
* Review the draft release on GitHub, then publish.
|
|
||||||
* Add the .rpm and .deb files to the official zrepl repos.
|
|
||||||
* Code for management of these repos: https://github.com/zrepl/package-repo-ops (private repo at this time)
|
|
||||||
* Update docs version list:
|
|
||||||
* Update `docs/_templates/versions.html` with the new release.
|
|
||||||
* Verify the link to `zrepl-noarch.tar` in the GitHub release works.
|
|
||||||
* Merge to `master` (docs auto-publish).
|
|
||||||
|
|
||||||
#### Patch releases, Go toolchain updates, APT/RPM Package rebuilds
|
|
||||||
|
|
||||||
`vMAJOR.MINOR.0` is typically a tagged commit on `master`, because development velocity isn't high
|
|
||||||
and thus release branches for stabilization aren't necessary.
|
|
||||||
|
|
||||||
Occasionally though there is a need for patch changes to a release, e.g.
|
|
||||||
- security issue in a dependency
|
|
||||||
- Go toolchain update (e.g. security issue in standard library)
|
|
||||||
|
|
||||||
The procedure for this is the following
|
|
||||||
- create a branch off the release tag we need to patch, named `releases/MAJOR.MINOR.X`
|
|
||||||
- that branch will never be merged into `master`, it'll be a dead-end for this specific patch
|
|
||||||
- make changes in that branch
|
|
||||||
- make the final commit that bumps version numbers
|
|
||||||
- create the git tag
|
|
||||||
- follow general procedure for publishing the release (previous sectino
|
|
||||||
|
|
||||||
For Go toolchain updates and package rebuilds with no source code changes, leverage the APT/RPM package revision field.
|
|
||||||
Control via the `ZREPL_PACKAGE_RELEASE` Makefile variable, see `origin/releases/0.6.1-2` for an example.
|
|
||||||
|
|
||||||
|
|
||||||
### Updating Dependencies
|
|
||||||
|
|
||||||
- Update the `go` directive and `toolchain` directive in `go.mod`
|
|
||||||
- `go` is the minimum supported version
|
|
||||||
- `toolchain` is the preferred toolchain version if `GOTOOLCHAIN` is not specified
|
|
||||||
|
|
||||||
Run `go mod tidy` to ensure consistency.
|
|
||||||
|
|
||||||
Update Go module dependencies:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Update all other dependencies
|
|
||||||
go get -u -t ./...
|
|
||||||
go mod tidy
|
|
||||||
|
|
||||||
# Above might fail if there are version selection conflicts.
|
|
||||||
# Figure out what's going on by updating packages from error messages first.
|
|
||||||
# Example:
|
|
||||||
go get -u google.golang.org/genproto google.golang.org/grpc google.golang.org/protobuf
|
|
||||||
```
|
|
||||||
|
|
||||||
Update codegen & lint tools
|
|
||||||
- `protoc` => `build/get_protoc.bash`
|
|
||||||
- GH releases publish sha256 sums
|
|
||||||
- `golangci-lint` => `build/get_golangci_lint.bash`
|
|
||||||
- bump versions in `build/tools.go`
|
|
||||||
- we use the tools.go trick:
|
|
||||||
- `go get -tags tools -u example.com/tool ; go mod tidy`
|
|
||||||
- review whether we're ready to switch to `go tool`: https://github.com/zrepl/zrepl/pull/909
|
|
||||||
|
|
||||||
Now run `make generate`.
|
|
||||||
|
|
||||||
Run `make lint` and `make vet`.
|
|
||||||
|
|
||||||
Update the CI configuration `.circleci/config.yml`:
|
|
||||||
- Update Go version references (we reference the minimum and max supported version)
|
|
||||||
- Set `Makefile` `RELEASE_GOVERSION` to the new Go version
|
|
||||||
- Update the pinned ZFS release tags in the `platformtest` matrix (`zfs_release` parameter) to the latest patch releases of each OpenZFS branch (currently 2.2, 2.3, 2.4)
|
|
||||||
- Update the pinned Ubuntu machine image for `platformtest` (e.g. `ubuntu-2404:2025.09.1`) — this is pinned rather than `current` so the ZFS build cache stays valid across runs
|
|
||||||
|
|
||||||
Update docs build tooling:
|
|
||||||
- Update `uv` version in `.circleci/config.yml` (search for `astral.sh/uv/` and cache keys containing the version)
|
|
||||||
- Check if there's now a CircleCI orb for uv that we could use
|
|
||||||
- Update Python version in `docs/.python-version`
|
|
||||||
|
|
||||||
Update docs dependencies (Sphinx, sphinx-rtd-theme):
|
|
||||||
- Check current versions in `docs/pyproject.toml`
|
|
||||||
- Review upstream changelogs for breaking changes
|
|
||||||
- Update version constraints in `pyproject.toml` and the `uv` lockfile (see [uv docs on dependencies](https://docs.astral.sh/uv/concepts/projects/dependencies/)):
|
|
||||||
- Test locally with `make docs`
|
|
||||||
|
|
||||||
Kick a full CI pipeline run (`do_ci=true` and `do_release=true`).
|
|
||||||
|
|
||||||
Merge PR with merge commit.
|
|
||||||
|
|
||||||
## Notes to Distro Package Maintainers
|
|
||||||
|
|
||||||
* The `Makefile` in this project is not suitable for builds in distros.
|
|
||||||
* Run the platform tests (Docs -> Usage -> Platform Tests) **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
|
|
||||||
* Ship a default config that adheres to your distro's `hier` and logging system.
|
* Ship a default config that adheres to your distro's `hier` and logging system.
|
||||||
* Ship a service manager file and _please_ try to upstream it to this repository.
|
* Ship a service manager file and _please_ try to upstream it to this repository.
|
||||||
* `dist/systemd` contains a Systemd unit template.
|
* `dist/systemd` contains a Systemd unit template.
|
||||||
* Ship other material provided in `./dist`, e.g. in `/usr/share/zrepl/`.
|
* Ship other material provided in `./dist`, e.g. in `/usr/share/zrepl/`.
|
||||||
* Have a look at the `Makefile`'s `ZREPL_VERSION` variable and how it passed to Go's `ldFlags`.
|
* Use `make release ZREPL_VERSION='mydistro-1.2.3_1'`
|
||||||
This is how `zrepl version` knows what version number to show.
|
* Your distro's name and any versioning supplemental to zrepl's (e.g. package revision) should be in this string
|
||||||
Your build system should set the `ldFlags` flags appropriately and add a prefix or suffix that indicates that the given zrepl binary is a distro build, not an official one.
|
* Use `make platformtest` **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
|
||||||
* Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
|
* Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
|
||||||
|
|
||||||
|
## Developer Documentation
|
||||||
|
|
||||||
|
zrepl is written in [Go](https://golang.org) and uses [Go modules](https://github.com/golang/go/wiki/Modules) to manage dependencies.
|
||||||
|
The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
|
||||||
|
|
||||||
|
To get started, run `./lazy.sh devsetup` to easily install build dependencies and read `docs/installation.rst -> Compiling from Source`.
|
||||||
|
|
||||||
|
### Overall Architecture
|
||||||
|
|
||||||
|
The application architecture is documented as part of the user docs in the *Implementation* section (`docs/content/impl`).
|
||||||
|
Make sure to develop an understanding how zrepl is typically used by studying the user docs first.
|
||||||
|
|
||||||
|
### Project Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
├── artifacts # build artifcats generate by make
|
||||||
|
├── cli # wrapper around CLI package cobra
|
||||||
|
├── client # all subcommands that are not `daemon`
|
||||||
|
├── config # config data types (=> package yaml-config)
|
||||||
|
│ └── samples
|
||||||
|
├── daemon # the implementation of `zrepl daemon` subcommand
|
||||||
|
│ ├── filters
|
||||||
|
│ ├── job # job implementations
|
||||||
|
│ ├── logging # logging outlets + formatters
|
||||||
|
│ ├── nethelpers
|
||||||
|
│ ├── prometheus
|
||||||
|
│ ├── pruner # pruner implementation
|
||||||
|
│ ├── snapper # snapshotter implementation
|
||||||
|
├── docs # sphinx-based documentation
|
||||||
|
├── dist # supplemental material for users & package maintainers
|
||||||
|
│ ├── **/*.rst # documentation in reStructuredText
|
||||||
|
│ ├── sphinxconf
|
||||||
|
│ │ └── conf.py # sphinx config (see commit 445a280 why its not in docs/)
|
||||||
|
│ ├── requirements.txt # pip3 requirements to build documentation
|
||||||
|
│ ├── publish.sh # shell script for automated rendering & deploy to zrepl.github.io repo
|
||||||
|
│ └── public_git # checkout of zrepl.github.io managed by above shell script
|
||||||
|
├── endpoint # implementation of replication endpoints (=> package replication)
|
||||||
|
├── logger # our own logger package
|
||||||
|
├── pruning # pruning rules (the logic, not the actual execution)
|
||||||
|
│ └── retentiongrid
|
||||||
|
├── replication
|
||||||
|
│ ├── driver # the driver of the replication logic (status reporting, error handling)
|
||||||
|
│ ├── logic # planning & executing replication steps via rpc
|
||||||
|
| | └── pdu # the generated gRPC & protobuf code used in replication (and endpoints)
|
||||||
|
│ └── report # the JSON-serializable report datastructures exposed to the client
|
||||||
|
├── rpc # the hybrid gRPC + ./dataconn RPC client: connects to a remote replication.Endpoint
|
||||||
|
│ ├── dataconn # Bulk data-transfer RPC protocol
|
||||||
|
│ ├── grpcclientidentity # adaptor to inject package transport's 'client identity' concept into gRPC contexts
|
||||||
|
│ ├── netadaptor # adaptor to convert a package transport's Connecter and Listener into net.* primitives
|
||||||
|
│ ├── transportmux # TCP connecter and listener used to split control & data traffic
|
||||||
|
│ └── versionhandshake # replication protocol version handshake perfomed on newly established connections
|
||||||
|
├── tlsconf # abstraction for Go TLS server + client config
|
||||||
|
├── transport # transports implementation
|
||||||
|
│ ├── fromconfig
|
||||||
|
│ ├── local
|
||||||
|
│ ├── ssh
|
||||||
|
│ ├── tcp
|
||||||
|
│ └── tls
|
||||||
|
├── util
|
||||||
|
├── vendor # managed by dep
|
||||||
|
├── version # abstraction for versions (filled during build by Makefile)
|
||||||
|
└── zfs # zfs(8) wrappers
|
||||||
|
```
|
||||||
|
|
||||||
|
### Coding Workflow
|
||||||
|
|
||||||
|
* Open an issue when starting to hack on a new feature
|
||||||
|
* Commits should reference the issue they are related to
|
||||||
|
* Docs improvements not documenting new features do not require an issue.
|
||||||
|
|
||||||
|
### Breaking Changes
|
||||||
|
|
||||||
|
Backward-incompatible changes must be documented in the git commit message and are listed in `docs/changelog.rst`.
|
||||||
|
|
||||||
|
* Config-breaking changes must contain a line `BREAK CONFIG` in the commit message
|
||||||
|
* Other breaking changes must contain a line `BREAK` in the commit message
|
||||||
|
|
||||||
|
### Glossary & Naming Inconsistencies
|
||||||
|
|
||||||
|
In ZFS, *dataset* refers to the objects *filesystem*, *ZVOL* and *snapshot*. <br />
|
||||||
|
However, we need a word for *filesystem* & *ZVOL* but not a snapshot, bookmark, etc.
|
||||||
|
|
||||||
|
Toward the user, the following terminology is used:
|
||||||
|
|
||||||
|
* **filesystem**: a ZFS filesystem or a ZVOL
|
||||||
|
* **filesystem version**: a ZFS snapshot or a bookmark
|
||||||
|
|
||||||
|
Sadly, the zrepl implementation is inconsistent in its use of these words:
|
||||||
|
variables and types are often named *dataset* when they in fact refer to a *filesystem*.
|
||||||
|
|
||||||
|
There will not be a big refactoring (an attempt was made, but it's destroying too much history without much gain).
|
||||||
|
|
||||||
|
However, new contributions & patches should fix naming without further notice in the commit message.
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
FROM golang:latest
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y \
|
||||||
|
python3-pip \
|
||||||
|
unzip
|
||||||
|
|
||||||
|
ADD build.installprotoc.bash ./
|
||||||
|
RUN bash build.installprotoc.bash
|
||||||
|
|
||||||
|
ADD lazy.sh /tmp/lazy.sh
|
||||||
|
ADD docs/requirements.txt /tmp/requirements.txt
|
||||||
|
ENV ZREPL_LAZY_DOCS_REQPATH=/tmp/requirements.txt
|
||||||
|
RUN /tmp/lazy.sh docdep
|
||||||
|
|
||||||
|
# prepare volume mount of git checkout to /zrepl
|
||||||
|
RUN mkdir -p /src/github.com/zrepl/zrepl
|
||||||
|
RUN mkdir -p /.cache && chmod -R 0777 /.cache
|
||||||
|
|
||||||
|
# $GOPATH is /go
|
||||||
|
# Go 1.12 doesn't use modules within GOPATH, but 1.13 and later do
|
||||||
|
# => store source outside of GOPATH
|
||||||
|
WORKDIR /src
|
||||||
|
|
||||||
|
# Install build tools (e.g. protobuf generator, stringer) into $GOPATH/bin
|
||||||
|
ADD go.mod go.sum ./
|
||||||
|
RUN /tmp/lazy.sh godep
|
||||||
|
|
||||||
|
RUN chmod -R 0777 /go
|
||||||
|
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
set -x
|
||||||
|
|
||||||
|
MACH=$(uname -m)
|
||||||
|
MACH="${MACH/aarch64/aarch_64}"
|
||||||
|
|
||||||
|
VERSION=3.6.1
|
||||||
|
FILENAME=protoc-"$VERSION"-linux-"$MACH".zip
|
||||||
|
|
||||||
|
if [ -e "$FILENAME" ]; then
|
||||||
|
echo "$FILENAME" already exists 1>&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
wget https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
|
||||||
|
|
||||||
|
stat "$FILENAME"
|
||||||
|
|
||||||
|
sha256sum -c --ignore-missing <<EOF
|
||||||
|
6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807 protoc-3.6.1-linux-x86_64.zip
|
||||||
|
af8e5aaaf39ddec62ec8dd2be1b8d9602c6da66564883a16393ade5f71170922 protoc-3.6.1-linux-aarch_64.zip
|
||||||
|
EOF
|
||||||
|
|
||||||
|
unzip -d /usr "$FILENAME"
|
||||||
@@ -1 +0,0 @@
|
|||||||
install/
|
|
||||||
@@ -1,30 +0,0 @@
|
|||||||
FROM !SUBSTITUTED_BY_MAKEFILE
|
|
||||||
|
|
||||||
ARG BUILD_UID=1000
|
|
||||||
ARG BUILD_GID=1000
|
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y \
|
|
||||||
python3 \
|
|
||||||
unzip \
|
|
||||||
gawk \
|
|
||||||
curl
|
|
||||||
|
|
||||||
# Create build user with the host's UID/GID before installing uv
|
|
||||||
RUN groupadd -g ${BUILD_GID} zrepl_build && \
|
|
||||||
useradd -u ${BUILD_UID} -g ${BUILD_GID} -m zrepl_build
|
|
||||||
|
|
||||||
# Go toolchain uses xdg-cache
|
|
||||||
RUN mkdir -p /.cache && chmod -R 0777 /.cache
|
|
||||||
|
|
||||||
# Install uv as the build user - version from .uv-version file
|
|
||||||
ADD .uv-version /tmp/.uv-version
|
|
||||||
USER zrepl_build
|
|
||||||
RUN UV_VERSION=$(cat /tmp/.uv-version) && \
|
|
||||||
curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh
|
|
||||||
|
|
||||||
# Go devtools are managed by Makefile
|
|
||||||
|
|
||||||
WORKDIR /src
|
|
||||||
ENV PATH="/home/zrepl_build/.local/bin:$PATH" \
|
|
||||||
GOCACHE="/.cache/go-build"
|
|
||||||
|
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
//+build windows
|
||||||
|
|
||||||
|
// This package cannot actually be built, and since the Travis
|
||||||
|
// tests run go test ./..., we have to avoid a build attempt.
|
||||||
|
// Windows is not supported atm, so this works ¯\_(ツ)_/¯
|
||||||
|
|
||||||
|
// This package is a pseudo-user of build-time dependencies for zrepl,
|
||||||
|
// mostly for various code-generation tools.
|
||||||
|
//
|
||||||
|
// The imports are necessary to satisfy go dep
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
_ "github.com/alvaroloes/enumer"
|
||||||
|
_ "github.com/golang/protobuf/protoc-gen-go"
|
||||||
|
_ "golang.org/x/tools/cmd/stringer"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
fmt.Println("just a placeholder")
|
||||||
|
}
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
set -x
|
|
||||||
|
|
||||||
# golangci-lint recommends against using `go get` and friends to install it
|
|
||||||
|
|
||||||
cd "$1"
|
|
||||||
|
|
||||||
VERSION=2.8.0
|
|
||||||
FILENAME=golangci-lint-"$VERSION"-linux-"$GOHOSTARCH".tar.gz
|
|
||||||
|
|
||||||
if [ -e "$FILENAME" ]; then
|
|
||||||
echo "$FILENAME" already exists 1>&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
wget --continue https://github.com/golangci/golangci-lint/releases/download/v"$VERSION"/"$FILENAME"
|
|
||||||
|
|
||||||
stat "$FILENAME"
|
|
||||||
|
|
||||||
# Select the correct checksum for the downloaded architecture
|
|
||||||
case "$GOHOSTARCH" in
|
|
||||||
arm64) EXPECTED_SHA256="2a58388db8af5ab9330791cea0ebdd4100723cd05ad7185d92febaaee272ec9a" ;;
|
|
||||||
amd64) EXPECTED_SHA256="7048bc6b25c9515ed092c83f9fa8709ca97937ead52d9ff317a143299ee97a50" ;;
|
|
||||||
*) echo "Unknown architecture: $GOHOSTARCH" >&2; exit 1 ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
# Verify checksum explicitly - fails if hash doesn't match
|
|
||||||
echo "$EXPECTED_SHA256 $FILENAME" | sha256sum -c -
|
|
||||||
|
|
||||||
tar -x --strip-components=1 -f "$FILENAME"
|
|
||||||
|
|
||||||
stat ./golangci-lint
|
|
||||||
@@ -1,32 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
set -x
|
|
||||||
|
|
||||||
cd "$1"
|
|
||||||
|
|
||||||
MACH=$(uname -m)
|
|
||||||
MACH="${MACH/aarch64/aarch_64}"
|
|
||||||
|
|
||||||
VERSION=33.5
|
|
||||||
FILENAME=protoc-"$VERSION"-linux-"$MACH".zip
|
|
||||||
|
|
||||||
if [ -e "$FILENAME" ]; then
|
|
||||||
echo "$FILENAME" already exists 1>&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
wget --continue https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
|
|
||||||
|
|
||||||
stat "$FILENAME"
|
|
||||||
|
|
||||||
# Select the correct checksum for the downloaded architecture
|
|
||||||
case "$MACH" in
|
|
||||||
aarch_64) EXPECTED_SHA256="2b0fcf9b2c32cbadccc0eb7a88b841fffecd4a06fc80acdba2b5be45e815c38a" ;;
|
|
||||||
x86_64) EXPECTED_SHA256="24e58fb231d50306ee28491f33a170301e99540f7e29ca461e0e80fd1239f8d1" ;;
|
|
||||||
*) echo "Unknown architecture: $MACH" >&2; exit 1 ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
# Verify checksum explicitly - fails if hash doesn't match
|
|
||||||
echo "$EXPECTED_SHA256 $FILENAME" | sha256sum -c -
|
|
||||||
|
|
||||||
unzip -d . "$FILENAME"
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
module github.com/zrepl/zrepl/build
|
|
||||||
|
|
||||||
go 1.24.13
|
|
||||||
|
|
||||||
toolchain go1.25.7
|
|
||||||
|
|
||||||
require (
|
|
||||||
github.com/alvaroloes/enumer v1.1.2
|
|
||||||
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad
|
|
||||||
golang.org/x/tools v0.41.0
|
|
||||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.1
|
|
||||||
google.golang.org/protobuf v1.36.11
|
|
||||||
)
|
|
||||||
|
|
||||||
require (
|
|
||||||
github.com/pascaldekloe/name v1.0.1 // indirect
|
|
||||||
golang.org/x/mod v0.32.0 // indirect
|
|
||||||
golang.org/x/sync v0.19.0 // indirect
|
|
||||||
golang.org/x/sys v0.41.0 // indirect
|
|
||||||
golang.org/x/telemetry v0.0.0-20260205145544-86a5c4bf3c8d // indirect
|
|
||||||
golang.org/x/text v0.31.0 // indirect
|
|
||||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c // indirect
|
|
||||||
google.golang.org/grpc v1.75.0 // indirect
|
|
||||||
)
|
|
||||||
@@ -1,37 +0,0 @@
|
|||||||
github.com/alvaroloes/enumer v1.1.2 h1:5khqHB33TZy1GWCO/lZwcroBFh7u+0j40T83VUbfAMY=
|
|
||||||
github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
|
|
||||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
|
||||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
|
||||||
github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ=
|
|
||||||
github.com/pascaldekloe/name v1.0.1 h1:9lnXOHeqeHHnWLbKfH6X98+4+ETVqFqxN09UXSjcMb0=
|
|
||||||
github.com/pascaldekloe/name v1.0.1/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM=
|
|
||||||
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad h1:W0LEBv82YCGEtcmPA3uNZBI33/qF//HAAs3MawDjRa0=
|
|
||||||
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad/go.mod h1:Hy8o65+MXnS6EwGElrSRjUzQDLXreJlzYLlWiHtt8hM=
|
|
||||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
|
||||||
golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
|
|
||||||
golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
|
|
||||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
|
||||||
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
|
|
||||||
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
|
|
||||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
|
||||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
|
||||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
|
||||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
|
||||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
|
||||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
|
||||||
golang.org/x/telemetry v0.0.0-20260205145544-86a5c4bf3c8d h1:xjbyaj6khjO0ocYYa7k9sQLbdi4R1KwHnsS1QGijbsM=
|
|
||||||
golang.org/x/telemetry v0.0.0-20260205145544-86a5c4bf3c8d/go.mod h1:b7fPSJ0pKZ3ccUh8gnTONJxhn3c/PS6tyzQvyqw4iA8=
|
|
||||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
|
||||||
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
|
|
||||||
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
|
|
||||||
golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
|
||||||
golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
|
|
||||||
golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
|
|
||||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c h1:qXWI/sQtv5UKboZ/zUk7h+mrf/lXORyI+n9DKDAusdg=
|
|
||||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c/go.mod h1:gw1tLEfykwDz2ET4a12jcXt4couGAm7IwsVaTy0Sflo=
|
|
||||||
google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4=
|
|
||||||
google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
|
|
||||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.1 h1:/WILD1UcXj/ujCxgoL/DvRgt2CP3txG8+FwkUbb9110=
|
|
||||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.1/go.mod h1:YNKnb2OAApgYn2oYY47Rn7alMr1zWjb2U8Q0aoGWiNc=
|
|
||||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
|
||||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
export GO111MODULE=on # otherwise, a checkout of this repo in GOPATH will disable modules on Go 1.12 and earlier
|
|
||||||
source <(go env)
|
|
||||||
# build tools for the host platform
|
|
||||||
export GOOS="$GOHOSTOS"
|
|
||||||
export GOARCH="$GOHOSTARCH"
|
|
||||||
# TODO GOARM=$GOHOSTARM?
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
set -euo pipefail
|
|
||||||
set -x
|
|
||||||
|
|
||||||
OUTDIR="$(readlink -f "$1")"
|
|
||||||
if [ -e "$OUTDIR" ]; then
|
|
||||||
echo "$OUTDIR" already exists 1>&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# go install command below will install tools to $GOBIN
|
|
||||||
export GOBIN="$OUTDIR"
|
|
||||||
|
|
||||||
cd "$(dirname "$0")"
|
|
||||||
|
|
||||||
source ./go_install_host_tool.source
|
|
||||||
|
|
||||||
cat tools.go | grep _ | awk -F'"' '{print $2}' | tee | xargs -tI '{}' go install '{}'
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
//go:build tools
|
|
||||||
// +build tools
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
// the lines are parsed by lazy.sh, do not edit
|
|
||||||
import (
|
|
||||||
_ "github.com/alvaroloes/enumer"
|
|
||||||
_ "github.com/wadey/gocovmerge"
|
|
||||||
_ "golang.org/x/tools/cmd/goimports"
|
|
||||||
_ "golang.org/x/tools/cmd/stringer"
|
|
||||||
_ "google.golang.org/grpc/cmd/protoc-gen-go-grpc"
|
|
||||||
_ "google.golang.org/protobuf/cmd/protoc-gen-go"
|
|
||||||
)
|
|
||||||
@@ -1,16 +1,13 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
"github.com/zrepl/zrepl/config"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var rootArgs struct {
|
var rootArgs struct {
|
||||||
@@ -22,55 +19,29 @@ var rootCmd = &cobra.Command{
|
|||||||
Short: "One-stop ZFS replication solution",
|
Short: "One-stop ZFS replication solution",
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
var bashcompCmd = &cobra.Command{
|
||||||
rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
|
Use: "bashcomp path/to/out/file",
|
||||||
}
|
Short: "generate bash completions",
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
var genCompletionCmd = &cobra.Command{
|
if len(args) != 1 {
|
||||||
Use: "gencompletion",
|
fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
|
||||||
Short: "generate shell auto-completions",
|
err := cmd.Usage()
|
||||||
}
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
type completionCmdInfo struct {
|
}
|
||||||
genFunc func(outpath string) error
|
os.Exit(1)
|
||||||
help string
|
}
|
||||||
}
|
if err := rootCmd.GenBashCompletionFile(args[0]); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "error generating bash completion: %s", err)
|
||||||
var completionCmdMap = map[string]completionCmdInfo{
|
os.Exit(1)
|
||||||
"zsh": {
|
}
|
||||||
rootCmd.GenZshCompletionFile,
|
|
||||||
" save to file `_zrepl` in your zsh's $fpath",
|
|
||||||
},
|
|
||||||
"bash": {
|
|
||||||
rootCmd.GenBashCompletionFile,
|
|
||||||
" save to a path and source that path in your .bashrc",
|
|
||||||
},
|
},
|
||||||
|
Hidden: true,
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
for sh, info := range completionCmdMap {
|
rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
|
||||||
sh, info := sh, info
|
rootCmd.AddCommand(bashcompCmd)
|
||||||
genCompletionCmd.AddCommand(&cobra.Command{
|
|
||||||
Use: fmt.Sprintf("%s path/to/out/file", sh),
|
|
||||||
Short: fmt.Sprintf("generate %s completions", sh),
|
|
||||||
Example: info.help,
|
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
|
||||||
if len(args) != 1 {
|
|
||||||
fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
|
|
||||||
err := cmd.Usage()
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
if err := info.genFunc(args[0]); err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "error generating %s completion: %s", sh, err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
rootCmd.AddCommand(genCompletionCmd)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type Subcommand struct {
|
type Subcommand struct {
|
||||||
@@ -78,7 +49,7 @@ type Subcommand struct {
|
|||||||
Short string
|
Short string
|
||||||
Example string
|
Example string
|
||||||
NoRequireConfig bool
|
NoRequireConfig bool
|
||||||
Run func(ctx context.Context, subcommand *Subcommand, args []string) error
|
Run func(subcommand *Subcommand, args []string) error
|
||||||
SetupFlags func(f *pflag.FlagSet)
|
SetupFlags func(f *pflag.FlagSet)
|
||||||
SetupSubcommands func() []*Subcommand
|
SetupSubcommands func() []*Subcommand
|
||||||
|
|
||||||
@@ -99,11 +70,7 @@ func (s *Subcommand) Config() *config.Config {
|
|||||||
|
|
||||||
func (s *Subcommand) run(cmd *cobra.Command, args []string) {
|
func (s *Subcommand) run(cmd *cobra.Command, args []string) {
|
||||||
s.tryParseConfig()
|
s.tryParseConfig()
|
||||||
ctx := context.Background()
|
err := s.Run(s, args)
|
||||||
endTask := trace.WithTaskFromStackUpdateCtx(&ctx)
|
|
||||||
defer endTask()
|
|
||||||
err := s.Run(ctx, s, args)
|
|
||||||
endTask()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "%s\n", err)
|
fmt.Fprintf(os.Stderr, "%s\n", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
package client
|
package client
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
@@ -11,17 +10,16 @@ import (
|
|||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
"github.com/zrepl/yaml-config"
|
"github.com/zrepl/yaml-config"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/cli"
|
"github.com/zrepl/zrepl/cli"
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
"github.com/zrepl/zrepl/daemon/job"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
"github.com/zrepl/zrepl/daemon/logging"
|
||||||
"github.com/zrepl/zrepl/internal/logger"
|
"github.com/zrepl/zrepl/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
var configcheckArgs struct {
|
var configcheckArgs struct {
|
||||||
format string
|
format string
|
||||||
what string
|
what string
|
||||||
skipCertCheck bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var ConfigcheckCmd = &cli.Subcommand{
|
var ConfigcheckCmd = &cli.Subcommand{
|
||||||
@@ -30,9 +28,8 @@ var ConfigcheckCmd = &cli.Subcommand{
|
|||||||
SetupFlags: func(f *pflag.FlagSet) {
|
SetupFlags: func(f *pflag.FlagSet) {
|
||||||
f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]")
|
f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]")
|
||||||
f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]")
|
f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]")
|
||||||
f.BoolVar(&configcheckArgs.skipCertCheck, "skip-cert-check", false, "skip checking cert files")
|
|
||||||
},
|
},
|
||||||
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
Run: func(subcommand *cli.Subcommand, args []string) error {
|
||||||
formatMap := map[string]func(interface{}){
|
formatMap := map[string]func(interface{}){
|
||||||
"": func(i interface{}) {},
|
"": func(i interface{}) {},
|
||||||
"pretty": func(i interface{}) {
|
"pretty": func(i interface{}) {
|
||||||
@@ -58,16 +55,8 @@ var ConfigcheckCmd = &cli.Subcommand{
|
|||||||
}
|
}
|
||||||
|
|
||||||
var hadErr bool
|
var hadErr bool
|
||||||
|
|
||||||
parseFlags := config.ParseFlagsNone
|
|
||||||
|
|
||||||
if configcheckArgs.skipCertCheck {
|
|
||||||
parseFlags |= config.ParseFlagsNoCertCheck
|
|
||||||
}
|
|
||||||
|
|
||||||
// further: try to build jobs
|
// further: try to build jobs
|
||||||
confJobs, err := job.JobsFromConfig(subcommand.Config(), parseFlags)
|
confJobs, err := job.JobsFromConfig(subcommand.Config())
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err := errors.Wrap(err, "cannot build jobs from config")
|
err := errors.Wrap(err, "cannot build jobs from config")
|
||||||
if configcheckArgs.what == "jobs" {
|
if configcheckArgs.what == "jobs" {
|
||||||
@@ -0,0 +1,104 @@
|
|||||||
|
package client
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"github.com/spf13/pflag"
|
||||||
|
|
||||||
|
"github.com/zrepl/zrepl/zfs"
|
||||||
|
|
||||||
|
"github.com/zrepl/zrepl/cli"
|
||||||
|
"github.com/zrepl/zrepl/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
MigrateCmd = &cli.Subcommand{
|
||||||
|
Use: "migrate",
|
||||||
|
Short: "perform migration of the on-disk / zfs properties",
|
||||||
|
SetupSubcommands: func() []*cli.Subcommand {
|
||||||
|
return migrations
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
var migrations = []*cli.Subcommand{
|
||||||
|
&cli.Subcommand{
|
||||||
|
Use: "0.0.X:0.1:placeholder",
|
||||||
|
Run: doMigratePlaceholder0_1,
|
||||||
|
SetupFlags: func(f *pflag.FlagSet) {
|
||||||
|
f.BoolVar(&migratePlaceholder0_1Args.dryRun, "dry-run", false, "dry run")
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var migratePlaceholder0_1Args struct {
|
||||||
|
dryRun bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func doMigratePlaceholder0_1(sc *cli.Subcommand, args []string) error {
|
||||||
|
if len(args) != 0 {
|
||||||
|
return fmt.Errorf("migration does not take arguments, got %v", args)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := sc.Config()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
allFSS, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrap(err, "cannot list filesystems")
|
||||||
|
}
|
||||||
|
|
||||||
|
type workItem struct {
|
||||||
|
jobName string
|
||||||
|
rootFS *zfs.DatasetPath
|
||||||
|
fss []*zfs.DatasetPath
|
||||||
|
}
|
||||||
|
var wis []workItem
|
||||||
|
for i, j := range cfg.Jobs {
|
||||||
|
var rfsS string
|
||||||
|
switch job := j.Ret.(type) {
|
||||||
|
case *config.SinkJob:
|
||||||
|
rfsS = job.RootFS
|
||||||
|
case *config.PullJob:
|
||||||
|
rfsS = job.RootFS
|
||||||
|
default:
|
||||||
|
fmt.Printf("ignoring job %q (%d/%d, type %T)\n", j.Name(), i, len(cfg.Jobs), j.Ret)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rfs, err := zfs.NewDatasetPath(rfsS)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "root fs for job %q is not a valid dataset path", j.Name())
|
||||||
|
}
|
||||||
|
var fss []*zfs.DatasetPath
|
||||||
|
for _, fs := range allFSS {
|
||||||
|
if fs.HasPrefix(rfs) {
|
||||||
|
fss = append(fss, fs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
wis = append(wis, workItem{j.Name(), rfs, fss})
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, wi := range wis {
|
||||||
|
fmt.Printf("job %q => migrate filesystems below root_fs %q\n", wi.jobName, wi.rootFS.ToString())
|
||||||
|
if len(wi.fss) == 0 {
|
||||||
|
fmt.Printf("\tno filesystems\n")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, fs := range wi.fss {
|
||||||
|
fmt.Printf("\t%q ... ", fs.ToString())
|
||||||
|
r, err := zfs.ZFSMigrateHashBasedPlaceholderToCurrent(fs, migratePlaceholder0_1Args.dryRun)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("error: %s\n", err)
|
||||||
|
} else if !r.NeedsModification {
|
||||||
|
fmt.Printf("unchanged (placeholder=%v)\n", r.OriginalState.IsPlaceholder)
|
||||||
|
} else {
|
||||||
|
fmt.Printf("migrate (placeholder=%v) (old value = %q)\n",
|
||||||
|
r.OriginalState.IsPlaceholder, r.OriginalState.RawLocalPropertyValue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -1,39 +1,38 @@
|
|||||||
package client
|
package client
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"errors"
|
"errors"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/cli"
|
"github.com/zrepl/zrepl/cli"
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon"
|
"github.com/zrepl/zrepl/daemon"
|
||||||
)
|
)
|
||||||
|
|
||||||
var pprofListenCmd struct {
|
var pprofArgs struct {
|
||||||
daemon.PprofServerControlMsg
|
daemon.PprofServerControlMsg
|
||||||
}
|
}
|
||||||
|
|
||||||
var PprofListenCmd = &cli.Subcommand{
|
var PprofCmd = &cli.Subcommand{
|
||||||
Use: "listen off | [on TCP_LISTEN_ADDRESS]",
|
Use: "pprof off | [on TCP_LISTEN_ADDRESS]",
|
||||||
Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
|
Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
|
||||||
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
Run: func(subcommand *cli.Subcommand, args []string) error {
|
||||||
if len(args) < 1 {
|
if len(args) < 1 {
|
||||||
goto enargs
|
goto enargs
|
||||||
}
|
}
|
||||||
switch args[0] {
|
switch args[0] {
|
||||||
case "on":
|
case "on":
|
||||||
pprofListenCmd.Run = true
|
pprofArgs.Run = true
|
||||||
if len(args) != 2 {
|
if len(args) != 2 {
|
||||||
return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
|
return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
|
||||||
}
|
}
|
||||||
pprofListenCmd.HttpListenAddress = args[1]
|
pprofArgs.HttpListenAddress = args[1]
|
||||||
case "off":
|
case "off":
|
||||||
if len(args) != 1 {
|
if len(args) != 1 {
|
||||||
goto enargs
|
goto enargs
|
||||||
}
|
}
|
||||||
pprofListenCmd.Run = false
|
pprofArgs.Run = false
|
||||||
}
|
}
|
||||||
|
|
||||||
RunPProf(subcommand.Config())
|
RunPProf(subcommand.Config())
|
||||||
@@ -59,7 +58,7 @@ func RunPProf(conf *config.Config) {
|
|||||||
log.Printf("error creating http client: %s", err)
|
log.Printf("error creating http client: %s", err)
|
||||||
die()
|
die()
|
||||||
}
|
}
|
||||||
err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofListenCmd.PprofServerControlMsg, struct{}{})
|
err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofArgs.PprofServerControlMsg, struct{}{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("error sending control message: %s", err)
|
log.Printf("error sending control message: %s", err)
|
||||||
die()
|
die()
|
||||||
@@ -1,19 +1,17 @@
|
|||||||
package client
|
package client
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/cli"
|
"github.com/zrepl/zrepl/cli"
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon"
|
"github.com/zrepl/zrepl/daemon"
|
||||||
)
|
)
|
||||||
|
|
||||||
var SignalCmd = &cli.Subcommand{
|
var SignalCmd = &cli.Subcommand{
|
||||||
Use: "signal [wakeup|reset] JOB",
|
Use: "signal [wakeup|reset] JOB",
|
||||||
Short: "wake up a job from wait state or abort its current invocation",
|
Short: "wake up a job from wait state or abort its current invocation",
|
||||||
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
Run: func(subcommand *cli.Subcommand, args []string) error {
|
||||||
return runSignalCmd(subcommand.Config(), args)
|
return runSignalCmd(subcommand.Config(), args)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,757 @@
|
|||||||
|
package client
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"math"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
// tcell is the termbox-compatbile library for abstracting away escape sequences, etc.
|
||||||
|
// as of tcell#252, the number of default distributed terminals is relatively limited
|
||||||
|
// additional terminal definitions can be included via side-effect import
|
||||||
|
// See https://github.com/gdamore/tcell/blob/master/terminfo/base/base.go
|
||||||
|
// See https://github.com/gdamore/tcell/issues/252#issuecomment-533836078
|
||||||
|
"github.com/gdamore/tcell/termbox"
|
||||||
|
_ "github.com/gdamore/tcell/terminfo/s/screen" // tmux on FreeBSD 11 & 12 without ncurses
|
||||||
|
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"github.com/spf13/pflag"
|
||||||
|
"github.com/zrepl/yaml-config"
|
||||||
|
|
||||||
|
"github.com/zrepl/zrepl/cli"
|
||||||
|
"github.com/zrepl/zrepl/daemon"
|
||||||
|
"github.com/zrepl/zrepl/daemon/job"
|
||||||
|
"github.com/zrepl/zrepl/daemon/pruner"
|
||||||
|
"github.com/zrepl/zrepl/daemon/snapper"
|
||||||
|
"github.com/zrepl/zrepl/replication/report"
|
||||||
|
)
|
||||||
|
|
||||||
|
type byteProgressMeasurement struct {
|
||||||
|
time time.Time
|
||||||
|
val int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type bytesProgressHistory struct {
|
||||||
|
last *byteProgressMeasurement // pointer as poor man's optional
|
||||||
|
changeCount int
|
||||||
|
lastChange time.Time
|
||||||
|
bpsAvg float64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *bytesProgressHistory) Update(currentVal int64) (bytesPerSecondAvg int64, changeCount int) {
|
||||||
|
|
||||||
|
if p.last == nil {
|
||||||
|
p.last = &byteProgressMeasurement{
|
||||||
|
time: time.Now(),
|
||||||
|
val: currentVal,
|
||||||
|
}
|
||||||
|
return 0, 0
|
||||||
|
}
|
||||||
|
|
||||||
|
if p.last.val != currentVal {
|
||||||
|
p.changeCount++
|
||||||
|
p.lastChange = time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
if time.Since(p.lastChange) > 3*time.Second {
|
||||||
|
p.last = nil
|
||||||
|
return 0, 0
|
||||||
|
}
|
||||||
|
|
||||||
|
deltaV := currentVal - p.last.val
|
||||||
|
deltaT := time.Since(p.last.time)
|
||||||
|
rate := float64(deltaV) / deltaT.Seconds()
|
||||||
|
|
||||||
|
factor := 0.3
|
||||||
|
p.bpsAvg = (1-factor)*p.bpsAvg + factor*rate
|
||||||
|
|
||||||
|
p.last.time = time.Now()
|
||||||
|
p.last.val = currentVal
|
||||||
|
|
||||||
|
return int64(p.bpsAvg), p.changeCount
|
||||||
|
}
|
||||||
|
|
||||||
|
type tui struct {
|
||||||
|
x, y int
|
||||||
|
indent int
|
||||||
|
|
||||||
|
lock sync.Mutex //For report and error
|
||||||
|
report map[string]job.Status
|
||||||
|
err error
|
||||||
|
|
||||||
|
jobFilter string
|
||||||
|
|
||||||
|
replicationProgress map[string]*bytesProgressHistory // by job name
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTui() tui {
|
||||||
|
return tui{
|
||||||
|
replicationProgress: make(map[string]*bytesProgressHistory),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const INDENT_MULTIPLIER = 4
|
||||||
|
|
||||||
|
func (t *tui) moveLine(dl int, col int) {
|
||||||
|
t.y += dl
|
||||||
|
t.x = t.indent*INDENT_MULTIPLIER + col
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *tui) write(text string) {
|
||||||
|
for _, c := range text {
|
||||||
|
if c == '\n' {
|
||||||
|
t.newline()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
termbox.SetCell(t.x, t.y, c, termbox.ColorDefault, termbox.ColorDefault)
|
||||||
|
t.x += 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *tui) printf(text string, a ...interface{}) {
|
||||||
|
t.write(fmt.Sprintf(text, a...))
|
||||||
|
}
|
||||||
|
|
||||||
|
func wrap(s string, width int) string {
|
||||||
|
var b strings.Builder
|
||||||
|
for len(s) > 0 {
|
||||||
|
rem := width
|
||||||
|
if rem > len(s) {
|
||||||
|
rem = len(s)
|
||||||
|
}
|
||||||
|
if idx := strings.IndexAny(s, "\n\r"); idx != -1 && idx < rem {
|
||||||
|
rem = idx + 1
|
||||||
|
}
|
||||||
|
untilNewline := strings.TrimRight(s[:rem], "\n\r")
|
||||||
|
s = s[rem:]
|
||||||
|
if len(untilNewline) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
b.WriteString(untilNewline)
|
||||||
|
b.WriteString("\n")
|
||||||
|
}
|
||||||
|
return strings.TrimRight(b.String(), "\n\r")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *tui) printfDrawIndentedAndWrappedIfMultiline(format string, a ...interface{}) {
|
||||||
|
whole := fmt.Sprintf(format, a...)
|
||||||
|
width, _ := termbox.Size()
|
||||||
|
if !strings.ContainsAny(whole, "\n\r") && t.x+len(whole) <= width {
|
||||||
|
t.printf(format, a...)
|
||||||
|
} else {
|
||||||
|
t.addIndent(1)
|
||||||
|
t.newline()
|
||||||
|
t.write(wrap(whole, width-INDENT_MULTIPLIER*t.indent))
|
||||||
|
t.addIndent(-1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *tui) newline() {
|
||||||
|
t.moveLine(1, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *tui) setIndent(indent int) {
|
||||||
|
t.indent = indent
|
||||||
|
t.moveLine(0, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *tui) addIndent(indent int) {
|
||||||
|
t.indent += indent
|
||||||
|
t.moveLine(0, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
var statusFlags struct {
|
||||||
|
Raw bool
|
||||||
|
Job string
|
||||||
|
}
|
||||||
|
|
||||||
|
var StatusCmd = &cli.Subcommand{
|
||||||
|
Use: "status",
|
||||||
|
Short: "show job activity or dump as JSON for monitoring",
|
||||||
|
SetupFlags: func(f *pflag.FlagSet) {
|
||||||
|
f.BoolVar(&statusFlags.Raw, "raw", false, "dump raw status description from zrepl daemon")
|
||||||
|
f.StringVar(&statusFlags.Job, "job", "", "only dump specified job")
|
||||||
|
},
|
||||||
|
Run: runStatus,
|
||||||
|
}
|
||||||
|
|
||||||
|
func runStatus(s *cli.Subcommand, args []string) error {
|
||||||
|
httpc, err := controlHttpClient(s.Config().Global.Control.SockPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if statusFlags.Raw {
|
||||||
|
resp, err := httpc.Get("http://unix" + daemon.ControlJobEndpointStatus)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
fmt.Fprintf(os.Stderr, "Received error response:\n")
|
||||||
|
_, err := io.CopyN(os.Stderr, resp.Body, 4096)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return errors.Errorf("exit")
|
||||||
|
}
|
||||||
|
if _, err := io.Copy(os.Stdout, resp.Body); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
t := newTui()
|
||||||
|
t.lock.Lock()
|
||||||
|
t.err = errors.New("Got no report yet")
|
||||||
|
t.lock.Unlock()
|
||||||
|
t.jobFilter = statusFlags.Job
|
||||||
|
|
||||||
|
err = termbox.Init()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer termbox.Close()
|
||||||
|
|
||||||
|
update := func() {
|
||||||
|
m := make(map[string]job.Status)
|
||||||
|
|
||||||
|
err2 := jsonRequestResponse(httpc, daemon.ControlJobEndpointStatus,
|
||||||
|
struct{}{},
|
||||||
|
&m,
|
||||||
|
)
|
||||||
|
|
||||||
|
t.lock.Lock()
|
||||||
|
t.err = err2
|
||||||
|
t.report = m
|
||||||
|
t.lock.Unlock()
|
||||||
|
t.draw()
|
||||||
|
}
|
||||||
|
update()
|
||||||
|
|
||||||
|
ticker := time.NewTicker(500 * time.Millisecond)
|
||||||
|
defer ticker.Stop()
|
||||||
|
go func() {
|
||||||
|
for range ticker.C {
|
||||||
|
update()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
termbox.HideCursor()
|
||||||
|
termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
|
||||||
|
|
||||||
|
loop:
|
||||||
|
for {
|
||||||
|
switch ev := termbox.PollEvent(); ev.Type {
|
||||||
|
case termbox.EventKey:
|
||||||
|
switch ev.Key {
|
||||||
|
case termbox.KeyEsc:
|
||||||
|
break loop
|
||||||
|
case termbox.KeyCtrlC:
|
||||||
|
break loop
|
||||||
|
}
|
||||||
|
case termbox.EventResize:
|
||||||
|
t.draw()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *tui) getReplicationProgresHistory(jobName string) *bytesProgressHistory {
|
||||||
|
p, ok := t.replicationProgress[jobName]
|
||||||
|
if !ok {
|
||||||
|
p = &bytesProgressHistory{}
|
||||||
|
t.replicationProgress[jobName] = p
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *tui) draw() {
|
||||||
|
t.lock.Lock()
|
||||||
|
defer t.lock.Unlock()
|
||||||
|
|
||||||
|
termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
|
||||||
|
t.x = 0
|
||||||
|
t.y = 0
|
||||||
|
t.indent = 0
|
||||||
|
|
||||||
|
if t.err != nil {
|
||||||
|
t.write(t.err.Error())
|
||||||
|
} else {
|
||||||
|
//Iterate over map in alphabetical order
|
||||||
|
keys := make([]string, 0, len(t.report))
|
||||||
|
for k := range t.report {
|
||||||
|
if len(k) == 0 || daemon.IsInternalJobName(k) { //Internal job
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if t.jobFilter != "" && k != t.jobFilter {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
|
||||||
|
if len(keys) == 0 {
|
||||||
|
t.setIndent(0)
|
||||||
|
t.printf("no jobs to display")
|
||||||
|
t.newline()
|
||||||
|
termbox.Flush()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, k := range keys {
|
||||||
|
v := t.report[k]
|
||||||
|
|
||||||
|
t.setIndent(0)
|
||||||
|
|
||||||
|
t.printf("Job: %s", k)
|
||||||
|
t.setIndent(1)
|
||||||
|
t.newline()
|
||||||
|
t.printf("Type: %s", v.Type)
|
||||||
|
t.setIndent(1)
|
||||||
|
t.newline()
|
||||||
|
|
||||||
|
if v.Type == job.TypePush || v.Type == job.TypePull {
|
||||||
|
activeStatus, ok := v.JobSpecific.(*job.ActiveSideStatus)
|
||||||
|
if !ok || activeStatus == nil {
|
||||||
|
t.printf("ActiveSideStatus is null")
|
||||||
|
t.newline()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
t.printf("Replication:")
|
||||||
|
t.newline()
|
||||||
|
t.addIndent(1)
|
||||||
|
t.renderReplicationReport(activeStatus.Replication, t.getReplicationProgresHistory(k))
|
||||||
|
t.addIndent(-1)
|
||||||
|
|
||||||
|
t.printf("Pruning Sender:")
|
||||||
|
t.newline()
|
||||||
|
t.addIndent(1)
|
||||||
|
t.renderPrunerReport(activeStatus.PruningSender)
|
||||||
|
t.addIndent(-1)
|
||||||
|
|
||||||
|
t.printf("Pruning Receiver:")
|
||||||
|
t.newline()
|
||||||
|
t.addIndent(1)
|
||||||
|
t.renderPrunerReport(activeStatus.PruningReceiver)
|
||||||
|
t.addIndent(-1)
|
||||||
|
|
||||||
|
if v.Type == job.TypePush {
|
||||||
|
t.printf("Snapshotting:")
|
||||||
|
t.newline()
|
||||||
|
t.addIndent(1)
|
||||||
|
t.renderSnapperReport(activeStatus.Snapshotting)
|
||||||
|
t.addIndent(-1)
|
||||||
|
}
|
||||||
|
|
||||||
|
} else if v.Type == job.TypeSnap {
|
||||||
|
snapStatus, ok := v.JobSpecific.(*job.SnapJobStatus)
|
||||||
|
if !ok || snapStatus == nil {
|
||||||
|
t.printf("SnapJobStatus is null")
|
||||||
|
t.newline()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
t.printf("Pruning snapshots:")
|
||||||
|
t.newline()
|
||||||
|
t.addIndent(1)
|
||||||
|
t.renderPrunerReport(snapStatus.Pruning)
|
||||||
|
t.addIndent(-1)
|
||||||
|
t.printf("Snapshotting:")
|
||||||
|
t.newline()
|
||||||
|
t.addIndent(1)
|
||||||
|
t.renderSnapperReport(snapStatus.Snapshotting)
|
||||||
|
t.addIndent(-1)
|
||||||
|
} else if v.Type == job.TypeSource {
|
||||||
|
|
||||||
|
st := v.JobSpecific.(*job.PassiveStatus)
|
||||||
|
t.printf("Snapshotting:\n")
|
||||||
|
t.addIndent(1)
|
||||||
|
t.renderSnapperReport(st.Snapper)
|
||||||
|
t.addIndent(-1)
|
||||||
|
|
||||||
|
} else {
|
||||||
|
t.printf("No status representation for job type '%s', dumping as YAML", v.Type)
|
||||||
|
t.newline()
|
||||||
|
asYaml, err := yaml.Marshal(v.JobSpecific)
|
||||||
|
if err != nil {
|
||||||
|
t.printf("Error marshaling status to YAML: %s", err)
|
||||||
|
t.newline()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
t.write(string(asYaml))
|
||||||
|
t.newline()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
termbox.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *tui) renderReplicationReport(rep *report.Report, history *bytesProgressHistory) {
|
||||||
|
if rep == nil {
|
||||||
|
t.printf("...\n")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if rep.WaitReconnectError != nil {
|
||||||
|
t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: %s", rep.WaitReconnectError)
|
||||||
|
t.newline()
|
||||||
|
}
|
||||||
|
if !rep.WaitReconnectSince.IsZero() {
|
||||||
|
delta := time.Until(rep.WaitReconnectUntil).Round(time.Second)
|
||||||
|
if rep.WaitReconnectUntil.IsZero() || delta > 0 {
|
||||||
|
var until string
|
||||||
|
if rep.WaitReconnectUntil.IsZero() {
|
||||||
|
until = "waiting indefinitely"
|
||||||
|
} else {
|
||||||
|
until = fmt.Sprintf("hard fail in %s @ %s", delta, rep.WaitReconnectUntil)
|
||||||
|
}
|
||||||
|
t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnecting with exponential backoff (since %s) (%s)",
|
||||||
|
rep.WaitReconnectSince, until)
|
||||||
|
} else {
|
||||||
|
t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnects reached hard-fail timeout @ %s", rep.WaitReconnectUntil)
|
||||||
|
}
|
||||||
|
t.newline()
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO visualize more than the latest attempt by folding all attempts into one
|
||||||
|
if len(rep.Attempts) == 0 {
|
||||||
|
t.printf("no attempts made yet")
|
||||||
|
return
|
||||||
|
} else {
|
||||||
|
t.printf("Attempt #%d", len(rep.Attempts))
|
||||||
|
if len(rep.Attempts) > 1 {
|
||||||
|
t.printf(". Previous attempts failed with the following statuses:")
|
||||||
|
t.newline()
|
||||||
|
t.addIndent(1)
|
||||||
|
for i, a := range rep.Attempts[:len(rep.Attempts)-1] {
|
||||||
|
t.printfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt))
|
||||||
|
t.newline()
|
||||||
|
}
|
||||||
|
t.addIndent(-1)
|
||||||
|
} else {
|
||||||
|
t.newline()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
latest := rep.Attempts[len(rep.Attempts)-1]
|
||||||
|
sort.Slice(latest.Filesystems, func(i, j int) bool {
|
||||||
|
return latest.Filesystems[i].Info.Name < latest.Filesystems[j].Info.Name
|
||||||
|
})
|
||||||
|
|
||||||
|
t.printf("Status: %s", latest.State)
|
||||||
|
t.newline()
|
||||||
|
if latest.State == report.AttemptPlanningError {
|
||||||
|
t.printf("Problem: ")
|
||||||
|
t.printfDrawIndentedAndWrappedIfMultiline("%s", latest.PlanError)
|
||||||
|
t.newline()
|
||||||
|
} else if latest.State == report.AttemptFanOutError {
|
||||||
|
t.printf("Problem: one or more of the filesystems encountered errors")
|
||||||
|
t.newline()
|
||||||
|
}
|
||||||
|
|
||||||
|
if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
|
||||||
|
// Draw global progress bar
|
||||||
|
// Progress: [---------------]
|
||||||
|
expected, replicated := latest.BytesSum()
|
||||||
|
rate, changeCount := history.Update(replicated)
|
||||||
|
t.write("Progress: ")
|
||||||
|
t.drawBar(50, replicated, expected, changeCount)
|
||||||
|
t.write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinary(replicated), ByteCountBinary(expected), ByteCountBinary(rate)))
|
||||||
|
t.newline()
|
||||||
|
|
||||||
|
var maxFSLen int
|
||||||
|
for _, fs := range latest.Filesystems {
|
||||||
|
if len(fs.Info.Name) > maxFSLen {
|
||||||
|
maxFSLen = len(fs.Info.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, fs := range latest.Filesystems {
|
||||||
|
t.printFilesystemStatus(fs, false, maxFSLen) // FIXME bring 'active' flag back
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *tui) renderPrunerReport(r *pruner.Report) {
|
||||||
|
if r == nil {
|
||||||
|
t.printf("...\n")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
state, err := pruner.StateString(r.State)
|
||||||
|
if err != nil {
|
||||||
|
t.printf("Status: %q (parse error: %q)\n", r.State, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
t.printf("Status: %s", state)
|
||||||
|
t.newline()
|
||||||
|
|
||||||
|
if r.Error != "" {
|
||||||
|
t.printf("Error: %s\n", r.Error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type commonFS struct {
|
||||||
|
*pruner.FSReport
|
||||||
|
completed bool
|
||||||
|
}
|
||||||
|
all := make([]commonFS, 0, len(r.Pending)+len(r.Completed))
|
||||||
|
for i := range r.Pending {
|
||||||
|
all = append(all, commonFS{&r.Pending[i], false})
|
||||||
|
}
|
||||||
|
for i := range r.Completed {
|
||||||
|
all = append(all, commonFS{&r.Completed[i], true})
|
||||||
|
}
|
||||||
|
|
||||||
|
switch state {
|
||||||
|
case pruner.Plan:
|
||||||
|
fallthrough
|
||||||
|
case pruner.PlanErr:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(all) == 0 {
|
||||||
|
t.printf("nothing to do\n")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var totalDestroyCount, completedDestroyCount int
|
||||||
|
var maxFSname int
|
||||||
|
for _, fs := range all {
|
||||||
|
totalDestroyCount += len(fs.DestroyList)
|
||||||
|
if fs.completed {
|
||||||
|
completedDestroyCount += len(fs.DestroyList)
|
||||||
|
}
|
||||||
|
if maxFSname < len(fs.Filesystem) {
|
||||||
|
maxFSname = len(fs.Filesystem)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// global progress bar
|
||||||
|
progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
|
||||||
|
t.write("Progress: ")
|
||||||
|
t.write("[")
|
||||||
|
t.write(times("=", progress))
|
||||||
|
t.write(">")
|
||||||
|
t.write(times("-", 80-progress))
|
||||||
|
t.write("]")
|
||||||
|
t.printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
|
||||||
|
t.newline()
|
||||||
|
|
||||||
|
sort.SliceStable(all, func(i, j int) bool {
|
||||||
|
return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
|
||||||
|
})
|
||||||
|
|
||||||
|
// Draw a table-like representation of 'all'
|
||||||
|
for _, fs := range all {
|
||||||
|
t.write(rightPad(fs.Filesystem, maxFSname, " "))
|
||||||
|
t.write(" ")
|
||||||
|
if !fs.SkipReason.NotSkipped() {
|
||||||
|
t.printf("skipped: %s\n", fs.SkipReason)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if fs.LastError != "" {
|
||||||
|
if strings.ContainsAny(fs.LastError, "\r\n") {
|
||||||
|
t.printf("ERROR:")
|
||||||
|
t.printfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError)
|
||||||
|
} else {
|
||||||
|
t.printfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError)
|
||||||
|
}
|
||||||
|
t.newline()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
pruneRuleActionStr := fmt.Sprintf("(destroy %d of %d snapshots)",
|
||||||
|
len(fs.DestroyList), len(fs.SnapshotList))
|
||||||
|
|
||||||
|
if fs.completed {
|
||||||
|
t.printf("Completed %s\n", pruneRuleActionStr)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
t.write("Pending ") // whitespace is padding 10
|
||||||
|
if len(fs.DestroyList) == 1 {
|
||||||
|
t.write(fs.DestroyList[0].Name)
|
||||||
|
} else {
|
||||||
|
t.write(pruneRuleActionStr)
|
||||||
|
}
|
||||||
|
t.newline()
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *tui) renderSnapperReport(r *snapper.Report) {
|
||||||
|
if r == nil {
|
||||||
|
t.printf("<snapshot type does not have a report>\n")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
t.printf("Status: %s", r.State)
|
||||||
|
t.newline()
|
||||||
|
|
||||||
|
if r.Error != "" {
|
||||||
|
t.printf("Error: %s\n", r.Error)
|
||||||
|
}
|
||||||
|
if !r.SleepUntil.IsZero() {
|
||||||
|
t.printf("Sleep until: %s\n", r.SleepUntil)
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Slice(r.Progress, func(i, j int) bool {
|
||||||
|
return strings.Compare(r.Progress[i].Path, r.Progress[j].Path) == -1
|
||||||
|
})
|
||||||
|
|
||||||
|
t.addIndent(1)
|
||||||
|
defer t.addIndent(-1)
|
||||||
|
dur := func(d time.Duration) string {
|
||||||
|
return d.Round(100 * time.Millisecond).String()
|
||||||
|
}
|
||||||
|
|
||||||
|
type row struct {
|
||||||
|
path, state, duration, remainder, hookReport string
|
||||||
|
}
|
||||||
|
var widths struct {
|
||||||
|
path, state, duration int
|
||||||
|
}
|
||||||
|
rows := make([]*row, len(r.Progress))
|
||||||
|
for i, fs := range r.Progress {
|
||||||
|
r := &row{
|
||||||
|
path: fs.Path,
|
||||||
|
state: fs.State.String(),
|
||||||
|
}
|
||||||
|
if fs.HooksHadError {
|
||||||
|
r.hookReport = fs.Hooks // FIXME render here, not in daemon
|
||||||
|
}
|
||||||
|
switch fs.State {
|
||||||
|
case snapper.SnapPending:
|
||||||
|
r.duration = "..."
|
||||||
|
r.remainder = ""
|
||||||
|
case snapper.SnapStarted:
|
||||||
|
r.duration = dur(time.Since(fs.StartAt))
|
||||||
|
r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
|
||||||
|
case snapper.SnapDone:
|
||||||
|
fallthrough
|
||||||
|
case snapper.SnapError:
|
||||||
|
r.duration = dur(fs.DoneAt.Sub(fs.StartAt))
|
||||||
|
r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
|
||||||
|
}
|
||||||
|
rows[i] = r
|
||||||
|
if len(r.path) > widths.path {
|
||||||
|
widths.path = len(r.path)
|
||||||
|
}
|
||||||
|
if len(r.state) > widths.state {
|
||||||
|
widths.state = len(r.state)
|
||||||
|
}
|
||||||
|
if len(r.duration) > widths.duration {
|
||||||
|
widths.duration = len(r.duration)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, r := range rows {
|
||||||
|
path := rightPad(r.path, widths.path, " ")
|
||||||
|
state := rightPad(r.state, widths.state, " ")
|
||||||
|
duration := rightPad(r.duration, widths.duration, " ")
|
||||||
|
t.printf("%s %s %s", path, state, duration)
|
||||||
|
t.printfDrawIndentedAndWrappedIfMultiline(" %s", r.remainder)
|
||||||
|
if r.hookReport != "" {
|
||||||
|
t.printfDrawIndentedAndWrappedIfMultiline("%s", r.hookReport)
|
||||||
|
}
|
||||||
|
t.newline()
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func times(str string, n int) (out string) {
|
||||||
|
for i := 0; i < n; i++ {
|
||||||
|
out += str
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func rightPad(str string, length int, pad string) string {
|
||||||
|
if len(str) > length {
|
||||||
|
return str[:length]
|
||||||
|
}
|
||||||
|
return str + strings.Repeat(pad, length-len(str))
|
||||||
|
}
|
||||||
|
|
||||||
|
var arrowPositions = `>\|/`
|
||||||
|
|
||||||
|
// changeCount = 0 indicates stall / no progresss
|
||||||
|
func (t *tui) drawBar(length int, bytes, totalBytes int64, changeCount int) {
|
||||||
|
var completedLength int
|
||||||
|
if totalBytes > 0 {
|
||||||
|
completedLength = int(int64(length) * bytes / totalBytes)
|
||||||
|
if completedLength > length {
|
||||||
|
completedLength = length
|
||||||
|
}
|
||||||
|
} else if totalBytes == bytes {
|
||||||
|
completedLength = length
|
||||||
|
}
|
||||||
|
|
||||||
|
t.write("[")
|
||||||
|
t.write(times("=", completedLength))
|
||||||
|
t.write(string(arrowPositions[changeCount%len(arrowPositions)]))
|
||||||
|
t.write(times("-", length-completedLength))
|
||||||
|
t.write("]")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *tui) printFilesystemStatus(rep *report.FilesystemReport, active bool, maxFS int) {
|
||||||
|
|
||||||
|
expected, replicated := rep.BytesSum()
|
||||||
|
status := fmt.Sprintf("%s (step %d/%d, %s/%s)",
|
||||||
|
strings.ToUpper(string(rep.State)),
|
||||||
|
rep.CurrentStep, len(rep.Steps),
|
||||||
|
ByteCountBinary(replicated), ByteCountBinary(expected),
|
||||||
|
)
|
||||||
|
|
||||||
|
activeIndicator := " "
|
||||||
|
if active {
|
||||||
|
activeIndicator = "*"
|
||||||
|
}
|
||||||
|
t.printf("%s %s %s ",
|
||||||
|
activeIndicator,
|
||||||
|
rightPad(rep.Info.Name, maxFS, " "),
|
||||||
|
status)
|
||||||
|
|
||||||
|
next := ""
|
||||||
|
if err := rep.Error(); err != nil {
|
||||||
|
next = err.Err
|
||||||
|
} else if rep.State != report.FilesystemDone {
|
||||||
|
if nextStep := rep.NextStep(); nextStep != nil {
|
||||||
|
if nextStep.IsIncremental() {
|
||||||
|
next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
|
||||||
|
} else {
|
||||||
|
next = fmt.Sprintf("next: %s (full)", nextStep.Info.To)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
next = "" // individual FSes may still be in planning state
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
t.printfDrawIndentedAndWrappedIfMultiline("%s", next)
|
||||||
|
|
||||||
|
t.newline()
|
||||||
|
}
|
||||||
|
|
||||||
|
func ByteCountBinary(b int64) string {
|
||||||
|
const unit = 1024
|
||||||
|
if b < unit {
|
||||||
|
return fmt.Sprintf("%d B", b)
|
||||||
|
}
|
||||||
|
div, exp := int64(unit), 0
|
||||||
|
for n := b / unit; n >= unit; n /= unit {
|
||||||
|
div *= unit
|
||||||
|
exp++
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
|
||||||
|
}
|
||||||
@@ -5,8 +5,8 @@ import (
|
|||||||
|
|
||||||
"github.com/problame/go-netssh"
|
"github.com/problame/go-netssh"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/cli"
|
"github.com/zrepl/zrepl/cli"
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
|
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
@@ -17,7 +17,7 @@ import (
|
|||||||
var StdinserverCmd = &cli.Subcommand{
|
var StdinserverCmd = &cli.Subcommand{
|
||||||
Use: "stdinserver CLIENT_IDENTITY",
|
Use: "stdinserver CLIENT_IDENTITY",
|
||||||
Short: "stdinserver transport mode (started from authorized_keys file as forced command)",
|
Short: "stdinserver transport mode (started from authorized_keys file as forced command)",
|
||||||
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
Run: func(subcommand *cli.Subcommand, args []string) error {
|
||||||
return runStdinserver(subcommand.Config(), args)
|
return runStdinserver(subcommand.Config(), args)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -1,24 +1,21 @@
|
|||||||
package client
|
package client
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/cli"
|
"github.com/zrepl/zrepl/cli"
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
"github.com/zrepl/zrepl/daemon/filters"
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
var TestCmd = &cli.Subcommand{
|
var TestCmd = &cli.Subcommand{
|
||||||
Use: "test",
|
Use: "test",
|
||||||
SetupSubcommands: func() []*cli.Subcommand {
|
SetupSubcommands: func() []*cli.Subcommand {
|
||||||
return []*cli.Subcommand{testFilter, testPlaceholder, testDecodeResumeToken}
|
return []*cli.Subcommand{testFilter, testPlaceholder}
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -39,7 +36,7 @@ var testFilter = &cli.Subcommand{
|
|||||||
Run: runTestFilterCmd,
|
Run: runTestFilterCmd,
|
||||||
}
|
}
|
||||||
|
|
||||||
func runTestFilterCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
|
||||||
|
|
||||||
if testFilterArgs.job == "" {
|
if testFilterArgs.job == "" {
|
||||||
return fmt.Errorf("must specify --job flag")
|
return fmt.Errorf("must specify --job flag")
|
||||||
@@ -60,8 +57,6 @@ func runTestFilterCmd(ctx context.Context, subcommand *cli.Subcommand, args []st
|
|||||||
confFilter = j.Filesystems
|
confFilter = j.Filesystems
|
||||||
case *config.PushJob:
|
case *config.PushJob:
|
||||||
confFilter = j.Filesystems
|
confFilter = j.Filesystems
|
||||||
case *config.SnapJob:
|
|
||||||
confFilter = j.Filesystems
|
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("job type %T does not have filesystems filter", j)
|
return fmt.Errorf("job type %T does not have filesystems filter", j)
|
||||||
}
|
}
|
||||||
@@ -75,7 +70,7 @@ func runTestFilterCmd(ctx context.Context, subcommand *cli.Subcommand, args []st
|
|||||||
if testFilterArgs.input != "" {
|
if testFilterArgs.input != "" {
|
||||||
fsnames = []string{testFilterArgs.input}
|
fsnames = []string{testFilterArgs.input}
|
||||||
} else {
|
} else {
|
||||||
out, err := zfs.ZFSList(ctx, []string{"name"})
|
out, err := zfs.ZFSList([]string{"name"})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not list ZFS filesystems: %s", err)
|
return fmt.Errorf("could not list ZFS filesystems: %s", err)
|
||||||
}
|
}
|
||||||
@@ -136,15 +131,13 @@ var testPlaceholder = &cli.Subcommand{
|
|||||||
Run: runTestPlaceholder,
|
Run: runTestPlaceholder,
|
||||||
}
|
}
|
||||||
|
|
||||||
func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
func runTestPlaceholder(subcommand *cli.Subcommand, args []string) error {
|
||||||
|
|
||||||
var checkDPs []*zfs.DatasetPath
|
var checkDPs []*zfs.DatasetPath
|
||||||
var datasetWasExplicitArgument bool
|
|
||||||
|
|
||||||
// all actions first
|
// all actions first
|
||||||
if testPlaceholderArgs.all {
|
if testPlaceholderArgs.all {
|
||||||
datasetWasExplicitArgument = false
|
out, err := zfs.ZFSList([]string{"name"})
|
||||||
out, err := zfs.ZFSList(ctx, []string{"name"})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, "could not list ZFS filesystems")
|
return errors.Wrap(err, "could not list ZFS filesystems")
|
||||||
}
|
}
|
||||||
@@ -156,7 +149,6 @@ func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []
|
|||||||
checkDPs = append(checkDPs, dp)
|
checkDPs = append(checkDPs, dp)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
datasetWasExplicitArgument = true
|
|
||||||
dp, err := zfs.NewDatasetPath(testPlaceholderArgs.ds)
|
dp, err := zfs.NewDatasetPath(testPlaceholderArgs.ds)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -169,17 +161,12 @@ func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []
|
|||||||
|
|
||||||
fmt.Printf("IS_PLACEHOLDER\tDATASET\tzrepl:placeholder\n")
|
fmt.Printf("IS_PLACEHOLDER\tDATASET\tzrepl:placeholder\n")
|
||||||
for _, dp := range checkDPs {
|
for _, dp := range checkDPs {
|
||||||
ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, dp)
|
ph, err := zfs.ZFSGetFilesystemPlaceholderState(dp)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, "cannot get placeholder state")
|
return errors.Wrap(err, "cannot get placeholder state")
|
||||||
}
|
}
|
||||||
if !ph.FSExists {
|
if !ph.FSExists {
|
||||||
if datasetWasExplicitArgument {
|
panic("placeholder state inconsistent: filesystem " + ph.FS + " must exist in this context")
|
||||||
return errors.Errorf("filesystem %q does not exist", ph.FS)
|
|
||||||
} else {
|
|
||||||
// got deleted between ZFSList and ZFSGetFilesystemPlaceholderState
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
is := "yes"
|
is := "yes"
|
||||||
if !ph.IsPlaceholder {
|
if !ph.IsPlaceholder {
|
||||||
@@ -189,32 +176,3 @@ func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var testDecodeResumeTokenArgs struct {
|
|
||||||
token string
|
|
||||||
}
|
|
||||||
|
|
||||||
var testDecodeResumeToken = &cli.Subcommand{
|
|
||||||
Use: "decoderesumetoken --token TOKEN",
|
|
||||||
Short: "decode resume token",
|
|
||||||
SetupFlags: func(f *pflag.FlagSet) {
|
|
||||||
f.StringVar(&testDecodeResumeTokenArgs.token, "token", "", "the resume token obtained from the receive_resume_token property")
|
|
||||||
},
|
|
||||||
Run: runTestDecodeResumeTokenCmd,
|
|
||||||
}
|
|
||||||
|
|
||||||
func runTestDecodeResumeTokenCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
|
||||||
if testDecodeResumeTokenArgs.token == "" {
|
|
||||||
return fmt.Errorf("token argument must be specified")
|
|
||||||
}
|
|
||||||
token, err := zfs.ParseResumeToken(ctx, testDecodeResumeTokenArgs.token)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
enc := json.NewEncoder(os.Stdout)
|
|
||||||
enc.SetIndent("", " ")
|
|
||||||
if err := enc.Encode(&token); err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,16 +1,15 @@
|
|||||||
package client
|
package client
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/cli"
|
"github.com/zrepl/zrepl/cli"
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon"
|
"github.com/zrepl/zrepl/daemon"
|
||||||
"github.com/zrepl/zrepl/internal/version"
|
"github.com/zrepl/zrepl/version"
|
||||||
)
|
)
|
||||||
|
|
||||||
var versionArgs struct {
|
var versionArgs struct {
|
||||||
@@ -26,7 +25,7 @@ var VersionCmd = &cli.Subcommand{
|
|||||||
SetupFlags: func(f *pflag.FlagSet) {
|
SetupFlags: func(f *pflag.FlagSet) {
|
||||||
f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)")
|
f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)")
|
||||||
},
|
},
|
||||||
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
Run: func(subcommand *cli.Subcommand, args []string) error {
|
||||||
versionArgs.Config = subcommand.Config()
|
versionArgs.Config = subcommand.Config()
|
||||||
versionArgs.ConfigErr = subcommand.ConfigParsingError()
|
versionArgs.ConfigErr = subcommand.ConfigParsingError()
|
||||||
return runVersionCmd()
|
return runVersionCmd()
|
||||||
@@ -2,32 +2,21 @@ package config
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
"log/syslog"
|
"log/syslog"
|
||||||
"os"
|
"os"
|
||||||
pathpkg "path"
|
"reflect"
|
||||||
"path/filepath"
|
"regexp"
|
||||||
"strings"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/robfig/cron/v3"
|
|
||||||
"github.com/zrepl/yaml-config"
|
"github.com/zrepl/yaml-config"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/util/datasizeunit"
|
|
||||||
zfsprop "github.com/zrepl/zrepl/internal/zfs/property"
|
|
||||||
)
|
|
||||||
|
|
||||||
type ParseFlags uint
|
|
||||||
|
|
||||||
const (
|
|
||||||
ParseFlagsNone ParseFlags = 0
|
|
||||||
ParseFlagsNoCertCheck ParseFlags = 1 << iota
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Jobs []JobEnum `yaml:"jobs,optional"`
|
Jobs []JobEnum `yaml:"jobs"`
|
||||||
Global *Global `yaml:"global,optional,fromdefaults"`
|
Global *Global `yaml:"global,optional,fromdefaults"`
|
||||||
Include []string `yaml:"include,optional"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Config) Job(name string) (*JobEnum, error) {
|
func (c *Config) Job(name string) (*JobEnum, error) {
|
||||||
@@ -63,110 +52,41 @@ func (j JobEnum) Name() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ActiveJob struct {
|
type ActiveJob struct {
|
||||||
Type string `yaml:"type"`
|
Type string `yaml:"type"`
|
||||||
Name string `yaml:"name"`
|
Name string `yaml:"name"`
|
||||||
Connect ConnectEnum `yaml:"connect"`
|
Connect ConnectEnum `yaml:"connect"`
|
||||||
Pruning PruningSenderReceiver `yaml:"pruning"`
|
Pruning PruningSenderReceiver `yaml:"pruning"`
|
||||||
Replication *Replication `yaml:"replication,optional,fromdefaults"`
|
Debug JobDebugSettings `yaml:"debug,optional"`
|
||||||
ConflictResolution *ConflictResolution `yaml:"conflict_resolution,optional,fromdefaults"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type ConflictResolution struct {
|
|
||||||
InitialReplication string `yaml:"initial_replication,optional,default=most_recent"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type PassiveJob struct {
|
type PassiveJob struct {
|
||||||
Type string `yaml:"type"`
|
Type string `yaml:"type"`
|
||||||
Name string `yaml:"name"`
|
Name string `yaml:"name"`
|
||||||
Serve ServeEnum `yaml:"serve"`
|
Serve ServeEnum `yaml:"serve"`
|
||||||
|
Debug JobDebugSettings `yaml:"debug,optional"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type SnapJob struct {
|
type SnapJob struct {
|
||||||
Type string `yaml:"type"`
|
Type string `yaml:"type"`
|
||||||
Name string `yaml:"name"`
|
Name string `yaml:"name"`
|
||||||
Pruning PruningLocal `yaml:"pruning"`
|
Pruning PruningLocal `yaml:"pruning"`
|
||||||
|
Debug JobDebugSettings `yaml:"debug,optional"`
|
||||||
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
|
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
|
||||||
Filesystems FilesystemsFilter `yaml:"filesystems"`
|
Filesystems FilesystemsFilter `yaml:"filesystems"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type SendOptions struct {
|
|
||||||
Encrypted bool `yaml:"encrypted,optional,default=false"`
|
|
||||||
Raw bool `yaml:"raw,optional,default=false"`
|
|
||||||
SendProperties bool `yaml:"send_properties,optional,default=false"`
|
|
||||||
BackupProperties bool `yaml:"backup_properties,optional,default=false"`
|
|
||||||
LargeBlocks bool `yaml:"large_blocks,optional,default=false"`
|
|
||||||
Compressed bool `yaml:"compressed,optional,default=false"`
|
|
||||||
EmbeddedData bool `yaml:"embedded_data,optional,default=false"`
|
|
||||||
Saved bool `yaml:"saved,optional,default=false"`
|
|
||||||
|
|
||||||
BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type RecvOptions struct {
|
|
||||||
// Note: we cannot enforce encrypted recv as the ZFS cli doesn't provide a mechanism for it
|
|
||||||
// Encrypted bool `yaml:"may_encrypted"`
|
|
||||||
// Future:
|
|
||||||
// Reencrypt bool `yaml:"reencrypt"`
|
|
||||||
|
|
||||||
Properties *PropertyRecvOptions `yaml:"properties,fromdefaults"`
|
|
||||||
|
|
||||||
BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
|
|
||||||
|
|
||||||
Placeholder *PlaceholderRecvOptions `yaml:"placeholder,fromdefaults"`
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ yaml.Unmarshaler = &datasizeunit.Bits{}
|
|
||||||
|
|
||||||
type BandwidthLimit struct {
|
|
||||||
Max datasizeunit.Bits `yaml:"max,default=-1 B"`
|
|
||||||
BucketCapacity datasizeunit.Bits `yaml:"bucket_capacity,default=128 KiB"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Replication struct {
|
|
||||||
Protection *ReplicationOptionsProtection `yaml:"protection,optional,fromdefaults"`
|
|
||||||
Concurrency *ReplicationOptionsConcurrency `yaml:"concurrency,optional,fromdefaults"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type ReplicationOptionsProtection struct {
|
|
||||||
Initial string `yaml:"initial,optional,default=guarantee_resumability"`
|
|
||||||
Incremental string `yaml:"incremental,optional,default=guarantee_resumability"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type ReplicationOptionsConcurrency struct {
|
|
||||||
Steps int `yaml:"steps,optional,default=1"`
|
|
||||||
SizeEstimates int `yaml:"size_estimates,optional,default=4"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type PropertyRecvOptions struct {
|
|
||||||
Inherit []zfsprop.Property `yaml:"inherit,optional"`
|
|
||||||
Override map[zfsprop.Property]string `yaml:"override,optional"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type PlaceholderRecvOptions struct {
|
|
||||||
Encryption string `yaml:"encryption,default=unspecified"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type PushJob struct {
|
type PushJob struct {
|
||||||
ActiveJob `yaml:",inline"`
|
ActiveJob `yaml:",inline"`
|
||||||
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
|
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
|
||||||
Filesystems FilesystemsFilter `yaml:"filesystems"`
|
Filesystems FilesystemsFilter `yaml:"filesystems"`
|
||||||
Send *SendOptions `yaml:"send,fromdefaults,optional"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (j *PushJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
|
|
||||||
func (j *PushJob) GetSendOptions() *SendOptions { return j.Send }
|
|
||||||
|
|
||||||
type PullJob struct {
|
type PullJob struct {
|
||||||
ActiveJob `yaml:",inline"`
|
ActiveJob `yaml:",inline"`
|
||||||
RootFS string `yaml:"root_fs"`
|
RootFS string `yaml:"root_fs"`
|
||||||
Interval PositiveDurationOrManual `yaml:"interval"`
|
Interval PositiveDurationOrManual `yaml:"interval"`
|
||||||
Recv *RecvOptions `yaml:"recv,fromdefaults,optional"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (j *PullJob) GetRootFS() string { return j.RootFS }
|
|
||||||
func (j *PullJob) GetAppendClientIdentity() bool { return false }
|
|
||||||
func (j *PullJob) GetRecvOptions() *RecvOptions { return j.Recv }
|
|
||||||
|
|
||||||
type PositiveDurationOrManual struct {
|
type PositiveDurationOrManual struct {
|
||||||
Interval time.Duration
|
Interval time.Duration
|
||||||
Manual bool
|
Manual bool
|
||||||
@@ -187,34 +107,28 @@ func (i *PositiveDurationOrManual) UnmarshalYAML(u func(interface{}, bool) error
|
|||||||
return fmt.Errorf("value must not be empty")
|
return fmt.Errorf("value must not be empty")
|
||||||
default:
|
default:
|
||||||
i.Manual = false
|
i.Manual = false
|
||||||
i.Interval, err = parsePositiveDuration(s)
|
i.Interval, err = time.ParseDuration(s)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if i.Interval <= 0 {
|
||||||
|
return fmt.Errorf("value must be a positive duration, got %q", s)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type SinkJob struct {
|
type SinkJob struct {
|
||||||
PassiveJob `yaml:",inline"`
|
PassiveJob `yaml:",inline"`
|
||||||
RootFS string `yaml:"root_fs"`
|
RootFS string `yaml:"root_fs"`
|
||||||
Recv *RecvOptions `yaml:"recv,optional,fromdefaults"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (j *SinkJob) GetRootFS() string { return j.RootFS }
|
|
||||||
func (j *SinkJob) GetAppendClientIdentity() bool { return true }
|
|
||||||
func (j *SinkJob) GetRecvOptions() *RecvOptions { return j.Recv }
|
|
||||||
|
|
||||||
type SourceJob struct {
|
type SourceJob struct {
|
||||||
PassiveJob `yaml:",inline"`
|
PassiveJob `yaml:",inline"`
|
||||||
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
|
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
|
||||||
Filesystems FilesystemsFilter `yaml:"filesystems"`
|
Filesystems FilesystemsFilter `yaml:"filesystems"`
|
||||||
Send *SendOptions `yaml:"send,optional,fromdefaults"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (j *SourceJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
|
|
||||||
func (j *SourceJob) GetSendOptions() *SendOptions { return j.Send }
|
|
||||||
|
|
||||||
type FilesystemsFilter map[string]bool
|
type FilesystemsFilter map[string]bool
|
||||||
|
|
||||||
type SnapshottingEnum struct {
|
type SnapshottingEnum struct {
|
||||||
@@ -222,49 +136,10 @@ type SnapshottingEnum struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type SnapshottingPeriodic struct {
|
type SnapshottingPeriodic struct {
|
||||||
Type string `yaml:"type"`
|
Type string `yaml:"type"`
|
||||||
Prefix string `yaml:"prefix"`
|
Prefix string `yaml:"prefix"`
|
||||||
Interval *PositiveDuration `yaml:"interval"`
|
Interval time.Duration `yaml:"interval,positive"`
|
||||||
Hooks HookList `yaml:"hooks,optional"`
|
Hooks HookList `yaml:"hooks,optional"`
|
||||||
TimestampFormattingSpec `yaml:",inline"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type TimestampFormattingSpec struct {
|
|
||||||
TimestampFormat string `yaml:"timestamp_format,optional,default=dense"`
|
|
||||||
TimestampLocation string `yaml:"timestamp_location,optional,default=UTC"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type CronSpec struct {
|
|
||||||
Schedule cron.Schedule
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ yaml.Unmarshaler = &CronSpec{}
|
|
||||||
|
|
||||||
func (s *CronSpec) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
|
|
||||||
var specString string
|
|
||||||
if err := unmarshal(&specString, false); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Use standard cron format.
|
|
||||||
// Disable the various "descriptors" (@daily, etc)
|
|
||||||
// They are just aliases to "top of hour", "midnight", etc.
|
|
||||||
parser := cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.SecondOptional)
|
|
||||||
|
|
||||||
sched, err := parser.Parse(specString)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrap(err, "cron syntax invalid")
|
|
||||||
}
|
|
||||||
s.Schedule = sched
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type SnapshottingCron struct {
|
|
||||||
Type string `yaml:"type"`
|
|
||||||
Prefix string `yaml:"prefix"`
|
|
||||||
Cron CronSpec `yaml:"cron"`
|
|
||||||
Hooks HookList `yaml:"hooks,optional"`
|
|
||||||
TimestampFormattingSpec `yaml:",inline"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type SnapshottingManual struct {
|
type SnapshottingManual struct {
|
||||||
@@ -306,6 +181,18 @@ type Global struct {
|
|||||||
Serve *GlobalServe `yaml:"serve,optional,fromdefaults"`
|
Serve *GlobalServe `yaml:"serve,optional,fromdefaults"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func Default(i interface{}) {
|
||||||
|
v := reflect.ValueOf(i)
|
||||||
|
if v.Kind() != reflect.Ptr {
|
||||||
|
panic(v)
|
||||||
|
}
|
||||||
|
y := `{}`
|
||||||
|
err := yaml.Unmarshal([]byte(y), v.Interface())
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type ConnectEnum struct {
|
type ConnectEnum struct {
|
||||||
Ret interface{}
|
Ret interface{}
|
||||||
}
|
}
|
||||||
@@ -358,16 +245,14 @@ type ServeCommon struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type TCPServe struct {
|
type TCPServe struct {
|
||||||
ServeCommon `yaml:",inline"`
|
ServeCommon `yaml:",inline"`
|
||||||
Listen string `yaml:"listen,hostport"`
|
Listen string `yaml:"listen,hostport"`
|
||||||
ListenFreeBind bool `yaml:"listen_freebind,default=false"`
|
Clients map[string]string `yaml:"clients"`
|
||||||
Clients map[string]string `yaml:"clients"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type TLSServe struct {
|
type TLSServe struct {
|
||||||
ServeCommon `yaml:",inline"`
|
ServeCommon `yaml:",inline"`
|
||||||
Listen string `yaml:"listen,hostport"`
|
Listen string `yaml:"listen,hostport"`
|
||||||
ListenFreeBind bool `yaml:"listen_freebind,default=false"`
|
|
||||||
Ca string `yaml:"ca"`
|
Ca string `yaml:"ca"`
|
||||||
Cert string `yaml:"cert"`
|
Cert string `yaml:"cert"`
|
||||||
Key string `yaml:"key"`
|
Key string `yaml:"key"`
|
||||||
@@ -397,7 +282,6 @@ type PruneKeepNotReplicated struct {
|
|||||||
type PruneKeepLastN struct {
|
type PruneKeepLastN struct {
|
||||||
Type string `yaml:"type"`
|
Type string `yaml:"type"`
|
||||||
Count int `yaml:"count"`
|
Count int `yaml:"count"`
|
||||||
Regex string `yaml:"regex,optional"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type PruneKeepRegex struct { // FIXME rename to KeepRegex
|
type PruneKeepRegex struct { // FIXME rename to KeepRegex
|
||||||
@@ -447,9 +331,8 @@ type MonitoringEnum struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type PrometheusMonitoring struct {
|
type PrometheusMonitoring struct {
|
||||||
Type string `yaml:"type"`
|
Type string `yaml:"type"`
|
||||||
Listen string `yaml:"listen,hostport"`
|
Listen string `yaml:"listen,hostport"`
|
||||||
ListenFreeBind bool `yaml:"listen_freebind,default=false"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type SyslogFacility syslog.Priority
|
type SyslogFacility syslog.Priority
|
||||||
@@ -472,6 +355,14 @@ type GlobalStdinServer struct {
|
|||||||
SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
|
SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type JobDebugSettings struct {
|
||||||
|
Conn *struct {
|
||||||
|
ReadDump string `yaml:"read_dump"`
|
||||||
|
WriteDump string `yaml:"write_dump"`
|
||||||
|
} `yaml:"conn,optional"`
|
||||||
|
RPCLog bool `yaml:"rpc_log,optional,default=false"`
|
||||||
|
}
|
||||||
|
|
||||||
type HookList []HookEnum
|
type HookList []HookEnum
|
||||||
|
|
||||||
type HookEnum struct {
|
type HookEnum struct {
|
||||||
@@ -570,7 +461,6 @@ func (t *SnapshottingEnum) UnmarshalYAML(u func(interface{}, bool) error) (err e
|
|||||||
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
||||||
"periodic": &SnapshottingPeriodic{},
|
"periodic": &SnapshottingPeriodic{},
|
||||||
"manual": &SnapshottingManual{},
|
"manual": &SnapshottingManual{},
|
||||||
"cron": &SnapshottingCron{},
|
|
||||||
})
|
})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -659,9 +549,8 @@ var ConfigFileDefaultLocations = []string{
|
|||||||
"/usr/local/etc/zrepl/zrepl.yml",
|
"/usr/local/etc/zrepl/zrepl.yml",
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseConfig(path string) (rootConfig *Config, err error) {
|
func ParseConfig(path string) (i *Config, err error) {
|
||||||
|
|
||||||
// Parse main configuration file
|
|
||||||
if path == "" {
|
if path == "" {
|
||||||
// Try default locations
|
// Try default locations
|
||||||
for _, l := range ConfigFileDefaultLocations {
|
for _, l := range ConfigFileDefaultLocations {
|
||||||
@@ -680,94 +569,11 @@ func ParseConfig(path string) (rootConfig *Config, err error) {
|
|||||||
|
|
||||||
var bytes []byte
|
var bytes []byte
|
||||||
|
|
||||||
if bytes, err = os.ReadFile(path); err != nil {
|
if bytes, err = ioutil.ReadFile(path); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
rootConfig, err = ParseConfigBytes(bytes)
|
return ParseConfigBytes(bytes)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = expandConfigInclude(path, rootConfig)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = validateJobNames(rootConfig); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return rootConfig, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func IsInternalJobName(s string) bool {
|
|
||||||
return strings.HasPrefix(s, "_")
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateJobNames(config *Config) error {
|
|
||||||
seen := make(map[string]struct{})
|
|
||||||
for _, job := range config.Jobs {
|
|
||||||
name := job.Name()
|
|
||||||
if IsInternalJobName(name) {
|
|
||||||
return errors.Errorf("job name %q is reserved for internal use (starts with _)", name)
|
|
||||||
}
|
|
||||||
if _, ok := seen[name]; ok {
|
|
||||||
return errors.Errorf("duplicate job name %q", name)
|
|
||||||
}
|
|
||||||
seen[name] = struct{}{}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func expandConfigInclude(configPath string, config *Config) (err error) {
|
|
||||||
var includeConfigPaths []string
|
|
||||||
for _, path := range config.Include {
|
|
||||||
if !pathpkg.IsAbs(configPath) {
|
|
||||||
path = pathpkg.Join(pathpkg.Dir(configPath), path)
|
|
||||||
}
|
|
||||||
|
|
||||||
stat, statErr := os.Stat(path)
|
|
||||||
if statErr != nil {
|
|
||||||
return errors.Wrapf(statErr, "stat path %q", path)
|
|
||||||
}
|
|
||||||
|
|
||||||
if stat.Mode().IsDir() {
|
|
||||||
directoryPaths, err := filepath.Glob(path + "/*.yml")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
includeConfigPaths = append(includeConfigPaths, directoryPaths...)
|
|
||||||
} else if stat.Mode().IsRegular() {
|
|
||||||
if extention := filepath.Ext(path); extention != ".yml" {
|
|
||||||
return fmt.Errorf("include config files must end with `.yml`: %s", path)
|
|
||||||
}
|
|
||||||
includeConfigPaths = append(includeConfigPaths, path)
|
|
||||||
} else {
|
|
||||||
return fmt.Errorf("not a file or directory: %s", path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, path := range includeConfigPaths {
|
|
||||||
var bytes []byte
|
|
||||||
if bytes, err = os.ReadFile(path); err != nil {
|
|
||||||
return errors.Wrapf(err, "read file: %q", path)
|
|
||||||
}
|
|
||||||
|
|
||||||
includedConfig, err := ParseConfigBytes(bytes)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(includedConfig.Include) > 0 {
|
|
||||||
return errors.Errorf("included configuration files must not include other files: %s", path)
|
|
||||||
}
|
|
||||||
|
|
||||||
config.Jobs = append(config.Jobs, includedConfig.Jobs...)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseConfigBytes(bytes []byte) (*Config, error) {
|
func ParseConfigBytes(bytes []byte) (*Config, error) {
|
||||||
@@ -775,16 +581,46 @@ func ParseConfigBytes(bytes []byte) (*Config, error) {
|
|||||||
if err := yaml.UnmarshalStrict(bytes, &c); err != nil {
|
if err := yaml.UnmarshalStrict(bytes, &c); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if c != nil {
|
|
||||||
return c, nil
|
|
||||||
}
|
|
||||||
// There was no yaml document in the file, deserialize from default.
|
|
||||||
// => See TestFromdefaultsEmptyDoc in yaml-config package.
|
|
||||||
if err := yaml.UnmarshalStrict([]byte("{}"), &c); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if c == nil {
|
if c == nil {
|
||||||
panic("the fallback to deserialize from `{}` should work")
|
return nil, fmt.Errorf("config is empty or only consists of comments")
|
||||||
}
|
}
|
||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
|
||||||
|
|
||||||
|
func parsePostitiveDuration(e string) (d time.Duration, err error) {
|
||||||
|
comps := durationStringRegex.FindStringSubmatch(e)
|
||||||
|
if len(comps) != 3 {
|
||||||
|
err = fmt.Errorf("does not match regex: %s %#v", e, comps)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if durationFactor <= 0 {
|
||||||
|
return 0, errors.New("duration must be positive integer")
|
||||||
|
}
|
||||||
|
|
||||||
|
var durationUnit time.Duration
|
||||||
|
switch comps[2] {
|
||||||
|
case "s":
|
||||||
|
durationUnit = time.Second
|
||||||
|
case "m":
|
||||||
|
durationUnit = time.Minute
|
||||||
|
case "h":
|
||||||
|
durationUnit = time.Hour
|
||||||
|
case "d":
|
||||||
|
durationUnit = 24 * time.Hour
|
||||||
|
case "w":
|
||||||
|
durationUnit = 24 * 7 * time.Hour
|
||||||
|
default:
|
||||||
|
err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
d = time.Duration(durationFactor) * durationUnit
|
||||||
|
return
|
||||||
|
}
|
||||||
@@ -21,7 +21,7 @@ jobs:
|
|||||||
clients: {
|
clients: {
|
||||||
"10.0.0.1":"foo"
|
"10.0.0.1":"foo"
|
||||||
}
|
}
|
||||||
root_fs: zroot/foo
|
root_fs: zoot/foo
|
||||||
`
|
`
|
||||||
_, err := ParseConfigBytes([]byte(jobdef))
|
_, err := ParseConfigBytes([]byte(jobdef))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -70,9 +70,9 @@ func TestPrometheusMonitoring(t *testing.T) {
|
|||||||
global:
|
global:
|
||||||
monitoring:
|
monitoring:
|
||||||
- type: prometheus
|
- type: prometheus
|
||||||
listen: ':9811'
|
listen: ':9091'
|
||||||
`)
|
`)
|
||||||
assert.Equal(t, ":9811", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
|
assert.Equal(t, ":9091", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSyslogLoggingOutletFacility(t *testing.T) {
|
func TestSyslogLoggingOutletFacility(t *testing.T) {
|
||||||
@@ -2,8 +2,16 @@ package config
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestConfigEmptyFails(t *testing.T) {
|
||||||
|
conf, err := testConfig(t, "\n")
|
||||||
|
assert.Nil(t, conf)
|
||||||
|
assert.Error(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
func TestJobsOnlyWorks(t *testing.T) {
|
func TestJobsOnlyWorks(t *testing.T) {
|
||||||
testValidConfig(t, `
|
testValidConfig(t, `
|
||||||
jobs:
|
jobs:
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSnapshotting(t *testing.T) {
|
||||||
|
tmpl := `
|
||||||
|
jobs:
|
||||||
|
- name: foo
|
||||||
|
type: push
|
||||||
|
connect:
|
||||||
|
type: local
|
||||||
|
listener_name: foo
|
||||||
|
client_identity: bar
|
||||||
|
filesystems: {"<": true}
|
||||||
|
%s
|
||||||
|
pruning:
|
||||||
|
keep_sender:
|
||||||
|
- type: last_n
|
||||||
|
count: 10
|
||||||
|
keep_receiver:
|
||||||
|
- type: last_n
|
||||||
|
count: 10
|
||||||
|
`
|
||||||
|
manual := `
|
||||||
|
snapshotting:
|
||||||
|
type: manual
|
||||||
|
`
|
||||||
|
periodic := `
|
||||||
|
snapshotting:
|
||||||
|
type: periodic
|
||||||
|
prefix: zrepl_
|
||||||
|
interval: 10m
|
||||||
|
`
|
||||||
|
|
||||||
|
hooks := `
|
||||||
|
snapshotting:
|
||||||
|
type: periodic
|
||||||
|
prefix: zrepl_
|
||||||
|
interval: 10m
|
||||||
|
hooks:
|
||||||
|
- type: command
|
||||||
|
path: /tmp/path/to/command
|
||||||
|
- type: command
|
||||||
|
path: /tmp/path/to/command
|
||||||
|
filesystems: { "zroot<": true, "<": false }
|
||||||
|
- type: postgres-checkpoint
|
||||||
|
dsn: "host=localhost port=5432 user=postgres sslmode=disable"
|
||||||
|
filesystems: {
|
||||||
|
"tank/postgres/data11": true
|
||||||
|
}
|
||||||
|
- type: mysql-lock-tables
|
||||||
|
dsn: "root@tcp(localhost)/"
|
||||||
|
filesystems: {
|
||||||
|
"tank/mysql": true
|
||||||
|
}
|
||||||
|
`
|
||||||
|
|
||||||
|
fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
|
||||||
|
var c *Config
|
||||||
|
|
||||||
|
t.Run("manual", func(t *testing.T) {
|
||||||
|
c = testValidConfig(t, fillSnapshotting(manual))
|
||||||
|
snm := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingManual)
|
||||||
|
assert.Equal(t, "manual", snm.Type)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("periodic", func(t *testing.T) {
|
||||||
|
c = testValidConfig(t, fillSnapshotting(periodic))
|
||||||
|
snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
|
||||||
|
assert.Equal(t, "periodic", snp.Type)
|
||||||
|
assert.Equal(t, 10*time.Minute, snp.Interval)
|
||||||
|
assert.Equal(t, "zrepl_", snp.Prefix)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("hooks", func(t *testing.T) {
|
||||||
|
c = testValidConfig(t, fillSnapshotting(hooks))
|
||||||
|
hs := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic).Hooks
|
||||||
|
assert.Equal(t, hs[0].Ret.(*HookCommand).Filesystems["<"], true)
|
||||||
|
assert.Equal(t, hs[1].Ret.(*HookCommand).Filesystems["zroot<"], true)
|
||||||
|
assert.Equal(t, hs[2].Ret.(*HookPostgresCheckpoint).Filesystems["tank/postgres/data11"], true)
|
||||||
|
assert.Equal(t, hs[3].Ret.(*HookMySQLLockTables).Filesystems["tank/mysql"], true)
|
||||||
|
})
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"text/template"
|
||||||
|
|
||||||
|
"github.com/kr/pretty"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
|
||||||
|
paths, err := filepath.Glob("./samples/*")
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("glob failed: %+v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, p := range paths {
|
||||||
|
|
||||||
|
if path.Ext(p) != ".yml" {
|
||||||
|
t.Logf("skipping file %s", p)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run(p, func(t *testing.T) {
|
||||||
|
c, err := ParseConfig(p)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("error parsing %s:\n%+v", p, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("file: %s", p)
|
||||||
|
t.Log(pretty.Sprint(c))
|
||||||
|
})
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// template must be a template/text template with a single '{{ . }}' as placehodler for val
|
||||||
|
//nolint[:deadcode,unused]
|
||||||
|
func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
|
||||||
|
tmp, err := template.New("master").Parse(tmpl)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
err = tmp.Execute(&buf, val)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return testValidConfig(t, buf.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
func testValidConfig(t *testing.T, input string) *Config {
|
||||||
|
t.Helper()
|
||||||
|
conf, err := testConfig(t, input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, conf)
|
||||||
|
return conf
|
||||||
|
}
|
||||||
|
|
||||||
|
func testConfig(t *testing.T, input string) (*Config, error) {
|
||||||
|
t.Helper()
|
||||||
|
return ParseConfigBytes([]byte(input))
|
||||||
|
}
|
||||||
|
|
||||||
|
func trimSpaceEachLineAndPad(s, pad string) string {
|
||||||
|
var out strings.Builder
|
||||||
|
scan := bufio.NewScanner(strings.NewReader(s))
|
||||||
|
for scan.Scan() {
|
||||||
|
fmt.Fprintf(&out, "%s%s\n", pad, bytes.TrimSpace(scan.Bytes()))
|
||||||
|
}
|
||||||
|
return out.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimSpaceEachLineAndPad(t *testing.T) {
|
||||||
|
foo := `
|
||||||
|
foo
|
||||||
|
bar baz
|
||||||
|
`
|
||||||
|
assert.Equal(t, " \n foo\n bar baz\n \n", trimSpaceEachLineAndPad(foo, " "))
|
||||||
|
}
|
||||||
@@ -37,7 +37,7 @@ func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
intervals, err := ParseRetentionIntervalSpec(in)
|
intervals, err := parseRetentionGridIntervalsString(in)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -64,7 +64,7 @@ func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval,
|
|||||||
return nil, fmt.Errorf("contains factor <= 0")
|
return nil, fmt.Errorf("contains factor <= 0")
|
||||||
}
|
}
|
||||||
|
|
||||||
duration, err := parsePositiveDuration(comps[2])
|
duration, err := parsePostitiveDuration(comps[2])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -102,7 +102,7 @@ func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval,
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseRetentionIntervalSpec(s string) (intervals []RetentionInterval, err error) {
|
func parseRetentionGridIntervalsString(s string) (intervals []RetentionInterval, err error) {
|
||||||
|
|
||||||
ges := strings.Split(s, "|")
|
ges := strings.Split(s, "|")
|
||||||
intervals = make([]RetentionInterval, 0, 7*len(ges))
|
intervals = make([]RetentionInterval, 0, 7*len(ges))
|
||||||
@@ -26,6 +26,7 @@ jobs:
|
|||||||
- type: last_n
|
- type: last_n
|
||||||
count: 10
|
count: 10
|
||||||
keep_receiver:
|
keep_receiver:
|
||||||
|
- type: most_recent_common_snapshot
|
||||||
- type: grid
|
- type: grid
|
||||||
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
|
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
|
||||||
regex: "zrepl_.*"
|
regex: "zrepl_.*"
|
||||||
@@ -9,7 +9,7 @@ jobs:
|
|||||||
port: 22
|
port: 22
|
||||||
identity_file: /etc/zrepl/ssh/identity
|
identity_file: /etc/zrepl/ssh/identity
|
||||||
options: # optional, default [], `-o` arguments passed to ssh
|
options: # optional, default [], `-o` arguments passed to ssh
|
||||||
- "Compression=yes"
|
- "Compression=on"
|
||||||
root_fs: "pool2/backup_servers"
|
root_fs: "pool2/backup_servers"
|
||||||
interval: 10m
|
interval: 10m
|
||||||
pruning:
|
pruning:
|
||||||
@@ -10,8 +10,6 @@ jobs:
|
|||||||
address: "backup-server.foo.bar:8888"
|
address: "backup-server.foo.bar:8888"
|
||||||
snapshotting:
|
snapshotting:
|
||||||
type: manual
|
type: manual
|
||||||
send:
|
|
||||||
encrypted: false
|
|
||||||
pruning:
|
pruning:
|
||||||
keep_sender:
|
keep_sender:
|
||||||
- type: not_replicated
|
- type: not_replicated
|
||||||
@@ -5,11 +5,7 @@ jobs:
|
|||||||
type: tcp
|
type: tcp
|
||||||
listen: "0.0.0.0:8888"
|
listen: "0.0.0.0:8888"
|
||||||
clients: {
|
clients: {
|
||||||
"192.168.122.123" : "mysql01",
|
"192.168.122.123" : "client1"
|
||||||
"192.168.122.42" : "mx01",
|
|
||||||
"2001:0db8:85a3::8a2e:0370:7334": "gateway",
|
|
||||||
"10.23.42.0/24": "cluster-*",
|
|
||||||
"fde4:8dba:82e1::/64": "san-*",
|
|
||||||
}
|
}
|
||||||
filesystems: {
|
filesystems: {
|
||||||
"<": true,
|
"<": true,
|
||||||
@@ -8,20 +8,17 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
"github.com/zrepl/zrepl/daemon/job"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/nethelpers"
|
"github.com/zrepl/zrepl/daemon/nethelpers"
|
||||||
"github.com/zrepl/zrepl/internal/endpoint"
|
"github.com/zrepl/zrepl/logger"
|
||||||
"github.com/zrepl/zrepl/internal/logger"
|
"github.com/zrepl/zrepl/util/envconst"
|
||||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
"github.com/zrepl/zrepl/version"
|
||||||
"github.com/zrepl/zrepl/internal/version"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
|
||||||
"github.com/zrepl/zrepl/internal/zfs/zfscmd"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type controlJob struct {
|
type controlJob struct {
|
||||||
@@ -47,8 +44,6 @@ func (j *controlJob) Status() *job.Status { return &job.Status{Type: job.TypeInt
|
|||||||
|
|
||||||
func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
|
func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
|
||||||
|
|
||||||
func (j *controlJob) SenderConfig() *endpoint.SenderConfig { return nil }
|
|
||||||
|
|
||||||
var promControl struct {
|
var promControl struct {
|
||||||
requestBegin *prometheus.CounterVec
|
requestBegin *prometheus.CounterVec
|
||||||
requestFinished *prometheus.HistogramVec
|
requestFinished *prometheus.HistogramVec
|
||||||
@@ -66,7 +61,7 @@ func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) {
|
|||||||
Namespace: "zrepl",
|
Namespace: "zrepl",
|
||||||
Subsystem: "control",
|
Subsystem: "control",
|
||||||
Name: "request_finished",
|
Name: "request_finished",
|
||||||
Help: "time it took a request to finish",
|
Help: "time it took a request to finih",
|
||||||
Buckets: []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3, 10e-3, 100e-3, 200e-3, 400e-3, 800e-3, 1, 10, 20},
|
Buckets: []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3, 10e-3, 100e-3, 200e-3, 400e-3, 800e-3, 1, 10, 20},
|
||||||
}, []string{"endpoint"})
|
}, []string{"endpoint"})
|
||||||
registerer.MustRegister(promControl.requestBegin)
|
registerer.MustRegister(promControl.requestBegin)
|
||||||
@@ -119,16 +114,7 @@ func (j *controlJob) Run(ctx context.Context) {
|
|||||||
mux.Handle(ControlJobEndpointStatus,
|
mux.Handle(ControlJobEndpointStatus,
|
||||||
// don't log requests to status endpoint, too spammy
|
// don't log requests to status endpoint, too spammy
|
||||||
jsonResponder{log, func() (interface{}, error) {
|
jsonResponder{log, func() (interface{}, error) {
|
||||||
jobs := j.jobs.status()
|
s := j.jobs.status()
|
||||||
globalZFS := zfscmd.GetReport()
|
|
||||||
envconstReport := envconst.GetReport()
|
|
||||||
s := Status{
|
|
||||||
Jobs: jobs,
|
|
||||||
Global: GlobalStatus{
|
|
||||||
ZFSCmds: globalZFS,
|
|
||||||
Envconst: envconstReport,
|
|
||||||
OsEnviron: os.Environ(),
|
|
||||||
}}
|
|
||||||
return s, nil
|
return s, nil
|
||||||
}})
|
}})
|
||||||
|
|
||||||
@@ -158,8 +144,8 @@ func (j *controlJob) Run(ctx context.Context) {
|
|||||||
server := http.Server{
|
server := http.Server{
|
||||||
Handler: mux,
|
Handler: mux,
|
||||||
// control socket is local, 1s timeout should be more than sufficient, even on a loaded system
|
// control socket is local, 1s timeout should be more than sufficient, even on a loaded system
|
||||||
WriteTimeout: envconst.Duration("ZREPL_DAEMON_CONTROL_SERVER_WRITE_TIMEOUT", 1*time.Second),
|
WriteTimeout: 1 * time.Second,
|
||||||
ReadTimeout: envconst.Duration("ZREPL_DAEMON_CONTROL_SERVER_READ_TIMEOUT", 1*time.Second),
|
ReadTimeout: 1 * time.Second,
|
||||||
}
|
}
|
||||||
|
|
||||||
outer:
|
outer:
|
||||||
@@ -261,7 +247,7 @@ func (j jsonRequestResponder) ServeHTTP(w http.ResponseWriter, r *http.Request)
|
|||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
encodeErr := json.NewEncoder(&buf).Encode(res)
|
encodeErr := json.NewEncoder(&buf).Encode(res)
|
||||||
if encodeErr != nil {
|
if encodeErr != nil {
|
||||||
j.log.WithError(producerErr).Error("control handler json marshal error")
|
j.log.WithError(producerErr).Error("control handler json marhsal error")
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
w.WriteHeader(http.StatusInternalServerError)
|
||||||
_, err := io.WriteString(w, encodeErr.Error())
|
_, err := io.WriteString(w, encodeErr.Error())
|
||||||
logIoErr(err)
|
logIoErr(err)
|
||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
@@ -12,22 +13,18 @@ import (
|
|||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/endpoint"
|
"github.com/zrepl/zrepl/daemon/job"
|
||||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
"github.com/zrepl/zrepl/daemon/job/reset"
|
||||||
|
"github.com/zrepl/zrepl/daemon/job/wakeup"
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/daemon/logging"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
"github.com/zrepl/zrepl/logger"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/job/reset"
|
"github.com/zrepl/zrepl/version"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/job/wakeup"
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
|
||||||
"github.com/zrepl/zrepl/internal/logger"
|
|
||||||
"github.com/zrepl/zrepl/internal/version"
|
|
||||||
"github.com/zrepl/zrepl/internal/zfs/zfscmd"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func Run(ctx context.Context, conf *config.Config) error {
|
func Run(conf *config.Config) error {
|
||||||
ctx, cancel := context.WithCancel(ctx)
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
|
||||||
defer cancel()
|
defer cancel()
|
||||||
sigChan := make(chan os.Signal, 1)
|
sigChan := make(chan os.Signal, 1)
|
||||||
@@ -41,9 +38,8 @@ func Run(ctx context.Context, conf *config.Config) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, "cannot build logging from config")
|
return errors.Wrap(err, "cannot build logging from config")
|
||||||
}
|
}
|
||||||
outlets.Add(newPrometheusLogOutlet(), logger.Debug)
|
|
||||||
|
|
||||||
confJobs, err := job.JobsFromConfig(conf, config.ParseFlagsNone)
|
confJobs, err := job.JobsFromConfig(conf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, "cannot build jobs from config")
|
return errors.Wrap(err, "cannot build jobs from config")
|
||||||
}
|
}
|
||||||
@@ -51,23 +47,14 @@ func Run(ctx context.Context, conf *config.Config) error {
|
|||||||
log := logger.NewLogger(outlets, 1*time.Second)
|
log := logger.NewLogger(outlets, 1*time.Second)
|
||||||
log.Info(version.NewZreplVersionInformation().String())
|
log.Info(version.NewZreplVersionInformation().String())
|
||||||
|
|
||||||
ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
|
|
||||||
trace.RegisterCallback(trace.Callback{
|
|
||||||
OnBegin: func(ctx context.Context) { logging.GetLogger(ctx, logging.SubsysTraceData).Debug("begin span") },
|
|
||||||
OnEnd: func(ctx context.Context, spanInfo trace.SpanInfo) {
|
|
||||||
logging.
|
|
||||||
GetLogger(ctx, logging.SubsysTraceData).
|
|
||||||
WithField("duration_s", spanInfo.EndedAt().Sub(spanInfo.StartedAt()).Seconds()).
|
|
||||||
Debug("finished span " + spanInfo.TaskAndSpanStack(trace.SpanStackKindAnnotation))
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
for _, job := range confJobs {
|
for _, job := range confJobs {
|
||||||
if config.IsInternalJobName(job.Name()) {
|
if IsInternalJobName(job.Name()) {
|
||||||
panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME
|
panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ctx = job.WithLogger(ctx, log)
|
||||||
|
|
||||||
jobs := newJobs()
|
jobs := newJobs()
|
||||||
|
|
||||||
// start control socket
|
// start control socket
|
||||||
@@ -89,17 +76,11 @@ func Run(ctx context.Context, conf *config.Config) error {
|
|||||||
return errors.Errorf("unknown monitoring job #%d (type %T)", i, v)
|
return errors.Errorf("unknown monitoring job #%d (type %T)", i, v)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrapf(err, "cannot build monitoring job #%d", i)
|
return errors.Wrapf(err, "cannot build monitorin gjob #%d", i)
|
||||||
}
|
}
|
||||||
jobs.start(ctx, job, true)
|
jobs.start(ctx, job, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
// register global (=non job-local) metrics
|
|
||||||
version.PrometheusRegister(prometheus.DefaultRegisterer)
|
|
||||||
zfscmd.RegisterMetrics(prometheus.DefaultRegisterer)
|
|
||||||
trace.RegisterMetrics(prometheus.DefaultRegisterer)
|
|
||||||
endpoint.RegisterMetrics(prometheus.DefaultRegisterer)
|
|
||||||
|
|
||||||
log.Info("starting daemon")
|
log.Info("starting daemon")
|
||||||
|
|
||||||
// start regular jobs
|
// start regular jobs
|
||||||
@@ -113,8 +94,6 @@ func Run(ctx context.Context, conf *config.Config) error {
|
|||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
log.WithError(ctx.Err()).Info("context finished")
|
log.WithError(ctx.Err()).Info("context finished")
|
||||||
}
|
}
|
||||||
log.Info("waiting for jobs to finish")
|
|
||||||
<-jobs.wait()
|
|
||||||
log.Info("daemon exiting")
|
log.Info("daemon exiting")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -137,26 +116,18 @@ func newJobs() *jobs {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
logJobField string = "job"
|
||||||
|
)
|
||||||
|
|
||||||
func (s *jobs) wait() <-chan struct{} {
|
func (s *jobs) wait() <-chan struct{} {
|
||||||
ch := make(chan struct{})
|
ch := make(chan struct{})
|
||||||
go func() {
|
go func() {
|
||||||
s.wg.Wait()
|
s.wg.Wait()
|
||||||
close(ch)
|
|
||||||
}()
|
}()
|
||||||
return ch
|
return ch
|
||||||
}
|
}
|
||||||
|
|
||||||
type Status struct {
|
|
||||||
Jobs map[string]*job.Status
|
|
||||||
Global GlobalStatus
|
|
||||||
}
|
|
||||||
|
|
||||||
type GlobalStatus struct {
|
|
||||||
ZFSCmds *zfscmd.Report
|
|
||||||
Envconst *envconst.Report
|
|
||||||
OsEnviron []string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *jobs) status() map[string]*job.Status {
|
func (s *jobs) status() map[string]*job.Status {
|
||||||
s.m.RLock()
|
s.m.RLock()
|
||||||
defer s.m.RUnlock()
|
defer s.m.RUnlock()
|
||||||
@@ -210,19 +181,22 @@ const (
|
|||||||
jobNameControl = "_control"
|
jobNameControl = "_control"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func IsInternalJobName(s string) bool {
|
||||||
|
return strings.HasPrefix(s, "_")
|
||||||
|
}
|
||||||
|
|
||||||
func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
|
func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
|
||||||
s.m.Lock()
|
s.m.Lock()
|
||||||
defer s.m.Unlock()
|
defer s.m.Unlock()
|
||||||
|
|
||||||
ctx = logging.WithInjectedField(ctx, logging.JobField, j.Name())
|
jobLog := job.GetLogger(ctx).
|
||||||
|
WithField(logJobField, j.Name()).
|
||||||
|
WithOutlet(newPrometheusLogOutlet(j.Name()), logger.Debug)
|
||||||
jobName := j.Name()
|
jobName := j.Name()
|
||||||
|
if !internal && IsInternalJobName(jobName) {
|
||||||
// package `config` enforces these with clean errors, these are just assertions
|
|
||||||
if !internal && config.IsInternalJobName(jobName) {
|
|
||||||
panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName))
|
panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName))
|
||||||
}
|
}
|
||||||
if internal && !config.IsInternalJobName(jobName) {
|
if internal && !IsInternalJobName(jobName) {
|
||||||
panic(fmt.Sprintf("internal job does not use internal job name %s", jobName))
|
panic(fmt.Sprintf("internal job does not use internal job name %s", jobName))
|
||||||
}
|
}
|
||||||
if _, ok := s.jobs[jobName]; ok {
|
if _, ok := s.jobs[jobName]; ok {
|
||||||
@@ -232,7 +206,7 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
|
|||||||
j.RegisterMetrics(prometheus.DefaultRegisterer)
|
j.RegisterMetrics(prometheus.DefaultRegisterer)
|
||||||
|
|
||||||
s.jobs[jobName] = j
|
s.jobs[jobName] = j
|
||||||
ctx = zfscmd.WithJobID(ctx, j.Name())
|
ctx = job.WithLogger(ctx, jobLog)
|
||||||
ctx, wakeup := wakeup.Context(ctx)
|
ctx, wakeup := wakeup.Context(ctx)
|
||||||
ctx, resetFunc := reset.Context(ctx)
|
ctx, resetFunc := reset.Context(ctx)
|
||||||
s.wakeups[jobName] = wakeup
|
s.wakeups[jobName] = wakeup
|
||||||
@@ -241,8 +215,8 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
|
|||||||
s.wg.Add(1)
|
s.wg.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
defer s.wg.Done()
|
defer s.wg.Done()
|
||||||
job.GetLogger(ctx).Info("starting job")
|
jobLog.Info("starting job")
|
||||||
defer job.GetLogger(ctx).Info("job exited")
|
defer jobLog.Info("job exited")
|
||||||
j.Run(ctx)
|
j.Run(ctx)
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
@@ -6,8 +6,8 @@ import (
|
|||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/endpoint"
|
"github.com/zrepl/zrepl/endpoint"
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
type DatasetMapFilter struct {
|
type DatasetMapFilter struct {
|
||||||
@@ -160,16 +160,8 @@ func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m DatasetMapFilter) UserSpecifiedDatasets() (datasets zfs.UserSpecifiedDatasetsSet) {
|
|
||||||
datasets = make(zfs.UserSpecifiedDatasetsSet)
|
|
||||||
for i := range m.entries {
|
|
||||||
datasets[m.entries[i].path.ToString()] = true
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Construct a new filter-only DatasetMapFilter from a mapping
|
// Construct a new filter-only DatasetMapFilter from a mapping
|
||||||
// The new filter allows exactly those paths that were not forbidden by the mapping.
|
// The new filter allows excactly those paths that were not forbidden by the mapping.
|
||||||
func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
|
func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
|
||||||
|
|
||||||
if m.filterMode {
|
if m.filterMode {
|
||||||
@@ -1,11 +1,11 @@
|
|||||||
package filters
|
package filters
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
func TestDatasetMapFilter(t *testing.T) {
|
func TestDatasetMapFilter(t *testing.T) {
|
||||||
|
|
||||||
type testCase struct {
|
type testCase struct {
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
package filters
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/zrepl/zrepl/zfs"
|
||||||
|
)
|
||||||
|
|
||||||
|
type AnyFSVFilter struct{}
|
||||||
|
|
||||||
|
func NewAnyFSVFilter() AnyFSVFilter {
|
||||||
|
return AnyFSVFilter{}
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ zfs.FilesystemVersionFilter = AnyFSVFilter{}
|
||||||
|
|
||||||
|
func (AnyFSVFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type PrefixFilter struct {
|
||||||
|
prefix string
|
||||||
|
fstype zfs.VersionType
|
||||||
|
fstypeSet bool // optionals anyone?
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ zfs.FilesystemVersionFilter = &PrefixFilter{}
|
||||||
|
|
||||||
|
func NewPrefixFilter(prefix string) *PrefixFilter {
|
||||||
|
return &PrefixFilter{prefix: prefix}
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewTypedPrefixFilter(prefix string, versionType zfs.VersionType) *PrefixFilter {
|
||||||
|
return &PrefixFilter{prefix, versionType, true}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *PrefixFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) {
|
||||||
|
fstypeMatches := (!f.fstypeSet || t == f.fstype)
|
||||||
|
prefixMatches := strings.HasPrefix(name, f.prefix)
|
||||||
|
return fstypeMatches && prefixMatches, nil
|
||||||
|
}
|
||||||
@@ -3,8 +3,8 @@ package hooks
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
type List []Hook
|
type List []Hook
|
||||||
@@ -5,7 +5,7 @@
|
|||||||
//
|
//
|
||||||
// This package also provides all supported hook type implementations and abstractions around them.
|
// This package also provides all supported hook type implementations and abstractions around them.
|
||||||
//
|
//
|
||||||
// # Use For Other Kinds Of ExpectStepReports
|
// Use For Other Kinds Of ExpectStepReports
|
||||||
//
|
//
|
||||||
// This package REQUIRES REFACTORING before it can be used for other activities than snapshots, e.g. pre- and post-replication:
|
// This package REQUIRES REFACTORING before it can be used for other activities than snapshots, e.g. pre- and post-replication:
|
||||||
//
|
//
|
||||||
@@ -15,7 +15,7 @@
|
|||||||
// The hook implementations should move out of this package.
|
// The hook implementations should move out of this package.
|
||||||
// However, there is a lot of tight coupling which to untangle isn't worth it ATM.
|
// However, there is a lot of tight coupling which to untangle isn't worth it ATM.
|
||||||
//
|
//
|
||||||
// # How This Package Is Used By Package Snapper
|
// How This Package Is Used By Package Snapper
|
||||||
//
|
//
|
||||||
// Deserialize a config.List using ListFromConfig().
|
// Deserialize a config.List using ListFromConfig().
|
||||||
// Then it MUST filter the list to only contain hooks for a particular filesystem using
|
// Then it MUST filter the list to only contain hooks for a particular filesystem using
|
||||||
@@ -30,4 +30,5 @@
|
|||||||
// Command hooks make it available in the environment variable ZREPL_DRYRUN.
|
// Command hooks make it available in the environment variable ZREPL_DRYRUN.
|
||||||
//
|
//
|
||||||
// Plan.Report() can be called while Plan.Run() is executing to give an overview of plan execution progress (future use in "zrepl status").
|
// Plan.Report() can be called while Plan.Run() is executing to give an overview of plan execution progress (future use in "zrepl status").
|
||||||
|
//
|
||||||
package hooks
|
package hooks
|
||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Re-export type here so that
|
// Re-export type here so that
|
||||||
@@ -6,17 +6,29 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
"github.com/zrepl/zrepl/logger"
|
||||||
"github.com/zrepl/zrepl/internal/logger"
|
"github.com/zrepl/zrepl/util/envconst"
|
||||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
)
|
||||||
|
|
||||||
|
type contextKey int
|
||||||
|
|
||||||
|
const (
|
||||||
|
contextKeyLog contextKey = 0
|
||||||
)
|
)
|
||||||
|
|
||||||
type Logger = logger.Logger
|
type Logger = logger.Logger
|
||||||
|
|
||||||
|
func WithLogger(ctx context.Context, log Logger) context.Context {
|
||||||
|
return context.WithValue(ctx, contextKeyLog, log)
|
||||||
|
}
|
||||||
|
|
||||||
func GetLogger(ctx context.Context) Logger { return getLogger(ctx) }
|
func GetLogger(ctx context.Context) Logger { return getLogger(ctx) }
|
||||||
|
|
||||||
func getLogger(ctx context.Context) Logger {
|
func getLogger(ctx context.Context) Logger {
|
||||||
return logging.GetLogger(ctx, logging.SubsysHooks)
|
if log, ok := ctx.Value(contextKeyLog).(Logger); ok {
|
||||||
|
return log
|
||||||
|
}
|
||||||
|
return logger.NewNullLogger()
|
||||||
}
|
}
|
||||||
|
|
||||||
const MAX_HOOK_LOG_SIZE_DEFAULT int = 1 << 20
|
const MAX_HOOK_LOG_SIZE_DEFAULT int = 1 << 20
|
||||||
@@ -4,8 +4,8 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
"github.com/zrepl/zrepl/daemon/filters"
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
type HookJobCallback func(ctx context.Context) error
|
type HookJobCallback func(ctx context.Context) error
|
||||||
@@ -12,11 +12,11 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
"github.com/zrepl/zrepl/daemon/filters"
|
||||||
"github.com/zrepl/zrepl/internal/logger"
|
"github.com/zrepl/zrepl/logger"
|
||||||
"github.com/zrepl/zrepl/internal/util/circlog"
|
"github.com/zrepl/zrepl/util/circlog"
|
||||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
"github.com/zrepl/zrepl/util/envconst"
|
||||||
)
|
)
|
||||||
|
|
||||||
type HookEnvVar string
|
type HookEnvVar string
|
||||||
@@ -93,14 +93,7 @@ func (r *CommandHookReport) String() string {
|
|||||||
cmdLine.WriteString(fmt.Sprintf("%s'%s'", sep, a))
|
cmdLine.WriteString(fmt.Sprintf("%s'%s'", sep, a))
|
||||||
}
|
}
|
||||||
|
|
||||||
var msg string
|
return fmt.Sprintf("command hook invocation: \"%s\"", cmdLine.String()) // no %q to make copy-pastable
|
||||||
if r.Err == nil {
|
|
||||||
msg = "command hook"
|
|
||||||
} else {
|
|
||||||
msg = fmt.Sprintf("command hook failed with %q", r.Err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Sprintf("%s: \"%s\"", msg, cmdLine.String()) // no %q to make copy-pastable
|
|
||||||
}
|
}
|
||||||
func (r *CommandHookReport) Error() string {
|
func (r *CommandHookReport) Error() string {
|
||||||
if r.Err == nil {
|
if r.Err == nil {
|
||||||
+11
-14
@@ -12,27 +12,24 @@ import (
|
|||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
"github.com/zrepl/zrepl/daemon/filters"
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Hook to implement the following recommmendation from MySQL docs
|
|
||||||
// https://dev.mysql.com/doc/mysql-backup-excerpt/5.7/en/backup-methods.html
|
// https://dev.mysql.com/doc/mysql-backup-excerpt/5.7/en/backup-methods.html
|
||||||
//
|
//
|
||||||
// Making Backups Using a File System Snapshot:
|
// Making Backups Using a File System Snapshot:
|
||||||
//
|
//
|
||||||
// If you are using a Veritas file system, you can make a backup like this:
|
// If you are using a Veritas file system, you can make a backup like this:
|
||||||
//
|
//
|
||||||
// From a client program, execute FLUSH TABLES WITH READ LOCK.
|
// From a client program, execute FLUSH TABLES WITH READ LOCK.
|
||||||
// From another shell, execute mount vxfs snapshot.
|
// From another shell, execute mount vxfs snapshot.
|
||||||
// From the first client, execute UNLOCK TABLES.
|
// From the first client, execute UNLOCK TABLES.
|
||||||
// Copy files from the snapshot.
|
// Copy files from the snapshot.
|
||||||
// Unmount the snapshot.
|
// Unmount the snapshot.
|
||||||
//
|
//
|
||||||
// Similar snapshot capabilities may be available in other file systems, such as LVM or ZFS.
|
// Similar snapshot capabilities may be available in other file systems, such as LVM or ZFS.
|
||||||
//
|
|
||||||
|
|
||||||
type MySQLLockTables struct {
|
type MySQLLockTables struct {
|
||||||
errIsFatal bool
|
errIsFatal bool
|
||||||
connector sqldriver.Connector
|
connector sqldriver.Connector
|
||||||
+3
-3
@@ -10,9 +10,9 @@ import (
|
|||||||
"github.com/lib/pq"
|
"github.com/lib/pq"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
"github.com/zrepl/zrepl/daemon/filters"
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
type PgChkptHook struct {
|
type PgChkptHook struct {
|
||||||
@@ -11,13 +11,10 @@ import (
|
|||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
"github.com/zrepl/zrepl/config"
|
||||||
|
"github.com/zrepl/zrepl/daemon/hooks"
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/logger"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/hooks"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
|
||||||
"github.com/zrepl/zrepl/internal/logger"
|
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type comparisonAssertionFunc func(require.TestingT, interface{}, interface{}, ...interface{})
|
type comparisonAssertionFunc func(require.TestingT, interface{}, interface{}, ...interface{})
|
||||||
@@ -72,9 +69,6 @@ func curry(f comparisonAssertionFunc, expected interface{}, right bool) (ret val
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHooks(t *testing.T) {
|
func TestHooks(t *testing.T) {
|
||||||
ctx, end := trace.WithTaskFromStack(context.Background())
|
|
||||||
defer end()
|
|
||||||
|
|
||||||
testFSName := "testpool/testdataset"
|
testFSName := "testpool/testdataset"
|
||||||
testSnapshotName := "testsnap"
|
testSnapshotName := "testsnap"
|
||||||
|
|
||||||
@@ -161,7 +155,7 @@ jobs:
|
|||||||
ExpectedEdge: hooks.Pre,
|
ExpectedEdge: hooks.Pre,
|
||||||
ExpectStatus: hooks.StepErr,
|
ExpectStatus: hooks.StepErr,
|
||||||
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
|
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
|
||||||
ErrorTest: regexpTest("^command hook failed.*exit status 1$"),
|
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
|
||||||
},
|
},
|
||||||
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
|
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
|
||||||
expectStep{
|
expectStep{
|
||||||
@@ -185,7 +179,7 @@ jobs:
|
|||||||
ExpectedEdge: hooks.Pre,
|
ExpectedEdge: hooks.Pre,
|
||||||
ExpectStatus: hooks.StepErr,
|
ExpectStatus: hooks.StepErr,
|
||||||
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
|
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
|
||||||
ErrorTest: regexpTest("^command hook failed.*exit status 1$"),
|
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
|
||||||
},
|
},
|
||||||
expectStep{
|
expectStep{
|
||||||
ExpectedEdge: hooks.Pre,
|
ExpectedEdge: hooks.Pre,
|
||||||
@@ -234,7 +228,7 @@ jobs:
|
|||||||
ExpectedEdge: hooks.Post,
|
ExpectedEdge: hooks.Post,
|
||||||
ExpectStatus: hooks.StepErr,
|
ExpectStatus: hooks.StepErr,
|
||||||
OutputTest: containsTest(fmt.Sprintf("TEST ERROR post_testing %s@%s", testFSName, testSnapshotName)),
|
OutputTest: containsTest(fmt.Sprintf("TEST ERROR post_testing %s@%s", testFSName, testSnapshotName)),
|
||||||
ErrorTest: regexpTest("^command hook failed.*exit status 1$"),
|
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -267,7 +261,7 @@ jobs:
|
|||||||
ExpectedEdge: hooks.Pre,
|
ExpectedEdge: hooks.Pre,
|
||||||
ExpectStatus: hooks.StepErr,
|
ExpectStatus: hooks.StepErr,
|
||||||
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
|
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
|
||||||
ErrorTest: regexpTest("^command hook failed.*exit status 1$"),
|
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
|
||||||
},
|
},
|
||||||
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
|
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
|
||||||
expectStep{
|
expectStep{
|
||||||
@@ -295,7 +289,7 @@ jobs:
|
|||||||
ExpectedEdge: hooks.Pre,
|
ExpectedEdge: hooks.Pre,
|
||||||
ExpectStatus: hooks.StepErr,
|
ExpectStatus: hooks.StepErr,
|
||||||
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
|
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
|
||||||
ErrorTest: regexpTest("^command hook failed.*exit status 1$"),
|
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
|
||||||
},
|
},
|
||||||
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
|
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
|
||||||
expectStep{
|
expectStep{
|
||||||
@@ -424,8 +418,9 @@ jobs:
|
|||||||
|
|
||||||
cbReached = false
|
cbReached = false
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
if testing.Verbose() && !tt.SuppressOutput {
|
if testing.Verbose() && !tt.SuppressOutput {
|
||||||
ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
|
ctx = hooks.WithLogger(ctx, log)
|
||||||
}
|
}
|
||||||
plan.Run(ctx, false)
|
plan.Run(ctx, false)
|
||||||
report := plan.Report()
|
report := plan.Report()
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
// Code generated by "enumer -type=StepStatus -trimprefix=Step"; DO NOT EDIT.
|
// Code generated by "enumer -type=StepStatus -trimprefix=Step"; DO NOT EDIT.
|
||||||
|
|
||||||
|
//
|
||||||
package hooks
|
package hooks
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -0,0 +1,434 @@
|
|||||||
|
package job
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
|
||||||
|
"github.com/zrepl/zrepl/config"
|
||||||
|
"github.com/zrepl/zrepl/daemon/filters"
|
||||||
|
"github.com/zrepl/zrepl/daemon/job/reset"
|
||||||
|
"github.com/zrepl/zrepl/daemon/job/wakeup"
|
||||||
|
"github.com/zrepl/zrepl/daemon/logging"
|
||||||
|
"github.com/zrepl/zrepl/daemon/pruner"
|
||||||
|
"github.com/zrepl/zrepl/daemon/snapper"
|
||||||
|
"github.com/zrepl/zrepl/endpoint"
|
||||||
|
"github.com/zrepl/zrepl/replication"
|
||||||
|
"github.com/zrepl/zrepl/replication/driver"
|
||||||
|
"github.com/zrepl/zrepl/replication/logic"
|
||||||
|
"github.com/zrepl/zrepl/replication/report"
|
||||||
|
"github.com/zrepl/zrepl/rpc"
|
||||||
|
"github.com/zrepl/zrepl/transport"
|
||||||
|
"github.com/zrepl/zrepl/transport/fromconfig"
|
||||||
|
"github.com/zrepl/zrepl/zfs"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ActiveSide struct {
|
||||||
|
mode activeMode
|
||||||
|
name string
|
||||||
|
connecter transport.Connecter
|
||||||
|
|
||||||
|
prunerFactory *pruner.PrunerFactory
|
||||||
|
|
||||||
|
promRepStateSecs *prometheus.HistogramVec // labels: state
|
||||||
|
promPruneSecs *prometheus.HistogramVec // labels: prune_side
|
||||||
|
promBytesReplicated *prometheus.CounterVec // labels: filesystem
|
||||||
|
|
||||||
|
tasksMtx sync.Mutex
|
||||||
|
tasks activeSideTasks
|
||||||
|
}
|
||||||
|
|
||||||
|
//go:generate enumer -type=ActiveSideState
|
||||||
|
type ActiveSideState int
|
||||||
|
|
||||||
|
const (
|
||||||
|
ActiveSideReplicating ActiveSideState = 1 << iota
|
||||||
|
ActiveSidePruneSender
|
||||||
|
ActiveSidePruneReceiver
|
||||||
|
ActiveSideDone // also errors
|
||||||
|
)
|
||||||
|
|
||||||
|
type activeSideTasks struct {
|
||||||
|
state ActiveSideState
|
||||||
|
|
||||||
|
// valid for state ActiveSideReplicating, ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
|
||||||
|
replicationReport driver.ReportFunc
|
||||||
|
replicationCancel context.CancelFunc
|
||||||
|
|
||||||
|
// valid for state ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
|
||||||
|
prunerSender, prunerReceiver *pruner.Pruner
|
||||||
|
|
||||||
|
// valid for state ActiveSidePruneReceiver, ActiveSideDone
|
||||||
|
prunerSenderCancel, prunerReceiverCancel context.CancelFunc
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *ActiveSide) updateTasks(u func(*activeSideTasks)) activeSideTasks {
|
||||||
|
a.tasksMtx.Lock()
|
||||||
|
defer a.tasksMtx.Unlock()
|
||||||
|
copy := a.tasks
|
||||||
|
if u == nil {
|
||||||
|
return copy
|
||||||
|
}
|
||||||
|
u(©)
|
||||||
|
a.tasks = copy
|
||||||
|
return copy
|
||||||
|
}
|
||||||
|
|
||||||
|
type activeMode interface {
|
||||||
|
ConnectEndpoints(rpcLoggers rpc.Loggers, connecter transport.Connecter)
|
||||||
|
DisconnectEndpoints()
|
||||||
|
SenderReceiver() (logic.Sender, logic.Receiver)
|
||||||
|
Type() Type
|
||||||
|
RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{})
|
||||||
|
SnapperReport() *snapper.Report
|
||||||
|
ResetConnectBackoff()
|
||||||
|
}
|
||||||
|
|
||||||
|
type modePush struct {
|
||||||
|
setupMtx sync.Mutex
|
||||||
|
sender *endpoint.Sender
|
||||||
|
receiver *rpc.Client
|
||||||
|
fsfilter endpoint.FSFilter
|
||||||
|
snapper *snapper.PeriodicOrManual
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *modePush) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Connecter) {
|
||||||
|
m.setupMtx.Lock()
|
||||||
|
defer m.setupMtx.Unlock()
|
||||||
|
if m.receiver != nil || m.sender != nil {
|
||||||
|
panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
|
||||||
|
}
|
||||||
|
m.sender = endpoint.NewSender(m.fsfilter)
|
||||||
|
m.receiver = rpc.NewClient(connecter, loggers)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *modePush) DisconnectEndpoints() {
|
||||||
|
m.setupMtx.Lock()
|
||||||
|
defer m.setupMtx.Unlock()
|
||||||
|
m.receiver.Close()
|
||||||
|
m.sender = nil
|
||||||
|
m.receiver = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *modePush) SenderReceiver() (logic.Sender, logic.Receiver) {
|
||||||
|
m.setupMtx.Lock()
|
||||||
|
defer m.setupMtx.Unlock()
|
||||||
|
return m.sender, m.receiver
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *modePush) Type() Type { return TypePush }
|
||||||
|
|
||||||
|
func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
|
||||||
|
m.snapper.Run(ctx, wakeUpCommon)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *modePush) SnapperReport() *snapper.Report {
|
||||||
|
return m.snapper.Report()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *modePush) ResetConnectBackoff() {
|
||||||
|
m.setupMtx.Lock()
|
||||||
|
defer m.setupMtx.Unlock()
|
||||||
|
if m.receiver != nil {
|
||||||
|
m.receiver.ResetConnectBackoff()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func modePushFromConfig(g *config.Global, in *config.PushJob) (*modePush, error) {
|
||||||
|
m := &modePush{}
|
||||||
|
fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "cannnot build filesystem filter")
|
||||||
|
}
|
||||||
|
m.fsfilter = fsf
|
||||||
|
|
||||||
|
if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
|
||||||
|
return nil, errors.Wrap(err, "cannot build snapper")
|
||||||
|
}
|
||||||
|
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type modePull struct {
|
||||||
|
setupMtx sync.Mutex
|
||||||
|
receiver *endpoint.Receiver
|
||||||
|
sender *rpc.Client
|
||||||
|
rootFS *zfs.DatasetPath
|
||||||
|
interval config.PositiveDurationOrManual
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *modePull) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Connecter) {
|
||||||
|
m.setupMtx.Lock()
|
||||||
|
defer m.setupMtx.Unlock()
|
||||||
|
if m.receiver != nil || m.sender != nil {
|
||||||
|
panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
|
||||||
|
}
|
||||||
|
m.receiver = endpoint.NewReceiver(m.rootFS, false)
|
||||||
|
m.sender = rpc.NewClient(connecter, loggers)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *modePull) DisconnectEndpoints() {
|
||||||
|
m.setupMtx.Lock()
|
||||||
|
defer m.setupMtx.Unlock()
|
||||||
|
m.sender.Close()
|
||||||
|
m.sender = nil
|
||||||
|
m.receiver = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *modePull) SenderReceiver() (logic.Sender, logic.Receiver) {
|
||||||
|
m.setupMtx.Lock()
|
||||||
|
defer m.setupMtx.Unlock()
|
||||||
|
return m.sender, m.receiver
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*modePull) Type() Type { return TypePull }
|
||||||
|
|
||||||
|
func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
|
||||||
|
if m.interval.Manual {
|
||||||
|
GetLogger(ctx).Info("manual pull configured, periodic pull disabled")
|
||||||
|
// "waiting for wakeups" is printed in common ActiveSide.do
|
||||||
|
return
|
||||||
|
}
|
||||||
|
t := time.NewTicker(m.interval.Interval)
|
||||||
|
defer t.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-t.C:
|
||||||
|
select {
|
||||||
|
case wakeUpCommon <- struct{}{}:
|
||||||
|
default:
|
||||||
|
GetLogger(ctx).
|
||||||
|
WithField("pull_interval", m.interval).
|
||||||
|
Warn("pull job took longer than pull interval")
|
||||||
|
wakeUpCommon <- struct{}{} // block anyways, to queue up the wakeup
|
||||||
|
}
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *modePull) SnapperReport() *snapper.Report {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *modePull) ResetConnectBackoff() {
|
||||||
|
m.setupMtx.Lock()
|
||||||
|
defer m.setupMtx.Unlock()
|
||||||
|
if m.sender != nil {
|
||||||
|
m.sender.ResetConnectBackoff()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func modePullFromConfig(g *config.Global, in *config.PullJob) (m *modePull, err error) {
|
||||||
|
m = &modePull{}
|
||||||
|
m.interval = in.Interval
|
||||||
|
|
||||||
|
m.rootFS, err = zfs.NewDatasetPath(in.RootFS)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.New("RootFS is not a valid zfs filesystem path")
|
||||||
|
}
|
||||||
|
if m.rootFS.Length() <= 0 {
|
||||||
|
return nil, errors.New("RootFS must not be empty") // duplicates error check of receiver
|
||||||
|
}
|
||||||
|
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func activeSide(g *config.Global, in *config.ActiveJob, mode activeMode) (j *ActiveSide, err error) {
|
||||||
|
|
||||||
|
j = &ActiveSide{mode: mode}
|
||||||
|
j.name = in.Name
|
||||||
|
j.promRepStateSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||||
|
Namespace: "zrepl",
|
||||||
|
Subsystem: "replication",
|
||||||
|
Name: "state_time",
|
||||||
|
Help: "seconds spent during replication",
|
||||||
|
ConstLabels: prometheus.Labels{"zrepl_job": j.name},
|
||||||
|
}, []string{"state"})
|
||||||
|
j.promBytesReplicated = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||||
|
Namespace: "zrepl",
|
||||||
|
Subsystem: "replication",
|
||||||
|
Name: "bytes_replicated",
|
||||||
|
Help: "number of bytes replicated from sender to receiver per filesystem",
|
||||||
|
ConstLabels: prometheus.Labels{"zrepl_job": j.name},
|
||||||
|
}, []string{"filesystem"})
|
||||||
|
|
||||||
|
j.connecter, err = fromconfig.ConnecterFromConfig(g, in.Connect)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "cannot build client")
|
||||||
|
}
|
||||||
|
|
||||||
|
j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||||
|
Namespace: "zrepl",
|
||||||
|
Subsystem: "pruning",
|
||||||
|
Name: "time",
|
||||||
|
Help: "seconds spent in pruner",
|
||||||
|
ConstLabels: prometheus.Labels{"zrepl_job": j.name},
|
||||||
|
}, []string{"prune_side"})
|
||||||
|
j.prunerFactory, err = pruner.NewPrunerFactory(in.Pruning, j.promPruneSecs)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return j, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) {
|
||||||
|
registerer.MustRegister(j.promRepStateSecs)
|
||||||
|
registerer.MustRegister(j.promPruneSecs)
|
||||||
|
registerer.MustRegister(j.promBytesReplicated)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (j *ActiveSide) Name() string { return j.name }
|
||||||
|
|
||||||
|
type ActiveSideStatus struct {
|
||||||
|
Replication *report.Report
|
||||||
|
PruningSender, PruningReceiver *pruner.Report
|
||||||
|
Snapshotting *snapper.Report
|
||||||
|
}
|
||||||
|
|
||||||
|
func (j *ActiveSide) Status() *Status {
|
||||||
|
tasks := j.updateTasks(nil)
|
||||||
|
|
||||||
|
s := &ActiveSideStatus{}
|
||||||
|
t := j.mode.Type()
|
||||||
|
if tasks.replicationReport != nil {
|
||||||
|
s.Replication = tasks.replicationReport()
|
||||||
|
}
|
||||||
|
if tasks.prunerSender != nil {
|
||||||
|
s.PruningSender = tasks.prunerSender.Report()
|
||||||
|
}
|
||||||
|
if tasks.prunerReceiver != nil {
|
||||||
|
s.PruningReceiver = tasks.prunerReceiver.Report()
|
||||||
|
}
|
||||||
|
s.Snapshotting = j.mode.SnapperReport()
|
||||||
|
return &Status{Type: t, JobSpecific: s}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (j *ActiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
|
||||||
|
pull, ok := j.mode.(*modePull)
|
||||||
|
if !ok {
|
||||||
|
_ = j.mode.(*modePush) // make sure we didn't introduce a new job type
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
return pull.rootFS.Copy(), true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (j *ActiveSide) Run(ctx context.Context) {
|
||||||
|
log := GetLogger(ctx)
|
||||||
|
ctx = logging.WithSubsystemLoggers(ctx, log)
|
||||||
|
|
||||||
|
defer log.Info("job exiting")
|
||||||
|
|
||||||
|
periodicDone := make(chan struct{})
|
||||||
|
ctx, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
go j.mode.RunPeriodic(ctx, periodicDone)
|
||||||
|
|
||||||
|
invocationCount := 0
|
||||||
|
outer:
|
||||||
|
for {
|
||||||
|
log.Info("wait for wakeups")
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
log.WithError(ctx.Err()).Info("context")
|
||||||
|
break outer
|
||||||
|
|
||||||
|
case <-wakeup.Wait(ctx):
|
||||||
|
j.mode.ResetConnectBackoff()
|
||||||
|
case <-periodicDone:
|
||||||
|
}
|
||||||
|
invocationCount++
|
||||||
|
invLog := log.WithField("invocation", invocationCount)
|
||||||
|
j.do(WithLogger(ctx, invLog))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (j *ActiveSide) do(ctx context.Context) {
|
||||||
|
|
||||||
|
log := GetLogger(ctx)
|
||||||
|
ctx = logging.WithSubsystemLoggers(ctx, log)
|
||||||
|
loggers := rpc.GetLoggersOrPanic(ctx) // filled by WithSubsystemLoggers
|
||||||
|
j.mode.ConnectEndpoints(loggers, j.connecter)
|
||||||
|
defer j.mode.DisconnectEndpoints()
|
||||||
|
|
||||||
|
// allow cancellation of an invocation (this function)
|
||||||
|
ctx, cancelThisRun := context.WithCancel(ctx)
|
||||||
|
defer cancelThisRun()
|
||||||
|
go func() {
|
||||||
|
select {
|
||||||
|
case <-reset.Wait(ctx):
|
||||||
|
log.Info("reset received, cancelling current invocation")
|
||||||
|
cancelThisRun()
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
sender, receiver := j.mode.SenderReceiver()
|
||||||
|
|
||||||
|
{
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
ctx, repCancel := context.WithCancel(ctx)
|
||||||
|
var repWait driver.WaitFunc
|
||||||
|
j.updateTasks(func(tasks *activeSideTasks) {
|
||||||
|
// reset it
|
||||||
|
*tasks = activeSideTasks{}
|
||||||
|
tasks.replicationCancel = repCancel
|
||||||
|
tasks.replicationReport, repWait = replication.Do(
|
||||||
|
ctx, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver),
|
||||||
|
)
|
||||||
|
tasks.state = ActiveSideReplicating
|
||||||
|
})
|
||||||
|
log.Info("start replication")
|
||||||
|
repWait(true) // wait blocking
|
||||||
|
repCancel() // always cancel to free up context resources
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
ctx, senderCancel := context.WithCancel(ctx)
|
||||||
|
tasks := j.updateTasks(func(tasks *activeSideTasks) {
|
||||||
|
tasks.prunerSender = j.prunerFactory.BuildSenderPruner(ctx, sender, sender)
|
||||||
|
tasks.prunerSenderCancel = senderCancel
|
||||||
|
tasks.state = ActiveSidePruneSender
|
||||||
|
})
|
||||||
|
log.Info("start pruning sender")
|
||||||
|
tasks.prunerSender.Prune()
|
||||||
|
log.Info("finished pruning sender")
|
||||||
|
senderCancel()
|
||||||
|
}
|
||||||
|
{
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
ctx, receiverCancel := context.WithCancel(ctx)
|
||||||
|
tasks := j.updateTasks(func(tasks *activeSideTasks) {
|
||||||
|
tasks.prunerReceiver = j.prunerFactory.BuildReceiverPruner(ctx, receiver, sender)
|
||||||
|
tasks.prunerReceiverCancel = receiverCancel
|
||||||
|
tasks.state = ActiveSidePruneReceiver
|
||||||
|
})
|
||||||
|
log.Info("start pruning receiver")
|
||||||
|
tasks.prunerReceiver.Prune()
|
||||||
|
log.Info("finished pruning receiver")
|
||||||
|
receiverCancel()
|
||||||
|
}
|
||||||
|
|
||||||
|
j.updateTasks(func(tasks *activeSideTasks) {
|
||||||
|
tasks.state = ActiveSideDone
|
||||||
|
})
|
||||||
|
|
||||||
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
// Code generated by "enumer -type=ActiveSideState"; DO NOT EDIT.
|
// Code generated by "enumer -type=ActiveSideState"; DO NOT EDIT.
|
||||||
|
|
||||||
|
//
|
||||||
package job
|
package job
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -7,14 +7,13 @@ import (
|
|||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/util/bandwidthlimit"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func JobsFromConfig(c *config.Config, parseFlags config.ParseFlags) ([]Job, error) {
|
func JobsFromConfig(c *config.Config) ([]Job, error) {
|
||||||
js := make([]Job, len(c.Jobs))
|
js := make([]Job, len(c.Jobs))
|
||||||
for i := range c.Jobs {
|
for i := range c.Jobs {
|
||||||
j, err := buildJob(c.Global, c.Jobs[i], parseFlags)
|
j, err := buildJob(c.Global, c.Jobs[i])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -24,22 +23,45 @@ func JobsFromConfig(c *config.Config, parseFlags config.ParseFlags) ([]Job, erro
|
|||||||
js[i] = j
|
js[i] = j
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// receiving-side root filesystems must not overlap
|
||||||
|
{
|
||||||
|
rfss := make([]string, 0, len(js))
|
||||||
|
for _, j := range js {
|
||||||
|
jrfs, ok := j.OwnedDatasetSubtreeRoot()
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rfss = append(rfss, jrfs.ToString())
|
||||||
|
}
|
||||||
|
if err := validateReceivingSidesDoNotOverlap(rfss); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return js, nil
|
return js, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildJob(c *config.Global, in config.JobEnum, parseFlags config.ParseFlags) (j Job, err error) {
|
func buildJob(c *config.Global, in config.JobEnum) (j Job, err error) {
|
||||||
cannotBuildJob := func(e error, name string) (Job, error) {
|
cannotBuildJob := func(e error, name string) (Job, error) {
|
||||||
return nil, errors.Wrapf(e, "cannot build job %q", name)
|
return nil, errors.Wrapf(e, "cannot build job %q", name)
|
||||||
}
|
}
|
||||||
// FIXME prettify this
|
// FIXME prettify this
|
||||||
switch v := in.Ret.(type) {
|
switch v := in.Ret.(type) {
|
||||||
case *config.SinkJob:
|
case *config.SinkJob:
|
||||||
j, err = passiveSideFromConfig(c, &v.PassiveJob, v, parseFlags)
|
m, err := modeSinkFromConfig(c, v)
|
||||||
|
if err != nil {
|
||||||
|
return cannotBuildJob(err, v.Name)
|
||||||
|
}
|
||||||
|
j, err = passiveSideFromConfig(c, &v.PassiveJob, m)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return cannotBuildJob(err, v.Name)
|
return cannotBuildJob(err, v.Name)
|
||||||
}
|
}
|
||||||
case *config.SourceJob:
|
case *config.SourceJob:
|
||||||
j, err = passiveSideFromConfig(c, &v.PassiveJob, v, parseFlags)
|
m, err := modeSourceFromConfig(c, v)
|
||||||
|
if err != nil {
|
||||||
|
return cannotBuildJob(err, v.Name)
|
||||||
|
}
|
||||||
|
j, err = passiveSideFromConfig(c, &v.PassiveJob, m)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return cannotBuildJob(err, v.Name)
|
return cannotBuildJob(err, v.Name)
|
||||||
}
|
}
|
||||||
@@ -49,12 +71,20 @@ func buildJob(c *config.Global, in config.JobEnum, parseFlags config.ParseFlags)
|
|||||||
return cannotBuildJob(err, v.Name)
|
return cannotBuildJob(err, v.Name)
|
||||||
}
|
}
|
||||||
case *config.PushJob:
|
case *config.PushJob:
|
||||||
j, err = activeSide(c, &v.ActiveJob, v, parseFlags)
|
m, err := modePushFromConfig(c, v)
|
||||||
|
if err != nil {
|
||||||
|
return cannotBuildJob(err, v.Name)
|
||||||
|
}
|
||||||
|
j, err = activeSide(c, &v.ActiveJob, m)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return cannotBuildJob(err, v.Name)
|
return cannotBuildJob(err, v.Name)
|
||||||
}
|
}
|
||||||
case *config.PullJob:
|
case *config.PullJob:
|
||||||
j, err = activeSide(c, &v.ActiveJob, v, parseFlags)
|
m, err := modePullFromConfig(c, v)
|
||||||
|
if err != nil {
|
||||||
|
return cannotBuildJob(err, v.Name)
|
||||||
|
}
|
||||||
|
j, err = activeSide(c, &v.ActiveJob, m)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return cannotBuildJob(err, v.Name)
|
return cannotBuildJob(err, v.Name)
|
||||||
}
|
}
|
||||||
@@ -74,11 +104,6 @@ func validateReceivingSidesDoNotOverlap(receivingRootFSs []string) error {
|
|||||||
sort.Slice(rfss, func(i, j int) bool {
|
sort.Slice(rfss, func(i, j int) bool {
|
||||||
return strings.Compare(rfss[i], rfss[j]) == -1
|
return strings.Compare(rfss[i], rfss[j]) == -1
|
||||||
})
|
})
|
||||||
// add tailing slash because of hierarchy-simulation
|
|
||||||
// rootfs/ is not root of rootfs2/
|
|
||||||
for i := 0; i < len(rfss); i++ {
|
|
||||||
rfss[i] += "/"
|
|
||||||
}
|
|
||||||
// idea:
|
// idea:
|
||||||
// no path in rfss must be prefix of another
|
// no path in rfss must be prefix of another
|
||||||
//
|
//
|
||||||
@@ -93,13 +118,3 @@ func validateReceivingSidesDoNotOverlap(receivingRootFSs []string) error {
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildBandwidthLimitConfig(in *config.BandwidthLimit) (c bandwidthlimit.Config, _ error) {
|
|
||||||
if in.Max.ToBytes() > 0 && int64(in.Max.ToBytes()) == 0 {
|
|
||||||
return c, fmt.Errorf("bandwidth limit `max` is too small, must at least specify one byte")
|
|
||||||
}
|
|
||||||
return bandwidthlimit.Config{
|
|
||||||
Max: int64(in.Max.ToBytes()),
|
|
||||||
BucketCapacity: int64(in.BucketCapacity.ToBytes()),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
package job
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
|
||||||
|
type testCase struct {
|
||||||
|
err bool
|
||||||
|
input []string
|
||||||
|
}
|
||||||
|
tcs := []testCase{
|
||||||
|
{false, nil},
|
||||||
|
{false, []string{}},
|
||||||
|
{false, []string{""}}, // not our job to determine valid paths
|
||||||
|
{false, []string{"a"}},
|
||||||
|
{false, []string{"some/path"}},
|
||||||
|
{false, []string{"zroot/sink1", "zroot/sink2", "zroot/sink3"}},
|
||||||
|
{true, []string{"zroot/b", "zroot/b"}},
|
||||||
|
{true, []string{"zroot/foo", "zroot/foo/bar", "zroot/baz"}},
|
||||||
|
{false, []string{"a/x", "b/x"}},
|
||||||
|
{false, []string{"a", "b"}},
|
||||||
|
{true, []string{"a", "a"}},
|
||||||
|
{true, []string{"a/x/y", "a/x"}},
|
||||||
|
{true, []string{"a/x", "a/x/y"}},
|
||||||
|
{true, []string{"a/x", "b/x", "a/x/y"}},
|
||||||
|
{true, []string{"a", "a/b", "a/c", "a/b"}},
|
||||||
|
{true, []string{"a/b", "a/c", "a/b", "a/d", "a/c"}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tcs {
|
||||||
|
t.Logf("input: %v", tc.input)
|
||||||
|
err := validateReceivingSidesDoNotOverlap(tc.input)
|
||||||
|
if tc.err {
|
||||||
|
assert.Error(t, err)
|
||||||
|
} else {
|
||||||
|
assert.NoError(t, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,16 +7,27 @@ import (
|
|||||||
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
"github.com/zrepl/zrepl/logger"
|
||||||
"github.com/zrepl/zrepl/internal/endpoint"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
"github.com/zrepl/zrepl/internal/logger"
|
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type Logger = logger.Logger
|
type Logger = logger.Logger
|
||||||
|
|
||||||
|
type contextKey int
|
||||||
|
|
||||||
|
const (
|
||||||
|
contextKeyLog contextKey = iota
|
||||||
|
)
|
||||||
|
|
||||||
func GetLogger(ctx context.Context) Logger {
|
func GetLogger(ctx context.Context) Logger {
|
||||||
return logging.GetLogger(ctx, logging.SubsysJob)
|
if l, ok := ctx.Value(contextKeyLog).(Logger); ok {
|
||||||
|
return l
|
||||||
|
}
|
||||||
|
return logger.NewNullLogger()
|
||||||
|
}
|
||||||
|
|
||||||
|
func WithLogger(ctx context.Context, l Logger) context.Context {
|
||||||
|
return context.WithValue(ctx, contextKeyLog, l)
|
||||||
}
|
}
|
||||||
|
|
||||||
type Job interface {
|
type Job interface {
|
||||||
@@ -27,7 +38,6 @@ type Job interface {
|
|||||||
// Jobs that return a subtree of the dataset hierarchy
|
// Jobs that return a subtree of the dataset hierarchy
|
||||||
// must return the root of that subtree as rfs and ok = true
|
// must return the root of that subtree as rfs and ok = true
|
||||||
OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool)
|
OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool)
|
||||||
SenderConfig() *endpoint.SenderConfig
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type Type string
|
type Type string
|
||||||
@@ -0,0 +1,158 @@
|
|||||||
|
package job
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
|
||||||
|
"github.com/zrepl/zrepl/config"
|
||||||
|
"github.com/zrepl/zrepl/daemon/filters"
|
||||||
|
"github.com/zrepl/zrepl/daemon/logging"
|
||||||
|
"github.com/zrepl/zrepl/daemon/snapper"
|
||||||
|
"github.com/zrepl/zrepl/endpoint"
|
||||||
|
"github.com/zrepl/zrepl/rpc"
|
||||||
|
"github.com/zrepl/zrepl/transport"
|
||||||
|
"github.com/zrepl/zrepl/transport/fromconfig"
|
||||||
|
"github.com/zrepl/zrepl/zfs"
|
||||||
|
)
|
||||||
|
|
||||||
|
type PassiveSide struct {
|
||||||
|
mode passiveMode
|
||||||
|
name string
|
||||||
|
listen transport.AuthenticatedListenerFactory
|
||||||
|
}
|
||||||
|
|
||||||
|
type passiveMode interface {
|
||||||
|
Handler() rpc.Handler
|
||||||
|
RunPeriodic(ctx context.Context)
|
||||||
|
SnapperReport() *snapper.Report // may be nil
|
||||||
|
Type() Type
|
||||||
|
}
|
||||||
|
|
||||||
|
type modeSink struct {
|
||||||
|
rootDataset *zfs.DatasetPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *modeSink) Type() Type { return TypeSink }
|
||||||
|
|
||||||
|
func (m *modeSink) Handler() rpc.Handler {
|
||||||
|
return endpoint.NewReceiver(m.rootDataset, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *modeSink) RunPeriodic(_ context.Context) {}
|
||||||
|
func (m *modeSink) SnapperReport() *snapper.Report { return nil }
|
||||||
|
|
||||||
|
func modeSinkFromConfig(g *config.Global, in *config.SinkJob) (m *modeSink, err error) {
|
||||||
|
m = &modeSink{}
|
||||||
|
m.rootDataset, err = zfs.NewDatasetPath(in.RootFS)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.New("root dataset is not a valid zfs filesystem path")
|
||||||
|
}
|
||||||
|
if m.rootDataset.Length() <= 0 {
|
||||||
|
return nil, errors.New("root dataset must not be empty") // duplicates error check of receiver
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type modeSource struct {
|
||||||
|
fsfilter zfs.DatasetFilter
|
||||||
|
snapper *snapper.PeriodicOrManual
|
||||||
|
}
|
||||||
|
|
||||||
|
func modeSourceFromConfig(g *config.Global, in *config.SourceJob) (m *modeSource, err error) {
|
||||||
|
// FIXME exact dedup of modePush
|
||||||
|
m = &modeSource{}
|
||||||
|
fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "cannnot build filesystem filter")
|
||||||
|
}
|
||||||
|
m.fsfilter = fsf
|
||||||
|
|
||||||
|
if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
|
||||||
|
return nil, errors.Wrap(err, "cannot build snapper")
|
||||||
|
}
|
||||||
|
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *modeSource) Type() Type { return TypeSource }
|
||||||
|
|
||||||
|
func (m *modeSource) Handler() rpc.Handler {
|
||||||
|
return endpoint.NewSender(m.fsfilter)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *modeSource) RunPeriodic(ctx context.Context) {
|
||||||
|
m.snapper.Run(ctx, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *modeSource) SnapperReport() *snapper.Report {
|
||||||
|
return m.snapper.Report()
|
||||||
|
}
|
||||||
|
|
||||||
|
func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, mode passiveMode) (s *PassiveSide, err error) {
|
||||||
|
|
||||||
|
s = &PassiveSide{mode: mode, name: in.Name}
|
||||||
|
if s.listen, err = fromconfig.ListenerFactoryFromConfig(g, in.Serve); err != nil {
|
||||||
|
return nil, errors.Wrap(err, "cannot build listener factory")
|
||||||
|
}
|
||||||
|
|
||||||
|
return s, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (j *PassiveSide) Name() string { return j.name }
|
||||||
|
|
||||||
|
type PassiveStatus struct {
|
||||||
|
Snapper *snapper.Report
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PassiveSide) Status() *Status {
|
||||||
|
st := &PassiveStatus{
|
||||||
|
Snapper: s.mode.SnapperReport(),
|
||||||
|
}
|
||||||
|
return &Status{Type: s.mode.Type(), JobSpecific: st}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (j *PassiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
|
||||||
|
sink, ok := j.mode.(*modeSink)
|
||||||
|
if !ok {
|
||||||
|
_ = j.mode.(*modeSource) // make sure we didn't introduce a new job type
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
return sink.rootDataset.Copy(), true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*PassiveSide) RegisterMetrics(registerer prometheus.Registerer) {}
|
||||||
|
|
||||||
|
func (j *PassiveSide) Run(ctx context.Context) {
|
||||||
|
|
||||||
|
log := GetLogger(ctx)
|
||||||
|
defer log.Info("job exiting")
|
||||||
|
ctx = logging.WithSubsystemLoggers(ctx, log)
|
||||||
|
{
|
||||||
|
ctx, cancel := context.WithCancel(ctx) // shadowing
|
||||||
|
defer cancel()
|
||||||
|
go j.mode.RunPeriodic(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
handler := j.mode.Handler()
|
||||||
|
if handler == nil {
|
||||||
|
panic(fmt.Sprintf("implementation error: j.mode.Handler() returned nil: %#v", j))
|
||||||
|
}
|
||||||
|
|
||||||
|
ctxInterceptor := func(handlerCtx context.Context) context.Context {
|
||||||
|
return logging.WithSubsystemLoggers(handlerCtx, log)
|
||||||
|
}
|
||||||
|
|
||||||
|
rpcLoggers := rpc.GetLoggersOrPanic(ctx) // WithSubsystemLoggers above
|
||||||
|
server := rpc.NewServer(handler, rpcLoggers, ctxInterceptor)
|
||||||
|
|
||||||
|
listener, err := j.listen()
|
||||||
|
if err != nil {
|
||||||
|
log.WithError(err).Error("cannot listen")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
server.Serve(ctx, listener)
|
||||||
|
}
|
||||||
@@ -4,39 +4,34 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
"sync"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/util/bandwidthlimit"
|
"github.com/zrepl/zrepl/daemon/filters"
|
||||||
"github.com/zrepl/zrepl/internal/util/nodefault"
|
"github.com/zrepl/zrepl/daemon/job/wakeup"
|
||||||
|
"github.com/zrepl/zrepl/daemon/logging"
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/daemon/pruner"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
"github.com/zrepl/zrepl/daemon/snapper"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/job/wakeup"
|
"github.com/zrepl/zrepl/endpoint"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/pruner"
|
"github.com/zrepl/zrepl/replication/logic/pdu"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/snapper"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
"github.com/zrepl/zrepl/internal/endpoint"
|
|
||||||
"github.com/zrepl/zrepl/internal/replication/logic/pdu"
|
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type SnapJob struct {
|
type SnapJob struct {
|
||||||
name endpoint.JobID
|
name string
|
||||||
fsfilter zfs.DatasetFilter
|
fsfilter zfs.DatasetFilter
|
||||||
snapper snapper.Snapper
|
snapper *snapper.PeriodicOrManual
|
||||||
|
|
||||||
prunerFactory *pruner.LocalPrunerFactory
|
prunerFactory *pruner.LocalPrunerFactory
|
||||||
|
|
||||||
promPruneSecs *prometheus.HistogramVec // labels: prune_side
|
promPruneSecs *prometheus.HistogramVec // labels: prune_side
|
||||||
|
|
||||||
prunerMtx sync.Mutex
|
pruner *pruner.Pruner
|
||||||
pruner *pruner.Pruner
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (j *SnapJob) Name() string { return j.name.String() }
|
func (j *SnapJob) Name() string { return j.name }
|
||||||
|
|
||||||
func (j *SnapJob) Type() Type { return TypeSnap }
|
func (j *SnapJob) Type() Type { return TypeSnap }
|
||||||
|
|
||||||
@@ -44,23 +39,20 @@ func snapJobFromConfig(g *config.Global, in *config.SnapJob) (j *SnapJob, err er
|
|||||||
j = &SnapJob{}
|
j = &SnapJob{}
|
||||||
fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
|
fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "cannot build filesystem filter")
|
return nil, errors.Wrap(err, "cannnot build filesystem filter")
|
||||||
}
|
}
|
||||||
j.fsfilter = fsf
|
j.fsfilter = fsf
|
||||||
|
|
||||||
if j.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
|
if j.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
|
||||||
return nil, errors.Wrap(err, "cannot build snapper")
|
return nil, errors.Wrap(err, "cannot build snapper")
|
||||||
}
|
}
|
||||||
j.name, err = endpoint.MakeJobID(in.Name)
|
j.name = in.Name
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "invalid job name")
|
|
||||||
}
|
|
||||||
j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||||
Namespace: "zrepl",
|
Namespace: "zrepl",
|
||||||
Subsystem: "pruning",
|
Subsystem: "pruning",
|
||||||
Name: "time",
|
Name: "time",
|
||||||
Help: "seconds spent in pruner",
|
Help: "seconds spent in pruner",
|
||||||
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
|
ConstLabels: prometheus.Labels{"zrepl_job": j.name},
|
||||||
}, []string{"prune_side"})
|
}, []string{"prune_side"})
|
||||||
j.prunerFactory, err = pruner.NewLocalPrunerFactory(in.Pruning, j.promPruneSecs)
|
j.prunerFactory, err = pruner.NewLocalPrunerFactory(in.Pruning, j.promPruneSecs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -81,13 +73,10 @@ type SnapJobStatus struct {
|
|||||||
func (j *SnapJob) Status() *Status {
|
func (j *SnapJob) Status() *Status {
|
||||||
s := &SnapJobStatus{}
|
s := &SnapJobStatus{}
|
||||||
t := j.Type()
|
t := j.Type()
|
||||||
j.prunerMtx.Lock()
|
|
||||||
if j.pruner != nil {
|
if j.pruner != nil {
|
||||||
s.Pruning = j.pruner.Report()
|
s.Pruning = j.pruner.Report()
|
||||||
}
|
}
|
||||||
j.prunerMtx.Unlock()
|
s.Snapshotting = j.snapper.Report()
|
||||||
r := j.snapper.Report()
|
|
||||||
s.Snapshotting = &r
|
|
||||||
return &Status{Type: t, JobSpecific: s}
|
return &Status{Type: t, JobSpecific: s}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -95,21 +84,16 @@ func (j *SnapJob) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
|
|||||||
return nil, false
|
return nil, false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (j *SnapJob) SenderConfig() *endpoint.SenderConfig { return nil }
|
|
||||||
|
|
||||||
func (j *SnapJob) Run(ctx context.Context) {
|
func (j *SnapJob) Run(ctx context.Context) {
|
||||||
ctx, endTask := trace.WithTaskAndSpan(ctx, "snap-job", j.Name())
|
|
||||||
defer endTask()
|
|
||||||
log := GetLogger(ctx)
|
log := GetLogger(ctx)
|
||||||
|
ctx = logging.WithSubsystemLoggers(ctx, log)
|
||||||
|
|
||||||
defer log.Info("job exiting")
|
defer log.Info("job exiting")
|
||||||
|
|
||||||
periodicDone := make(chan struct{})
|
periodicDone := make(chan struct{})
|
||||||
ctx, cancel := context.WithCancel(ctx)
|
ctx, cancel := context.WithCancel(ctx)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
periodicCtx, endTask := trace.WithTask(ctx, "snapshotting")
|
go j.snapper.Run(ctx, periodicDone)
|
||||||
defer endTask()
|
|
||||||
go j.snapper.Run(periodicCtx, periodicDone)
|
|
||||||
|
|
||||||
invocationCount := 0
|
invocationCount := 0
|
||||||
outer:
|
outer:
|
||||||
@@ -124,10 +108,8 @@ outer:
|
|||||||
case <-periodicDone:
|
case <-periodicDone:
|
||||||
}
|
}
|
||||||
invocationCount++
|
invocationCount++
|
||||||
|
invLog := log.WithField("invocation", invocationCount)
|
||||||
invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", invocationCount))
|
j.doPrune(WithLogger(ctx, invLog))
|
||||||
j.doPrune(invocationCtx)
|
|
||||||
endSpan()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -138,7 +120,7 @@ outer:
|
|||||||
// TODO:
|
// TODO:
|
||||||
// This is a work-around for the current package daemon/pruner
|
// This is a work-around for the current package daemon/pruner
|
||||||
// and package pruning.Snapshot limitation: they require the
|
// and package pruning.Snapshot limitation: they require the
|
||||||
// `Replicated` getter method be present, but obviously,
|
// `Replicated` getter method be present, but obviously,
|
||||||
// a local job like SnapJob can't deliver on that.
|
// a local job like SnapJob can't deliver on that.
|
||||||
// But the pruner.Pruner gives up on an FS if no replication
|
// But the pruner.Pruner gives up on an FS if no replication
|
||||||
// cursor is present, which is why this pruner returns the
|
// cursor is present, which is why this pruner returns the
|
||||||
@@ -148,9 +130,12 @@ type alwaysUpToDateReplicationCursorHistory struct {
|
|||||||
target pruner.Target
|
target pruner.Target
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ pruner.Sender = (*alwaysUpToDateReplicationCursorHistory)(nil)
|
var _ pruner.History = (*alwaysUpToDateReplicationCursorHistory)(nil)
|
||||||
|
|
||||||
func (h alwaysUpToDateReplicationCursorHistory) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
|
func (h alwaysUpToDateReplicationCursorHistory) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
|
||||||
|
if req.GetGet() == nil {
|
||||||
|
return nil, fmt.Errorf("unsupported ReplicationCursor request: SnapJob only supports GETting a (faked) cursor")
|
||||||
|
}
|
||||||
fsvReq := &pdu.ListFilesystemVersionsReq{
|
fsvReq := &pdu.ListFilesystemVersionsReq{
|
||||||
Filesystem: req.GetFilesystem(),
|
Filesystem: req.GetFilesystem(),
|
||||||
}
|
}
|
||||||
@@ -175,21 +160,10 @@ func (h alwaysUpToDateReplicationCursorHistory) ListFilesystems(ctx context.Cont
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (j *SnapJob) doPrune(ctx context.Context) {
|
func (j *SnapJob) doPrune(ctx context.Context) {
|
||||||
ctx, endSpan := trace.WithSpan(ctx, "snap-job-do-prune")
|
|
||||||
defer endSpan()
|
|
||||||
log := GetLogger(ctx)
|
log := GetLogger(ctx)
|
||||||
sender := endpoint.NewSender(endpoint.SenderConfig{
|
ctx = logging.WithSubsystemLoggers(ctx, log)
|
||||||
JobID: j.name,
|
sender := endpoint.NewSender(j.fsfilter)
|
||||||
FSF: j.fsfilter,
|
|
||||||
// FIXME the following config fields are irrelevant for SnapJob
|
|
||||||
// because the endpoint is only used as pruner.Target.
|
|
||||||
// However, the implementation requires them to be set.
|
|
||||||
Encrypt: &nodefault.Bool{B: true},
|
|
||||||
BandwidthLimit: bandwidthlimit.NoLimitConfig(),
|
|
||||||
})
|
|
||||||
j.prunerMtx.Lock()
|
|
||||||
j.pruner = j.prunerFactory.BuildLocalPruner(ctx, sender, alwaysUpToDateReplicationCursorHistory{sender})
|
j.pruner = j.prunerFactory.BuildLocalPruner(ctx, sender, alwaysUpToDateReplicationCursorHistory{sender})
|
||||||
j.prunerMtx.Unlock()
|
|
||||||
log.Info("start pruning")
|
log.Info("start pruning")
|
||||||
j.pruner.Prune()
|
j.pruner.Prune()
|
||||||
log.Info("finished pruning")
|
log.Info("finished pruning")
|
||||||
@@ -10,10 +10,18 @@ import (
|
|||||||
"github.com/mattn/go-isatty"
|
"github.com/mattn/go-isatty"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
"github.com/zrepl/zrepl/daemon/hooks"
|
||||||
"github.com/zrepl/zrepl/internal/logger"
|
"github.com/zrepl/zrepl/daemon/pruner"
|
||||||
"github.com/zrepl/zrepl/internal/tlsconf"
|
"github.com/zrepl/zrepl/daemon/snapper"
|
||||||
|
"github.com/zrepl/zrepl/endpoint"
|
||||||
|
"github.com/zrepl/zrepl/logger"
|
||||||
|
"github.com/zrepl/zrepl/replication/driver"
|
||||||
|
"github.com/zrepl/zrepl/replication/logic"
|
||||||
|
"github.com/zrepl/zrepl/rpc"
|
||||||
|
"github.com/zrepl/zrepl/rpc/transportmux"
|
||||||
|
"github.com/zrepl/zrepl/tlsconf"
|
||||||
|
"github.com/zrepl/zrepl/transport"
|
||||||
)
|
)
|
||||||
|
|
||||||
func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error) {
|
func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error) {
|
||||||
@@ -61,10 +69,8 @@ func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error)
|
|||||||
type Subsystem string
|
type Subsystem string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
SubsysMeta Subsystem = "meta"
|
|
||||||
SubsysJob Subsystem = "job"
|
|
||||||
SubsysReplication Subsystem = "repl"
|
SubsysReplication Subsystem = "repl"
|
||||||
SubsysEndpoint Subsystem = "endpoint"
|
SubsyEndpoint Subsystem = "endpoint"
|
||||||
SubsysPruning Subsystem = "pruning"
|
SubsysPruning Subsystem = "pruning"
|
||||||
SubsysSnapshot Subsystem = "snapshot"
|
SubsysSnapshot Subsystem = "snapshot"
|
||||||
SubsysHooks Subsystem = "hook"
|
SubsysHooks Subsystem = "hook"
|
||||||
@@ -73,104 +79,29 @@ const (
|
|||||||
SubsysRPC Subsystem = "rpc"
|
SubsysRPC Subsystem = "rpc"
|
||||||
SubsysRPCControl Subsystem = "rpc.ctrl"
|
SubsysRPCControl Subsystem = "rpc.ctrl"
|
||||||
SubsysRPCData Subsystem = "rpc.data"
|
SubsysRPCData Subsystem = "rpc.data"
|
||||||
SubsysZFSCmd Subsystem = "zfs.cmd"
|
|
||||||
SubsysTraceData Subsystem = "trace.data"
|
|
||||||
SubsysPlatformtest Subsystem = "platformtest"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var AllSubsystems = []Subsystem{
|
func WithSubsystemLoggers(ctx context.Context, log logger.Logger) context.Context {
|
||||||
SubsysMeta,
|
ctx = logic.WithLogger(ctx, log.WithField(SubsysField, SubsysReplication))
|
||||||
SubsysJob,
|
ctx = driver.WithLogger(ctx, log.WithField(SubsysField, SubsysReplication))
|
||||||
SubsysReplication,
|
ctx = endpoint.WithLogger(ctx, log.WithField(SubsysField, SubsyEndpoint))
|
||||||
SubsysEndpoint,
|
ctx = pruner.WithLogger(ctx, log.WithField(SubsysField, SubsysPruning))
|
||||||
SubsysPruning,
|
ctx = snapper.WithLogger(ctx, log.WithField(SubsysField, SubsysSnapshot))
|
||||||
SubsysSnapshot,
|
ctx = hooks.WithLogger(ctx, log.WithField(SubsysField, SubsysHooks))
|
||||||
SubsysHooks,
|
ctx = transport.WithLogger(ctx, log.WithField(SubsysField, SubsysTransport))
|
||||||
SubsysTransport,
|
ctx = transportmux.WithLogger(ctx, log.WithField(SubsysField, SubsysTransportMux))
|
||||||
SubsysTransportMux,
|
ctx = rpc.WithLoggers(ctx,
|
||||||
SubsysRPC,
|
rpc.Loggers{
|
||||||
SubsysRPCControl,
|
General: log.WithField(SubsysField, SubsysRPC),
|
||||||
SubsysRPCData,
|
Control: log.WithField(SubsysField, SubsysRPCControl),
|
||||||
SubsysZFSCmd,
|
Data: log.WithField(SubsysField, SubsysRPCData),
|
||||||
SubsysTraceData,
|
},
|
||||||
SubsysPlatformtest,
|
)
|
||||||
|
return ctx
|
||||||
}
|
}
|
||||||
|
|
||||||
type injectedField struct {
|
func LogSubsystem(log logger.Logger, subsys Subsystem) logger.Logger {
|
||||||
field string
|
return log.ReplaceField(SubsysField, subsys)
|
||||||
value interface{}
|
|
||||||
parent *injectedField
|
|
||||||
}
|
|
||||||
|
|
||||||
func WithInjectedField(ctx context.Context, field string, value interface{}) context.Context {
|
|
||||||
var parent *injectedField
|
|
||||||
parentI := ctx.Value(contextKeyInjectedField)
|
|
||||||
if parentI != nil {
|
|
||||||
parent = parentI.(*injectedField)
|
|
||||||
}
|
|
||||||
// TODO sanity-check `field` now
|
|
||||||
this := &injectedField{field, value, parent}
|
|
||||||
return context.WithValue(ctx, contextKeyInjectedField, this)
|
|
||||||
}
|
|
||||||
|
|
||||||
func iterInjectedFields(ctx context.Context, cb func(field string, value interface{})) {
|
|
||||||
injI := ctx.Value(contextKeyInjectedField)
|
|
||||||
if injI == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
inj := injI.(*injectedField)
|
|
||||||
for ; inj != nil; inj = inj.parent {
|
|
||||||
cb(inj.field, inj.value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type SubsystemLoggers map[Subsystem]logger.Logger
|
|
||||||
|
|
||||||
func SubsystemLoggersWithUniversalLogger(l logger.Logger) SubsystemLoggers {
|
|
||||||
loggers := make(SubsystemLoggers)
|
|
||||||
for _, s := range AllSubsystems {
|
|
||||||
loggers[s] = l
|
|
||||||
}
|
|
||||||
return loggers
|
|
||||||
}
|
|
||||||
|
|
||||||
func WithLoggers(ctx context.Context, loggers SubsystemLoggers) context.Context {
|
|
||||||
return context.WithValue(ctx, contextKeyLoggers, loggers)
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetLoggers(ctx context.Context) SubsystemLoggers {
|
|
||||||
loggers, ok := ctx.Value(contextKeyLoggers).(SubsystemLoggers)
|
|
||||||
if !ok {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return loggers
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetLogger(ctx context.Context, subsys Subsystem) logger.Logger {
|
|
||||||
return getLoggerImpl(ctx, subsys, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
func getLoggerImpl(ctx context.Context, subsys Subsystem, panicIfEnded bool) logger.Logger {
|
|
||||||
loggers, ok := ctx.Value(contextKeyLoggers).(SubsystemLoggers)
|
|
||||||
if !ok || loggers == nil {
|
|
||||||
return logger.NewNullLogger()
|
|
||||||
}
|
|
||||||
l, ok := loggers[subsys]
|
|
||||||
if !ok {
|
|
||||||
return logger.NewNullLogger()
|
|
||||||
}
|
|
||||||
|
|
||||||
l = l.WithField(SubsysField, subsys)
|
|
||||||
|
|
||||||
l = l.WithField(SpanField, trace.GetSpanStackOrDefault(ctx, *trace.StackKindId, "NOSPAN"))
|
|
||||||
|
|
||||||
fields := make(logger.Fields)
|
|
||||||
iterInjectedFields(ctx, func(field string, value interface{}) {
|
|
||||||
fields[field] = value
|
|
||||||
})
|
|
||||||
l = l.WithFields(fields)
|
|
||||||
|
|
||||||
return l
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
|
func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
|
||||||
+5
-6
@@ -10,7 +10,7 @@ import (
|
|||||||
"github.com/go-logfmt/logfmt"
|
"github.com/go-logfmt/logfmt"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/logger"
|
"github.com/zrepl/zrepl/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -22,7 +22,6 @@ const (
|
|||||||
const (
|
const (
|
||||||
JobField string = "job"
|
JobField string = "job"
|
||||||
SubsysField string = "subsystem"
|
SubsysField string = "subsystem"
|
||||||
SpanField string = "span"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type MetadataFlags int64
|
type MetadataFlags int64
|
||||||
@@ -86,7 +85,7 @@ func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
|
|||||||
fmt.Fprintf(&line, "[%s]", col.Sprint(e.Level.Short()))
|
fmt.Fprintf(&line, "[%s]", col.Sprint(e.Level.Short()))
|
||||||
}
|
}
|
||||||
|
|
||||||
prefixFields := []string{JobField, SubsysField, SpanField}
|
prefixFields := []string{JobField, SubsysField}
|
||||||
prefixed := make(map[string]bool, len(prefixFields)+2)
|
prefixed := make(map[string]bool, len(prefixFields)+2)
|
||||||
for _, field := range prefixFields {
|
for _, field := range prefixFields {
|
||||||
val, ok := e.Fields[field]
|
val, ok := e.Fields[field]
|
||||||
@@ -175,8 +174,8 @@ func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// at least try and put job and task in front
|
// at least try and put job and task in front
|
||||||
prefixed := make(map[string]bool, 3)
|
prefixed := make(map[string]bool, 2)
|
||||||
prefix := []string{JobField, SubsysField, SpanField}
|
prefix := []string{JobField, SubsysField}
|
||||||
for _, pf := range prefix {
|
for _, pf := range prefix {
|
||||||
v, ok := e.Fields[pf]
|
v, ok := e.Fields[pf]
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -212,7 +211,7 @@ func logfmtTryEncodeKeyval(enc *logfmt.Encoder, field, value interface{}) error
|
|||||||
case logfmt.ErrUnsupportedValueType:
|
case logfmt.ErrUnsupportedValueType:
|
||||||
err := enc.EncodeKeyval(field, fmt.Sprintf("<%T>", value))
|
err := enc.EncodeKeyval(field, fmt.Sprintf("<%T>", value))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, "cannot encode unsupported value type Go type")
|
return errors.Wrap(err, "cannot encode unsuuported value type Go type")
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/logger"
|
"github.com/zrepl/zrepl/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
type EntryFormatter interface {
|
type EntryFormatter interface {
|
||||||
@@ -63,7 +63,7 @@ func NewTCPOutlet(formatter EntryFormatter, network, address string, tlsConfig *
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
entryChan := make(chan *bytes.Buffer, 1) // allow one message in flight while previous is in io.Copy()
|
entryChan := make(chan *bytes.Buffer, 1) // allow one message in flight while previos is in io.Copy()
|
||||||
|
|
||||||
o := &TCPOutlet{
|
o := &TCPOutlet{
|
||||||
formatter: formatter,
|
formatter: formatter,
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
package daemon
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/zrepl/zrepl/cli"
|
||||||
|
"github.com/zrepl/zrepl/logger"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Logger = logger.Logger
|
||||||
|
|
||||||
|
var DaemonCmd = &cli.Subcommand{
|
||||||
|
Use: "daemon",
|
||||||
|
Short: "run the zrepl daemon",
|
||||||
|
Run: func(subcommand *cli.Subcommand, args []string) error {
|
||||||
|
return Run(subcommand.Config())
|
||||||
|
},
|
||||||
|
}
|
||||||
@@ -8,11 +8,7 @@ import (
|
|||||||
"net"
|
"net"
|
||||||
"net/http/pprof"
|
"net/http/pprof"
|
||||||
|
|
||||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
"github.com/zrepl/zrepl/daemon/job"
|
||||||
"golang.org/x/net/websocket"
|
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type pprofServer struct {
|
type pprofServer struct {
|
||||||
@@ -68,8 +64,6 @@ outer:
|
|||||||
mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
|
mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
|
||||||
mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
|
mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
|
||||||
mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
|
mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
|
||||||
mux.Handle("/metrics", promhttp.Handler())
|
|
||||||
mux.Handle("/debug/zrepl/activity-trace", websocket.Handler(trace.ChrometraceClientWebsocketHandler))
|
|
||||||
go func() {
|
go func() {
|
||||||
err := http.Serve(s.listener, mux)
|
err := http.Serve(s.listener, mux)
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
@@ -8,26 +8,22 @@ import (
|
|||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
"github.com/zrepl/zrepl/daemon/job"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
"github.com/zrepl/zrepl/logger"
|
||||||
"github.com/zrepl/zrepl/internal/endpoint"
|
"github.com/zrepl/zrepl/rpc/dataconn/frameconn"
|
||||||
"github.com/zrepl/zrepl/internal/logger"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
"github.com/zrepl/zrepl/internal/rpc/dataconn/frameconn"
|
|
||||||
"github.com/zrepl/zrepl/internal/util/tcpsock"
|
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type prometheusJob struct {
|
type prometheusJob struct {
|
||||||
listen string
|
listen string
|
||||||
freeBind bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func newPrometheusJobFromConfig(in *config.PrometheusMonitoring) (*prometheusJob, error) {
|
func newPrometheusJobFromConfig(in *config.PrometheusMonitoring) (*prometheusJob, error) {
|
||||||
if _, _, err := net.SplitHostPort(in.Listen); err != nil {
|
if _, _, err := net.SplitHostPort(in.Listen); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return &prometheusJob{in.Listen, in.ListenFreeBind}, nil
|
return &prometheusJob{in.Listen}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var prom struct {
|
var prom struct {
|
||||||
@@ -50,8 +46,6 @@ func (j *prometheusJob) Status() *job.Status { return &job.Status{Type: job.Type
|
|||||||
|
|
||||||
func (j *prometheusJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
|
func (j *prometheusJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
|
||||||
|
|
||||||
func (j *prometheusJob) SenderConfig() *endpoint.SenderConfig { return nil }
|
|
||||||
|
|
||||||
func (j *prometheusJob) RegisterMetrics(registerer prometheus.Registerer) {}
|
func (j *prometheusJob) RegisterMetrics(registerer prometheus.Registerer) {}
|
||||||
|
|
||||||
func (j *prometheusJob) Run(ctx context.Context) {
|
func (j *prometheusJob) Run(ctx context.Context) {
|
||||||
@@ -66,10 +60,9 @@ func (j *prometheusJob) Run(ctx context.Context) {
|
|||||||
|
|
||||||
log := job.GetLogger(ctx)
|
log := job.GetLogger(ctx)
|
||||||
|
|
||||||
l, err := tcpsock.Listen(j.listen, j.freeBind)
|
l, err := net.Listen("tcp", j.listen)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.WithError(err).Error("cannot listen")
|
log.WithError(err).Error("cannot listen")
|
||||||
return
|
|
||||||
}
|
}
|
||||||
go func() {
|
go func() {
|
||||||
<-ctx.Done()
|
<-ctx.Done()
|
||||||
@@ -87,19 +80,16 @@ func (j *prometheusJob) Run(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type prometheusJobOutlet struct {
|
type prometheusJobOutlet struct {
|
||||||
|
jobName string
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ logger.Outlet = prometheusJobOutlet{}
|
var _ logger.Outlet = prometheusJobOutlet{}
|
||||||
|
|
||||||
func newPrometheusLogOutlet() prometheusJobOutlet {
|
func newPrometheusLogOutlet(jobName string) prometheusJobOutlet {
|
||||||
return prometheusJobOutlet{}
|
return prometheusJobOutlet{jobName}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (o prometheusJobOutlet) WriteEntry(entry logger.Entry) error {
|
func (o prometheusJobOutlet) WriteEntry(entry logger.Entry) error {
|
||||||
jobFieldVal, ok := entry.Fields[logging.JobField].(string)
|
prom.taskLogEntries.WithLabelValues(o.jobName, entry.Level.String()).Inc()
|
||||||
if !ok {
|
|
||||||
jobFieldVal = "_nojobid"
|
|
||||||
}
|
|
||||||
prom.taskLogEntries.WithLabelValues(jobFieldVal, entry.Level.String()).Inc()
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -11,29 +11,20 @@ import (
|
|||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
"github.com/zrepl/zrepl/logger"
|
||||||
"github.com/zrepl/zrepl/internal/logger"
|
"github.com/zrepl/zrepl/pruning"
|
||||||
"github.com/zrepl/zrepl/internal/pruning"
|
"github.com/zrepl/zrepl/replication/logic/pdu"
|
||||||
"github.com/zrepl/zrepl/internal/replication/logic/pdu"
|
"github.com/zrepl/zrepl/util/envconst"
|
||||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// The sender in the replication setup.
|
// Try to keep it compatible with gitub.com/zrepl/zrepl/endpoint.Endpoint
|
||||||
// The pruner uses the Sender to determine which of the Target's filesystems need to be pruned.
|
type History interface {
|
||||||
// Also, it asks the Sender about the replication cursor of each filesystem
|
|
||||||
// to enable the 'not_replicated' pruning rule.
|
|
||||||
//
|
|
||||||
// Try to keep it compatible with github.com/zrepl/zrepl/endpoint.Endpoint
|
|
||||||
type Sender interface {
|
|
||||||
ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error)
|
ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error)
|
||||||
ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
|
ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The pruning target, i.e., on which snapshots are destroyed.
|
// Try to keep it compatible with gitub.com/zrepl/zrepl/endpoint.Endpoint
|
||||||
// This can be a replication sender or receiver.
|
|
||||||
//
|
|
||||||
// Try to keep it compatible with github.com/zrepl/zrepl/endpoint.Endpoint
|
|
||||||
type Target interface {
|
type Target interface {
|
||||||
ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
|
ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
|
||||||
ListFilesystemVersions(ctx context.Context, req *pdu.ListFilesystemVersionsReq) (*pdu.ListFilesystemVersionsRes, error)
|
ListFilesystemVersions(ctx context.Context, req *pdu.ListFilesystemVersionsReq) (*pdu.ListFilesystemVersionsRes, error)
|
||||||
@@ -44,19 +35,23 @@ type Logger = logger.Logger
|
|||||||
|
|
||||||
type contextKey int
|
type contextKey int
|
||||||
|
|
||||||
const (
|
const contextKeyLogger contextKey = 0
|
||||||
contextKeyPruneSide contextKey = 1 + iota
|
|
||||||
)
|
func WithLogger(ctx context.Context, log Logger) context.Context {
|
||||||
|
return context.WithValue(ctx, contextKeyLogger, log)
|
||||||
|
}
|
||||||
|
|
||||||
func GetLogger(ctx context.Context) Logger {
|
func GetLogger(ctx context.Context) Logger {
|
||||||
pruneSide := ctx.Value(contextKeyPruneSide).(string)
|
if l, ok := ctx.Value(contextKeyLogger).(Logger); ok {
|
||||||
return logging.GetLogger(ctx, logging.SubsysPruning).WithField("prune_side", pruneSide)
|
return l
|
||||||
|
}
|
||||||
|
return logger.NewNullLogger()
|
||||||
}
|
}
|
||||||
|
|
||||||
type args struct {
|
type args struct {
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
target Target
|
target Target
|
||||||
sender Sender
|
receiver History
|
||||||
rules []pruning.KeepRule
|
rules []pruning.KeepRule
|
||||||
retryWait time.Duration
|
retryWait time.Duration
|
||||||
considerSnapAtCursorReplicated bool
|
considerSnapAtCursorReplicated bool
|
||||||
@@ -140,12 +135,12 @@ func NewPrunerFactory(in config.PruningSenderReceiver, promPruneSecs *prometheus
|
|||||||
return f, nil
|
return f, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, sender Sender) *Pruner {
|
func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, receiver History) *Pruner {
|
||||||
p := &Pruner{
|
p := &Pruner{
|
||||||
args: args{
|
args: args{
|
||||||
context.WithValue(ctx, contextKeyPruneSide, "sender"),
|
WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "sender")),
|
||||||
target,
|
target,
|
||||||
sender,
|
receiver,
|
||||||
f.senderRules,
|
f.senderRules,
|
||||||
f.retryWait,
|
f.retryWait,
|
||||||
f.considerSnapAtCursorReplicated,
|
f.considerSnapAtCursorReplicated,
|
||||||
@@ -156,12 +151,12 @@ func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, se
|
|||||||
return p
|
return p
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target, sender Sender) *Pruner {
|
func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target, receiver History) *Pruner {
|
||||||
p := &Pruner{
|
p := &Pruner{
|
||||||
args: args{
|
args: args{
|
||||||
context.WithValue(ctx, contextKeyPruneSide, "receiver"),
|
WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "receiver")),
|
||||||
target,
|
target,
|
||||||
sender,
|
receiver,
|
||||||
f.receiverRules,
|
f.receiverRules,
|
||||||
f.retryWait,
|
f.retryWait,
|
||||||
false, // senseless here anyways
|
false, // senseless here anyways
|
||||||
@@ -172,12 +167,12 @@ func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target,
|
|||||||
return p
|
return p
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *LocalPrunerFactory) BuildLocalPruner(ctx context.Context, target Target, history Sender) *Pruner {
|
func (f *LocalPrunerFactory) BuildLocalPruner(ctx context.Context, target Target, receiver History) *Pruner {
|
||||||
p := &Pruner{
|
p := &Pruner{
|
||||||
args: args{
|
args: args{
|
||||||
context.WithValue(ctx, contextKeyPruneSide, "local"),
|
ctx,
|
||||||
target,
|
target,
|
||||||
history,
|
receiver,
|
||||||
f.keepRules,
|
f.keepRules,
|
||||||
f.retryWait,
|
f.retryWait,
|
||||||
false, // considerSnapAtCursorReplicated is not relevant for local pruning
|
false, // considerSnapAtCursorReplicated is not relevant for local pruning
|
||||||
@@ -199,16 +194,6 @@ const (
|
|||||||
Done
|
Done
|
||||||
)
|
)
|
||||||
|
|
||||||
// Returns true in case the State is a terminal state(PlanErr, ExecErr, Done)
|
|
||||||
func (s State) IsTerminal() bool {
|
|
||||||
switch s {
|
|
||||||
case PlanErr, ExecErr, Done:
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type updater func(func(*Pruner))
|
type updater func(func(*Pruner))
|
||||||
|
|
||||||
func (p *Pruner) Prune() {
|
func (p *Pruner) Prune() {
|
||||||
@@ -361,9 +346,9 @@ func (s snapshot) Date() time.Time { return s.date }
|
|||||||
|
|
||||||
func doOneAttempt(a *args, u updater) {
|
func doOneAttempt(a *args, u updater) {
|
||||||
|
|
||||||
ctx, target, sender := a.ctx, a.target, a.sender
|
ctx, target, receiver := a.ctx, a.target, a.receiver
|
||||||
|
|
||||||
sfssres, err := sender.ListFilesystems(ctx, &pdu.ListFilesystemReq{})
|
sfssres, err := receiver.ListFilesystems(ctx, &pdu.ListFilesystemReq{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
u(func(p *Pruner) {
|
u(func(p *Pruner) {
|
||||||
p.state = PlanErr
|
p.state = PlanErr
|
||||||
@@ -427,8 +412,11 @@ tfss_loop:
|
|||||||
|
|
||||||
rcReq := &pdu.ReplicationCursorReq{
|
rcReq := &pdu.ReplicationCursorReq{
|
||||||
Filesystem: tfs.Path,
|
Filesystem: tfs.Path,
|
||||||
|
Op: &pdu.ReplicationCursorReq_Get{
|
||||||
|
Get: &pdu.ReplicationCursorReq_GetOp{},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
rc, err := sender.ReplicationCursor(ctx, rcReq)
|
rc, err := receiver.ReplicationCursor(ctx, rcReq)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
pfsPlanErrAndLog(err, "cannot get replication cursor bookmark")
|
pfsPlanErrAndLog(err, "cannot get replication cursor bookmark")
|
||||||
continue tfss_loop
|
continue tfss_loop
|
||||||
@@ -474,7 +462,7 @@ tfss_loop:
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
if preCursor {
|
if preCursor {
|
||||||
pfsPlanErrAndLog(fmt.Errorf("prune target has no snapshot that corresponds to sender replication cursor bookmark"), "")
|
pfsPlanErrAndLog(fmt.Errorf("replication cursor not found in prune target filesystem versions"), "")
|
||||||
continue tfss_loop
|
continue tfss_loop
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
// Code generated by "enumer -type=State"; DO NOT EDIT.
|
// Code generated by "enumer -type=State"; DO NOT EDIT.
|
||||||
|
|
||||||
|
//
|
||||||
package pruner
|
package pruner
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -0,0 +1,465 @@
|
|||||||
|
package snapper
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
|
"github.com/zrepl/zrepl/config"
|
||||||
|
"github.com/zrepl/zrepl/daemon/filters"
|
||||||
|
"github.com/zrepl/zrepl/daemon/hooks"
|
||||||
|
"github.com/zrepl/zrepl/logger"
|
||||||
|
"github.com/zrepl/zrepl/zfs"
|
||||||
|
)
|
||||||
|
|
||||||
|
//go:generate stringer -type=SnapState
|
||||||
|
type SnapState uint
|
||||||
|
|
||||||
|
const (
|
||||||
|
SnapPending SnapState = 1 << iota
|
||||||
|
SnapStarted
|
||||||
|
SnapDone
|
||||||
|
SnapError
|
||||||
|
)
|
||||||
|
|
||||||
|
// All fields protected by Snapper.mtx
|
||||||
|
type snapProgress struct {
|
||||||
|
state SnapState
|
||||||
|
|
||||||
|
// SnapStarted, SnapDone, SnapError
|
||||||
|
name string
|
||||||
|
startAt time.Time
|
||||||
|
hookPlan *hooks.Plan
|
||||||
|
|
||||||
|
// SnapDone
|
||||||
|
doneAt time.Time
|
||||||
|
|
||||||
|
// SnapErr TODO disambiguate state
|
||||||
|
runResults hooks.PlanReport
|
||||||
|
}
|
||||||
|
|
||||||
|
type args struct {
|
||||||
|
ctx context.Context
|
||||||
|
log Logger
|
||||||
|
prefix string
|
||||||
|
interval time.Duration
|
||||||
|
fsf *filters.DatasetMapFilter
|
||||||
|
snapshotsTaken chan<- struct{}
|
||||||
|
hooks *hooks.List
|
||||||
|
dryRun bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type Snapper struct {
|
||||||
|
args args
|
||||||
|
|
||||||
|
mtx sync.Mutex
|
||||||
|
state State
|
||||||
|
|
||||||
|
// set in state Plan, used in Waiting
|
||||||
|
lastInvocation time.Time
|
||||||
|
|
||||||
|
// valid for state Snapshotting
|
||||||
|
plan map[*zfs.DatasetPath]*snapProgress
|
||||||
|
|
||||||
|
// valid for state SyncUp and Waiting
|
||||||
|
sleepUntil time.Time
|
||||||
|
|
||||||
|
// valid for state Err
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
//go:generate stringer -type=State
|
||||||
|
type State uint
|
||||||
|
|
||||||
|
const (
|
||||||
|
SyncUp State = 1 << iota
|
||||||
|
SyncUpErrWait
|
||||||
|
Planning
|
||||||
|
Snapshotting
|
||||||
|
Waiting
|
||||||
|
ErrorWait
|
||||||
|
Stopped
|
||||||
|
)
|
||||||
|
|
||||||
|
func (s State) sf() state {
|
||||||
|
m := map[State]state{
|
||||||
|
SyncUp: syncUp,
|
||||||
|
SyncUpErrWait: wait,
|
||||||
|
Planning: plan,
|
||||||
|
Snapshotting: snapshot,
|
||||||
|
Waiting: wait,
|
||||||
|
ErrorWait: wait,
|
||||||
|
Stopped: nil,
|
||||||
|
}
|
||||||
|
return m[s]
|
||||||
|
}
|
||||||
|
|
||||||
|
type updater func(u func(*Snapper)) State
|
||||||
|
type state func(a args, u updater) state
|
||||||
|
|
||||||
|
type contextKey int
|
||||||
|
|
||||||
|
const (
|
||||||
|
contextKeyLog contextKey = 0
|
||||||
|
)
|
||||||
|
|
||||||
|
type Logger = logger.Logger
|
||||||
|
|
||||||
|
func WithLogger(ctx context.Context, log Logger) context.Context {
|
||||||
|
return context.WithValue(ctx, contextKeyLog, log)
|
||||||
|
}
|
||||||
|
|
||||||
|
func getLogger(ctx context.Context) Logger {
|
||||||
|
if log, ok := ctx.Value(contextKeyLog).(Logger); ok {
|
||||||
|
return log
|
||||||
|
}
|
||||||
|
return logger.NewNullLogger()
|
||||||
|
}
|
||||||
|
|
||||||
|
func PeriodicFromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in *config.SnapshottingPeriodic) (*Snapper, error) {
|
||||||
|
if in.Prefix == "" {
|
||||||
|
return nil, errors.New("prefix must not be empty")
|
||||||
|
}
|
||||||
|
if in.Interval <= 0 {
|
||||||
|
return nil, errors.New("interval must be positive")
|
||||||
|
}
|
||||||
|
|
||||||
|
hookList, err := hooks.ListFromConfig(&in.Hooks)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "hook config error")
|
||||||
|
}
|
||||||
|
|
||||||
|
args := args{
|
||||||
|
prefix: in.Prefix,
|
||||||
|
interval: in.Interval,
|
||||||
|
fsf: fsf,
|
||||||
|
hooks: hookList,
|
||||||
|
// ctx and log is set in Run()
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Snapper{state: SyncUp, args: args}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Snapper) Run(ctx context.Context, snapshotsTaken chan<- struct{}) {
|
||||||
|
|
||||||
|
getLogger(ctx).Debug("start")
|
||||||
|
defer getLogger(ctx).Debug("stop")
|
||||||
|
|
||||||
|
s.args.snapshotsTaken = snapshotsTaken
|
||||||
|
s.args.ctx = ctx
|
||||||
|
s.args.log = getLogger(ctx)
|
||||||
|
s.args.dryRun = false // for future expansion
|
||||||
|
|
||||||
|
u := func(u func(*Snapper)) State {
|
||||||
|
s.mtx.Lock()
|
||||||
|
defer s.mtx.Unlock()
|
||||||
|
if u != nil {
|
||||||
|
u(s)
|
||||||
|
}
|
||||||
|
return s.state
|
||||||
|
}
|
||||||
|
|
||||||
|
var st state = syncUp
|
||||||
|
|
||||||
|
for st != nil {
|
||||||
|
pre := u(nil)
|
||||||
|
st = st(s.args, u)
|
||||||
|
post := u(nil)
|
||||||
|
getLogger(ctx).
|
||||||
|
WithField("transition", fmt.Sprintf("%s=>%s", pre, post)).
|
||||||
|
Debug("state transition")
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func onErr(err error, u updater) state {
|
||||||
|
return u(func(s *Snapper) {
|
||||||
|
s.err = err
|
||||||
|
preState := s.state
|
||||||
|
switch s.state {
|
||||||
|
case SyncUp:
|
||||||
|
s.state = SyncUpErrWait
|
||||||
|
case Planning:
|
||||||
|
fallthrough
|
||||||
|
case Snapshotting:
|
||||||
|
s.state = ErrorWait
|
||||||
|
}
|
||||||
|
s.args.log.WithError(err).WithField("pre_state", preState).WithField("post_state", s.state).Error("snapshotting error")
|
||||||
|
}).sf()
|
||||||
|
}
|
||||||
|
|
||||||
|
func onMainCtxDone(ctx context.Context, u updater) state {
|
||||||
|
return u(func(s *Snapper) {
|
||||||
|
s.err = ctx.Err()
|
||||||
|
s.state = Stopped
|
||||||
|
}).sf()
|
||||||
|
}
|
||||||
|
|
||||||
|
func syncUp(a args, u updater) state {
|
||||||
|
u(func(snapper *Snapper) {
|
||||||
|
snapper.lastInvocation = time.Now()
|
||||||
|
})
|
||||||
|
fss, err := listFSes(a.ctx, a.fsf)
|
||||||
|
if err != nil {
|
||||||
|
return onErr(err, u)
|
||||||
|
}
|
||||||
|
syncPoint, err := findSyncPoint(a.log, fss, a.prefix, a.interval)
|
||||||
|
if err != nil {
|
||||||
|
return onErr(err, u)
|
||||||
|
}
|
||||||
|
u(func(s *Snapper) {
|
||||||
|
s.sleepUntil = syncPoint
|
||||||
|
})
|
||||||
|
t := time.NewTimer(time.Until(syncPoint))
|
||||||
|
defer t.Stop()
|
||||||
|
select {
|
||||||
|
case <-t.C:
|
||||||
|
return u(func(s *Snapper) {
|
||||||
|
s.state = Planning
|
||||||
|
}).sf()
|
||||||
|
case <-a.ctx.Done():
|
||||||
|
return onMainCtxDone(a.ctx, u)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func plan(a args, u updater) state {
|
||||||
|
u(func(snapper *Snapper) {
|
||||||
|
snapper.lastInvocation = time.Now()
|
||||||
|
})
|
||||||
|
fss, err := listFSes(a.ctx, a.fsf)
|
||||||
|
if err != nil {
|
||||||
|
return onErr(err, u)
|
||||||
|
}
|
||||||
|
|
||||||
|
plan := make(map[*zfs.DatasetPath]*snapProgress, len(fss))
|
||||||
|
for _, fs := range fss {
|
||||||
|
plan[fs] = &snapProgress{state: SnapPending}
|
||||||
|
}
|
||||||
|
return u(func(s *Snapper) {
|
||||||
|
s.state = Snapshotting
|
||||||
|
s.plan = plan
|
||||||
|
s.err = nil
|
||||||
|
}).sf()
|
||||||
|
}
|
||||||
|
|
||||||
|
func snapshot(a args, u updater) state {
|
||||||
|
|
||||||
|
var plan map[*zfs.DatasetPath]*snapProgress
|
||||||
|
u(func(snapper *Snapper) {
|
||||||
|
plan = snapper.plan
|
||||||
|
})
|
||||||
|
|
||||||
|
hookMatchCount := make(map[hooks.Hook]int, len(*a.hooks))
|
||||||
|
for _, h := range *a.hooks {
|
||||||
|
hookMatchCount[h] = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
anyFsHadErr := false
|
||||||
|
// TODO channel programs -> allow a little jitter?
|
||||||
|
for fs, progress := range plan {
|
||||||
|
suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
|
||||||
|
snapname := fmt.Sprintf("%s%s", a.prefix, suffix)
|
||||||
|
|
||||||
|
l := a.log.
|
||||||
|
WithField("fs", fs.ToString()).
|
||||||
|
WithField("snap", snapname)
|
||||||
|
|
||||||
|
hookEnvExtra := hooks.Env{
|
||||||
|
hooks.EnvFS: fs.ToString(),
|
||||||
|
hooks.EnvSnapshot: snapname,
|
||||||
|
}
|
||||||
|
|
||||||
|
jobCallback := hooks.NewCallbackHookForFilesystem("snapshot", fs, func(_ context.Context) (err error) {
|
||||||
|
l.Debug("create snapshot")
|
||||||
|
err = zfs.ZFSSnapshot(fs, snapname, false) // TODO propagagte context to ZFSSnapshot
|
||||||
|
if err != nil {
|
||||||
|
l.WithError(err).Error("cannot create snapshot")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
})
|
||||||
|
|
||||||
|
fsHadErr := false
|
||||||
|
var planReport hooks.PlanReport
|
||||||
|
var plan *hooks.Plan
|
||||||
|
{
|
||||||
|
filteredHooks, err := a.hooks.CopyFilteredForFilesystem(fs)
|
||||||
|
if err != nil {
|
||||||
|
l.WithError(err).Error("unexpected filter error")
|
||||||
|
fsHadErr = true
|
||||||
|
goto updateFSState
|
||||||
|
}
|
||||||
|
// account for running hooks
|
||||||
|
for _, h := range filteredHooks {
|
||||||
|
hookMatchCount[h] = hookMatchCount[h] + 1
|
||||||
|
}
|
||||||
|
|
||||||
|
var planErr error
|
||||||
|
plan, planErr = hooks.NewPlan(&filteredHooks, hooks.PhaseSnapshot, jobCallback, hookEnvExtra)
|
||||||
|
if planErr != nil {
|
||||||
|
fsHadErr = true
|
||||||
|
l.WithError(planErr).Error("cannot create job hook plan")
|
||||||
|
goto updateFSState
|
||||||
|
}
|
||||||
|
}
|
||||||
|
u(func(snapper *Snapper) {
|
||||||
|
progress.name = snapname
|
||||||
|
progress.startAt = time.Now()
|
||||||
|
progress.hookPlan = plan
|
||||||
|
progress.state = SnapStarted
|
||||||
|
})
|
||||||
|
{
|
||||||
|
l := hooks.GetLogger(a.ctx).WithField("fs", fs.ToString()).WithField("snap", snapname)
|
||||||
|
l.WithField("report", plan.Report().String()).Debug("begin run job plan")
|
||||||
|
plan.Run(hooks.WithLogger(a.ctx, l), a.dryRun)
|
||||||
|
planReport = plan.Report()
|
||||||
|
fsHadErr = planReport.HadError() // not just fatal errors
|
||||||
|
if fsHadErr {
|
||||||
|
l.WithField("report", planReport.String()).Error("end run job plan with error")
|
||||||
|
} else {
|
||||||
|
l.WithField("report", planReport.String()).Info("end run job plan successful")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
updateFSState:
|
||||||
|
anyFsHadErr = anyFsHadErr || fsHadErr
|
||||||
|
u(func(snapper *Snapper) {
|
||||||
|
progress.doneAt = time.Now()
|
||||||
|
progress.state = SnapDone
|
||||||
|
if fsHadErr {
|
||||||
|
progress.state = SnapError
|
||||||
|
}
|
||||||
|
progress.runResults = planReport
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case a.snapshotsTaken <- struct{}{}:
|
||||||
|
default:
|
||||||
|
if a.snapshotsTaken != nil {
|
||||||
|
a.log.Warn("callback channel is full, discarding snapshot update event")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for h, mc := range hookMatchCount {
|
||||||
|
if mc == 0 {
|
||||||
|
hookIdx := -1
|
||||||
|
for idx, ah := range *a.hooks {
|
||||||
|
if ah == h {
|
||||||
|
hookIdx = idx
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.log.WithField("hook", h.String()).WithField("hook_number", hookIdx+1).Warn("hook did not match any snapshotted filesystems")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return u(func(snapper *Snapper) {
|
||||||
|
if anyFsHadErr {
|
||||||
|
snapper.state = ErrorWait
|
||||||
|
snapper.err = errors.New("one or more snapshots could not be created, check logs for details")
|
||||||
|
} else {
|
||||||
|
snapper.state = Waiting
|
||||||
|
snapper.err = nil
|
||||||
|
}
|
||||||
|
}).sf()
|
||||||
|
}
|
||||||
|
|
||||||
|
func wait(a args, u updater) state {
|
||||||
|
var sleepUntil time.Time
|
||||||
|
u(func(snapper *Snapper) {
|
||||||
|
lastTick := snapper.lastInvocation
|
||||||
|
snapper.sleepUntil = lastTick.Add(a.interval)
|
||||||
|
sleepUntil = snapper.sleepUntil
|
||||||
|
log := a.log.WithField("sleep_until", sleepUntil).WithField("duration", a.interval)
|
||||||
|
logFunc := log.Debug
|
||||||
|
if snapper.state == ErrorWait || snapper.state == SyncUpErrWait {
|
||||||
|
logFunc = log.Error
|
||||||
|
}
|
||||||
|
logFunc("enter wait-state after error")
|
||||||
|
})
|
||||||
|
|
||||||
|
t := time.NewTimer(time.Until(sleepUntil))
|
||||||
|
defer t.Stop()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-t.C:
|
||||||
|
return u(func(snapper *Snapper) {
|
||||||
|
snapper.state = Planning
|
||||||
|
}).sf()
|
||||||
|
case <-a.ctx.Done():
|
||||||
|
return onMainCtxDone(a.ctx, u)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func listFSes(ctx context.Context, mf *filters.DatasetMapFilter) (fss []*zfs.DatasetPath, err error) {
|
||||||
|
return zfs.ZFSListMapping(ctx, mf)
|
||||||
|
}
|
||||||
|
|
||||||
|
func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval time.Duration) (syncPoint time.Time, err error) {
|
||||||
|
type snapTime struct {
|
||||||
|
ds *zfs.DatasetPath
|
||||||
|
time time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(fss) == 0 {
|
||||||
|
return time.Now(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
snaptimes := make([]snapTime, 0, len(fss))
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
log.Debug("examine filesystem state")
|
||||||
|
for _, d := range fss {
|
||||||
|
|
||||||
|
l := log.WithField("fs", d.ToString())
|
||||||
|
|
||||||
|
fsvs, err := zfs.ZFSListFilesystemVersions(d, filters.NewTypedPrefixFilter(prefix, zfs.Snapshot))
|
||||||
|
if err != nil {
|
||||||
|
l.WithError(err).Error("cannot list filesystem versions")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(fsvs) <= 0 {
|
||||||
|
l.WithField("prefix", prefix).Debug("no filesystem versions with prefix")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort versions by creation
|
||||||
|
sort.SliceStable(fsvs, func(i, j int) bool {
|
||||||
|
return fsvs[i].CreateTXG < fsvs[j].CreateTXG
|
||||||
|
})
|
||||||
|
|
||||||
|
latest := fsvs[len(fsvs)-1]
|
||||||
|
l.WithField("creation", latest.Creation).
|
||||||
|
Debug("found latest snapshot")
|
||||||
|
|
||||||
|
since := now.Sub(latest.Creation)
|
||||||
|
if since < 0 {
|
||||||
|
l.WithField("snapshot", latest.Name).
|
||||||
|
WithField("creation", latest.Creation).
|
||||||
|
Error("snapshot is from the future")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
next := now
|
||||||
|
if since < interval {
|
||||||
|
next = latest.Creation.Add(interval)
|
||||||
|
}
|
||||||
|
snaptimes = append(snaptimes, snapTime{d, next})
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(snaptimes) == 0 {
|
||||||
|
snaptimes = append(snaptimes, snapTime{nil, now})
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Slice(snaptimes, func(i, j int) bool {
|
||||||
|
return snaptimes[i].time.Before(snaptimes[j].time)
|
||||||
|
})
|
||||||
|
|
||||||
|
return snaptimes[0].time, nil
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
package snapper
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/zrepl/zrepl/config"
|
||||||
|
"github.com/zrepl/zrepl/daemon/filters"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FIXME: properly abstract snapshotting:
|
||||||
|
// - split up things that trigger snapshotting from the mechanism
|
||||||
|
// - timer-based trigger (periodic)
|
||||||
|
// - call from control socket (manual)
|
||||||
|
// - mixed modes?
|
||||||
|
// - support a `zrepl snapshot JOBNAME` subcommand for config.SnapshottingManual
|
||||||
|
type PeriodicOrManual struct {
|
||||||
|
s *Snapper
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PeriodicOrManual) Run(ctx context.Context, wakeUpCommon chan<- struct{}) {
|
||||||
|
if s.s != nil {
|
||||||
|
s.s.Run(ctx, wakeUpCommon)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Returns nil if manual
|
||||||
|
func (s *PeriodicOrManual) Report() *Report {
|
||||||
|
if s.s != nil {
|
||||||
|
return s.s.Report()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func FromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in config.SnapshottingEnum) (*PeriodicOrManual, error) {
|
||||||
|
switch v := in.Ret.(type) {
|
||||||
|
case *config.SnapshottingPeriodic:
|
||||||
|
snapper, err := PeriodicFromConfig(g, fsf, v)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &PeriodicOrManual{snapper}, nil
|
||||||
|
case *config.SnapshottingManual:
|
||||||
|
return &PeriodicOrManual{}, nil
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unknown snapshotting type %T", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
package snapper
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/zrepl/zrepl/daemon/hooks"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Report struct {
|
||||||
|
State State
|
||||||
|
// valid in state SyncUp and Waiting
|
||||||
|
SleepUntil time.Time
|
||||||
|
// valid in state Err
|
||||||
|
Error string
|
||||||
|
// valid in state Snapshotting
|
||||||
|
Progress []*ReportFilesystem
|
||||||
|
}
|
||||||
|
|
||||||
|
type ReportFilesystem struct {
|
||||||
|
Path string
|
||||||
|
State SnapState
|
||||||
|
|
||||||
|
// Valid in SnapStarted and later
|
||||||
|
SnapName string
|
||||||
|
StartAt time.Time
|
||||||
|
Hooks string
|
||||||
|
HooksHadError bool
|
||||||
|
|
||||||
|
// Valid in SnapDone | SnapError
|
||||||
|
DoneAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func errOrEmptyString(e error) string {
|
||||||
|
if e != nil {
|
||||||
|
return e.Error()
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Snapper) Report() *Report {
|
||||||
|
s.mtx.Lock()
|
||||||
|
defer s.mtx.Unlock()
|
||||||
|
|
||||||
|
pReps := make([]*ReportFilesystem, 0, len(s.plan))
|
||||||
|
for fs, p := range s.plan {
|
||||||
|
var hooksStr string
|
||||||
|
var hooksHadError bool
|
||||||
|
if p.hookPlan != nil {
|
||||||
|
hr := p.hookPlan.Report()
|
||||||
|
// FIXME: technically this belongs into client
|
||||||
|
// but we can't serialize hooks.Step ATM
|
||||||
|
rightPad := func(str string, length int, pad string) string {
|
||||||
|
if len(str) > length {
|
||||||
|
return str[:length]
|
||||||
|
}
|
||||||
|
return str + strings.Repeat(pad, length-len(str))
|
||||||
|
}
|
||||||
|
hooksHadError = hr.HadError()
|
||||||
|
rows := make([][]string, len(hr))
|
||||||
|
const numCols = 4
|
||||||
|
lens := make([]int, numCols)
|
||||||
|
for i, e := range hr {
|
||||||
|
rows[i] = make([]string, numCols)
|
||||||
|
rows[i][0] = fmt.Sprintf("%d", i+1)
|
||||||
|
rows[i][1] = e.Status.String()
|
||||||
|
runTime := "..."
|
||||||
|
if e.Status != hooks.StepPending {
|
||||||
|
runTime = e.End.Sub(e.Begin).Round(time.Millisecond).String()
|
||||||
|
}
|
||||||
|
rows[i][2] = runTime
|
||||||
|
rows[i][3] = ""
|
||||||
|
if e.Report != nil {
|
||||||
|
rows[i][3] = e.Report.String()
|
||||||
|
}
|
||||||
|
for j, col := range lens {
|
||||||
|
if len(rows[i][j]) > col {
|
||||||
|
lens[j] = len(rows[i][j])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rowsFlat := make([]string, len(hr))
|
||||||
|
for i, r := range rows {
|
||||||
|
colsPadded := make([]string, len(r))
|
||||||
|
for j, c := range r[:len(r)-1] {
|
||||||
|
colsPadded[j] = rightPad(c, lens[j], " ")
|
||||||
|
}
|
||||||
|
colsPadded[len(r)-1] = r[len(r)-1]
|
||||||
|
rowsFlat[i] = strings.Join(colsPadded, " ")
|
||||||
|
}
|
||||||
|
hooksStr = strings.Join(rowsFlat, "\n")
|
||||||
|
}
|
||||||
|
pReps = append(pReps, &ReportFilesystem{
|
||||||
|
Path: fs.ToString(),
|
||||||
|
State: p.state,
|
||||||
|
SnapName: p.name,
|
||||||
|
StartAt: p.startAt,
|
||||||
|
DoneAt: p.doneAt,
|
||||||
|
Hooks: hooksStr,
|
||||||
|
HooksHadError: hooksHadError,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Slice(pReps, func(i, j int) bool {
|
||||||
|
return strings.Compare(pReps[i].Path, pReps[j].Path) == -1
|
||||||
|
})
|
||||||
|
|
||||||
|
r := &Report{
|
||||||
|
State: s.state,
|
||||||
|
SleepUntil: s.sleepUntil,
|
||||||
|
Error: errOrEmptyString(s.err),
|
||||||
|
Progress: pReps,
|
||||||
|
}
|
||||||
|
|
||||||
|
return r
|
||||||
|
}
|
||||||
+595
@@ -0,0 +1,595 @@
|
|||||||
|
{
|
||||||
|
"annotations": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"$$hashKey": "object:3351",
|
||||||
|
"builtIn": 1,
|
||||||
|
"datasource": "-- Grafana --",
|
||||||
|
"enable": true,
|
||||||
|
"hide": true,
|
||||||
|
"iconColor": "rgba(0, 211, 255, 1)",
|
||||||
|
"name": "Annotations & Alerts",
|
||||||
|
"type": "dashboard"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"editable": true,
|
||||||
|
"gnetId": null,
|
||||||
|
"graphTooltip": 1,
|
||||||
|
"id": 7,
|
||||||
|
"iteration": 1552746418826,
|
||||||
|
"links": [],
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"content": "# zrepl Prometheus Metrics\n\nzrepl exposes Prometheus metrics and ships with this Grafana dashboard.\nThe exported metrics are suitable for health checks:\n\n* The log should generally be warning & error-free\n * The `Log Messages that require attention` graph visualizes log message counts indicating problems.\n* The number of goroutines should not grow unboundedly over time.\n * During replication, the number of goroutines can be way higher than during idle time.\n * If the goroutine count grows with each replication, there is clearly a goroutine leak. Please open a bug report.\n* The sys memory consumption should not grow unboundedly over time.\n * Note that the Go runtime pre-allocates some of its heap from the OS.\n * zrepl actually uses much less memory than allocated from the OS.\n * Since Go 1.11, Go pre-allocates more aggressively.\n* Monitor that some data is replicated, although that metric does not guarantee that replication was successful.\n\n**In general, note that the exported metrics are not stable unless declared otherwise.**",
|
||||||
|
"gridPos": {
|
||||||
|
"h": 9,
|
||||||
|
"w": 24,
|
||||||
|
"x": 0,
|
||||||
|
"y": 0
|
||||||
|
},
|
||||||
|
"id": 35,
|
||||||
|
"mode": "markdown",
|
||||||
|
"title": "Panel Title",
|
||||||
|
"type": "text"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"aliasColors": {},
|
||||||
|
"bars": false,
|
||||||
|
"dashLength": 10,
|
||||||
|
"dashes": false,
|
||||||
|
"datasource": null,
|
||||||
|
"fill": 1,
|
||||||
|
"gridPos": {
|
||||||
|
"h": 9,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 9
|
||||||
|
},
|
||||||
|
"id": 15,
|
||||||
|
"legend": {
|
||||||
|
"avg": false,
|
||||||
|
"current": false,
|
||||||
|
"max": false,
|
||||||
|
"min": false,
|
||||||
|
"show": true,
|
||||||
|
"total": false,
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"lines": true,
|
||||||
|
"linewidth": 1,
|
||||||
|
"links": [],
|
||||||
|
"nullPointMode": "null",
|
||||||
|
"percentage": false,
|
||||||
|
"pointradius": 5,
|
||||||
|
"points": false,
|
||||||
|
"renderer": "flot",
|
||||||
|
"seriesOverrides": [],
|
||||||
|
"spaceLength": 10,
|
||||||
|
"stack": true,
|
||||||
|
"steppedLine": false,
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"$$hashKey": "object:3436",
|
||||||
|
"expr": "up{job='$prom_job_name'}",
|
||||||
|
"format": "time_series",
|
||||||
|
"intervalFactor": 1,
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"thresholds": [],
|
||||||
|
"timeFrom": null,
|
||||||
|
"timeShift": null,
|
||||||
|
"title": "zrepl Instances Up",
|
||||||
|
"tooltip": {
|
||||||
|
"shared": true,
|
||||||
|
"sort": 0,
|
||||||
|
"value_type": "individual"
|
||||||
|
},
|
||||||
|
"type": "graph",
|
||||||
|
"xaxis": {
|
||||||
|
"buckets": null,
|
||||||
|
"mode": "time",
|
||||||
|
"name": null,
|
||||||
|
"show": true,
|
||||||
|
"values": []
|
||||||
|
},
|
||||||
|
"yaxes": [
|
||||||
|
{
|
||||||
|
"format": "short",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": "5",
|
||||||
|
"min": "0",
|
||||||
|
"show": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"format": "short",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"aliasColors": {},
|
||||||
|
"bars": false,
|
||||||
|
"dashLength": 10,
|
||||||
|
"dashes": false,
|
||||||
|
"datasource": null,
|
||||||
|
"fill": 1,
|
||||||
|
"gridPos": {
|
||||||
|
"h": 9,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 9
|
||||||
|
},
|
||||||
|
"id": 22,
|
||||||
|
"legend": {
|
||||||
|
"avg": false,
|
||||||
|
"current": false,
|
||||||
|
"max": false,
|
||||||
|
"min": false,
|
||||||
|
"show": true,
|
||||||
|
"total": false,
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"lines": true,
|
||||||
|
"linewidth": 1,
|
||||||
|
"links": [],
|
||||||
|
"nullPointMode": "null",
|
||||||
|
"percentage": false,
|
||||||
|
"pointradius": 5,
|
||||||
|
"points": false,
|
||||||
|
"renderer": "flot",
|
||||||
|
"seriesOverrides": [],
|
||||||
|
"spaceLength": 10,
|
||||||
|
"stack": false,
|
||||||
|
"steppedLine": false,
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "increase(zrepl_daemon_log_entries{job='$prom_job_name',level=~'warn|error'}[$__interval])",
|
||||||
|
"format": "time_series",
|
||||||
|
"intervalFactor": 1,
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"thresholds": [],
|
||||||
|
"timeFrom": null,
|
||||||
|
"timeShift": null,
|
||||||
|
"title": "Log Messages that require attention",
|
||||||
|
"tooltip": {
|
||||||
|
"shared": true,
|
||||||
|
"sort": 0,
|
||||||
|
"value_type": "individual"
|
||||||
|
},
|
||||||
|
"type": "graph",
|
||||||
|
"xaxis": {
|
||||||
|
"buckets": null,
|
||||||
|
"mode": "time",
|
||||||
|
"name": null,
|
||||||
|
"show": true,
|
||||||
|
"values": []
|
||||||
|
},
|
||||||
|
"yaxes": [
|
||||||
|
{
|
||||||
|
"format": "short",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": "0",
|
||||||
|
"show": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"format": "short",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"aliasColors": {},
|
||||||
|
"bars": false,
|
||||||
|
"dashLength": 10,
|
||||||
|
"dashes": false,
|
||||||
|
"datasource": null,
|
||||||
|
"fill": 1,
|
||||||
|
"gridPos": {
|
||||||
|
"h": 9,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 18
|
||||||
|
},
|
||||||
|
"id": 33,
|
||||||
|
"legend": {
|
||||||
|
"avg": false,
|
||||||
|
"current": false,
|
||||||
|
"max": false,
|
||||||
|
"min": false,
|
||||||
|
"show": true,
|
||||||
|
"total": false,
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"lines": true,
|
||||||
|
"linewidth": 1,
|
||||||
|
"links": [],
|
||||||
|
"nullPointMode": "null",
|
||||||
|
"percentage": false,
|
||||||
|
"pointradius": 5,
|
||||||
|
"points": false,
|
||||||
|
"renderer": "flot",
|
||||||
|
"seriesOverrides": [
|
||||||
|
{
|
||||||
|
"alias": "/replicated bytes in last.*/",
|
||||||
|
"yaxis": 2
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"spaceLength": 10,
|
||||||
|
"stack": false,
|
||||||
|
"steppedLine": false,
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum(rate(zrepl_replication_bytes_replicated{job='$prom_job_name'}[$__interval])) by (zrepl_job)",
|
||||||
|
"format": "time_series",
|
||||||
|
"hide": false,
|
||||||
|
"interval": "",
|
||||||
|
"intervalFactor": 1,
|
||||||
|
"legendFormat": "replication data rate zrepl_job={{zrepl_job}}",
|
||||||
|
"refId": "A"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "sum(increase(zrepl_replication_bytes_replicated{job='$prom_job_name'}[10m])) by (zrepl_job)",
|
||||||
|
"format": "time_series",
|
||||||
|
"hide": false,
|
||||||
|
"interval": "",
|
||||||
|
"intervalFactor": 1,
|
||||||
|
"legendFormat": "replicated bytes in last 10min zrepl_job={{zrepl_job}}",
|
||||||
|
"refId": "B"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"thresholds": [],
|
||||||
|
"timeFrom": null,
|
||||||
|
"timeShift": null,
|
||||||
|
"title": "Replication Data Rate and Volume(integrates last 10min)",
|
||||||
|
"tooltip": {
|
||||||
|
"shared": true,
|
||||||
|
"sort": 0,
|
||||||
|
"value_type": "individual"
|
||||||
|
},
|
||||||
|
"type": "graph",
|
||||||
|
"xaxis": {
|
||||||
|
"buckets": null,
|
||||||
|
"mode": "time",
|
||||||
|
"name": null,
|
||||||
|
"show": true,
|
||||||
|
"values": []
|
||||||
|
},
|
||||||
|
"yaxes": [
|
||||||
|
{
|
||||||
|
"format": "Bps",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"format": "bytes",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"aliasColors": {},
|
||||||
|
"bars": false,
|
||||||
|
"dashLength": 10,
|
||||||
|
"dashes": false,
|
||||||
|
"datasource": null,
|
||||||
|
"fill": 1,
|
||||||
|
"gridPos": {
|
||||||
|
"h": 9,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 18
|
||||||
|
},
|
||||||
|
"id": 23,
|
||||||
|
"legend": {
|
||||||
|
"avg": false,
|
||||||
|
"current": false,
|
||||||
|
"max": false,
|
||||||
|
"min": false,
|
||||||
|
"show": true,
|
||||||
|
"total": false,
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"lines": true,
|
||||||
|
"linewidth": 1,
|
||||||
|
"links": [],
|
||||||
|
"nullPointMode": "null",
|
||||||
|
"percentage": false,
|
||||||
|
"pointradius": 5,
|
||||||
|
"points": false,
|
||||||
|
"renderer": "flot",
|
||||||
|
"seriesOverrides": [],
|
||||||
|
"spaceLength": 10,
|
||||||
|
"stack": false,
|
||||||
|
"steppedLine": false,
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum(increase(zrepl_daemon_log_entries{job='$prom_job_name',zrepl_job=~\"^[^_].*\"}[$__interval])) by (instance,zrepl_job)",
|
||||||
|
"format": "time_series",
|
||||||
|
"intervalFactor": 1,
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"thresholds": [],
|
||||||
|
"timeFrom": null,
|
||||||
|
"timeShift": null,
|
||||||
|
"title": "Log Activity (without internal jobs)",
|
||||||
|
"tooltip": {
|
||||||
|
"shared": true,
|
||||||
|
"sort": 0,
|
||||||
|
"value_type": "individual"
|
||||||
|
},
|
||||||
|
"type": "graph",
|
||||||
|
"xaxis": {
|
||||||
|
"buckets": null,
|
||||||
|
"mode": "time",
|
||||||
|
"name": null,
|
||||||
|
"show": true,
|
||||||
|
"values": []
|
||||||
|
},
|
||||||
|
"yaxes": [
|
||||||
|
{
|
||||||
|
"format": "short",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": "0",
|
||||||
|
"show": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"format": "short",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"aliasColors": {},
|
||||||
|
"bars": false,
|
||||||
|
"dashLength": 10,
|
||||||
|
"dashes": false,
|
||||||
|
"datasource": null,
|
||||||
|
"fill": 1,
|
||||||
|
"gridPos": {
|
||||||
|
"h": 9,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 27
|
||||||
|
},
|
||||||
|
"id": 17,
|
||||||
|
"legend": {
|
||||||
|
"avg": false,
|
||||||
|
"current": false,
|
||||||
|
"max": false,
|
||||||
|
"min": false,
|
||||||
|
"show": true,
|
||||||
|
"total": false,
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"lines": true,
|
||||||
|
"linewidth": 1,
|
||||||
|
"links": [],
|
||||||
|
"nullPointMode": "null",
|
||||||
|
"percentage": false,
|
||||||
|
"pointradius": 5,
|
||||||
|
"points": false,
|
||||||
|
"renderer": "flot",
|
||||||
|
"seriesOverrides": [],
|
||||||
|
"spaceLength": 10,
|
||||||
|
"stack": false,
|
||||||
|
"steppedLine": false,
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"$$hashKey": "object:3535",
|
||||||
|
"expr": "go_memstats_sys_bytes{job='$prom_job_name'}",
|
||||||
|
"format": "time_series",
|
||||||
|
"hide": false,
|
||||||
|
"intervalFactor": 1,
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"thresholds": [],
|
||||||
|
"timeFrom": null,
|
||||||
|
"timeShift": null,
|
||||||
|
"title": "Memory Allocated by the Go runtime from the OS (should not grow unboundedly)",
|
||||||
|
"tooltip": {
|
||||||
|
"shared": true,
|
||||||
|
"sort": 0,
|
||||||
|
"value_type": "individual"
|
||||||
|
},
|
||||||
|
"type": "graph",
|
||||||
|
"xaxis": {
|
||||||
|
"buckets": null,
|
||||||
|
"mode": "time",
|
||||||
|
"name": null,
|
||||||
|
"show": true,
|
||||||
|
"values": []
|
||||||
|
},
|
||||||
|
"yaxes": [
|
||||||
|
{
|
||||||
|
"format": "bytes",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": "0",
|
||||||
|
"show": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"format": "short",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"aliasColors": {},
|
||||||
|
"bars": false,
|
||||||
|
"dashLength": 10,
|
||||||
|
"dashes": false,
|
||||||
|
"datasource": null,
|
||||||
|
"fill": 1,
|
||||||
|
"gridPos": {
|
||||||
|
"h": 9,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 27
|
||||||
|
},
|
||||||
|
"id": 19,
|
||||||
|
"legend": {
|
||||||
|
"avg": false,
|
||||||
|
"current": false,
|
||||||
|
"max": false,
|
||||||
|
"min": false,
|
||||||
|
"show": true,
|
||||||
|
"total": false,
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"lines": true,
|
||||||
|
"linewidth": 1,
|
||||||
|
"links": [],
|
||||||
|
"nullPointMode": "null",
|
||||||
|
"percentage": false,
|
||||||
|
"pointradius": 5,
|
||||||
|
"points": false,
|
||||||
|
"renderer": "flot",
|
||||||
|
"seriesOverrides": [],
|
||||||
|
"spaceLength": 10,
|
||||||
|
"stack": false,
|
||||||
|
"steppedLine": false,
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "go_goroutines{job='$prom_job_name'}",
|
||||||
|
"format": "time_series",
|
||||||
|
"intervalFactor": 1,
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"thresholds": [],
|
||||||
|
"timeFrom": null,
|
||||||
|
"timeShift": null,
|
||||||
|
"title": "number of goroutines (should not grow unboundedly)",
|
||||||
|
"tooltip": {
|
||||||
|
"shared": true,
|
||||||
|
"sort": 0,
|
||||||
|
"value_type": "individual"
|
||||||
|
},
|
||||||
|
"type": "graph",
|
||||||
|
"xaxis": {
|
||||||
|
"buckets": null,
|
||||||
|
"mode": "time",
|
||||||
|
"name": null,
|
||||||
|
"show": true,
|
||||||
|
"values": []
|
||||||
|
},
|
||||||
|
"yaxes": [
|
||||||
|
{
|
||||||
|
"format": "short",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": "0",
|
||||||
|
"show": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"format": "short",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"refresh": "1m",
|
||||||
|
"schemaVersion": 16,
|
||||||
|
"style": "dark",
|
||||||
|
"tags": [],
|
||||||
|
"templating": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"allValue": null,
|
||||||
|
"current": {
|
||||||
|
"text": "zrepl",
|
||||||
|
"value": "zrepl"
|
||||||
|
},
|
||||||
|
"datasource": "prometheus",
|
||||||
|
"hide": 0,
|
||||||
|
"includeAll": false,
|
||||||
|
"label": "Prometheus Job Name",
|
||||||
|
"multi": false,
|
||||||
|
"name": "prom_job_name",
|
||||||
|
"options": [],
|
||||||
|
"query": "label_values(up, job)",
|
||||||
|
"refresh": 1,
|
||||||
|
"regex": "",
|
||||||
|
"sort": 1,
|
||||||
|
"tagValuesQuery": "",
|
||||||
|
"tags": [],
|
||||||
|
"tagsQuery": "",
|
||||||
|
"type": "query",
|
||||||
|
"useTags": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"time": {
|
||||||
|
"from": "now-2d",
|
||||||
|
"to": "now"
|
||||||
|
},
|
||||||
|
"timepicker": {
|
||||||
|
"refresh_intervals": [
|
||||||
|
"5s",
|
||||||
|
"10s",
|
||||||
|
"30s",
|
||||||
|
"1m",
|
||||||
|
"5m",
|
||||||
|
"15m",
|
||||||
|
"30m",
|
||||||
|
"1h",
|
||||||
|
"2h",
|
||||||
|
"1d"
|
||||||
|
],
|
||||||
|
"time_options": [
|
||||||
|
"5m",
|
||||||
|
"15m",
|
||||||
|
"1h",
|
||||||
|
"6h",
|
||||||
|
"12h",
|
||||||
|
"24h",
|
||||||
|
"2d",
|
||||||
|
"7d",
|
||||||
|
"30d"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"timezone": "",
|
||||||
|
"title": "zrepl 0.1",
|
||||||
|
"uid": "xTljn4qmk",
|
||||||
|
"version": 6
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user