Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Christian Schwarz
|
cd47103669 |
+89
-365
@@ -1,386 +1,110 @@
|
||||
version: 2.1
|
||||
orbs:
|
||||
# NB: this is not the Go version, but the Orb version
|
||||
# https://circleci.com/developer/orbs/orb/circleci/go#usage-go-modules-cache
|
||||
go: circleci/go@1.11.0
|
||||
|
||||
commands:
|
||||
setup-home-local-bin:
|
||||
steps:
|
||||
- run:
|
||||
shell: /bin/bash -euo pipefail
|
||||
command: |
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
line='export PATH="$HOME/.local/bin:$PATH"'
|
||||
if grep "$line" $BASH_ENV >/dev/null; then
|
||||
echo "$line" >> $BASH_ENV
|
||||
fi
|
||||
|
||||
# NOTE: UV version is defined in .uv-version file at repository root
|
||||
install-docdep:
|
||||
steps:
|
||||
- run:
|
||||
name: Read UV version from .uv-version
|
||||
command: |
|
||||
UV_VERSION=$(cat .uv-version)
|
||||
echo "export UV_VERSION=$UV_VERSION" >> $BASH_ENV
|
||||
# Python is managed by uv - it will automatically download the version
|
||||
# specified in docs/.python-version when needed
|
||||
- run:
|
||||
name: Install uv
|
||||
command: curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh
|
||||
- run:
|
||||
name: Add uv to PATH and set cache dir
|
||||
command: |
|
||||
echo 'export PATH="$HOME/.local/bin:$PATH"' >> $BASH_ENV
|
||||
echo 'export UV_CACHE_DIR="$HOME/.cache/uv"' >> $BASH_ENV
|
||||
- restore_cache:
|
||||
name: Restore uv cache
|
||||
keys:
|
||||
- uv-cache-v1-${UV_VERSION}-{{ checksum "docs/uv.lock" }}
|
||||
- uv-cache-v1-${UV_VERSION}-
|
||||
|
||||
save-uv-cache:
|
||||
steps:
|
||||
- run:
|
||||
name: Prune uv cache for CI
|
||||
command: uv cache prune --ci
|
||||
- save_cache:
|
||||
name: Save uv cache
|
||||
key: uv-cache-v1-${UV_VERSION}-{{ checksum "docs/uv.lock" }}
|
||||
paths:
|
||||
- ~/.cache/uv
|
||||
|
||||
install-zfs-from-source:
|
||||
parameters:
|
||||
zfs_release:
|
||||
type: string
|
||||
steps:
|
||||
- run:
|
||||
name: Record kernel version for cache key
|
||||
command: uname -r > /tmp/kernel-version
|
||||
- restore_cache:
|
||||
name: Restore ZFS native debs cache
|
||||
keys:
|
||||
- zfs-debs-v2-<<parameters.zfs_release>>-{{ checksum "/tmp/kernel-version" }}
|
||||
- run:
|
||||
# https://openzfs.github.io/openzfs-docs/Developer%20Resources/Building%20ZFS.html
|
||||
name: Build ZFS <<parameters.zfs_release>> native debs (if not cached)
|
||||
no_output_timeout: 20m
|
||||
command: |
|
||||
# CircleCI machine images have pyenv Python 3.13 shadowing the system
|
||||
# Python 3.12. The apt python3-* packages (setuptools, cffi, etc.) only
|
||||
# install for the system Python, and ZFS's dpkg-buildpackage needs them
|
||||
# for --enable-pyzfs. Use the system Python so apt packages are visible.
|
||||
export PYENV_VERSION=system
|
||||
if [ -d /tmp/zfs-debs ]; then
|
||||
echo "ZFS debs cache hit, skipping build"
|
||||
exit 0
|
||||
fi
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y \
|
||||
alien autoconf automake build-essential debhelper-compat dh-autoreconf \
|
||||
dh-dkms dh-python dkms fakeroot gawk git libaio-dev libattr1-dev \
|
||||
libblkid-dev libcurl4-openssl-dev libelf-dev libffi-dev libpam0g-dev \
|
||||
libssl-dev libtirpc-dev libtool libudev-dev linux-headers-$(uname -r) lsb-release \
|
||||
parallel po-debconf python3 python3-all-dev python3-cffi python3-dev \
|
||||
python3-packaging python3-setuptools python3-sphinx uuid-dev zlib1g-dev
|
||||
mkdir -p /tmp/zfs-build
|
||||
cd /tmp/zfs-build
|
||||
git clone --depth 1 --branch <<parameters.zfs_release>> https://github.com/openzfs/zfs.git zfs-src
|
||||
cd zfs-src
|
||||
sh autogen.sh
|
||||
./configure
|
||||
make native-deb
|
||||
mkdir -p /tmp/zfs-debs
|
||||
find /tmp/zfs-build -name '*.deb' -exec mv -t /tmp/zfs-debs/ {} +
|
||||
- save_cache:
|
||||
name: Save ZFS native debs cache
|
||||
key: zfs-debs-v2-<<parameters.zfs_release>>-{{ checksum "/tmp/kernel-version" }}
|
||||
paths:
|
||||
- /tmp/zfs-debs
|
||||
- run:
|
||||
name: Install ZFS <<parameters.zfs_release>>
|
||||
command: |
|
||||
ls /tmp/zfs-debs/
|
||||
# Only install the debs we need. Exclude dracut (conflicts with
|
||||
# initramfs-tools), dkms (we have prebuilt modules), and packages
|
||||
# we don't need (pyzfs, test, doc).
|
||||
sudo apt-get install -y $(find /tmp/zfs-debs -name '*.deb' \
|
||||
! -name '*dracut*' \
|
||||
! -name '*dkms*' \
|
||||
! -name '*initramfs*' \
|
||||
! -name '*pyzfs*' \
|
||||
! -name '*doc*' \
|
||||
! -name '*test*' \
|
||||
-print)
|
||||
sudo modprobe zfs
|
||||
- run:
|
||||
name: Verify ZFS version
|
||||
command: |
|
||||
sudo zfs version
|
||||
sudo zpool version
|
||||
|
||||
docs-publish-sh:
|
||||
parameters:
|
||||
push:
|
||||
type: boolean
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
command: |
|
||||
git config --global user.email "zreplbot@cschwarz.com"
|
||||
git config --global user.name "zrepl-github-io-ci"
|
||||
|
||||
# Configure git to use the GitHub token for HTTPS authentication.
|
||||
# The token is stored in the 'zrepl-github-io-deploy' context.
|
||||
- when:
|
||||
condition: << parameters.push >>
|
||||
steps:
|
||||
- run:
|
||||
name: Configure git to use GitHub token for push
|
||||
# GITHUB_PAGES_TOKEN is from the 'zrepl-github-io-deploy' context.
|
||||
# CircleCI's secret masking automatically redacts context variables in logs.
|
||||
command: |
|
||||
# Unset CircleCI's SSH URL rewriting that checkout step configured
|
||||
git config --global --unset-all url."ssh://git@github.com".insteadOf || true
|
||||
# Set up credential helper with GitHub token
|
||||
git config --global credential.helper store
|
||||
echo "https://x-access-token:${GITHUB_PAGES_TOKEN}@github.com" > ~/.git-credentials
|
||||
chmod 600 ~/.git-credentials
|
||||
|
||||
# caller must install-docdep
|
||||
- when:
|
||||
condition: << parameters.push >>
|
||||
steps:
|
||||
- run: bash -x docs/publish.sh -c -a -P
|
||||
- when:
|
||||
condition:
|
||||
not: << parameters.push >>
|
||||
steps:
|
||||
- run: bash -x docs/publish.sh -c -a
|
||||
|
||||
parameters:
|
||||
do_ci:
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
do_release:
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
version: 2.0
|
||||
workflows:
|
||||
version: 2
|
||||
|
||||
ci:
|
||||
when: << pipeline.parameters.do_ci >>
|
||||
build:
|
||||
jobs:
|
||||
- run-docs-publish-sh:
|
||||
name: quickcheck-docs
|
||||
push: false
|
||||
- quickcheck-go:
|
||||
name: quickcheck-go-amd64-linux-1.25.7
|
||||
goversion: &latest-go-release "1.25.7"
|
||||
goos: linux
|
||||
goarch: amd64
|
||||
- test-go:
|
||||
goversion: *latest-go-release
|
||||
- quickcheck-go:
|
||||
requires:
|
||||
- quickcheck-go-amd64-linux-1.25.7 #quickcheck-go-smoketest.name
|
||||
matrix:
|
||||
alias: quickcheck-go-matrix
|
||||
parameters:
|
||||
goversion: [*latest-go-release, "1.24.13"]
|
||||
goos: ["linux", "freebsd"]
|
||||
goarch: ["amd64", "arm64"]
|
||||
exclude:
|
||||
# don't re-do quickcheck-go-smoketest
|
||||
- goversion: *latest-go-release
|
||||
goos: linux
|
||||
goarch: amd64
|
||||
- platformtest:
|
||||
matrix:
|
||||
parameters:
|
||||
goversion: [*latest-go-release]
|
||||
goos: ["linux"]
|
||||
goarch: ["amd64"]
|
||||
zfs_release: ["zfs-2.2.9", "zfs-2.3.5", "zfs-2.4.0"]
|
||||
requires:
|
||||
- test-go
|
||||
- quickcheck-go-<< matrix.goarch >>-<< matrix.goos >>-<< matrix.goversion >>
|
||||
|
||||
release:
|
||||
when: << pipeline.parameters.do_release >>
|
||||
jobs:
|
||||
- release-build
|
||||
- release-deb:
|
||||
requires:
|
||||
- release-build
|
||||
- release-rpm:
|
||||
requires:
|
||||
- release-build
|
||||
- release-upload:
|
||||
requires:
|
||||
- release-build
|
||||
- release-deb
|
||||
- release-rpm
|
||||
|
||||
publish-zrepl.github.io:
|
||||
jobs:
|
||||
- run-docs-publish-sh:
|
||||
name: publish-zrepl.github.io
|
||||
push: true
|
||||
context:
|
||||
- zrepl-github-io-deploy
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
|
||||
- build-1.11
|
||||
- build-1.12
|
||||
- build-1.13
|
||||
- build-latest
|
||||
jobs:
|
||||
run-docs-publish-sh:
|
||||
# build-latest serves as the template
|
||||
# we use YAML anchors & aliases to exchange the docker image (and hence Go version used for the build)
|
||||
build-latest: &build-latest
|
||||
description: Builds zrepl
|
||||
parameters:
|
||||
push:
|
||||
type: boolean
|
||||
docker:
|
||||
- image: cimg/base:current
|
||||
steps:
|
||||
- checkout
|
||||
- install-docdep
|
||||
- docs-publish-sh:
|
||||
push: << parameters.push >>
|
||||
- save-uv-cache
|
||||
|
||||
quickcheck-go:
|
||||
parameters:
|
||||
goversion:
|
||||
type: string
|
||||
goos:
|
||||
type: string
|
||||
goarch:
|
||||
image:
|
||||
description: "the docker image that the job should use"
|
||||
type: string
|
||||
docker:
|
||||
- image: &cimg_with_modern_go cimg/go:1.25
|
||||
- image: circleci/golang:latest
|
||||
environment:
|
||||
GOOS: <<parameters.goos>>
|
||||
GOARCH: <<parameters.goarch>>
|
||||
GOTOOLCHAIN: "go<<parameters.goversion>>"
|
||||
|
||||
# required by lazy.sh
|
||||
TERM: xterm
|
||||
working_directory: /go/src/github.com/zrepl/zrepl
|
||||
steps:
|
||||
- run:
|
||||
name: Setup environment variables
|
||||
command: |
|
||||
# used by pip (for docs)
|
||||
echo 'export PATH="$HOME/.local/bin:$PATH"' >> $BASH_ENV
|
||||
# we use modules
|
||||
echo 'export GO111MODULE=on' >> $BASH_ENV
|
||||
|
||||
- restore_cache:
|
||||
keys:
|
||||
- source
|
||||
- protobuf
|
||||
|
||||
- checkout
|
||||
|
||||
- go/load-cache:
|
||||
key: quickcheck-<<parameters.goversion>>
|
||||
- run: make build/install
|
||||
- run: go mod download
|
||||
- run: cd build && go mod download
|
||||
- go/save-cache:
|
||||
key: quickcheck-<<parameters.goversion>>
|
||||
- save_cache:
|
||||
key: source
|
||||
paths:
|
||||
- ".git"
|
||||
|
||||
# install deps
|
||||
- run: wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
|
||||
- run: echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807 protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
|
||||
- run: sudo unzip -d /usr protoc-3.6.1-linux-x86_64.zip
|
||||
- save_cache:
|
||||
key: protobuf
|
||||
paths:
|
||||
- "/usr/include/google/protobuf"
|
||||
|
||||
- run: sudo apt update && sudo apt install python3 python3-pip libgirepository1.0-dev gawk
|
||||
- run: ./lazy.sh devsetup
|
||||
|
||||
# ensure all code has been generated
|
||||
- run: make generate
|
||||
- run: |
|
||||
if output=$(git status --porcelain) && [ -z "$output" ]; then
|
||||
echo "Working directory clean"
|
||||
else
|
||||
echo "Uncommitted changes"
|
||||
echo ""
|
||||
echo "$output"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# other checks
|
||||
- run: make zrepl-bin test-platform-bin
|
||||
- run: make zrepl-bin
|
||||
- run: make vet
|
||||
- run: make test
|
||||
- run: make lint
|
||||
- run: make release
|
||||
|
||||
|
||||
- store_artifacts:
|
||||
path: artifacts
|
||||
- persist_to_workspace:
|
||||
root: .
|
||||
paths: [.]
|
||||
path: ./artifacts/release
|
||||
when: always
|
||||
|
||||
platformtest:
|
||||
parameters:
|
||||
goversion:
|
||||
type: string
|
||||
goos:
|
||||
type: string
|
||||
goarch:
|
||||
type: string
|
||||
zfs_release:
|
||||
type: string
|
||||
machine:
|
||||
# pinned (not :current) to keep ZFS build cache valid across runs
|
||||
image: ubuntu-2404:2025.09.1
|
||||
resource_class: medium
|
||||
environment:
|
||||
GOOS: <<parameters.goos>>
|
||||
GOARCH: <<parameters.goarch>>
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- install-zfs-from-source:
|
||||
zfs_release: <<parameters.zfs_release>>
|
||||
- run: sudo make test-platform GOOS="$GOOS" GOARCH="$GOARCH"
|
||||
- run:
|
||||
shell: /bin/bash -eo pipefail
|
||||
when: always
|
||||
command: |
|
||||
if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
|
||||
echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
|
||||
exit 0
|
||||
fi
|
||||
set -u # from now on
|
||||
|
||||
# Download and install minio
|
||||
curl -sSL https://dl.minio.io/client/mc/release/linux-amd64/mc -o ${GOPATH}/bin/mc
|
||||
chmod +x ${GOPATH}/bin/mc
|
||||
mc config host add --api s3v4 zrepl-minio https://minio.cschwarz.com ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
|
||||
|
||||
# Upload artifacts
|
||||
echo "$CIRCLE_BUILD_URL" > ./artifacts/release/cirlceci_build_url
|
||||
mc cp -r artifacts/release "zrepl-minio/zrepl-ci-artifacts/${CIRCLE_SHA1}/${CIRCLE_JOB}/"
|
||||
|
||||
test-go:
|
||||
parameters:
|
||||
goversion:
|
||||
type: string
|
||||
# Push Artifact Link to GitHub
|
||||
REPO="zrepl/zrepl"
|
||||
COMMIT="${CIRCLE_SHA1}"
|
||||
JOB_NAME="${CIRCLE_JOB}"
|
||||
curl "https://api.github.com/repos/$REPO/statuses/$COMMIT?access_token=$GITHUB_COMMIT_STATUS_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST \
|
||||
-d '{"context":"zrepl/publish-ci-artifacts", "state": "success", "description":"CI Build Artifacts for '"$JOB_NAME"'", "target_url":"https://minio.cschwarz.com/minio/zrepl-ci-artifacts/'"$COMMIT"'/"}'
|
||||
|
||||
build-1.11:
|
||||
<<: *build-latest
|
||||
docker:
|
||||
- image: *cimg_with_modern_go
|
||||
environment:
|
||||
GOTOOLCHAIN: "go<<parameters.goversion>>"
|
||||
steps:
|
||||
- checkout
|
||||
- go/load-cache:
|
||||
key: make-test-go
|
||||
- run: make test-go
|
||||
- go/save-cache:
|
||||
key: make-test-go
|
||||
- image: circleci/golang:1.11
|
||||
|
||||
release-build:
|
||||
machine:
|
||||
image: &release-vm-image "ubuntu-2404:current"
|
||||
resource_class: large
|
||||
steps:
|
||||
- checkout
|
||||
- run: make release-docker
|
||||
- persist_to_workspace:
|
||||
root: .
|
||||
paths: [.]
|
||||
release-deb:
|
||||
machine:
|
||||
image: *release-vm-image
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- run: make debs-docker
|
||||
- persist_to_workspace:
|
||||
root: .
|
||||
paths:
|
||||
- "artifacts/*.deb"
|
||||
|
||||
release-rpm:
|
||||
machine:
|
||||
image: *release-vm-image
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- run: make rpms-docker
|
||||
- persist_to_workspace:
|
||||
root: .
|
||||
paths:
|
||||
- "artifacts/*.rpm"
|
||||
|
||||
release-upload:
|
||||
build-1.12:
|
||||
<<: *build-latest
|
||||
docker:
|
||||
- image: cimg/base:2024.09
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- run: make wrapup-and-checksum
|
||||
- store_artifacts:
|
||||
path: artifacts/release
|
||||
- image: circleci/golang:1.12
|
||||
|
||||
build-1.13:
|
||||
<<: *build-latest
|
||||
docker:
|
||||
- image: circleci/golang:1.13
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
import requests
|
||||
|
||||
import time
|
||||
|
||||
import os
|
||||
import yaml
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
circle_token = os.environ.get('CIRCLE_TOKEN')
|
||||
if not circle_token:
|
||||
cli_yml = Path.home() / ".circleci" / "cli.yml"
|
||||
if cli_yml.exists():
|
||||
with open(cli_yml) as f:
|
||||
data = yaml.safe_load(f)
|
||||
if data:
|
||||
circle_token = data.get("token")
|
||||
if not circle_token:
|
||||
raise ValueError('CIRCLE_TOKEN not set and no token found in ~/.circleci/cli.yml')
|
||||
|
||||
parser = argparse.ArgumentParser(description='Download artifacts from CircleCI')
|
||||
parser.add_argument('build_num', type=str, help='Build number')
|
||||
parser.add_argument('dst', type=Path, help='Destination directory')
|
||||
parser.add_argument('--prefix', type=str, default='', help='Filter for prefix')
|
||||
parser.add_argument('--match', type=str, default='.*', help='Only include paths matching the given regex')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
res = requests.get(
|
||||
f"https://circleci.com/api/v1.1/project/github/zrepl/zrepl/{args.build_num}/artifacts",
|
||||
headers={
|
||||
"Circle-Token": circle_token,
|
||||
},
|
||||
)
|
||||
res.raise_for_status()
|
||||
|
||||
# https://circleci.com/docs/api/v1/index.html#artifacts-of-a-job
|
||||
# [ {
|
||||
# "path" : "raw-test-output/go-test-report.xml",
|
||||
# "pretty_path" : "raw-test-output/go-test-report.xml",
|
||||
# "node_index" : 0,
|
||||
# "url" : "https://24-88881093-gh.circle-artifacts.com/0/raw-test-output/go-test-report.xml"
|
||||
# }, {
|
||||
# "path" : "raw-test-output/go-test.out",
|
||||
# "pretty_path" : "raw-test-output/go-test.out",
|
||||
# "node_index" : 0,
|
||||
# "url" : "https://24-88881093-gh.circle-artifacts.com/0/raw-test-output/go-test.out"
|
||||
# } ]
|
||||
res = res.json()
|
||||
|
||||
for artifact in res:
|
||||
if not artifact["pretty_path"].startswith(args.prefix):
|
||||
continue
|
||||
if not re.match(args.match, artifact["pretty_path"]):
|
||||
continue
|
||||
stripped = artifact["pretty_path"][len(args.prefix):]
|
||||
print(f"Downloading {artifact['pretty_path']} to {args.dst / stripped}")
|
||||
artifact_rel = Path(stripped)
|
||||
artifact_dst = args.dst / artifact_rel
|
||||
artifact_dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
res = requests.get(
|
||||
artifact["url"],
|
||||
headers={
|
||||
"Circle-Token": circle_token,
|
||||
},
|
||||
stream=True,
|
||||
)
|
||||
res.raise_for_status()
|
||||
|
||||
total_size = int(res.headers.get("Content-Length", 0))
|
||||
block_size = 128 * 1024
|
||||
with open(artifact_dst, "wb") as f:
|
||||
progress = 0
|
||||
start_time = time.time()
|
||||
for chunk in res.iter_content(chunk_size=block_size):
|
||||
f.write(chunk)
|
||||
progress += len(chunk)
|
||||
percent = progress / total_size * 100
|
||||
elapsed_time = time.time() - start_time
|
||||
if elapsed_time >= 5:
|
||||
print(f"Downloaded {progress}/{total_size} bytes ({percent:.2f}%)", end="\r")
|
||||
start_time = time.time()
|
||||
print(f"Downloaded {progress}/{total_size} bytes ({percent:.2f}%)")
|
||||
print("Download complete!")
|
||||
|
||||
print("All files downloaded")
|
||||
@@ -1,95 +0,0 @@
|
||||
---
|
||||
description: Generate GitHub release notes from docs/changelog.rst and create a draft release with artifacts
|
||||
argument-hint: <version-tag, e.g. v0.7.0>
|
||||
model: sonnet
|
||||
allowed-tools: Read, Write
|
||||
---
|
||||
|
||||
# Task
|
||||
|
||||
Create a draft GitHub release for zrepl version $ARGUMENTS and upload release artifacts.
|
||||
|
||||
## Inputs
|
||||
|
||||
Here is the current changelog RST source:
|
||||
|
||||
!`cat docs/changelog.rst`
|
||||
|
||||
## Instructions
|
||||
|
||||
### Phase 1: Pre-flight Sanity Checks
|
||||
|
||||
Before creating the release, verify that the previous build steps have been completed.
|
||||
Run these checks using bash commands:
|
||||
|
||||
```bash
|
||||
test -d artifacts/release && echo "✓ artifacts/release directory exists" || echo "✗ Missing artifacts/release directory - run: make download-circleci-release JOB_NUM=<num>"
|
||||
test -f artifacts/release/sha512sum.txt && echo "✓ sha512sum.txt exists" || echo "✗ Missing sha512sum.txt - run: make download-circleci-release JOB_NUM=<num>"
|
||||
test -f artifacts/release/sha512sum.txt.asc && echo "✓ sha512sum.txt.asc exists (signed)" || echo "✗ Missing signature - run: make verify-and-sign"
|
||||
ls artifacts/release/zrepl-* >/dev/null 2>&1 && echo "✓ Release artifacts present" || echo "✗ No release artifacts found"
|
||||
```
|
||||
|
||||
If ANY check fails (shows ✗), **STOP** and print the error messages. Do not proceed with release creation.
|
||||
|
||||
### Phase 2: Generate Release Notes
|
||||
|
||||
2. Extract the changelog section for version $ARGUMENTS from the RST source above.
|
||||
3. Extract all GitHub usernames mentioned in that changelog section (look for @username patterns or other contributor attributions).
|
||||
4. Write a short "Highlights" section (2-4 bullets) summarizing the most impactful user-facing changes in plain language. Do NOT include commit/issue links.
|
||||
5. Write a "Contributors" thank you line for all GitHub users found in step 3. Format as: "Thanks to @user1, @user2, and @user3 for their contributions to this release!"
|
||||
6. Write a "Breaking Changes" section. Convert RST formatting to plain Markdown text (no commit or issue links):
|
||||
- `:commit:\`abc123\`` → just remove it or describe what it changed
|
||||
- `:issue:\`123\`` → just remove it or describe the issue in plain text
|
||||
- `:ref:\`display text <anchor>\`` → just use `display text` as plain text
|
||||
- `:repomasterlink:\`path\`` → just mention the path without a link
|
||||
- RST inline code ` ``code`` ` → markdown `` `code` ``
|
||||
- RST links `` `text <url>`_ `` → just use `text` without the link
|
||||
If there are no breaking changes, write "No breaking changes. vX.Y.Z-1 is interoperable with vX.Y.Z." (fill in actual versions).
|
||||
7. Assemble the full release notes using the template below. **IMPORTANT**: Do NOT include a detailed changelog list. Only include what is specified in the template.
|
||||
8. Write the result to `artifacts/release-notes.md`.
|
||||
9. Validate all links in the release notes by checking HTTP status codes with curl:
|
||||
- Extract all URLs from the markdown file
|
||||
- Check each URL with `curl -sI -w "%{http_code}" -o /dev/null <url>`
|
||||
- Report any broken links (non-200 status codes)
|
||||
- If any links are broken, stop and notify the user before creating the release
|
||||
|
||||
### Phase 3: Create Draft Release and Upload Artifacts
|
||||
|
||||
10. Run: `gh release create $ARGUMENTS --title "$ARGUMENTS" --notes-file artifacts/release-notes.md --draft`
|
||||
11. Upload all artifacts: `gh release upload $ARGUMENTS artifacts/release/*`
|
||||
12. Verify upload succeeded: `gh release view $ARGUMENTS --json assets --jq '.assets | length'`
|
||||
13. Print the release URL and confirm that artifacts were uploaded successfully.
|
||||
|
||||
## Template
|
||||
|
||||
```
|
||||
The full changelog is [on the docs site](https://zrepl.github.io/changelog.html).
|
||||
|
||||
## Highlights
|
||||
|
||||
{2-4 bullet plain-language summary of the most impactful changes}
|
||||
|
||||
{Thank all GitHub users mentioned in the changelog, e.g., "Thanks to @user1, @user2, and @user3 for their contributions to this release!"}
|
||||
|
||||
## Breaking Changes
|
||||
|
||||
{breaking changes, or "No breaking changes."}
|
||||
|
||||
## New Users
|
||||
|
||||
We provide [quick-start guides](https://zrepl.github.io/quickstart.html) for different usage scenarios.
|
||||
We also recommend studying the [overview section of the configuration chapter](https://zrepl.github.io/configuration/overview.html).
|
||||
|
||||
## Testing & Upgrading
|
||||
|
||||
* Read the [Changelog](https://zrepl.github.io/changelog.html)
|
||||
* [Run the platform tests](https://zrepl.github.io/usage.html#platform-tests) on a test system.
|
||||
* Download & deploy the `zrepl` binary / distro package.
|
||||
|
||||
## Donations
|
||||
|
||||
zrepl is a spare-time project primarily developed by [Christian Schwarz](https://cschwarz.com).
|
||||
Express your support through a donation to keep maintenance and feature development going.
|
||||
|
||||
[](https://patreon.com/zrepl) [](https://github.com/sponsors/problame) [](https://liberapay.com/zrepl/donate) [](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
|
||||
```
|
||||
@@ -1,149 +0,0 @@
|
||||
---
|
||||
description: Update supporters list from git history and changelog contributors
|
||||
argument-hint: <time-range-or-release>
|
||||
model: sonnet
|
||||
---
|
||||
|
||||
# Task
|
||||
|
||||
Automatically update `docs/supporters.rst` by extracting GitHub contributors from:
|
||||
1. Git commit history (code contributors)
|
||||
2. `docs/changelog.rst` (all mentioned GitHub users)
|
||||
|
||||
## Arguments
|
||||
|
||||
Specify a time range or release range to filter contributors:
|
||||
|
||||
**Time range examples:**
|
||||
- `--since="2024-01-01"` - Contributors since a specific date
|
||||
- `--since="1 year ago"` - Contributors in the last year
|
||||
- `--since="6 months ago"` - Contributors in the last 6 months
|
||||
|
||||
**Release range examples:**
|
||||
- `v0.6.0..HEAD` - Contributors between v0.6.0 and current HEAD
|
||||
- `v0.6.0..v0.7.0` - Contributors between two releases
|
||||
- `--all` - All contributors (default if no argument provided)
|
||||
|
||||
## Instructions
|
||||
|
||||
### Step 1: Extract Contributors from Git History
|
||||
|
||||
Parse the arguments to determine the git log range:
|
||||
- If `--since="..."` is provided, use: `git log --since="..." --format='%aN <%aE>' | sort -u`
|
||||
- If a commit range like `v0.6.0..HEAD` is provided, use: `git log v0.6.0..HEAD --format='%aN <%aE>' | sort -u`
|
||||
- If `--all` or no argument, use: `git log --format='%aN <%aE>' | sort -u`
|
||||
|
||||
Parse the output to identify:
|
||||
- GitHub users (look for @users.noreply.github.com emails or known GitHub patterns)
|
||||
- Real names of code contributors
|
||||
- Filter out the main maintainer (Christian Schwarz / problame) as they're already credited
|
||||
|
||||
### Step 2: Extract GitHub Users from Changelog
|
||||
|
||||
Read `docs/changelog.rst` and extract all GitHub usernames mentioned:
|
||||
- Pattern: `` `@username <https://github.com/username>`_ ``
|
||||
- Pattern: `@username` in text
|
||||
- Collect all unique usernames
|
||||
|
||||
### Step 3: Categorize Contributors
|
||||
|
||||
For each contributor, determine their tier:
|
||||
|
||||
**Code contributors** (`|supporter-code|`):
|
||||
- Anyone who appears in git history as a commit author
|
||||
- Prioritize those with substantial contributions (multiple commits)
|
||||
|
||||
**Other contributors** (keep existing `|supporter-gold|` and `|supporter-std|`):
|
||||
- Keep all existing gold/std supporters as-is (monetary supporters)
|
||||
- Only update code contributors
|
||||
|
||||
### Step 4: Read Current Supporters File
|
||||
|
||||
Read `docs/supporters.rst` and parse existing entries to:
|
||||
- Preserve all `|supporter-gold|` entries (monetary supporters - don't touch these)
|
||||
- Preserve all `|supporter-std|` entries (monetary supporters - don't touch these)
|
||||
- Preserve ALL existing `|supporter-code|` entries in their current order
|
||||
- Extract list of existing code contributor names/GitHub usernames to avoid duplicates
|
||||
|
||||
### Step 5: Identify Contributors to Promote/Add
|
||||
|
||||
**IMPORTANT: Find contributors in the range, and either ADD them (if new) or MOVE them up (if recurring).**
|
||||
|
||||
For each GitHub user found in changelog or git history for the specified range:
|
||||
1. Check if they're already listed as `|supporter-code|` in the existing file
|
||||
2. Categorize them:
|
||||
- **NEW**: Not currently in the file → will be added
|
||||
- **RECURRING**: Already in the file with new contributions in this range → will be moved to top
|
||||
- **UNCHANGED**: Not in this range → stays in original position
|
||||
3. For both NEW and RECURRING contributors:
|
||||
- Get their latest contribution date from the range: `git log <range> --author="email" --format="%ai" -1`
|
||||
- Format as: `* |supporter-code| `Name <https://github.com/username>`_`
|
||||
- Use their real name if available from git history, otherwise use GitHub username
|
||||
4. Sort NEW + RECURRING contributors together by their contribution date (newest first)
|
||||
5. Keep list of UNCHANGED contributors in their original order
|
||||
|
||||
**The final order will be: [NEW + RECURRING from range, sorted newest first] + [UNCHANGED in original order]**
|
||||
|
||||
### Step 6: Generate Updated File
|
||||
|
||||
Create the updated supporters list:
|
||||
- Keep all `|supporter-gold|` entries unchanged and in their exact positions
|
||||
- Keep all `|supporter-std|` entries unchanged and in their exact positions
|
||||
- For `|supporter-code|` entries, reorganize as follows:
|
||||
1. **Top section**: Contributors from the analyzed range (NEW + RECURRING), sorted newest first
|
||||
2. **Bottom section**: UNCHANGED contributors (not in this range), in their original order
|
||||
- Update the spacer comment showing the range that was just processed:
|
||||
```rst
|
||||
..
|
||||
↓ claude --permission-mode default /update-supporters <actual-range-used>
|
||||
```
|
||||
Replace `<actual-range-used>` with the actual argument passed (e.g., `v0.6.1..v0.7.0`)
|
||||
This comment should appear immediately before the first code contributor entry
|
||||
If there's an existing comment in this format, replace it; otherwise add it
|
||||
- Maintain all other file structure and RST formatting, including the "Before /update-supporters" marker comment
|
||||
|
||||
### Step 7: Apply Changes
|
||||
|
||||
- Use Edit tool to update the `docs/supporters.rst` file
|
||||
- Display summary of changes made:
|
||||
- Time/release range analyzed
|
||||
- Number of contributors found
|
||||
- New code contributors added
|
||||
- Final counts by tier
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- **DO NOT modify, remove, or re-sort** any `|supporter-gold|` or `|supporter-std|` entries
|
||||
- **For code contributors**: ADD new ones and MOVE UP recurring ones (those with contributions in the analyzed range)
|
||||
- When extracting from git history, focus on substantial contributors (filter out single-commit or trivial changes if needed)
|
||||
- **CRITICAL:** Contributors from the analyzed range go to the top
|
||||
- Use `git log <range> --author="<email>" --format="%ai" -1` to get latest contribution date in the range
|
||||
- Both NEW and RECURRING contributors from this range are sorted together (newest first) and placed at the TOP
|
||||
- UNCHANGED contributors (not in this range) stay in their original relative order below
|
||||
- This creates a chronological history: most recent release's contributors appear first
|
||||
- Add/update the spacer comment showing the processed range
|
||||
- Handle edge cases:
|
||||
- Contributors mentioned in changelog but not in git history (external contributors, reporters)
|
||||
- Contributors in git history but not mentioned in changelog (decide whether to include)
|
||||
|
||||
## Example Output
|
||||
|
||||
```
|
||||
Analyzing contributors for range: v0.6.0..v0.7.0
|
||||
- Found 8 code contributors in git history
|
||||
- Found 12 GitHub users mentioned in changelog.rst (v0.7.0 section)
|
||||
- Current supporters.rst has 8 gold, 12 std, 10 code contributors
|
||||
|
||||
New contributors:
|
||||
+ Jane Doe (@janedoe)
|
||||
+ John Smith (@jsmith)
|
||||
|
||||
Promoted (recurring) contributors:
|
||||
↑ Alice Developer (@alice) - new contributions in v0.6.0..v0.7.0
|
||||
|
||||
Updated docs/supporters.rst:
|
||||
- Gold supporters: 8 (unchanged)
|
||||
- Std supporters: 12 (unchanged)
|
||||
- Code contributors: 10 → 12
|
||||
- Promoted to top: 3 contributors (2 new + 1 recurring)
|
||||
```
|
||||
@@ -1,4 +1,3 @@
|
||||
github: problame
|
||||
patreon: zrepl
|
||||
liberapay: zrepl
|
||||
custom: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
version: 2
|
||||
updates:
|
||||
# Docs use Python only for static site generation.
|
||||
- package-ecosystem: "pip"
|
||||
directory: /docs
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
ignore:
|
||||
- dependency-name: "*"
|
||||
@@ -1,6 +1,5 @@
|
||||
# Build
|
||||
artifacts/
|
||||
build/install.tmp
|
||||
|
||||
# Golang
|
||||
vendor/
|
||||
|
||||
+8
-40
@@ -1,42 +1,10 @@
|
||||
version: "2"
|
||||
linters:
|
||||
enable:
|
||||
- revive
|
||||
settings:
|
||||
revive:
|
||||
rules:
|
||||
- name: time-equal
|
||||
exclusions:
|
||||
generated: lax
|
||||
presets:
|
||||
- comments
|
||||
- common-false-positives
|
||||
- legacy
|
||||
- std-error-handling
|
||||
rules:
|
||||
- linters:
|
||||
- errcheck
|
||||
path: _test\.go
|
||||
- linters:
|
||||
- staticcheck
|
||||
text: 'SA9003:'
|
||||
- linters:
|
||||
- staticcheck
|
||||
text: '(QF1001|QF1011|ST1012|QF1008|ST1005|ST1023|QF1003|ST1006|ST1001|QF1004):'
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
formatters:
|
||||
enable:
|
||||
- goimports
|
||||
settings:
|
||||
goimports:
|
||||
local-prefixes:
|
||||
- github.com/zrepl/zrepl
|
||||
exclusions:
|
||||
generated: lax
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
- goimports
|
||||
|
||||
issues:
|
||||
exclude-rules:
|
||||
- path: _test\.go
|
||||
linters:
|
||||
- errcheck
|
||||
|
||||
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
dist: xenial
|
||||
services:
|
||||
- docker
|
||||
|
||||
env: # for allow_failures: https://docs.travis-ci.com/user/customizing-the-build/
|
||||
matrix:
|
||||
include:
|
||||
|
||||
- language: go
|
||||
name: "Build in Docker (docs/installation.rst)"
|
||||
script:
|
||||
- sudo docker build -t zrepl_build -f build.Dockerfile .
|
||||
- |
|
||||
sudo docker run -it --rm \
|
||||
-v "${PWD}:/go/src/github.com/zrepl/zrepl" \
|
||||
--user "$(id -u):$(id -g)" \
|
||||
zrepl_build make vendordeps release
|
||||
|
||||
- &zrepl_build_template
|
||||
language: go
|
||||
go_import_path: github.com/zrepl/zrepl
|
||||
before_install:
|
||||
- wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
|
||||
- echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807 protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
|
||||
- sudo unzip -d /usr protoc-3.6.1-linux-x86_64.zip
|
||||
- ./lazy.sh godep
|
||||
- make vendordeps
|
||||
script:
|
||||
- make
|
||||
- make vet
|
||||
- make test
|
||||
- make lint
|
||||
- make artifacts/zrepl-freebsd-amd64
|
||||
- make artifacts/zrepl-linux-amd64
|
||||
- make artifacts/zrepl-darwin-amd64
|
||||
go:
|
||||
- "1.11"
|
||||
|
||||
- <<: *zrepl_build_template
|
||||
go:
|
||||
- "1.12"
|
||||
|
||||
- <<: *zrepl_build_template
|
||||
go:
|
||||
- "master"
|
||||
|
||||
- &zrepl_docs_template
|
||||
language: python
|
||||
python:
|
||||
- "3.4"
|
||||
install:
|
||||
- sudo apt-get install libgirepository1.0-dev
|
||||
- pip install -r docs/requirements.txt
|
||||
script:
|
||||
- make docs
|
||||
- <<: *zrepl_docs_template
|
||||
python:
|
||||
- "3.5"
|
||||
- <<: *zrepl_docs_template
|
||||
python:
|
||||
- "3.6"
|
||||
- <<: *zrepl_docs_template
|
||||
python:
|
||||
- "3.7"
|
||||
|
||||
|
||||
allow_failures:
|
||||
- <<: *zrepl_build_template
|
||||
go:
|
||||
- "master"
|
||||
@@ -1 +0,0 @@
|
||||
0.9.30
|
||||
@@ -1,5 +1,5 @@
|
||||
.PHONY: generate build test vet cover release docs docs-clean clean format lint platformtest
|
||||
.PHONY: release release-noarch
|
||||
.PHONY: release bins-all release-noarch
|
||||
.DEFAULT_GOAL := zrepl-bin
|
||||
|
||||
ARTIFACTDIR := artifacts
|
||||
@@ -14,182 +14,56 @@ ifndef _ZREPL_VERSION
|
||||
endif
|
||||
endif
|
||||
|
||||
ZREPL_PACKAGE_RELEASE := 1
|
||||
|
||||
GO := go
|
||||
GOOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOOS"')
|
||||
GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"')
|
||||
GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"')
|
||||
GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"')
|
||||
GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
|
||||
GO_ENV_VARS := CGO_ENABLED=0
|
||||
GO_LDFLAGS := "-X github.com/zrepl/zrepl/internal/version.zreplVersion=$(_ZREPL_VERSION)"
|
||||
GO_ENV_VARS := GO111MODULE=on
|
||||
GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
|
||||
GO_MOD_READONLY := -mod=readonly
|
||||
GO_EXTRA_BUILDFLAGS :=
|
||||
GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
|
||||
GO_BUILDFLAGS := $(GO_MOD_READONLY)
|
||||
GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
|
||||
GOCOVMERGE := gocovmerge
|
||||
RELEASE_GOVERSION ?= go1.25.7
|
||||
STRIPPED_GOVERSION := $(subst go,,$(RELEASE_GOVERSION))
|
||||
RELEASE_DOCKER_BASEIMAGE ?= golang:$(STRIPPED_GOVERSION)
|
||||
RELEASE_DOCKER_CACHEMOUNT :=
|
||||
GOLANGCI_LINT := golangci-lint
|
||||
|
||||
ifneq ($(GOARM),)
|
||||
ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM)
|
||||
else
|
||||
ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)
|
||||
endif
|
||||
.PHONY: printvars
|
||||
printvars:
|
||||
@echo GOOS=$(GOOS)
|
||||
@echo GOARCH=$(GOARCH)
|
||||
|
||||
|
||||
ifneq ($(RELEASE_DOCKER_CACHEMOUNT),)
|
||||
_RELEASE_DOCKER_CACHEMOUNT := -v $(RELEASE_DOCKER_CACHEMOUNT)/mod:/go/pkg/mod -v $(RELEASE_DOCKER_CACHEMOUNT)/xdg-cache:/.cache/go-build
|
||||
.PHONY: release-docker-mkcachemount
|
||||
release-docker-mkcachemount:
|
||||
mkdir -p $(RELEASE_DOCKER_CACHEMOUNT)
|
||||
mkdir -p $(RELEASE_DOCKER_CACHEMOUNT)/mod
|
||||
mkdir -p $(RELEASE_DOCKER_CACHEMOUNT)/xdg-cache
|
||||
else
|
||||
_RELEASE_DOCKER_CACHEMOUNT :=
|
||||
.PHONY: release-docker-mkcachemount
|
||||
release-docker-mkcachemount:
|
||||
# nothing to do
|
||||
endif
|
||||
|
||||
##################### PRODUCING A RELEASE #############
|
||||
.PHONY: release wrapup-and-checksum check-git-clean verify-and-sign clean ensure-release-toolchain
|
||||
.PHONY: release wrapup-and-checksum check-git-clean sign clean
|
||||
|
||||
ensure-release-toolchain:
|
||||
# ensure the toolchain is actually the one we expect
|
||||
test $(RELEASE_GOVERSION) = "$$($(GO_ENV_VARS) $(GO) env GOVERSION)"
|
||||
|
||||
release: ensure-release-toolchain
|
||||
$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="vet"
|
||||
$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="lint"
|
||||
$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="zrepl-bin"
|
||||
$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="test-platform-bin"
|
||||
release: clean
|
||||
# no cross-platform support for target test
|
||||
$(MAKE) test
|
||||
$(MAKE) bins-all
|
||||
$(MAKE) noarch
|
||||
|
||||
release-docker: $(ARTIFACTDIR) release-docker-mkcachemount
|
||||
sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build/build.Dockerfile > $(ARTIFACTDIR)/build.Dockerfile
|
||||
docker build -t zrepl_release --pull \
|
||||
--build-arg BUILD_UID=$$(id -u) \
|
||||
--build-arg BUILD_GID=$$(id -g) \
|
||||
-f $(ARTIFACTDIR)/build.Dockerfile .
|
||||
docker run --rm -i$$(test -t 0 && echo t) \
|
||||
$(_RELEASE_DOCKER_CACHEMOUNT) \
|
||||
-v $(CURDIR):/src \
|
||||
zrepl_release \
|
||||
make release \
|
||||
GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
|
||||
ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE) \
|
||||
RELEASE_GOVERSION=$(RELEASE_GOVERSION)
|
||||
|
||||
debs-docker:
|
||||
$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=deb
|
||||
rpms-docker:
|
||||
$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=rpm
|
||||
_debs_or_rpms_docker: # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
|
||||
$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=amd64
|
||||
$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=arm64
|
||||
$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=arm GOARM=7
|
||||
$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=386
|
||||
|
||||
rpm: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
|
||||
$(eval _ZREPL_RPM_VERSION := $(subst -,.,$(_ZREPL_VERSION)))
|
||||
$(eval _ZREPL_RPM_TOPDIR_ABS := $(CURDIR)/$(ARTIFACTDIR)/rpmbuild)
|
||||
rm -rf "$(_ZREPL_RPM_TOPDIR_ABS)"
|
||||
mkdir "$(_ZREPL_RPM_TOPDIR_ABS)"
|
||||
for d in BUILD BUILDROOT RPMS SOURCES SPECS SRPMS; do \
|
||||
mkdir -p "$(_ZREPL_RPM_TOPDIR_ABS)/$$d"; \
|
||||
done
|
||||
sed \
|
||||
-e "s/^Version:.*/Version: $(_ZREPL_RPM_VERSION)/g" \
|
||||
-e "s/^Release:.*/Release: $(ZREPL_PACKAGE_RELEASE)/g" \
|
||||
packaging/rpm/zrepl.spec \
|
||||
> $(_ZREPL_RPM_TOPDIR_ABS)/SPECS/zrepl.spec
|
||||
|
||||
# see /usr/lib/rpm/platform
|
||||
ifeq ($(GOARCH),amd64)
|
||||
$(eval _ZREPL_RPMBUILD_TARGET := x86_64)
|
||||
else ifeq ($(GOARCH), 386)
|
||||
$(eval _ZREPL_RPMBUILD_TARGET := i386)
|
||||
else ifeq ($(GOARCH), arm64)
|
||||
$(eval _ZREPL_RPMBUILD_TARGET := aarch64)
|
||||
else ifeq ($(GOARCH), arm)
|
||||
$(eval _ZREPL_RPMBUILD_TARGET := armv7hl)
|
||||
else
|
||||
$(eval _ZREPL_RPMBUILD_TARGET := $(GOARCH))
|
||||
$(MAKE) wrapup-and-checksum
|
||||
$(MAKE) check-git-clean
|
||||
ifeq (SIGN, 1)
|
||||
$(make) sign
|
||||
endif
|
||||
rpmbuild \
|
||||
--build-in-place \
|
||||
--define "_sourcedir $(CURDIR)" \
|
||||
--define "_topdir $(_ZREPL_RPM_TOPDIR_ABS)" \
|
||||
--define "_zrepl_binary_filename zrepl-$(ZREPL_TARGET_TUPLE)" \
|
||||
--target $(_ZREPL_RPMBUILD_TARGET) \
|
||||
-bb "$(_ZREPL_RPM_TOPDIR_ABS)"/SPECS/zrepl.spec
|
||||
cp "$(_ZREPL_RPM_TOPDIR_ABS)"/RPMS/$(_ZREPL_RPMBUILD_TARGET)/zrepl-$(_ZREPL_RPM_VERSION)*.rpm $(ARTIFACTDIR)/
|
||||
@echo "ZREPL RELEASE ARTIFACTS AVAILABLE IN artifacts/release"
|
||||
|
||||
rpm-docker:
|
||||
docker build -t zrepl_rpm_pkg --pull -f packaging/rpm/Dockerfile .
|
||||
docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
|
||||
zrepl_rpm_pkg \
|
||||
make rpm \
|
||||
GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
|
||||
ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE)
|
||||
|
||||
|
||||
deb: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
|
||||
|
||||
cp packaging/deb/debian/changelog.template packaging/deb/debian/changelog
|
||||
sed -i 's/DATE_DASH_R_OUTPUT/$(shell date -R)/' packaging/deb/debian/changelog
|
||||
VERSION="$(subst -,.,$(_ZREPL_VERSION))-$(ZREPL_PACKAGE_RELEASE)"; \
|
||||
export VERSION="$${VERSION#v}"; \
|
||||
sed -i 's/VERSION/'"$$VERSION"'/' packaging/deb/debian/changelog
|
||||
|
||||
ifeq ($(GOARCH), arm)
|
||||
$(eval DEB_HOST_ARCH := armhf)
|
||||
else ifeq ($(GOARCH), 386)
|
||||
$(eval DEB_HOST_ARCH := i386)
|
||||
else
|
||||
$(eval DEB_HOST_ARCH := $(GOARCH))
|
||||
endif
|
||||
|
||||
export ZREPL_DPKG_ZREPL_BINARY_FILENAME=zrepl-$(ZREPL_TARGET_TUPLE); \
|
||||
dpkg-buildpackage -b --no-sign --host-arch $(DEB_HOST_ARCH)
|
||||
cp ../*.deb artifacts/
|
||||
|
||||
deb-docker:
|
||||
docker build -t zrepl_debian_pkg --pull -f packaging/deb/Dockerfile .
|
||||
# Use a small open file limit to make fakeroot work. If we don't
|
||||
# specify it, docker daemon will use its file limit. I don't know
|
||||
# what changed (Docker, its systemd service, its Go version). But I
|
||||
# observed fakeroot iterating close(i) up to i > 1000000, which costs
|
||||
# a good amount of CPU time and makes the build slow.
|
||||
docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
|
||||
--ulimit nofile=1024:1024 \
|
||||
zrepl_debian_pkg \
|
||||
make deb \
|
||||
GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
|
||||
ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE)
|
||||
|
||||
# expects `release`, `deb` & `rpm` targets to have run before
|
||||
# expects `release` target to have run before
|
||||
NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
|
||||
wrapup-and-checksum:
|
||||
rm -f $(NOARCH_TARBALL)
|
||||
tar --mtime='1970-01-01' --sort=name \
|
||||
--transform 's/$(ARTIFACTDIR)/zrepl-$(_ZREPL_VERSION)-noarch/' \
|
||||
--transform 's#dist#zrepl-$(_ZREPL_VERSION)-noarch/dist#' \
|
||||
--transform 's#internal/config/samples#zrepl-$(_ZREPL_VERSION)-noarch/config#' \
|
||||
--transform 's#config/samples#zrepl-$(_ZREPL_VERSION)-noarch/config#' \
|
||||
-acf $(NOARCH_TARBALL) \
|
||||
$(ARTIFACTDIR)/docs/html \
|
||||
$(ARTIFACTDIR)/bash_completion \
|
||||
$(ARTIFACTDIR)/_zrepl.zsh_completion \
|
||||
$(ARTIFACTDIR)/go_env.txt \
|
||||
dist \
|
||||
internal/config/samples
|
||||
config/samples
|
||||
rm -rf "$(ARTIFACTDIR)/release"
|
||||
mkdir -p "$(ARTIFACTDIR)/release"
|
||||
cp -l $(ARTIFACTDIR)/zrepl* \
|
||||
cp -l $(ARTIFACTDIR)/zrepl-* \
|
||||
$(ARTIFACTDIR)/platformtest-* \
|
||||
"$(ARTIFACTDIR)/release"
|
||||
cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
|
||||
@@ -207,159 +81,66 @@ check-git-clean:
|
||||
fi; \
|
||||
fi;
|
||||
|
||||
tag-release:
|
||||
test -n "$(ZREPL_TAG_VERSION)" || exit 1
|
||||
git tag -u '328A6627FA98061D!' -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)"
|
||||
|
||||
verify-and-sign:
|
||||
cd "$(ARTIFACTDIR)/release" && sha512sum -c sha512sum.txt
|
||||
gpg -u '328A6627FA98061D!' \
|
||||
sign:
|
||||
gpg -u "89BC 5D89 C845 568B F578 B306 CDBD 8EC8 E27C A5FC" \
|
||||
--armor \
|
||||
--detach-sign $(ARTIFACTDIR)/release/sha512sum.txt
|
||||
|
||||
clean: docs-clean
|
||||
rm -rf "$(ARTIFACTDIR)"
|
||||
|
||||
download-circleci-release:
|
||||
rm -rf "$(ARTIFACTDIR)"
|
||||
mkdir -p "$(ARTIFACTDIR)/release"
|
||||
python3 .circleci/download_artifacts.py --prefix 'artifacts/release/' "$(JOB_NUM)" "$(ARTIFACTDIR)/release"
|
||||
##################### BINARIES #####################
|
||||
.PHONY: bins-all lint test vet zrepl-bin platformtest-bin
|
||||
|
||||
##################### MULTI-ARCH HELPERS #####################
|
||||
|
||||
_run_make_foreach_target_tuple:
|
||||
if [ "$(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG)" = "" ]; then \
|
||||
echo "RUN_MAKE_FOREACH_TARGET_TUPLE_ARG must be set"; \
|
||||
exit 1; \
|
||||
fi
|
||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd GOARCH=amd64
|
||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd GOARCH=386
|
||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd GOARCH=arm GOARM=7
|
||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd GOARCH=arm64
|
||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux GOARCH=amd64
|
||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux GOARCH=arm64
|
||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux GOARCH=arm GOARM=7
|
||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux GOARCH=386
|
||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=darwin GOARCH=amd64
|
||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=solaris GOARCH=amd64
|
||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=illumos GOARCH=amd64
|
||||
|
||||
##################### REGULAR TARGETS #####################
|
||||
.PHONY: lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin
|
||||
|
||||
lint: build/install
|
||||
$(GO_ENV_VARS) build/install/golangci_lint/golangci-lint run ./...
|
||||
BINS_ALL_TARGETS := zrepl-bin platformtest-bin vet lint
|
||||
GO_SUPPORTS_ILLUMOS := $(shell $(GO) version | gawk -F '.' '/^go version /{split($$0, comps, " "); split(comps[3], v, "."); if (v[1] == "go1" && v[2] >= 13) { print "illumos"; } else { print "noillumos"; }}')
|
||||
bins-all:
|
||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd GOARCH=amd64
|
||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd GOARCH=386
|
||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux GOARCH=amd64
|
||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux GOARCH=arm64
|
||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux GOARCH=386
|
||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=darwin GOARCH=amd64
|
||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=solaris GOARCH=amd64
|
||||
ifeq ($(GO_SUPPORTS_ILLUMOS), illumos)
|
||||
$(MAKE) $(BINS_ALL_TARGETS) GOOS=illumos GOARCH=amd64
|
||||
else ifeq ($(GO_SUPPORTS_ILLUMOS), noillumos)
|
||||
@echo "SKIPPING ILLUMOS BUILD BECAUSE GO VERSION DOESN'T SUPPORT IT"
|
||||
else
|
||||
@echo "CANNOT DETERMINE WHETHER GO VERSION SUPPORTS GOOS=illumos"; exit 1
|
||||
endif
|
||||
|
||||
lint:
|
||||
$(GO_ENV_VARS) $(GOLANGCI_LINT) run ./...
|
||||
test:
|
||||
$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) ./...
|
||||
vet:
|
||||
$(GO_ENV_VARS) $(GO) vet $(GO_BUILDFLAGS) ./...
|
||||
|
||||
test-go: $(ARTIFACTDIR)
|
||||
rm -f "$(ARTIFACTDIR)/gotest.cover"
|
||||
ifeq ($(COVER),1)
|
||||
$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
|
||||
-coverpkg github.com/zrepl/zrepl/... \
|
||||
-covermode atomic \
|
||||
-coverprofile "$(ARTIFACTDIR)/gotest.cover" \
|
||||
./...
|
||||
else
|
||||
$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
|
||||
./...
|
||||
endif
|
||||
|
||||
zrepl-bin:
|
||||
$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(ZREPL_TARGET_TUPLE)"
|
||||
$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(GOOS)-$(GOARCH)"
|
||||
|
||||
COVER_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-cover-$(ZREPL_TARGET_TUPLE)
|
||||
cover-platform-bin:
|
||||
$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
|
||||
-c -o "$(COVER_PLATFORM_BIN_PATH)" \
|
||||
-covermode=atomic -cover -coverpkg github.com/zrepl/zrepl/... \
|
||||
./platformtest/harness
|
||||
cover-platform:
|
||||
# do not track dependency on cover-platform-bin to allow build of binary outside of test VM
|
||||
export _TEST_PLATFORM_CMD="$(COVER_PLATFORM_BIN_PATH) \
|
||||
-test.coverprofile \"$(ARTIFACTDIR)/platformtest.cover\" \
|
||||
-test.v \
|
||||
__DEVEL--i-heard-you-like-tests"; \
|
||||
$(MAKE) _test-or-cover-platform-impl
|
||||
|
||||
TEST_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)
|
||||
test-platform-bin:
|
||||
$(GO_BUILD) -o "$(TEST_PLATFORM_BIN_PATH)" ./internal/platformtest/harness
|
||||
test-platform:
|
||||
export _TEST_PLATFORM_CMD="\"$(TEST_PLATFORM_BIN_PATH)\""; \
|
||||
$(MAKE) _test-or-cover-platform-impl
|
||||
|
||||
ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
|
||||
ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
|
||||
ZREPL_PLATFORMTEST_MOUNTPOINT := /tmp/zreplplatformtest.pool
|
||||
ZREPL_PLATFORMTEST_ZFS_LOG := /tmp/zreplplatformtest.zfs.log
|
||||
# ZREPL_PLATFORMTEST_STOP_AND_KEEP := -failure.stop-and-keep-pool
|
||||
ZREPL_PLATFORMTEST_ARGS :=
|
||||
_test-or-cover-platform-impl: $(ARTIFACTDIR)
|
||||
ifndef _TEST_PLATFORM_CMD
|
||||
$(error _TEST_PLATFORM_CMD is undefined, caller 'cover-platform' or 'test-platform' should have defined it)
|
||||
endif
|
||||
rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
|
||||
rm -f "$(ARTIFACTDIR)/platformtest.cover"
|
||||
internal/platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
|
||||
$(_TEST_PLATFORM_CMD) \
|
||||
-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
|
||||
-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
|
||||
-mountpoint "$(ZREPL_PLATFORMTEST_MOUNTPOINT)" \
|
||||
$(ZREPL_PLATFORMTEST_STOP_AND_KEEP) \
|
||||
$(ZREPL_PLATFORMTEST_ARGS)
|
||||
|
||||
cover-merge: $(ARTIFACTDIR)
|
||||
$(GOCOVMERGE) $(ARTIFACTDIR)/platformtest.cover $(ARTIFACTDIR)/gotest.cover > $(ARTIFACTDIR)/merged.cover
|
||||
cover-html: cover-merge
|
||||
$(GO) tool cover -html "$(ARTIFACTDIR)/merged.cover" -o "$(ARTIFACTDIR)/merged.cover.html"
|
||||
|
||||
cover-full:
|
||||
test "$$(id -u)" = "0" || echo "MUST RUN AS ROOT" 1>&2
|
||||
$(MAKE) test-go COVER=1
|
||||
$(MAKE) cover-platform-bin
|
||||
$(MAKE) cover-platform
|
||||
$(MAKE) cover-html
|
||||
platformtest-bin:
|
||||
$(GO_BUILD) -o "$(ARTIFACTDIR)/platformtest-$(GOOS)-$(GOARCH)" ./platformtest/harness
|
||||
|
||||
##################### DEV TARGETS #####################
|
||||
# not part of the build, must do that manually
|
||||
.PHONY: generate format
|
||||
.PHONY: generate format platformtest
|
||||
|
||||
build/install:
|
||||
rm -rf build/install.tmp
|
||||
mkdir build/install.tmp
|
||||
generate:
|
||||
protoc -I=replication/logic/pdu --go_out=plugins=grpc:replication/logic/pdu replication/logic/pdu/pdu.proto
|
||||
$(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./...
|
||||
|
||||
-echo "installing protoc"
|
||||
mkdir build/install.tmp/protoc
|
||||
bash -x build/get_protoc.bash build/install.tmp/protoc
|
||||
format:
|
||||
goimports -srcdir . -local 'github.com/zrepl/zrepl' -w $(shell find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go')
|
||||
|
||||
-echo "installing golangci-lint"
|
||||
mkdir -p build/install.tmp/golangci_lint
|
||||
GOHOSTARCH=$(GOHOSTARCH) \
|
||||
build/get_golangci_lint.bash build/install.tmp/golangci_lint
|
||||
|
||||
-echo "installing go tools"
|
||||
build/go_install_tools.bash build/install.tmp/gobin
|
||||
|
||||
mv build/install.tmp build/install
|
||||
|
||||
|
||||
generate: build/install
|
||||
# TODO: would be nice to run with a pure path here
|
||||
PATH="$(CURDIR)/build/install/gobin:$(CURDIR)/build/install/protoc/bin:$$PATH" && \
|
||||
build/install/protoc/bin/protoc -I=internal/replication/logic/pdu --go_out=internal/replication/logic/pdu --go-grpc_out=internal/replication/logic/pdu internal/replication/logic/pdu/pdu.proto && \
|
||||
build/install/protoc/bin/protoc -I=internal/rpc/grpcclientidentity/example --go_out=internal/rpc/grpcclientidentity/example/pdu --go-grpc_out=internal/rpc/grpcclientidentity/example/pdu internal/rpc/grpcclientidentity/example/grpcauth.proto && \
|
||||
$(GO) generate $(GO_BUILDFLAGS) -x ./... && \
|
||||
true
|
||||
|
||||
GOIMPORTS := goimports -srcdir . -local 'github.com/zrepl/zrepl'
|
||||
|
||||
format: build/install
|
||||
@ build/install/gobin/goimports -w -local 'github.com/zrepl/zrepl' $$(find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go')
|
||||
ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
|
||||
ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
|
||||
platformtest: # do not track dependency on platformtest-bin to allow build of platformtest outside of test VM
|
||||
"$(ARTIFACTDIR)/platformtest-$(GOOS)-$(GOARCH)" -poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" -imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)"
|
||||
|
||||
##################### NOARCH #####################
|
||||
.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
|
||||
.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
|
||||
|
||||
|
||||
$(ARTIFACTDIR):
|
||||
@@ -367,26 +148,22 @@ $(ARTIFACTDIR):
|
||||
$(ARTIFACTDIR)/docs: $(ARTIFACTDIR)
|
||||
mkdir -p "$@"
|
||||
|
||||
noarch: $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs
|
||||
noarch: $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_env.txt docs
|
||||
# pass
|
||||
|
||||
$(ARTIFACTDIR)/bash_completion:
|
||||
$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
|
||||
artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion bash "$@"
|
||||
|
||||
$(ARTIFACTDIR)/_zrepl.zsh_completion:
|
||||
$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
|
||||
artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion zsh "$@"
|
||||
artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) bashcomp "$@"
|
||||
|
||||
$(ARTIFACTDIR)/go_env.txt:
|
||||
$(GO_ENV_VARS) $(GO) env > $@
|
||||
$(GO) version >> $@
|
||||
|
||||
docs: $(ARTIFACTDIR)/docs
|
||||
# https://www.sphinx-doc.org/en/master/man/sphinx-build.html
|
||||
cd docs && uv sync --frozen
|
||||
cd docs && uv run sphinx-build -W --keep-going -n . ../artifacts/docs/html
|
||||
make -C docs \
|
||||
html \
|
||||
BUILDDIR=../artifacts/docs \
|
||||
|
||||
docs-clean:
|
||||
rm -rf artifacts/docs
|
||||
rm -rf docs/.venv
|
||||
make -C docs \
|
||||
clean \
|
||||
BUILDDIR=../artifacts/docs
|
||||
|
||||
@@ -1,12 +1,9 @@
|
||||
[](https://github.com/zrepl/zrepl/blob/master/LICENSE)
|
||||
[](https://golang.org/)
|
||||
[](https://golang.org/)
|
||||
[](https://zrepl.github.io)
|
||||
[](https://patreon.com/zrepl)
|
||||
[](https://github.com/sponsors/problame)
|
||||
[](https://liberapay.com/zrepl/donate)
|
||||
[](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
|
||||
[](https://liberapay.com/zrepl/donate)
|
||||
[](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl)
|
||||
[](https://matrix.to/#/#zrepl:matrix.org)
|
||||
|
||||
# zrepl
|
||||
zrepl is a one-stop ZFS backup & replication solution.
|
||||
@@ -22,203 +19,118 @@ zrepl is a one-stop ZFS backup & replication solution.
|
||||
|
||||
## Feature Requests
|
||||
|
||||
1. Does your feature request require default values / some kind of configuration?
|
||||
1. Does you feature request require default values / some kind of configuration?
|
||||
If so, think of an expressive configuration example.
|
||||
2. Think of at least one use case that generalizes from your concrete application.
|
||||
3. Open an issue on GitHub with example conf & use case attached.
|
||||
4. **Optional**: [Contact Christian Schwarz](https://cschwarz.com) for contract work.
|
||||
|
||||
The above does not apply if you already implemented everything.
|
||||
Check out the *Coding Workflow* section below for details.
|
||||
|
||||
## Development
|
||||
## Package Maintainer Information
|
||||
|
||||
zrepl is written in [Go](https://golang.org) and uses [Go modules](https://github.com/golang/go/wiki/Modules) to manage dependencies.
|
||||
The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
|
||||
|
||||
### Building
|
||||
|
||||
#### Go Code
|
||||
|
||||
Dependencies:
|
||||
|
||||
* Go
|
||||
* GNU Make
|
||||
* Git
|
||||
* wget (`make generate`)
|
||||
* unzip (`make generate`)
|
||||
|
||||
Some Go code is **generated**, and generated code is committed to the source tree.
|
||||
Therefore, building does not require having code generation tools set up.
|
||||
When making changes that require code to be (re-)generated, run `make generate`.
|
||||
I downloads and installs pinned versions of the code generation tools into `./build/install`.
|
||||
There is a CI check that ensures Git state is clean, i.e., code generation has been done by a PR and is deterministic.
|
||||
|
||||
#### Docs
|
||||
|
||||
Install [uv](https://docs.astral.sh/uv/getting-started/installation/), then run `make docs`.
|
||||
uv automatically manages Python and dependencies.
|
||||
|
||||
### Testing
|
||||
|
||||
The **test suite** is split into pure **Go tests** (`make test-go`) and **platform tests** that interact with ZFS and thus generally **require root privileges** (`sudo make test-platform`).
|
||||
Platform tests run on their own pool with the name `zreplplatformtest`, which is created using the file vdev in `/tmp`.
|
||||
|
||||
For a full **code coverage** profile, run `make test-go COVER=1 && sudo make test-platform && make cover-merge`.
|
||||
An HTML report can be generated using `make cover-html`.
|
||||
|
||||
### Circle CI
|
||||
|
||||
We use CircleCI for automated build & test pre- and post-merge.
|
||||
|
||||
There are two workflows:
|
||||
|
||||
* `ci` runs for every commit / branch / tag pushed to GitHub.
|
||||
It is supposed to run very fast (<5min and provides quick feedback to developers).
|
||||
It runs formatting checks, lints and tests on the most important OSes / architectures.
|
||||
|
||||
* `release` runs
|
||||
* on manual triggers through the CircleCI API (in order to produce a release)
|
||||
* periodically on `master`
|
||||
|
||||
Artifacts are stored in CircleCI.
|
||||
|
||||
### Releasing
|
||||
|
||||
All zrepl releases are git-tagged and then published as a GitHub Release.
|
||||
There is a git tag for each zrepl release, usually `vMAJOR.MINOR.0`.
|
||||
We don't move git tags once the release has been published.
|
||||
|
||||
The procedure to issue a release is as follows:
|
||||
|
||||
* Prepare the release (as a PR to `master`):
|
||||
* Finalize `docs/changelog.rst` for the release.
|
||||
* Update the supporters list based on contributors for this release:
|
||||
```bash
|
||||
claude --permission-mode default '/update-supporters v0.6.1..v0.7.0'
|
||||
```
|
||||
Replace version tags with the appropriate range for your release.
|
||||
* Merge the PR. Docs are auto-published to zrepl.github.io on merge.
|
||||
* Tag the release:
|
||||
* Git tag the release on the `master` branch (e.g., `vMAJOR.MINOR.0`).
|
||||
```
|
||||
make tag-release ZREPL_TAG_VERSION=v0.7.0
|
||||
```
|
||||
* Push the tag.
|
||||
* Build and publish:
|
||||
* Run the `release` pipeline against the `master` branch (trigger via CircleCI UI).
|
||||
This URL: https://app.circleci.com/pipelines/github/zrepl/zrepl?branch=master.
|
||||
Example pipeline: https://app.circleci.com/pipelines/github/zrepl/zrepl/8547
|
||||
* Download artifacts using this handy makefile target.
|
||||
Note: `JOB_NUM` must be the **job number** from the `release-upload` job, **not the pipeline number**.
|
||||
Find it via: pipeline → `release` workflow → `release-upload` job number.
|
||||
Example URL: https://app.circleci.com/pipelines/github/zrepl/zrepl/8547/workflows/65feb2c9-15d7-46ab-a551-46d62a5769b0/jobs/66079/steps
|
||||
|
||||
```
|
||||
make download-circleci-release JOB_NUM=66079
|
||||
```
|
||||
* Verify checksums and sign the checksums file:
|
||||
```
|
||||
make verify-and-sign
|
||||
```
|
||||
* Create GitHub draft release and upload artifacts:
|
||||
```bash
|
||||
claude --permission-mode default '/draft-release v0.7.0'
|
||||
```
|
||||
This command will verify that artifacts are ready, create the draft release, and upload all artifacts.
|
||||
* Review the draft release on GitHub, then publish.
|
||||
* Add the .rpm and .deb files to the official zrepl repos.
|
||||
* Code for management of these repos: https://github.com/zrepl/package-repo-ops (private repo at this time)
|
||||
* Update docs version list:
|
||||
* Update `docs/_templates/versions.html` with the new release.
|
||||
* Verify the link to `zrepl-noarch.tar` in the GitHub release works.
|
||||
* Merge to `master` (docs auto-publish).
|
||||
|
||||
#### Patch releases, Go toolchain updates, APT/RPM Package rebuilds
|
||||
|
||||
`vMAJOR.MINOR.0` is typically a tagged commit on `master`, because development velocity isn't high
|
||||
and thus release branches for stabilization aren't necessary.
|
||||
|
||||
Occasionally though there is a need for patch changes to a release, e.g.
|
||||
- security issue in a dependency
|
||||
- Go toolchain update (e.g. security issue in standard library)
|
||||
|
||||
The procedure for this is the following
|
||||
- create a branch off the release tag we need to patch, named `releases/MAJOR.MINOR.X`
|
||||
- that branch will never be merged into `master`, it'll be a dead-end for this specific patch
|
||||
- make changes in that branch
|
||||
- make the final commit that bumps version numbers
|
||||
- create the git tag
|
||||
- follow general procedure for publishing the release (previous sectino
|
||||
|
||||
For Go toolchain updates and package rebuilds with no source code changes, leverage the APT/RPM package revision field.
|
||||
Control via the `ZREPL_PACKAGE_RELEASE` Makefile variable, see `origin/releases/0.6.1-2` for an example.
|
||||
|
||||
|
||||
### Updating Dependencies
|
||||
|
||||
- Update the `go` directive and `toolchain` directive in `go.mod`
|
||||
- `go` is the minimum supported version
|
||||
- `toolchain` is the preferred toolchain version if `GOTOOLCHAIN` is not specified
|
||||
|
||||
Run `go mod tidy` to ensure consistency.
|
||||
|
||||
Update Go module dependencies:
|
||||
|
||||
```bash
|
||||
# Update all other dependencies
|
||||
go get -u -t ./...
|
||||
go mod tidy
|
||||
|
||||
# Above might fail if there are version selection conflicts.
|
||||
# Figure out what's going on by updating packages from error messages first.
|
||||
# Example:
|
||||
go get -u google.golang.org/genproto google.golang.org/grpc google.golang.org/protobuf
|
||||
```
|
||||
|
||||
Update codegen & lint tools
|
||||
- `protoc` => `build/get_protoc.bash`
|
||||
- GH releases publish sha256 sums
|
||||
- `golangci-lint` => `build/get_golangci_lint.bash`
|
||||
- bump versions in `build/tools.go`
|
||||
- we use the tools.go trick:
|
||||
- `go get -tags tools -u example.com/tool ; go mod tidy`
|
||||
- review whether we're ready to switch to `go tool`: https://github.com/zrepl/zrepl/pull/909
|
||||
|
||||
Now run `make generate`.
|
||||
|
||||
Run `make lint` and `make vet`.
|
||||
|
||||
Update the CI configuration `.circleci/config.yml`:
|
||||
- Update Go version references (we reference the minimum and max supported version)
|
||||
- Set `Makefile` `RELEASE_GOVERSION` to the new Go version
|
||||
- Update the pinned ZFS release tags in the `platformtest` matrix (`zfs_release` parameter) to the latest patch releases of each OpenZFS branch (currently 2.2, 2.3, 2.4)
|
||||
- Update the pinned Ubuntu machine image for `platformtest` (e.g. `ubuntu-2404:2025.09.1`) — this is pinned rather than `current` so the ZFS build cache stays valid across runs
|
||||
|
||||
Update docs build tooling:
|
||||
- Update `uv` version in `.circleci/config.yml` (search for `astral.sh/uv/` and cache keys containing the version)
|
||||
- Check if there's now a CircleCI orb for uv that we could use
|
||||
- Update Python version in `docs/.python-version`
|
||||
|
||||
Update docs dependencies (Sphinx, sphinx-rtd-theme):
|
||||
- Check current versions in `docs/pyproject.toml`
|
||||
- Review upstream changelogs for breaking changes
|
||||
- Update version constraints in `pyproject.toml` and the `uv` lockfile (see [uv docs on dependencies](https://docs.astral.sh/uv/concepts/projects/dependencies/)):
|
||||
- Test locally with `make docs`
|
||||
|
||||
Kick a full CI pipeline run (`do_ci=true` and `do_release=true`).
|
||||
|
||||
Merge PR with merge commit.
|
||||
|
||||
## Notes to Distro Package Maintainers
|
||||
|
||||
* The `Makefile` in this project is not suitable for builds in distros.
|
||||
* Run the platform tests (Docs -> Usage -> Platform Tests) **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
|
||||
* Follow the steps in `docs/installation.rst -> Compiling from Source` and read the Makefile / shell scripts used in this process.
|
||||
* Make sure your distro is compatible with the paths in `docs/installation.rst`.
|
||||
* Ship a default config that adheres to your distro's `hier` and logging system.
|
||||
* Ship a service manager file and _please_ try to upstream it to this repository.
|
||||
* `dist/systemd` contains a Systemd unit template.
|
||||
* Ship other material provided in `./dist`, e.g. in `/usr/share/zrepl/`.
|
||||
* Have a look at the `Makefile`'s `ZREPL_VERSION` variable and how it passed to Go's `ldFlags`.
|
||||
This is how `zrepl version` knows what version number to show.
|
||||
Your build system should set the `ldFlags` flags appropriately and add a prefix or suffix that indicates that the given zrepl binary is a distro build, not an official one.
|
||||
* Use `make release ZREPL_VERSION='mydistro-1.2.3_1'`
|
||||
* Your distro's name and any versioning supplemental to zrepl's (e.g. package revision) should be in this string
|
||||
* Use `make platformtest` **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
|
||||
* Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
|
||||
|
||||
## Developer Documentation
|
||||
|
||||
zrepl is written in [Go](https://golang.org) and uses [Go modules](https://github.com/golang/go/wiki/Modules) to manage dependencies.
|
||||
The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
|
||||
|
||||
To get started, run `./lazy.sh devsetup` to easily install build dependencies and read `docs/installation.rst -> Compiling from Source`.
|
||||
|
||||
### Overall Architecture
|
||||
|
||||
The application architecture is documented as part of the user docs in the *Implementation* section (`docs/content/impl`).
|
||||
Make sure to develop an understanding how zrepl is typically used by studying the user docs first.
|
||||
|
||||
### Project Structure
|
||||
|
||||
```
|
||||
├── artifacts # build artifcats generate by make
|
||||
├── cli # wrapper around CLI package cobra
|
||||
├── client # all subcommands that are not `daemon`
|
||||
├── config # config data types (=> package yaml-config)
|
||||
│ └── samples
|
||||
├── daemon # the implementation of `zrepl daemon` subcommand
|
||||
│ ├── filters
|
||||
│ ├── job # job implementations
|
||||
│ ├── logging # logging outlets + formatters
|
||||
│ ├── nethelpers
|
||||
│ ├── prometheus
|
||||
│ ├── pruner # pruner implementation
|
||||
│ ├── snapper # snapshotter implementation
|
||||
├── docs # sphinx-based documentation
|
||||
├── dist # supplemental material for users & package maintainers
|
||||
│ ├── **/*.rst # documentation in reStructuredText
|
||||
│ ├── sphinxconf
|
||||
│ │ └── conf.py # sphinx config (see commit 445a280 why its not in docs/)
|
||||
│ ├── requirements.txt # pip3 requirements to build documentation
|
||||
│ ├── publish.sh # shell script for automated rendering & deploy to zrepl.github.io repo
|
||||
│ └── public_git # checkout of zrepl.github.io managed by above shell script
|
||||
├── endpoint # implementation of replication endpoints (=> package replication)
|
||||
├── logger # our own logger package
|
||||
├── pruning # pruning rules (the logic, not the actual execution)
|
||||
│ └── retentiongrid
|
||||
├── replication
|
||||
│ ├── driver # the driver of the replication logic (status reporting, error handling)
|
||||
│ ├── logic # planning & executing replication steps via rpc
|
||||
| | └── pdu # the generated gRPC & protobuf code used in replication (and endpoints)
|
||||
│ └── report # the JSON-serializable report datastructures exposed to the client
|
||||
├── rpc # the hybrid gRPC + ./dataconn RPC client: connects to a remote replication.Endpoint
|
||||
│ ├── dataconn # Bulk data-transfer RPC protocol
|
||||
│ ├── grpcclientidentity # adaptor to inject package transport's 'client identity' concept into gRPC contexts
|
||||
│ ├── netadaptor # adaptor to convert a package transport's Connecter and Listener into net.* primitives
|
||||
│ ├── transportmux # TCP connecter and listener used to split control & data traffic
|
||||
│ └── versionhandshake # replication protocol version handshake perfomed on newly established connections
|
||||
├── tlsconf # abstraction for Go TLS server + client config
|
||||
├── transport # transports implementation
|
||||
│ ├── fromconfig
|
||||
│ ├── local
|
||||
│ ├── ssh
|
||||
│ ├── tcp
|
||||
│ └── tls
|
||||
├── util
|
||||
├── vendor # managed by dep
|
||||
├── version # abstraction for versions (filled during build by Makefile)
|
||||
└── zfs # zfs(8) wrappers
|
||||
```
|
||||
|
||||
### Coding Workflow
|
||||
|
||||
* Open an issue when starting to hack on a new feature
|
||||
* Commits should reference the issue they are related to
|
||||
* Docs improvements not documenting new features do not require an issue.
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
Backward-incompatible changes must be documented in the git commit message and are listed in `docs/changelog.rst`.
|
||||
|
||||
* Config-breaking changes must contain a line `BREAK CONFIG` in the commit message
|
||||
* Other breaking changes must contain a line `BREAK` in the commit message
|
||||
|
||||
### Glossary & Naming Inconsistencies
|
||||
|
||||
In ZFS, *dataset* refers to the objects *filesystem*, *ZVOL* and *snapshot*. <br />
|
||||
However, we need a word for *filesystem* & *ZVOL* but not a snapshot, bookmark, etc.
|
||||
|
||||
Toward the user, the following terminology is used:
|
||||
|
||||
* **filesystem**: a ZFS filesystem or a ZVOL
|
||||
* **filesystem version**: a ZFS snapshot or a bookmark
|
||||
|
||||
Sadly, the zrepl implementation is inconsistent in its use of these words:
|
||||
variables and types are often named *dataset* when they in fact refer to a *filesystem*.
|
||||
|
||||
There will not be a big refactoring (an attempt was made, but it's destroying too much history without much gain).
|
||||
|
||||
However, new contributions & patches should fix naming without further notice in the commit message.
|
||||
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
FROM golang:latest
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
python3-pip \
|
||||
unzip
|
||||
|
||||
ADD build.installprotoc.bash ./
|
||||
RUN bash build.installprotoc.bash
|
||||
|
||||
ADD lazy.sh /tmp/lazy.sh
|
||||
ADD docs/requirements.txt /tmp/requirements.txt
|
||||
ENV ZREPL_LAZY_DOCS_REQPATH=/tmp/requirements.txt
|
||||
RUN /tmp/lazy.sh docdep
|
||||
|
||||
# prepare volume mount of git checkout to /zrepl
|
||||
RUN mkdir -p /src/github.com/zrepl/zrepl
|
||||
RUN mkdir -p /.cache && chmod -R 0777 /.cache
|
||||
|
||||
# $GOPATH is /go
|
||||
# Go 1.12 doesn't use modules within GOPATH, but 1.13 and later do
|
||||
# => store source outside of GOPATH
|
||||
WORKDIR /src
|
||||
|
||||
# Install build tools (e.g. protobuf generator, stringer) into $GOPATH/bin
|
||||
ADD go.mod go.sum ./
|
||||
RUN /tmp/lazy.sh godep
|
||||
|
||||
RUN chmod -R 0777 /go
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
set -x
|
||||
|
||||
MACH=$(uname -m)
|
||||
MACH="${MACH/aarch64/aarch_64}"
|
||||
|
||||
VERSION=3.6.1
|
||||
FILENAME=protoc-"$VERSION"-linux-"$MACH".zip
|
||||
|
||||
if [ -e "$FILENAME" ]; then
|
||||
echo "$FILENAME" already exists 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
wget https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
|
||||
|
||||
stat "$FILENAME"
|
||||
|
||||
sha256sum -c --ignore-missing <<EOF
|
||||
6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807 protoc-3.6.1-linux-x86_64.zip
|
||||
af8e5aaaf39ddec62ec8dd2be1b8d9602c6da66564883a16393ade5f71170922 protoc-3.6.1-linux-aarch_64.zip
|
||||
EOF
|
||||
|
||||
unzip -d /usr "$FILENAME"
|
||||
@@ -1 +0,0 @@
|
||||
install/
|
||||
@@ -1,30 +0,0 @@
|
||||
FROM !SUBSTITUTED_BY_MAKEFILE
|
||||
|
||||
ARG BUILD_UID=1000
|
||||
ARG BUILD_GID=1000
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
python3 \
|
||||
unzip \
|
||||
gawk \
|
||||
curl
|
||||
|
||||
# Create build user with the host's UID/GID before installing uv
|
||||
RUN groupadd -g ${BUILD_GID} zrepl_build && \
|
||||
useradd -u ${BUILD_UID} -g ${BUILD_GID} -m zrepl_build
|
||||
|
||||
# Go toolchain uses xdg-cache
|
||||
RUN mkdir -p /.cache && chmod -R 0777 /.cache
|
||||
|
||||
# Install uv as the build user - version from .uv-version file
|
||||
ADD .uv-version /tmp/.uv-version
|
||||
USER zrepl_build
|
||||
RUN UV_VERSION=$(cat /tmp/.uv-version) && \
|
||||
curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh
|
||||
|
||||
# Go devtools are managed by Makefile
|
||||
|
||||
WORKDIR /src
|
||||
ENV PATH="/home/zrepl_build/.local/bin:$PATH" \
|
||||
GOCACHE="/.cache/go-build"
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
set -x
|
||||
|
||||
# golangci-lint recommends against using `go get` and friends to install it
|
||||
|
||||
cd "$1"
|
||||
|
||||
VERSION=2.8.0
|
||||
FILENAME=golangci-lint-"$VERSION"-linux-"$GOHOSTARCH".tar.gz
|
||||
|
||||
if [ -e "$FILENAME" ]; then
|
||||
echo "$FILENAME" already exists 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
wget --continue https://github.com/golangci/golangci-lint/releases/download/v"$VERSION"/"$FILENAME"
|
||||
|
||||
stat "$FILENAME"
|
||||
|
||||
# Select the correct checksum for the downloaded architecture
|
||||
case "$GOHOSTARCH" in
|
||||
arm64) EXPECTED_SHA256="2a58388db8af5ab9330791cea0ebdd4100723cd05ad7185d92febaaee272ec9a" ;;
|
||||
amd64) EXPECTED_SHA256="7048bc6b25c9515ed092c83f9fa8709ca97937ead52d9ff317a143299ee97a50" ;;
|
||||
*) echo "Unknown architecture: $GOHOSTARCH" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
# Verify checksum explicitly - fails if hash doesn't match
|
||||
echo "$EXPECTED_SHA256 $FILENAME" | sha256sum -c -
|
||||
|
||||
tar -x --strip-components=1 -f "$FILENAME"
|
||||
|
||||
stat ./golangci-lint
|
||||
@@ -1,32 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
set -x
|
||||
|
||||
cd "$1"
|
||||
|
||||
MACH=$(uname -m)
|
||||
MACH="${MACH/aarch64/aarch_64}"
|
||||
|
||||
VERSION=33.5
|
||||
FILENAME=protoc-"$VERSION"-linux-"$MACH".zip
|
||||
|
||||
if [ -e "$FILENAME" ]; then
|
||||
echo "$FILENAME" already exists 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
wget --continue https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
|
||||
|
||||
stat "$FILENAME"
|
||||
|
||||
# Select the correct checksum for the downloaded architecture
|
||||
case "$MACH" in
|
||||
aarch_64) EXPECTED_SHA256="2b0fcf9b2c32cbadccc0eb7a88b841fffecd4a06fc80acdba2b5be45e815c38a" ;;
|
||||
x86_64) EXPECTED_SHA256="24e58fb231d50306ee28491f33a170301e99540f7e29ca461e0e80fd1239f8d1" ;;
|
||||
*) echo "Unknown architecture: $MACH" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
# Verify checksum explicitly - fails if hash doesn't match
|
||||
echo "$EXPECTED_SHA256 $FILENAME" | sha256sum -c -
|
||||
|
||||
unzip -d . "$FILENAME"
|
||||
+44
-17
@@ -1,24 +1,51 @@
|
||||
module github.com/zrepl/zrepl/build
|
||||
|
||||
go 1.24.13
|
||||
|
||||
toolchain go1.25.7
|
||||
go 1.12
|
||||
|
||||
require (
|
||||
github.com/alvaroloes/enumer v1.1.2
|
||||
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad
|
||||
golang.org/x/tools v0.41.0
|
||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.1
|
||||
google.golang.org/protobuf v1.36.11
|
||||
github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 // indirect
|
||||
github.com/alvaroloes/enumer v1.1.1
|
||||
github.com/fatih/color v1.7.0 // indirect
|
||||
github.com/go-critic/go-critic v0.3.4 // indirect
|
||||
github.com/gogo/protobuf v1.2.1 // indirect
|
||||
github.com/golang/mock v1.2.0 // indirect
|
||||
github.com/golang/protobuf v1.2.0
|
||||
github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 // indirect
|
||||
github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 // indirect
|
||||
github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d // indirect
|
||||
github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 // indirect
|
||||
github.com/golangci/golangci-lint v1.17.1
|
||||
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb // indirect
|
||||
github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 // indirect
|
||||
github.com/golangci/misspell v0.3.4 // indirect
|
||||
github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 // indirect
|
||||
github.com/google/go-cmp v0.3.0 // indirect
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/pkg/errors v0.8.1 // indirect
|
||||
github.com/sirupsen/logrus v1.4.2 // indirect
|
||||
github.com/spf13/afero v1.2.2 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
github.com/spf13/viper v1.3.2 // indirect
|
||||
github.com/stretchr/testify v1.4.0 // indirect
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 // indirect
|
||||
golang.org/x/sys v0.0.0-20191010194322-b09406accb47 // indirect
|
||||
golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b
|
||||
mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe // indirect
|
||||
sourcegraph.com/sqs/pbtypes v1.0.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/pascaldekloe/name v1.0.1 // indirect
|
||||
golang.org/x/mod v0.32.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20260205145544-86a5c4bf3c8d // indirect
|
||||
golang.org/x/text v0.31.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c // indirect
|
||||
google.golang.org/grpc v1.75.0 // indirect
|
||||
// invalid dates in transitive dependencies (first validated in Go 1.13, didn't fail in earlier Go versions)
|
||||
replace (
|
||||
github.com/go-critic/go-critic v0.0.0-20181204210945-1df300866540 => github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540
|
||||
github.com/go-critic/go-critic v0.0.0-20181204210945-ee9bf5809ead => github.com/go-critic/go-critic v0.3.5-0.20190210220443-ee9bf5809ead
|
||||
github.com/golangci/errcheck v0.0.0-20181003203344-ef45e06d44b6 => github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6
|
||||
github.com/golangci/go-tools v0.0.0-20180109140146-35a9f45a5db0 => github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0
|
||||
github.com/golangci/go-tools v0.0.0-20180109140146-af6baa5dc196 => github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196
|
||||
github.com/golangci/gofmt v0.0.0-20181105071733-0b8337e80d98 => github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98
|
||||
github.com/golangci/gosec v0.0.0-20180901114220-66fb7fc33547 => github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547
|
||||
github.com/golangci/ineffassign v0.0.0-20180808204949-42439a7714cc => github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc
|
||||
github.com/golangci/lint-1 v0.0.0-20180610141402-ee948d087217 => github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217
|
||||
golang.org/x/tools v0.0.0-20190125232054-379209517ffe => golang.org/x/tools v0.0.0-20190205201329-379209517ffe
|
||||
mvdan.cc/unparam v0.0.0-20190124213536-fbb59629db34 => mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34
|
||||
)
|
||||
|
||||
+273
-30
@@ -1,37 +1,280 @@
|
||||
github.com/alvaroloes/enumer v1.1.2 h1:5khqHB33TZy1GWCO/lZwcroBFh7u+0j40T83VUbfAMY=
|
||||
github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/OpenPeeDeeP/depguard v0.0.0-20180806142446-a69c782687b2/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
|
||||
github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 h1:pFks+oaqVWDFq0KsLQZFB2pQGB5Us0HfMSBENtieXOs=
|
||||
github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
|
||||
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
github.com/alvaroloes/enumer v1.1.1 h1:v+1scNqEhSFAPb9tLwblQegeTXJhnw8hf9O0amTbOvQ=
|
||||
github.com/alvaroloes/enumer v1.1.1/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
|
||||
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/go-critic/go-critic v0.3.4 h1:FYaiaLjX0Nqei80KPhm4CyFQUBbmJwSrHxQ73taaGBc=
|
||||
github.com/go-critic/go-critic v0.3.4/go.mod h1:AHR42Lk/E/aOznsrYdMYeIQS5RH10HZHSqP+rD6AJrc=
|
||||
github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
|
||||
github.com/go-lintpack/lintpack v0.5.1/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
|
||||
github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0=
|
||||
github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
|
||||
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
|
||||
github.com/go-toolsmith/astcast v0.0.0-20181028201508-b7a89ed70af1/go.mod h1:TEo3Ghaj7PsZawQHxT/oBvo4HK/sl1RcuUHDKTTju+o=
|
||||
github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=
|
||||
github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
|
||||
github.com/go-toolsmith/astcopy v0.0.0-20180903214859-79b422d080c4/go.mod h1:c9CPdq2AzM8oPomdlPniEfPAC6g1s7NqZzODt8y6ib8=
|
||||
github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8=
|
||||
github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
|
||||
github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
|
||||
github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ=
|
||||
github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
|
||||
github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
|
||||
github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k=
|
||||
github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
|
||||
github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
|
||||
github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
|
||||
github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg=
|
||||
github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
|
||||
github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8 h1:vVouagbdmqTVlCIAxpyYsNNTbkKZ3V66VpKOLU/s6W4=
|
||||
github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
|
||||
github.com/go-toolsmith/pkgload v1.0.0 h1:4DFWWMXVfbcN5So1sBNW9+yeiMqLFGl1wFLTL5R0Tgg=
|
||||
github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
|
||||
github.com/go-toolsmith/strparse v0.0.0-20180903215201-830b6daa1241/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
|
||||
github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4=
|
||||
github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
|
||||
github.com/go-toolsmith/typep v0.0.0-20181030061450-d63dc7650676/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
|
||||
github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA=
|
||||
github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
|
||||
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
|
||||
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
|
||||
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
||||
github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0=
|
||||
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
|
||||
github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
|
||||
github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
|
||||
github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w=
|
||||
github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
|
||||
github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw=
|
||||
github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
|
||||
github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 h1:MRhC9XbUjE6XDOInSJ8pwHuPagqsyO89QDU9IdVhe3o=
|
||||
github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
|
||||
github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
|
||||
github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8=
|
||||
github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
|
||||
github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
|
||||
github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d h1:pXTK/gkVNs7Zyy7WKgLXmpQ5bHTrq5GDsp8R9Qs67g0=
|
||||
github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
|
||||
github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU=
|
||||
github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
|
||||
github.com/golangci/golangci-lint v1.17.1 h1:lc8Hf9GPCjIr0hg3S/xhvFT1+Hydass8F1xchr8jkME=
|
||||
github.com/golangci/golangci-lint v1.17.1/go.mod h1:+5sJSl2h3aly+fpmL2meSP8CaSKua2E4Twi9LPy7b1g=
|
||||
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb h1:Bi7BYmZVg4C+mKGi8LeohcP2GGUl2XJD4xCkJoZSaYc=
|
||||
github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb/go.mod h1:ON/c2UR0VAAv6ZEAFKhjCLplESSmRFfZcDLASbI1GWo=
|
||||
github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
|
||||
github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
|
||||
github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
|
||||
github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 h1:utua3L2IbQJmauC5IXdEA547bcoU5dozgQAfc8Onsg4=
|
||||
github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
|
||||
github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
|
||||
github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
|
||||
github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
|
||||
github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
|
||||
github.com/golangci/misspell v0.3.4 h1:kAD5J0611Zo2VirUn9KFP15RjEqkmfEfnFxyIVxZCYg=
|
||||
github.com/golangci/misspell v0.3.4/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
|
||||
github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us=
|
||||
github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
|
||||
github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
|
||||
github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 h1:XQKc8IYQOeRwVs36tDrEmTgDgP88d5iEURwpmtiAlOM=
|
||||
github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
|
||||
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
|
||||
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
|
||||
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw=
|
||||
github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
|
||||
github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
|
||||
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM=
|
||||
github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
|
||||
github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
|
||||
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
|
||||
github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
|
||||
github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
|
||||
github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663 h1:Ri1EhipkbhWsffPJ3IPlrb4SkTOPa2PfRXp3jchBczw=
|
||||
github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
|
||||
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
|
||||
github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
|
||||
github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 h1:/I3lTljEEDNYLho3/FUB7iD/oc2cEFgVmbHzV+O0PtU=
|
||||
github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ=
|
||||
github.com/pascaldekloe/name v1.0.1 h1:9lnXOHeqeHHnWLbKfH6X98+4+ETVqFqxN09UXSjcMb0=
|
||||
github.com/pascaldekloe/name v1.0.1/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM=
|
||||
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad h1:W0LEBv82YCGEtcmPA3uNZBI33/qF//HAAs3MawDjRa0=
|
||||
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad/go.mod h1:Hy8o65+MXnS6EwGElrSRjUzQDLXreJlzYLlWiHtt8hM=
|
||||
github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
|
||||
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.2.1/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
|
||||
github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
|
||||
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM=
|
||||
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
|
||||
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc=
|
||||
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
|
||||
github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
|
||||
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs=
|
||||
github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
|
||||
github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
|
||||
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
|
||||
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
|
||||
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v0.0.2 h1:NfkwRbgViGoyjBKsLI0QMDcuMnhM+SBg3T0cGfpvKDE=
|
||||
github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
|
||||
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
|
||||
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
|
||||
github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
|
||||
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb h1:lI9ufgFfvuqRctP9Ny8lDDLbSWCMxBPletcSqrnyFYM=
|
||||
github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
|
||||
github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
|
||||
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
|
||||
golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
|
||||
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
|
||||
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
|
||||
golang.org/x/net v0.0.0-20190313220215-9f648a60d977 h1:actzWV6iWn3GLqN8dZjzsB+CLt+gaV2+wsxroxiQI8I=
|
||||
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/telemetry v0.0.0-20260205145544-86a5c4bf3c8d h1:xjbyaj6khjO0ocYYa7k9sQLbdi4R1KwHnsS1QGijbsM=
|
||||
golang.org/x/telemetry v0.0.0-20260205145544-86a5c4bf3c8d/go.mod h1:b7fPSJ0pKZ3ccUh8gnTONJxhn3c/PS6tyzQvyqw4iA8=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
|
||||
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
|
||||
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
|
||||
golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181205014116-22934f0fdb62/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8 h1:b0PLhFjEMqrIqsD4rS5sp0YxBYgdqKzX0KkkGGKLV8I=
|
||||
golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b h1:iEAPfYPbYbxG/2lNN4cMOHkmgKNsCuUwkxlDCK46UlU=
|
||||
golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
|
||||
golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c h1:qXWI/sQtv5UKboZ/zUk7h+mrf/lXORyI+n9DKDAusdg=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c/go.mod h1:gw1tLEfykwDz2ET4a12jcXt4couGAm7IwsVaTy0Sflo=
|
||||
google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4=
|
||||
google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
|
||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.1 h1:/WILD1UcXj/ujCxgoL/DvRgt2CP3txG8+FwkUbb9110=
|
||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.1/go.mod h1:YNKnb2OAApgYn2oYY47Rn7alMr1zWjb2U8Q0aoGWiNc=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
|
||||
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
|
||||
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=
|
||||
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
|
||||
mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
|
||||
mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe h1:Ekmnp+NcP2joadI9pbK4Bva87QKZSeY7le//oiMrc9g=
|
||||
mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe/go.mod h1:BnhuWBAqxH3+J5bDybdxgw5ZfS+DsVd4iylsKQePN8o=
|
||||
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
|
||||
sourcegraph.com/sqs/pbtypes v1.0.0 h1:f7lAwqviDEGvON4kRv0o5V7FT/IQK+tbkF664XMbP3o=
|
||||
sourcegraph.com/sqs/pbtypes v1.0.0/go.mod h1:3AciMUv4qUuRHRHhOG4TZOB+72GdPVz5k+c648qsFS4=
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
export GO111MODULE=on # otherwise, a checkout of this repo in GOPATH will disable modules on Go 1.12 and earlier
|
||||
source <(go env)
|
||||
# build tools for the host platform
|
||||
export GOOS="$GOHOSTOS"
|
||||
export GOARCH="$GOHOSTARCH"
|
||||
# TODO GOARM=$GOHOSTARM?
|
||||
@@ -1,19 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
set -x
|
||||
|
||||
OUTDIR="$(readlink -f "$1")"
|
||||
if [ -e "$OUTDIR" ]; then
|
||||
echo "$OUTDIR" already exists 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# go install command below will install tools to $GOBIN
|
||||
export GOBIN="$OUTDIR"
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
source ./go_install_host_tool.source
|
||||
|
||||
cat tools.go | grep _ | awk -F'"' '{print $2}' | tee | xargs -tI '{}' go install '{}'
|
||||
+4
-7
@@ -1,14 +1,11 @@
|
||||
//go:build tools
|
||||
// +build tools
|
||||
|
||||
package main
|
||||
|
||||
// the lines are parsed by lazy.sh, do not edit
|
||||
import (
|
||||
_ "github.com/alvaroloes/enumer"
|
||||
_ "github.com/wadey/gocovmerge"
|
||||
_ "golang.org/x/tools/cmd/goimports"
|
||||
_ "golang.org/x/tools/cmd/stringer"
|
||||
_ "google.golang.org/grpc/cmd/protoc-gen-go-grpc"
|
||||
_ "google.golang.org/protobuf/cmd/protoc-gen-go"
|
||||
_ "github.com/golang/protobuf/protoc-gen-go"
|
||||
_ "github.com/alvaroloes/enumer"
|
||||
_ "golang.org/x/tools/cmd/goimports"
|
||||
_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
|
||||
)
|
||||
|
||||
@@ -1,16 +1,13 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
)
|
||||
|
||||
var rootArgs struct {
|
||||
@@ -22,55 +19,29 @@ var rootCmd = &cobra.Command{
|
||||
Short: "One-stop ZFS replication solution",
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
|
||||
}
|
||||
|
||||
var genCompletionCmd = &cobra.Command{
|
||||
Use: "gencompletion",
|
||||
Short: "generate shell auto-completions",
|
||||
}
|
||||
|
||||
type completionCmdInfo struct {
|
||||
genFunc func(outpath string) error
|
||||
help string
|
||||
}
|
||||
|
||||
var completionCmdMap = map[string]completionCmdInfo{
|
||||
"zsh": {
|
||||
rootCmd.GenZshCompletionFile,
|
||||
" save to file `_zrepl` in your zsh's $fpath",
|
||||
},
|
||||
"bash": {
|
||||
rootCmd.GenBashCompletionFile,
|
||||
" save to a path and source that path in your .bashrc",
|
||||
var bashcompCmd = &cobra.Command{
|
||||
Use: "bashcomp path/to/out/file",
|
||||
Short: "generate bash completions",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
if len(args) != 1 {
|
||||
fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
|
||||
err := cmd.Usage()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := rootCmd.GenBashCompletionFile(args[0]); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error generating bash completion: %s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
},
|
||||
Hidden: true,
|
||||
}
|
||||
|
||||
func init() {
|
||||
for sh, info := range completionCmdMap {
|
||||
sh, info := sh, info
|
||||
genCompletionCmd.AddCommand(&cobra.Command{
|
||||
Use: fmt.Sprintf("%s path/to/out/file", sh),
|
||||
Short: fmt.Sprintf("generate %s completions", sh),
|
||||
Example: info.help,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
if len(args) != 1 {
|
||||
fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
|
||||
err := cmd.Usage()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := info.genFunc(args[0]); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error generating %s completion: %s", sh, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
rootCmd.AddCommand(genCompletionCmd)
|
||||
rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
|
||||
rootCmd.AddCommand(bashcompCmd)
|
||||
}
|
||||
|
||||
type Subcommand struct {
|
||||
@@ -78,7 +49,7 @@ type Subcommand struct {
|
||||
Short string
|
||||
Example string
|
||||
NoRequireConfig bool
|
||||
Run func(ctx context.Context, subcommand *Subcommand, args []string) error
|
||||
Run func(subcommand *Subcommand, args []string) error
|
||||
SetupFlags func(f *pflag.FlagSet)
|
||||
SetupSubcommands func() []*Subcommand
|
||||
|
||||
@@ -99,11 +70,7 @@ func (s *Subcommand) Config() *config.Config {
|
||||
|
||||
func (s *Subcommand) run(cmd *cobra.Command, args []string) {
|
||||
s.tryParseConfig()
|
||||
ctx := context.Background()
|
||||
endTask := trace.WithTaskFromStackUpdateCtx(&ctx)
|
||||
defer endTask()
|
||||
err := s.Run(ctx, s, args)
|
||||
endTask()
|
||||
err := s.Run(s, args)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s\n", err)
|
||||
os.Exit(1)
|
||||
@@ -1,7 +1,6 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -11,17 +10,16 @@ import (
|
||||
"github.com/spf13/pflag"
|
||||
"github.com/zrepl/yaml-config"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/cli"
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
||||
"github.com/zrepl/zrepl/internal/logger"
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/job"
|
||||
"github.com/zrepl/zrepl/daemon/logging"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
)
|
||||
|
||||
var configcheckArgs struct {
|
||||
format string
|
||||
what string
|
||||
skipCertCheck bool
|
||||
format string
|
||||
what string
|
||||
}
|
||||
|
||||
var ConfigcheckCmd = &cli.Subcommand{
|
||||
@@ -30,9 +28,8 @@ var ConfigcheckCmd = &cli.Subcommand{
|
||||
SetupFlags: func(f *pflag.FlagSet) {
|
||||
f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]")
|
||||
f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]")
|
||||
f.BoolVar(&configcheckArgs.skipCertCheck, "skip-cert-check", false, "skip checking cert files")
|
||||
},
|
||||
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
||||
Run: func(subcommand *cli.Subcommand, args []string) error {
|
||||
formatMap := map[string]func(interface{}){
|
||||
"": func(i interface{}) {},
|
||||
"pretty": func(i interface{}) {
|
||||
@@ -58,16 +55,8 @@ var ConfigcheckCmd = &cli.Subcommand{
|
||||
}
|
||||
|
||||
var hadErr bool
|
||||
|
||||
parseFlags := config.ParseFlagsNone
|
||||
|
||||
if configcheckArgs.skipCertCheck {
|
||||
parseFlags |= config.ParseFlagsNoCertCheck
|
||||
}
|
||||
|
||||
// further: try to build jobs
|
||||
confJobs, err := job.JobsFromConfig(subcommand.Config(), parseFlags)
|
||||
|
||||
confJobs, err := job.JobsFromConfig(subcommand.Config())
|
||||
if err != nil {
|
||||
err := errors.Wrap(err, "cannot build jobs from config")
|
||||
if configcheckArgs.what == "jobs" {
|
||||
@@ -0,0 +1,104 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
)
|
||||
|
||||
var (
|
||||
MigrateCmd = &cli.Subcommand{
|
||||
Use: "migrate",
|
||||
Short: "perform migration of the on-disk / zfs properties",
|
||||
SetupSubcommands: func() []*cli.Subcommand {
|
||||
return migrations
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
var migrations = []*cli.Subcommand{
|
||||
&cli.Subcommand{
|
||||
Use: "0.0.X:0.1:placeholder",
|
||||
Run: doMigratePlaceholder0_1,
|
||||
SetupFlags: func(f *pflag.FlagSet) {
|
||||
f.BoolVar(&migratePlaceholder0_1Args.dryRun, "dry-run", false, "dry run")
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
var migratePlaceholder0_1Args struct {
|
||||
dryRun bool
|
||||
}
|
||||
|
||||
func doMigratePlaceholder0_1(sc *cli.Subcommand, args []string) error {
|
||||
if len(args) != 0 {
|
||||
return fmt.Errorf("migration does not take arguments, got %v", args)
|
||||
}
|
||||
|
||||
cfg := sc.Config()
|
||||
|
||||
ctx := context.Background()
|
||||
allFSS, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "cannot list filesystems")
|
||||
}
|
||||
|
||||
type workItem struct {
|
||||
jobName string
|
||||
rootFS *zfs.DatasetPath
|
||||
fss []*zfs.DatasetPath
|
||||
}
|
||||
var wis []workItem
|
||||
for i, j := range cfg.Jobs {
|
||||
var rfsS string
|
||||
switch job := j.Ret.(type) {
|
||||
case *config.SinkJob:
|
||||
rfsS = job.RootFS
|
||||
case *config.PullJob:
|
||||
rfsS = job.RootFS
|
||||
default:
|
||||
fmt.Printf("ignoring job %q (%d/%d, type %T)\n", j.Name(), i, len(cfg.Jobs), j.Ret)
|
||||
continue
|
||||
}
|
||||
rfs, err := zfs.NewDatasetPath(rfsS)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "root fs for job %q is not a valid dataset path", j.Name())
|
||||
}
|
||||
var fss []*zfs.DatasetPath
|
||||
for _, fs := range allFSS {
|
||||
if fs.HasPrefix(rfs) {
|
||||
fss = append(fss, fs)
|
||||
}
|
||||
}
|
||||
wis = append(wis, workItem{j.Name(), rfs, fss})
|
||||
}
|
||||
|
||||
for _, wi := range wis {
|
||||
fmt.Printf("job %q => migrate filesystems below root_fs %q\n", wi.jobName, wi.rootFS.ToString())
|
||||
if len(wi.fss) == 0 {
|
||||
fmt.Printf("\tno filesystems\n")
|
||||
continue
|
||||
}
|
||||
for _, fs := range wi.fss {
|
||||
fmt.Printf("\t%q ... ", fs.ToString())
|
||||
r, err := zfs.ZFSMigrateHashBasedPlaceholderToCurrent(fs, migratePlaceholder0_1Args.dryRun)
|
||||
if err != nil {
|
||||
fmt.Printf("error: %s\n", err)
|
||||
} else if !r.NeedsModification {
|
||||
fmt.Printf("unchanged (placeholder=%v)\n", r.OriginalState.IsPlaceholder)
|
||||
} else {
|
||||
fmt.Printf("migrate (placeholder=%v) (old value = %q)\n",
|
||||
r.OriginalState.IsPlaceholder, r.OriginalState.RawLocalPropertyValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,39 +1,38 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/cli"
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon"
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon"
|
||||
)
|
||||
|
||||
var pprofListenCmd struct {
|
||||
var pprofArgs struct {
|
||||
daemon.PprofServerControlMsg
|
||||
}
|
||||
|
||||
var PprofListenCmd = &cli.Subcommand{
|
||||
Use: "listen off | [on TCP_LISTEN_ADDRESS]",
|
||||
var PprofCmd = &cli.Subcommand{
|
||||
Use: "pprof off | [on TCP_LISTEN_ADDRESS]",
|
||||
Short: "start a http server exposing go-tool-compatible profiling endpoints at TCP_LISTEN_ADDRESS",
|
||||
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
||||
Run: func(subcommand *cli.Subcommand, args []string) error {
|
||||
if len(args) < 1 {
|
||||
goto enargs
|
||||
}
|
||||
switch args[0] {
|
||||
case "on":
|
||||
pprofListenCmd.Run = true
|
||||
pprofArgs.Run = true
|
||||
if len(args) != 2 {
|
||||
return errors.New("must specify TCP_LISTEN_ADDRESS as second positional argument")
|
||||
}
|
||||
pprofListenCmd.HttpListenAddress = args[1]
|
||||
pprofArgs.HttpListenAddress = args[1]
|
||||
case "off":
|
||||
if len(args) != 1 {
|
||||
goto enargs
|
||||
}
|
||||
pprofListenCmd.Run = false
|
||||
pprofArgs.Run = false
|
||||
}
|
||||
|
||||
RunPProf(subcommand.Config())
|
||||
@@ -59,7 +58,7 @@ func RunPProf(conf *config.Config) {
|
||||
log.Printf("error creating http client: %s", err)
|
||||
die()
|
||||
}
|
||||
err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofListenCmd.PprofServerControlMsg, struct{}{})
|
||||
err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPProf, pprofArgs.PprofServerControlMsg, struct{}{})
|
||||
if err != nil {
|
||||
log.Printf("error sending control message: %s", err)
|
||||
die()
|
||||
@@ -1,19 +1,17 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/cli"
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon"
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon"
|
||||
)
|
||||
|
||||
var SignalCmd = &cli.Subcommand{
|
||||
Use: "signal [wakeup|reset] JOB",
|
||||
Short: "wake up a job from wait state or abort its current invocation",
|
||||
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
||||
Run: func(subcommand *cli.Subcommand, args []string) error {
|
||||
return runSignalCmd(subcommand.Config(), args)
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,757 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
// tcell is the termbox-compatbile library for abstracting away escape sequences, etc.
|
||||
// as of tcell#252, the number of default distributed terminals is relatively limited
|
||||
// additional terminal definitions can be included via side-effect import
|
||||
// See https://github.com/gdamore/tcell/blob/master/terminfo/base/base.go
|
||||
// See https://github.com/gdamore/tcell/issues/252#issuecomment-533836078
|
||||
"github.com/gdamore/tcell/termbox"
|
||||
_ "github.com/gdamore/tcell/terminfo/s/screen" // tmux on FreeBSD 11 & 12 without ncurses
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
"github.com/zrepl/yaml-config"
|
||||
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/daemon"
|
||||
"github.com/zrepl/zrepl/daemon/job"
|
||||
"github.com/zrepl/zrepl/daemon/pruner"
|
||||
"github.com/zrepl/zrepl/daemon/snapper"
|
||||
"github.com/zrepl/zrepl/replication/report"
|
||||
)
|
||||
|
||||
type byteProgressMeasurement struct {
|
||||
time time.Time
|
||||
val int64
|
||||
}
|
||||
|
||||
type bytesProgressHistory struct {
|
||||
last *byteProgressMeasurement // pointer as poor man's optional
|
||||
changeCount int
|
||||
lastChange time.Time
|
||||
bpsAvg float64
|
||||
}
|
||||
|
||||
func (p *bytesProgressHistory) Update(currentVal int64) (bytesPerSecondAvg int64, changeCount int) {
|
||||
|
||||
if p.last == nil {
|
||||
p.last = &byteProgressMeasurement{
|
||||
time: time.Now(),
|
||||
val: currentVal,
|
||||
}
|
||||
return 0, 0
|
||||
}
|
||||
|
||||
if p.last.val != currentVal {
|
||||
p.changeCount++
|
||||
p.lastChange = time.Now()
|
||||
}
|
||||
|
||||
if time.Since(p.lastChange) > 3*time.Second {
|
||||
p.last = nil
|
||||
return 0, 0
|
||||
}
|
||||
|
||||
deltaV := currentVal - p.last.val
|
||||
deltaT := time.Since(p.last.time)
|
||||
rate := float64(deltaV) / deltaT.Seconds()
|
||||
|
||||
factor := 0.3
|
||||
p.bpsAvg = (1-factor)*p.bpsAvg + factor*rate
|
||||
|
||||
p.last.time = time.Now()
|
||||
p.last.val = currentVal
|
||||
|
||||
return int64(p.bpsAvg), p.changeCount
|
||||
}
|
||||
|
||||
type tui struct {
|
||||
x, y int
|
||||
indent int
|
||||
|
||||
lock sync.Mutex //For report and error
|
||||
report map[string]job.Status
|
||||
err error
|
||||
|
||||
jobFilter string
|
||||
|
||||
replicationProgress map[string]*bytesProgressHistory // by job name
|
||||
}
|
||||
|
||||
func newTui() tui {
|
||||
return tui{
|
||||
replicationProgress: make(map[string]*bytesProgressHistory),
|
||||
}
|
||||
}
|
||||
|
||||
const INDENT_MULTIPLIER = 4
|
||||
|
||||
func (t *tui) moveLine(dl int, col int) {
|
||||
t.y += dl
|
||||
t.x = t.indent*INDENT_MULTIPLIER + col
|
||||
}
|
||||
|
||||
func (t *tui) write(text string) {
|
||||
for _, c := range text {
|
||||
if c == '\n' {
|
||||
t.newline()
|
||||
continue
|
||||
}
|
||||
termbox.SetCell(t.x, t.y, c, termbox.ColorDefault, termbox.ColorDefault)
|
||||
t.x += 1
|
||||
}
|
||||
}
|
||||
|
||||
func (t *tui) printf(text string, a ...interface{}) {
|
||||
t.write(fmt.Sprintf(text, a...))
|
||||
}
|
||||
|
||||
func wrap(s string, width int) string {
|
||||
var b strings.Builder
|
||||
for len(s) > 0 {
|
||||
rem := width
|
||||
if rem > len(s) {
|
||||
rem = len(s)
|
||||
}
|
||||
if idx := strings.IndexAny(s, "\n\r"); idx != -1 && idx < rem {
|
||||
rem = idx + 1
|
||||
}
|
||||
untilNewline := strings.TrimRight(s[:rem], "\n\r")
|
||||
s = s[rem:]
|
||||
if len(untilNewline) == 0 {
|
||||
continue
|
||||
}
|
||||
b.WriteString(untilNewline)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return strings.TrimRight(b.String(), "\n\r")
|
||||
}
|
||||
|
||||
func (t *tui) printfDrawIndentedAndWrappedIfMultiline(format string, a ...interface{}) {
|
||||
whole := fmt.Sprintf(format, a...)
|
||||
width, _ := termbox.Size()
|
||||
if !strings.ContainsAny(whole, "\n\r") && t.x+len(whole) <= width {
|
||||
t.printf(format, a...)
|
||||
} else {
|
||||
t.addIndent(1)
|
||||
t.newline()
|
||||
t.write(wrap(whole, width-INDENT_MULTIPLIER*t.indent))
|
||||
t.addIndent(-1)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *tui) newline() {
|
||||
t.moveLine(1, 0)
|
||||
}
|
||||
|
||||
func (t *tui) setIndent(indent int) {
|
||||
t.indent = indent
|
||||
t.moveLine(0, 0)
|
||||
}
|
||||
|
||||
func (t *tui) addIndent(indent int) {
|
||||
t.indent += indent
|
||||
t.moveLine(0, 0)
|
||||
}
|
||||
|
||||
var statusFlags struct {
|
||||
Raw bool
|
||||
Job string
|
||||
}
|
||||
|
||||
var StatusCmd = &cli.Subcommand{
|
||||
Use: "status",
|
||||
Short: "show job activity or dump as JSON for monitoring",
|
||||
SetupFlags: func(f *pflag.FlagSet) {
|
||||
f.BoolVar(&statusFlags.Raw, "raw", false, "dump raw status description from zrepl daemon")
|
||||
f.StringVar(&statusFlags.Job, "job", "", "only dump specified job")
|
||||
},
|
||||
Run: runStatus,
|
||||
}
|
||||
|
||||
func runStatus(s *cli.Subcommand, args []string) error {
|
||||
httpc, err := controlHttpClient(s.Config().Global.Control.SockPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if statusFlags.Raw {
|
||||
resp, err := httpc.Get("http://unix" + daemon.ControlJobEndpointStatus)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
fmt.Fprintf(os.Stderr, "Received error response:\n")
|
||||
_, err := io.CopyN(os.Stderr, resp.Body, 4096)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return errors.Errorf("exit")
|
||||
}
|
||||
if _, err := io.Copy(os.Stdout, resp.Body); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
t := newTui()
|
||||
t.lock.Lock()
|
||||
t.err = errors.New("Got no report yet")
|
||||
t.lock.Unlock()
|
||||
t.jobFilter = statusFlags.Job
|
||||
|
||||
err = termbox.Init()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer termbox.Close()
|
||||
|
||||
update := func() {
|
||||
m := make(map[string]job.Status)
|
||||
|
||||
err2 := jsonRequestResponse(httpc, daemon.ControlJobEndpointStatus,
|
||||
struct{}{},
|
||||
&m,
|
||||
)
|
||||
|
||||
t.lock.Lock()
|
||||
t.err = err2
|
||||
t.report = m
|
||||
t.lock.Unlock()
|
||||
t.draw()
|
||||
}
|
||||
update()
|
||||
|
||||
ticker := time.NewTicker(500 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
go func() {
|
||||
for range ticker.C {
|
||||
update()
|
||||
}
|
||||
}()
|
||||
|
||||
termbox.HideCursor()
|
||||
termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
|
||||
|
||||
loop:
|
||||
for {
|
||||
switch ev := termbox.PollEvent(); ev.Type {
|
||||
case termbox.EventKey:
|
||||
switch ev.Key {
|
||||
case termbox.KeyEsc:
|
||||
break loop
|
||||
case termbox.KeyCtrlC:
|
||||
break loop
|
||||
}
|
||||
case termbox.EventResize:
|
||||
t.draw()
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func (t *tui) getReplicationProgresHistory(jobName string) *bytesProgressHistory {
|
||||
p, ok := t.replicationProgress[jobName]
|
||||
if !ok {
|
||||
p = &bytesProgressHistory{}
|
||||
t.replicationProgress[jobName] = p
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func (t *tui) draw() {
|
||||
t.lock.Lock()
|
||||
defer t.lock.Unlock()
|
||||
|
||||
termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
|
||||
t.x = 0
|
||||
t.y = 0
|
||||
t.indent = 0
|
||||
|
||||
if t.err != nil {
|
||||
t.write(t.err.Error())
|
||||
} else {
|
||||
//Iterate over map in alphabetical order
|
||||
keys := make([]string, 0, len(t.report))
|
||||
for k := range t.report {
|
||||
if len(k) == 0 || daemon.IsInternalJobName(k) { //Internal job
|
||||
continue
|
||||
}
|
||||
if t.jobFilter != "" && k != t.jobFilter {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
if len(keys) == 0 {
|
||||
t.setIndent(0)
|
||||
t.printf("no jobs to display")
|
||||
t.newline()
|
||||
termbox.Flush()
|
||||
return
|
||||
}
|
||||
|
||||
for _, k := range keys {
|
||||
v := t.report[k]
|
||||
|
||||
t.setIndent(0)
|
||||
|
||||
t.printf("Job: %s", k)
|
||||
t.setIndent(1)
|
||||
t.newline()
|
||||
t.printf("Type: %s", v.Type)
|
||||
t.setIndent(1)
|
||||
t.newline()
|
||||
|
||||
if v.Type == job.TypePush || v.Type == job.TypePull {
|
||||
activeStatus, ok := v.JobSpecific.(*job.ActiveSideStatus)
|
||||
if !ok || activeStatus == nil {
|
||||
t.printf("ActiveSideStatus is null")
|
||||
t.newline()
|
||||
continue
|
||||
}
|
||||
|
||||
t.printf("Replication:")
|
||||
t.newline()
|
||||
t.addIndent(1)
|
||||
t.renderReplicationReport(activeStatus.Replication, t.getReplicationProgresHistory(k))
|
||||
t.addIndent(-1)
|
||||
|
||||
t.printf("Pruning Sender:")
|
||||
t.newline()
|
||||
t.addIndent(1)
|
||||
t.renderPrunerReport(activeStatus.PruningSender)
|
||||
t.addIndent(-1)
|
||||
|
||||
t.printf("Pruning Receiver:")
|
||||
t.newline()
|
||||
t.addIndent(1)
|
||||
t.renderPrunerReport(activeStatus.PruningReceiver)
|
||||
t.addIndent(-1)
|
||||
|
||||
if v.Type == job.TypePush {
|
||||
t.printf("Snapshotting:")
|
||||
t.newline()
|
||||
t.addIndent(1)
|
||||
t.renderSnapperReport(activeStatus.Snapshotting)
|
||||
t.addIndent(-1)
|
||||
}
|
||||
|
||||
} else if v.Type == job.TypeSnap {
|
||||
snapStatus, ok := v.JobSpecific.(*job.SnapJobStatus)
|
||||
if !ok || snapStatus == nil {
|
||||
t.printf("SnapJobStatus is null")
|
||||
t.newline()
|
||||
continue
|
||||
}
|
||||
t.printf("Pruning snapshots:")
|
||||
t.newline()
|
||||
t.addIndent(1)
|
||||
t.renderPrunerReport(snapStatus.Pruning)
|
||||
t.addIndent(-1)
|
||||
t.printf("Snapshotting:")
|
||||
t.newline()
|
||||
t.addIndent(1)
|
||||
t.renderSnapperReport(snapStatus.Snapshotting)
|
||||
t.addIndent(-1)
|
||||
} else if v.Type == job.TypeSource {
|
||||
|
||||
st := v.JobSpecific.(*job.PassiveStatus)
|
||||
t.printf("Snapshotting:\n")
|
||||
t.addIndent(1)
|
||||
t.renderSnapperReport(st.Snapper)
|
||||
t.addIndent(-1)
|
||||
|
||||
} else {
|
||||
t.printf("No status representation for job type '%s', dumping as YAML", v.Type)
|
||||
t.newline()
|
||||
asYaml, err := yaml.Marshal(v.JobSpecific)
|
||||
if err != nil {
|
||||
t.printf("Error marshaling status to YAML: %s", err)
|
||||
t.newline()
|
||||
continue
|
||||
}
|
||||
t.write(string(asYaml))
|
||||
t.newline()
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
termbox.Flush()
|
||||
}
|
||||
|
||||
func (t *tui) renderReplicationReport(rep *report.Report, history *bytesProgressHistory) {
|
||||
if rep == nil {
|
||||
t.printf("...\n")
|
||||
return
|
||||
}
|
||||
|
||||
if rep.WaitReconnectError != nil {
|
||||
t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: %s", rep.WaitReconnectError)
|
||||
t.newline()
|
||||
}
|
||||
if !rep.WaitReconnectSince.IsZero() {
|
||||
delta := time.Until(rep.WaitReconnectUntil).Round(time.Second)
|
||||
if rep.WaitReconnectUntil.IsZero() || delta > 0 {
|
||||
var until string
|
||||
if rep.WaitReconnectUntil.IsZero() {
|
||||
until = "waiting indefinitely"
|
||||
} else {
|
||||
until = fmt.Sprintf("hard fail in %s @ %s", delta, rep.WaitReconnectUntil)
|
||||
}
|
||||
t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnecting with exponential backoff (since %s) (%s)",
|
||||
rep.WaitReconnectSince, until)
|
||||
} else {
|
||||
t.printfDrawIndentedAndWrappedIfMultiline("Connectivity: reconnects reached hard-fail timeout @ %s", rep.WaitReconnectUntil)
|
||||
}
|
||||
t.newline()
|
||||
}
|
||||
|
||||
// TODO visualize more than the latest attempt by folding all attempts into one
|
||||
if len(rep.Attempts) == 0 {
|
||||
t.printf("no attempts made yet")
|
||||
return
|
||||
} else {
|
||||
t.printf("Attempt #%d", len(rep.Attempts))
|
||||
if len(rep.Attempts) > 1 {
|
||||
t.printf(". Previous attempts failed with the following statuses:")
|
||||
t.newline()
|
||||
t.addIndent(1)
|
||||
for i, a := range rep.Attempts[:len(rep.Attempts)-1] {
|
||||
t.printfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt))
|
||||
t.newline()
|
||||
}
|
||||
t.addIndent(-1)
|
||||
} else {
|
||||
t.newline()
|
||||
}
|
||||
}
|
||||
|
||||
latest := rep.Attempts[len(rep.Attempts)-1]
|
||||
sort.Slice(latest.Filesystems, func(i, j int) bool {
|
||||
return latest.Filesystems[i].Info.Name < latest.Filesystems[j].Info.Name
|
||||
})
|
||||
|
||||
t.printf("Status: %s", latest.State)
|
||||
t.newline()
|
||||
if latest.State == report.AttemptPlanningError {
|
||||
t.printf("Problem: ")
|
||||
t.printfDrawIndentedAndWrappedIfMultiline("%s", latest.PlanError)
|
||||
t.newline()
|
||||
} else if latest.State == report.AttemptFanOutError {
|
||||
t.printf("Problem: one or more of the filesystems encountered errors")
|
||||
t.newline()
|
||||
}
|
||||
|
||||
if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
|
||||
// Draw global progress bar
|
||||
// Progress: [---------------]
|
||||
expected, replicated := latest.BytesSum()
|
||||
rate, changeCount := history.Update(replicated)
|
||||
t.write("Progress: ")
|
||||
t.drawBar(50, replicated, expected, changeCount)
|
||||
t.write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinary(replicated), ByteCountBinary(expected), ByteCountBinary(rate)))
|
||||
t.newline()
|
||||
|
||||
var maxFSLen int
|
||||
for _, fs := range latest.Filesystems {
|
||||
if len(fs.Info.Name) > maxFSLen {
|
||||
maxFSLen = len(fs.Info.Name)
|
||||
}
|
||||
}
|
||||
for _, fs := range latest.Filesystems {
|
||||
t.printFilesystemStatus(fs, false, maxFSLen) // FIXME bring 'active' flag back
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (t *tui) renderPrunerReport(r *pruner.Report) {
|
||||
if r == nil {
|
||||
t.printf("...\n")
|
||||
return
|
||||
}
|
||||
|
||||
state, err := pruner.StateString(r.State)
|
||||
if err != nil {
|
||||
t.printf("Status: %q (parse error: %q)\n", r.State, err)
|
||||
return
|
||||
}
|
||||
|
||||
t.printf("Status: %s", state)
|
||||
t.newline()
|
||||
|
||||
if r.Error != "" {
|
||||
t.printf("Error: %s\n", r.Error)
|
||||
}
|
||||
|
||||
type commonFS struct {
|
||||
*pruner.FSReport
|
||||
completed bool
|
||||
}
|
||||
all := make([]commonFS, 0, len(r.Pending)+len(r.Completed))
|
||||
for i := range r.Pending {
|
||||
all = append(all, commonFS{&r.Pending[i], false})
|
||||
}
|
||||
for i := range r.Completed {
|
||||
all = append(all, commonFS{&r.Completed[i], true})
|
||||
}
|
||||
|
||||
switch state {
|
||||
case pruner.Plan:
|
||||
fallthrough
|
||||
case pruner.PlanErr:
|
||||
return
|
||||
}
|
||||
|
||||
if len(all) == 0 {
|
||||
t.printf("nothing to do\n")
|
||||
return
|
||||
}
|
||||
|
||||
var totalDestroyCount, completedDestroyCount int
|
||||
var maxFSname int
|
||||
for _, fs := range all {
|
||||
totalDestroyCount += len(fs.DestroyList)
|
||||
if fs.completed {
|
||||
completedDestroyCount += len(fs.DestroyList)
|
||||
}
|
||||
if maxFSname < len(fs.Filesystem) {
|
||||
maxFSname = len(fs.Filesystem)
|
||||
}
|
||||
}
|
||||
|
||||
// global progress bar
|
||||
progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
|
||||
t.write("Progress: ")
|
||||
t.write("[")
|
||||
t.write(times("=", progress))
|
||||
t.write(">")
|
||||
t.write(times("-", 80-progress))
|
||||
t.write("]")
|
||||
t.printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
|
||||
t.newline()
|
||||
|
||||
sort.SliceStable(all, func(i, j int) bool {
|
||||
return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
|
||||
})
|
||||
|
||||
// Draw a table-like representation of 'all'
|
||||
for _, fs := range all {
|
||||
t.write(rightPad(fs.Filesystem, maxFSname, " "))
|
||||
t.write(" ")
|
||||
if !fs.SkipReason.NotSkipped() {
|
||||
t.printf("skipped: %s\n", fs.SkipReason)
|
||||
continue
|
||||
}
|
||||
if fs.LastError != "" {
|
||||
if strings.ContainsAny(fs.LastError, "\r\n") {
|
||||
t.printf("ERROR:")
|
||||
t.printfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError)
|
||||
} else {
|
||||
t.printfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError)
|
||||
}
|
||||
t.newline()
|
||||
continue
|
||||
}
|
||||
|
||||
pruneRuleActionStr := fmt.Sprintf("(destroy %d of %d snapshots)",
|
||||
len(fs.DestroyList), len(fs.SnapshotList))
|
||||
|
||||
if fs.completed {
|
||||
t.printf("Completed %s\n", pruneRuleActionStr)
|
||||
continue
|
||||
}
|
||||
|
||||
t.write("Pending ") // whitespace is padding 10
|
||||
if len(fs.DestroyList) == 1 {
|
||||
t.write(fs.DestroyList[0].Name)
|
||||
} else {
|
||||
t.write(pruneRuleActionStr)
|
||||
}
|
||||
t.newline()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (t *tui) renderSnapperReport(r *snapper.Report) {
|
||||
if r == nil {
|
||||
t.printf("<snapshot type does not have a report>\n")
|
||||
return
|
||||
}
|
||||
|
||||
t.printf("Status: %s", r.State)
|
||||
t.newline()
|
||||
|
||||
if r.Error != "" {
|
||||
t.printf("Error: %s\n", r.Error)
|
||||
}
|
||||
if !r.SleepUntil.IsZero() {
|
||||
t.printf("Sleep until: %s\n", r.SleepUntil)
|
||||
}
|
||||
|
||||
sort.Slice(r.Progress, func(i, j int) bool {
|
||||
return strings.Compare(r.Progress[i].Path, r.Progress[j].Path) == -1
|
||||
})
|
||||
|
||||
t.addIndent(1)
|
||||
defer t.addIndent(-1)
|
||||
dur := func(d time.Duration) string {
|
||||
return d.Round(100 * time.Millisecond).String()
|
||||
}
|
||||
|
||||
type row struct {
|
||||
path, state, duration, remainder, hookReport string
|
||||
}
|
||||
var widths struct {
|
||||
path, state, duration int
|
||||
}
|
||||
rows := make([]*row, len(r.Progress))
|
||||
for i, fs := range r.Progress {
|
||||
r := &row{
|
||||
path: fs.Path,
|
||||
state: fs.State.String(),
|
||||
}
|
||||
if fs.HooksHadError {
|
||||
r.hookReport = fs.Hooks // FIXME render here, not in daemon
|
||||
}
|
||||
switch fs.State {
|
||||
case snapper.SnapPending:
|
||||
r.duration = "..."
|
||||
r.remainder = ""
|
||||
case snapper.SnapStarted:
|
||||
r.duration = dur(time.Since(fs.StartAt))
|
||||
r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
|
||||
case snapper.SnapDone:
|
||||
fallthrough
|
||||
case snapper.SnapError:
|
||||
r.duration = dur(fs.DoneAt.Sub(fs.StartAt))
|
||||
r.remainder = fmt.Sprintf("snap name: %q", fs.SnapName)
|
||||
}
|
||||
rows[i] = r
|
||||
if len(r.path) > widths.path {
|
||||
widths.path = len(r.path)
|
||||
}
|
||||
if len(r.state) > widths.state {
|
||||
widths.state = len(r.state)
|
||||
}
|
||||
if len(r.duration) > widths.duration {
|
||||
widths.duration = len(r.duration)
|
||||
}
|
||||
}
|
||||
|
||||
for _, r := range rows {
|
||||
path := rightPad(r.path, widths.path, " ")
|
||||
state := rightPad(r.state, widths.state, " ")
|
||||
duration := rightPad(r.duration, widths.duration, " ")
|
||||
t.printf("%s %s %s", path, state, duration)
|
||||
t.printfDrawIndentedAndWrappedIfMultiline(" %s", r.remainder)
|
||||
if r.hookReport != "" {
|
||||
t.printfDrawIndentedAndWrappedIfMultiline("%s", r.hookReport)
|
||||
}
|
||||
t.newline()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func times(str string, n int) (out string) {
|
||||
for i := 0; i < n; i++ {
|
||||
out += str
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func rightPad(str string, length int, pad string) string {
|
||||
if len(str) > length {
|
||||
return str[:length]
|
||||
}
|
||||
return str + strings.Repeat(pad, length-len(str))
|
||||
}
|
||||
|
||||
var arrowPositions = `>\|/`
|
||||
|
||||
// changeCount = 0 indicates stall / no progresss
|
||||
func (t *tui) drawBar(length int, bytes, totalBytes int64, changeCount int) {
|
||||
var completedLength int
|
||||
if totalBytes > 0 {
|
||||
completedLength = int(int64(length) * bytes / totalBytes)
|
||||
if completedLength > length {
|
||||
completedLength = length
|
||||
}
|
||||
} else if totalBytes == bytes {
|
||||
completedLength = length
|
||||
}
|
||||
|
||||
t.write("[")
|
||||
t.write(times("=", completedLength))
|
||||
t.write(string(arrowPositions[changeCount%len(arrowPositions)]))
|
||||
t.write(times("-", length-completedLength))
|
||||
t.write("]")
|
||||
}
|
||||
|
||||
func (t *tui) printFilesystemStatus(rep *report.FilesystemReport, active bool, maxFS int) {
|
||||
|
||||
expected, replicated := rep.BytesSum()
|
||||
status := fmt.Sprintf("%s (step %d/%d, %s/%s)",
|
||||
strings.ToUpper(string(rep.State)),
|
||||
rep.CurrentStep, len(rep.Steps),
|
||||
ByteCountBinary(replicated), ByteCountBinary(expected),
|
||||
)
|
||||
|
||||
activeIndicator := " "
|
||||
if active {
|
||||
activeIndicator = "*"
|
||||
}
|
||||
t.printf("%s %s %s ",
|
||||
activeIndicator,
|
||||
rightPad(rep.Info.Name, maxFS, " "),
|
||||
status)
|
||||
|
||||
next := ""
|
||||
if err := rep.Error(); err != nil {
|
||||
next = err.Err
|
||||
} else if rep.State != report.FilesystemDone {
|
||||
if nextStep := rep.NextStep(); nextStep != nil {
|
||||
if nextStep.IsIncremental() {
|
||||
next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
|
||||
} else {
|
||||
next = fmt.Sprintf("next: %s (full)", nextStep.Info.To)
|
||||
}
|
||||
} else {
|
||||
next = "" // individual FSes may still be in planning state
|
||||
}
|
||||
|
||||
}
|
||||
t.printfDrawIndentedAndWrappedIfMultiline("%s", next)
|
||||
|
||||
t.newline()
|
||||
}
|
||||
|
||||
func ByteCountBinary(b int64) string {
|
||||
const unit = 1024
|
||||
if b < unit {
|
||||
return fmt.Sprintf("%d B", b)
|
||||
}
|
||||
div, exp := int64(unit), 0
|
||||
for n := b / unit; n >= unit; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
@@ -5,8 +5,8 @@ import (
|
||||
|
||||
"github.com/problame/go-netssh"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/cli"
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
|
||||
"context"
|
||||
"errors"
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
var StdinserverCmd = &cli.Subcommand{
|
||||
Use: "stdinserver CLIENT_IDENTITY",
|
||||
Short: "stdinserver transport mode (started from authorized_keys file as forced command)",
|
||||
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
||||
Run: func(subcommand *cli.Subcommand, args []string) error {
|
||||
return runStdinserver(subcommand.Config(), args)
|
||||
},
|
||||
}
|
||||
@@ -1,24 +1,21 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/cli"
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
var TestCmd = &cli.Subcommand{
|
||||
Use: "test",
|
||||
SetupSubcommands: func() []*cli.Subcommand {
|
||||
return []*cli.Subcommand{testFilter, testPlaceholder, testDecodeResumeToken}
|
||||
return []*cli.Subcommand{testFilter, testPlaceholder}
|
||||
},
|
||||
}
|
||||
|
||||
@@ -39,7 +36,7 @@ var testFilter = &cli.Subcommand{
|
||||
Run: runTestFilterCmd,
|
||||
}
|
||||
|
||||
func runTestFilterCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
||||
func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
|
||||
|
||||
if testFilterArgs.job == "" {
|
||||
return fmt.Errorf("must specify --job flag")
|
||||
@@ -60,8 +57,6 @@ func runTestFilterCmd(ctx context.Context, subcommand *cli.Subcommand, args []st
|
||||
confFilter = j.Filesystems
|
||||
case *config.PushJob:
|
||||
confFilter = j.Filesystems
|
||||
case *config.SnapJob:
|
||||
confFilter = j.Filesystems
|
||||
default:
|
||||
return fmt.Errorf("job type %T does not have filesystems filter", j)
|
||||
}
|
||||
@@ -75,7 +70,7 @@ func runTestFilterCmd(ctx context.Context, subcommand *cli.Subcommand, args []st
|
||||
if testFilterArgs.input != "" {
|
||||
fsnames = []string{testFilterArgs.input}
|
||||
} else {
|
||||
out, err := zfs.ZFSList(ctx, []string{"name"})
|
||||
out, err := zfs.ZFSList([]string{"name"})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not list ZFS filesystems: %s", err)
|
||||
}
|
||||
@@ -136,15 +131,13 @@ var testPlaceholder = &cli.Subcommand{
|
||||
Run: runTestPlaceholder,
|
||||
}
|
||||
|
||||
func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
||||
func runTestPlaceholder(subcommand *cli.Subcommand, args []string) error {
|
||||
|
||||
var checkDPs []*zfs.DatasetPath
|
||||
var datasetWasExplicitArgument bool
|
||||
|
||||
// all actions first
|
||||
if testPlaceholderArgs.all {
|
||||
datasetWasExplicitArgument = false
|
||||
out, err := zfs.ZFSList(ctx, []string{"name"})
|
||||
out, err := zfs.ZFSList([]string{"name"})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not list ZFS filesystems")
|
||||
}
|
||||
@@ -156,7 +149,6 @@ func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []
|
||||
checkDPs = append(checkDPs, dp)
|
||||
}
|
||||
} else {
|
||||
datasetWasExplicitArgument = true
|
||||
dp, err := zfs.NewDatasetPath(testPlaceholderArgs.ds)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -169,17 +161,12 @@ func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []
|
||||
|
||||
fmt.Printf("IS_PLACEHOLDER\tDATASET\tzrepl:placeholder\n")
|
||||
for _, dp := range checkDPs {
|
||||
ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, dp)
|
||||
ph, err := zfs.ZFSGetFilesystemPlaceholderState(dp)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "cannot get placeholder state")
|
||||
}
|
||||
if !ph.FSExists {
|
||||
if datasetWasExplicitArgument {
|
||||
return errors.Errorf("filesystem %q does not exist", ph.FS)
|
||||
} else {
|
||||
// got deleted between ZFSList and ZFSGetFilesystemPlaceholderState
|
||||
continue
|
||||
}
|
||||
panic("placeholder state inconsistent: filesystem " + ph.FS + " must exist in this context")
|
||||
}
|
||||
is := "yes"
|
||||
if !ph.IsPlaceholder {
|
||||
@@ -189,32 +176,3 @@ func runTestPlaceholder(ctx context.Context, subcommand *cli.Subcommand, args []
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var testDecodeResumeTokenArgs struct {
|
||||
token string
|
||||
}
|
||||
|
||||
var testDecodeResumeToken = &cli.Subcommand{
|
||||
Use: "decoderesumetoken --token TOKEN",
|
||||
Short: "decode resume token",
|
||||
SetupFlags: func(f *pflag.FlagSet) {
|
||||
f.StringVar(&testDecodeResumeTokenArgs.token, "token", "", "the resume token obtained from the receive_resume_token property")
|
||||
},
|
||||
Run: runTestDecodeResumeTokenCmd,
|
||||
}
|
||||
|
||||
func runTestDecodeResumeTokenCmd(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
||||
if testDecodeResumeTokenArgs.token == "" {
|
||||
return fmt.Errorf("token argument must be specified")
|
||||
}
|
||||
token, err := zfs.ParseResumeToken(ctx, testDecodeResumeTokenArgs.token)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
if err := enc.Encode(&token); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,16 +1,15 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/cli"
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon"
|
||||
"github.com/zrepl/zrepl/internal/version"
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon"
|
||||
"github.com/zrepl/zrepl/version"
|
||||
)
|
||||
|
||||
var versionArgs struct {
|
||||
@@ -26,7 +25,7 @@ var VersionCmd = &cli.Subcommand{
|
||||
SetupFlags: func(f *pflag.FlagSet) {
|
||||
f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)")
|
||||
},
|
||||
Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
|
||||
Run: func(subcommand *cli.Subcommand, args []string) error {
|
||||
versionArgs.Config = subcommand.Config()
|
||||
versionArgs.ConfigErr = subcommand.ConfigParsingError()
|
||||
return runVersionCmd()
|
||||
@@ -2,32 +2,21 @@ package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log/syslog"
|
||||
"os"
|
||||
pathpkg "path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/robfig/cron/v3"
|
||||
"github.com/zrepl/yaml-config"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/util/datasizeunit"
|
||||
zfsprop "github.com/zrepl/zrepl/internal/zfs/property"
|
||||
)
|
||||
|
||||
type ParseFlags uint
|
||||
|
||||
const (
|
||||
ParseFlagsNone ParseFlags = 0
|
||||
ParseFlagsNoCertCheck ParseFlags = 1 << iota
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Jobs []JobEnum `yaml:"jobs,optional"`
|
||||
Global *Global `yaml:"global,optional,fromdefaults"`
|
||||
Include []string `yaml:"include,optional"`
|
||||
Jobs []JobEnum `yaml:"jobs"`
|
||||
Global *Global `yaml:"global,optional,fromdefaults"`
|
||||
}
|
||||
|
||||
func (c *Config) Job(name string) (*JobEnum, error) {
|
||||
@@ -63,110 +52,41 @@ func (j JobEnum) Name() string {
|
||||
}
|
||||
|
||||
type ActiveJob struct {
|
||||
Type string `yaml:"type"`
|
||||
Name string `yaml:"name"`
|
||||
Connect ConnectEnum `yaml:"connect"`
|
||||
Pruning PruningSenderReceiver `yaml:"pruning"`
|
||||
Replication *Replication `yaml:"replication,optional,fromdefaults"`
|
||||
ConflictResolution *ConflictResolution `yaml:"conflict_resolution,optional,fromdefaults"`
|
||||
}
|
||||
|
||||
type ConflictResolution struct {
|
||||
InitialReplication string `yaml:"initial_replication,optional,default=most_recent"`
|
||||
Type string `yaml:"type"`
|
||||
Name string `yaml:"name"`
|
||||
Connect ConnectEnum `yaml:"connect"`
|
||||
Pruning PruningSenderReceiver `yaml:"pruning"`
|
||||
Debug JobDebugSettings `yaml:"debug,optional"`
|
||||
}
|
||||
|
||||
type PassiveJob struct {
|
||||
Type string `yaml:"type"`
|
||||
Name string `yaml:"name"`
|
||||
Serve ServeEnum `yaml:"serve"`
|
||||
Type string `yaml:"type"`
|
||||
Name string `yaml:"name"`
|
||||
Serve ServeEnum `yaml:"serve"`
|
||||
Debug JobDebugSettings `yaml:"debug,optional"`
|
||||
}
|
||||
|
||||
type SnapJob struct {
|
||||
Type string `yaml:"type"`
|
||||
Name string `yaml:"name"`
|
||||
Pruning PruningLocal `yaml:"pruning"`
|
||||
Debug JobDebugSettings `yaml:"debug,optional"`
|
||||
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
|
||||
Filesystems FilesystemsFilter `yaml:"filesystems"`
|
||||
}
|
||||
|
||||
type SendOptions struct {
|
||||
Encrypted bool `yaml:"encrypted,optional,default=false"`
|
||||
Raw bool `yaml:"raw,optional,default=false"`
|
||||
SendProperties bool `yaml:"send_properties,optional,default=false"`
|
||||
BackupProperties bool `yaml:"backup_properties,optional,default=false"`
|
||||
LargeBlocks bool `yaml:"large_blocks,optional,default=false"`
|
||||
Compressed bool `yaml:"compressed,optional,default=false"`
|
||||
EmbeddedData bool `yaml:"embedded_data,optional,default=false"`
|
||||
Saved bool `yaml:"saved,optional,default=false"`
|
||||
|
||||
BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
|
||||
}
|
||||
|
||||
type RecvOptions struct {
|
||||
// Note: we cannot enforce encrypted recv as the ZFS cli doesn't provide a mechanism for it
|
||||
// Encrypted bool `yaml:"may_encrypted"`
|
||||
// Future:
|
||||
// Reencrypt bool `yaml:"reencrypt"`
|
||||
|
||||
Properties *PropertyRecvOptions `yaml:"properties,fromdefaults"`
|
||||
|
||||
BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
|
||||
|
||||
Placeholder *PlaceholderRecvOptions `yaml:"placeholder,fromdefaults"`
|
||||
}
|
||||
|
||||
var _ yaml.Unmarshaler = &datasizeunit.Bits{}
|
||||
|
||||
type BandwidthLimit struct {
|
||||
Max datasizeunit.Bits `yaml:"max,default=-1 B"`
|
||||
BucketCapacity datasizeunit.Bits `yaml:"bucket_capacity,default=128 KiB"`
|
||||
}
|
||||
|
||||
type Replication struct {
|
||||
Protection *ReplicationOptionsProtection `yaml:"protection,optional,fromdefaults"`
|
||||
Concurrency *ReplicationOptionsConcurrency `yaml:"concurrency,optional,fromdefaults"`
|
||||
}
|
||||
|
||||
type ReplicationOptionsProtection struct {
|
||||
Initial string `yaml:"initial,optional,default=guarantee_resumability"`
|
||||
Incremental string `yaml:"incremental,optional,default=guarantee_resumability"`
|
||||
}
|
||||
|
||||
type ReplicationOptionsConcurrency struct {
|
||||
Steps int `yaml:"steps,optional,default=1"`
|
||||
SizeEstimates int `yaml:"size_estimates,optional,default=4"`
|
||||
}
|
||||
|
||||
type PropertyRecvOptions struct {
|
||||
Inherit []zfsprop.Property `yaml:"inherit,optional"`
|
||||
Override map[zfsprop.Property]string `yaml:"override,optional"`
|
||||
}
|
||||
|
||||
type PlaceholderRecvOptions struct {
|
||||
Encryption string `yaml:"encryption,default=unspecified"`
|
||||
}
|
||||
|
||||
type PushJob struct {
|
||||
ActiveJob `yaml:",inline"`
|
||||
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
|
||||
Filesystems FilesystemsFilter `yaml:"filesystems"`
|
||||
Send *SendOptions `yaml:"send,fromdefaults,optional"`
|
||||
}
|
||||
|
||||
func (j *PushJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
|
||||
func (j *PushJob) GetSendOptions() *SendOptions { return j.Send }
|
||||
|
||||
type PullJob struct {
|
||||
ActiveJob `yaml:",inline"`
|
||||
RootFS string `yaml:"root_fs"`
|
||||
Interval PositiveDurationOrManual `yaml:"interval"`
|
||||
Recv *RecvOptions `yaml:"recv,fromdefaults,optional"`
|
||||
}
|
||||
|
||||
func (j *PullJob) GetRootFS() string { return j.RootFS }
|
||||
func (j *PullJob) GetAppendClientIdentity() bool { return false }
|
||||
func (j *PullJob) GetRecvOptions() *RecvOptions { return j.Recv }
|
||||
|
||||
type PositiveDurationOrManual struct {
|
||||
Interval time.Duration
|
||||
Manual bool
|
||||
@@ -187,34 +107,28 @@ func (i *PositiveDurationOrManual) UnmarshalYAML(u func(interface{}, bool) error
|
||||
return fmt.Errorf("value must not be empty")
|
||||
default:
|
||||
i.Manual = false
|
||||
i.Interval, err = parsePositiveDuration(s)
|
||||
i.Interval, err = time.ParseDuration(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if i.Interval <= 0 {
|
||||
return fmt.Errorf("value must be a positive duration, got %q", s)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type SinkJob struct {
|
||||
PassiveJob `yaml:",inline"`
|
||||
RootFS string `yaml:"root_fs"`
|
||||
Recv *RecvOptions `yaml:"recv,optional,fromdefaults"`
|
||||
RootFS string `yaml:"root_fs"`
|
||||
}
|
||||
|
||||
func (j *SinkJob) GetRootFS() string { return j.RootFS }
|
||||
func (j *SinkJob) GetAppendClientIdentity() bool { return true }
|
||||
func (j *SinkJob) GetRecvOptions() *RecvOptions { return j.Recv }
|
||||
|
||||
type SourceJob struct {
|
||||
PassiveJob `yaml:",inline"`
|
||||
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
|
||||
Filesystems FilesystemsFilter `yaml:"filesystems"`
|
||||
Send *SendOptions `yaml:"send,optional,fromdefaults"`
|
||||
}
|
||||
|
||||
func (j *SourceJob) GetFilesystems() FilesystemsFilter { return j.Filesystems }
|
||||
func (j *SourceJob) GetSendOptions() *SendOptions { return j.Send }
|
||||
|
||||
type FilesystemsFilter map[string]bool
|
||||
|
||||
type SnapshottingEnum struct {
|
||||
@@ -222,49 +136,10 @@ type SnapshottingEnum struct {
|
||||
}
|
||||
|
||||
type SnapshottingPeriodic struct {
|
||||
Type string `yaml:"type"`
|
||||
Prefix string `yaml:"prefix"`
|
||||
Interval *PositiveDuration `yaml:"interval"`
|
||||
Hooks HookList `yaml:"hooks,optional"`
|
||||
TimestampFormattingSpec `yaml:",inline"`
|
||||
}
|
||||
|
||||
type TimestampFormattingSpec struct {
|
||||
TimestampFormat string `yaml:"timestamp_format,optional,default=dense"`
|
||||
TimestampLocation string `yaml:"timestamp_location,optional,default=UTC"`
|
||||
}
|
||||
|
||||
type CronSpec struct {
|
||||
Schedule cron.Schedule
|
||||
}
|
||||
|
||||
var _ yaml.Unmarshaler = &CronSpec{}
|
||||
|
||||
func (s *CronSpec) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
|
||||
var specString string
|
||||
if err := unmarshal(&specString, false); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Use standard cron format.
|
||||
// Disable the various "descriptors" (@daily, etc)
|
||||
// They are just aliases to "top of hour", "midnight", etc.
|
||||
parser := cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.SecondOptional)
|
||||
|
||||
sched, err := parser.Parse(specString)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "cron syntax invalid")
|
||||
}
|
||||
s.Schedule = sched
|
||||
return nil
|
||||
}
|
||||
|
||||
type SnapshottingCron struct {
|
||||
Type string `yaml:"type"`
|
||||
Prefix string `yaml:"prefix"`
|
||||
Cron CronSpec `yaml:"cron"`
|
||||
Hooks HookList `yaml:"hooks,optional"`
|
||||
TimestampFormattingSpec `yaml:",inline"`
|
||||
Type string `yaml:"type"`
|
||||
Prefix string `yaml:"prefix"`
|
||||
Interval time.Duration `yaml:"interval,positive"`
|
||||
Hooks HookList `yaml:"hooks,optional"`
|
||||
}
|
||||
|
||||
type SnapshottingManual struct {
|
||||
@@ -306,6 +181,18 @@ type Global struct {
|
||||
Serve *GlobalServe `yaml:"serve,optional,fromdefaults"`
|
||||
}
|
||||
|
||||
func Default(i interface{}) {
|
||||
v := reflect.ValueOf(i)
|
||||
if v.Kind() != reflect.Ptr {
|
||||
panic(v)
|
||||
}
|
||||
y := `{}`
|
||||
err := yaml.Unmarshal([]byte(y), v.Interface())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
type ConnectEnum struct {
|
||||
Ret interface{}
|
||||
}
|
||||
@@ -358,16 +245,14 @@ type ServeCommon struct {
|
||||
}
|
||||
|
||||
type TCPServe struct {
|
||||
ServeCommon `yaml:",inline"`
|
||||
Listen string `yaml:"listen,hostport"`
|
||||
ListenFreeBind bool `yaml:"listen_freebind,default=false"`
|
||||
Clients map[string]string `yaml:"clients"`
|
||||
ServeCommon `yaml:",inline"`
|
||||
Listen string `yaml:"listen,hostport"`
|
||||
Clients map[string]string `yaml:"clients"`
|
||||
}
|
||||
|
||||
type TLSServe struct {
|
||||
ServeCommon `yaml:",inline"`
|
||||
Listen string `yaml:"listen,hostport"`
|
||||
ListenFreeBind bool `yaml:"listen_freebind,default=false"`
|
||||
Ca string `yaml:"ca"`
|
||||
Cert string `yaml:"cert"`
|
||||
Key string `yaml:"key"`
|
||||
@@ -397,7 +282,6 @@ type PruneKeepNotReplicated struct {
|
||||
type PruneKeepLastN struct {
|
||||
Type string `yaml:"type"`
|
||||
Count int `yaml:"count"`
|
||||
Regex string `yaml:"regex,optional"`
|
||||
}
|
||||
|
||||
type PruneKeepRegex struct { // FIXME rename to KeepRegex
|
||||
@@ -447,9 +331,8 @@ type MonitoringEnum struct {
|
||||
}
|
||||
|
||||
type PrometheusMonitoring struct {
|
||||
Type string `yaml:"type"`
|
||||
Listen string `yaml:"listen,hostport"`
|
||||
ListenFreeBind bool `yaml:"listen_freebind,default=false"`
|
||||
Type string `yaml:"type"`
|
||||
Listen string `yaml:"listen,hostport"`
|
||||
}
|
||||
|
||||
type SyslogFacility syslog.Priority
|
||||
@@ -472,6 +355,14 @@ type GlobalStdinServer struct {
|
||||
SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
|
||||
}
|
||||
|
||||
type JobDebugSettings struct {
|
||||
Conn *struct {
|
||||
ReadDump string `yaml:"read_dump"`
|
||||
WriteDump string `yaml:"write_dump"`
|
||||
} `yaml:"conn,optional"`
|
||||
RPCLog bool `yaml:"rpc_log,optional,default=false"`
|
||||
}
|
||||
|
||||
type HookList []HookEnum
|
||||
|
||||
type HookEnum struct {
|
||||
@@ -570,7 +461,6 @@ func (t *SnapshottingEnum) UnmarshalYAML(u func(interface{}, bool) error) (err e
|
||||
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
||||
"periodic": &SnapshottingPeriodic{},
|
||||
"manual": &SnapshottingManual{},
|
||||
"cron": &SnapshottingCron{},
|
||||
})
|
||||
return
|
||||
}
|
||||
@@ -659,9 +549,8 @@ var ConfigFileDefaultLocations = []string{
|
||||
"/usr/local/etc/zrepl/zrepl.yml",
|
||||
}
|
||||
|
||||
func ParseConfig(path string) (rootConfig *Config, err error) {
|
||||
func ParseConfig(path string) (i *Config, err error) {
|
||||
|
||||
// Parse main configuration file
|
||||
if path == "" {
|
||||
// Try default locations
|
||||
for _, l := range ConfigFileDefaultLocations {
|
||||
@@ -680,94 +569,11 @@ func ParseConfig(path string) (rootConfig *Config, err error) {
|
||||
|
||||
var bytes []byte
|
||||
|
||||
if bytes, err = os.ReadFile(path); err != nil {
|
||||
if bytes, err = ioutil.ReadFile(path); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
rootConfig, err = ParseConfigBytes(bytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = expandConfigInclude(path, rootConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err = validateJobNames(rootConfig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return rootConfig, err
|
||||
}
|
||||
|
||||
func IsInternalJobName(s string) bool {
|
||||
return strings.HasPrefix(s, "_")
|
||||
}
|
||||
|
||||
func validateJobNames(config *Config) error {
|
||||
seen := make(map[string]struct{})
|
||||
for _, job := range config.Jobs {
|
||||
name := job.Name()
|
||||
if IsInternalJobName(name) {
|
||||
return errors.Errorf("job name %q is reserved for internal use (starts with _)", name)
|
||||
}
|
||||
if _, ok := seen[name]; ok {
|
||||
return errors.Errorf("duplicate job name %q", name)
|
||||
}
|
||||
seen[name] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func expandConfigInclude(configPath string, config *Config) (err error) {
|
||||
var includeConfigPaths []string
|
||||
for _, path := range config.Include {
|
||||
if !pathpkg.IsAbs(configPath) {
|
||||
path = pathpkg.Join(pathpkg.Dir(configPath), path)
|
||||
}
|
||||
|
||||
stat, statErr := os.Stat(path)
|
||||
if statErr != nil {
|
||||
return errors.Wrapf(statErr, "stat path %q", path)
|
||||
}
|
||||
|
||||
if stat.Mode().IsDir() {
|
||||
directoryPaths, err := filepath.Glob(path + "/*.yml")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
includeConfigPaths = append(includeConfigPaths, directoryPaths...)
|
||||
} else if stat.Mode().IsRegular() {
|
||||
if extention := filepath.Ext(path); extention != ".yml" {
|
||||
return fmt.Errorf("include config files must end with `.yml`: %s", path)
|
||||
}
|
||||
includeConfigPaths = append(includeConfigPaths, path)
|
||||
} else {
|
||||
return fmt.Errorf("not a file or directory: %s", path)
|
||||
}
|
||||
}
|
||||
|
||||
for _, path := range includeConfigPaths {
|
||||
var bytes []byte
|
||||
if bytes, err = os.ReadFile(path); err != nil {
|
||||
return errors.Wrapf(err, "read file: %q", path)
|
||||
}
|
||||
|
||||
includedConfig, err := ParseConfigBytes(bytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(includedConfig.Include) > 0 {
|
||||
return errors.Errorf("included configuration files must not include other files: %s", path)
|
||||
}
|
||||
|
||||
config.Jobs = append(config.Jobs, includedConfig.Jobs...)
|
||||
}
|
||||
|
||||
return nil
|
||||
return ParseConfigBytes(bytes)
|
||||
}
|
||||
|
||||
func ParseConfigBytes(bytes []byte) (*Config, error) {
|
||||
@@ -775,16 +581,46 @@ func ParseConfigBytes(bytes []byte) (*Config, error) {
|
||||
if err := yaml.UnmarshalStrict(bytes, &c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c != nil {
|
||||
return c, nil
|
||||
}
|
||||
// There was no yaml document in the file, deserialize from default.
|
||||
// => See TestFromdefaultsEmptyDoc in yaml-config package.
|
||||
if err := yaml.UnmarshalStrict([]byte("{}"), &c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c == nil {
|
||||
panic("the fallback to deserialize from `{}` should work")
|
||||
return nil, fmt.Errorf("config is empty or only consists of comments")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
|
||||
|
||||
func parsePostitiveDuration(e string) (d time.Duration, err error) {
|
||||
comps := durationStringRegex.FindStringSubmatch(e)
|
||||
if len(comps) != 3 {
|
||||
err = fmt.Errorf("does not match regex: %s %#v", e, comps)
|
||||
return
|
||||
}
|
||||
|
||||
durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if durationFactor <= 0 {
|
||||
return 0, errors.New("duration must be positive integer")
|
||||
}
|
||||
|
||||
var durationUnit time.Duration
|
||||
switch comps[2] {
|
||||
case "s":
|
||||
durationUnit = time.Second
|
||||
case "m":
|
||||
durationUnit = time.Minute
|
||||
case "h":
|
||||
durationUnit = time.Hour
|
||||
case "d":
|
||||
durationUnit = 24 * time.Hour
|
||||
case "w":
|
||||
durationUnit = 24 * 7 * time.Hour
|
||||
default:
|
||||
err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
|
||||
return
|
||||
}
|
||||
|
||||
d = time.Duration(durationFactor) * durationUnit
|
||||
return
|
||||
}
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
clients: {
|
||||
"10.0.0.1":"foo"
|
||||
}
|
||||
root_fs: zroot/foo
|
||||
root_fs: zoot/foo
|
||||
`
|
||||
_, err := ParseConfigBytes([]byte(jobdef))
|
||||
require.NoError(t, err)
|
||||
@@ -70,9 +70,9 @@ func TestPrometheusMonitoring(t *testing.T) {
|
||||
global:
|
||||
monitoring:
|
||||
- type: prometheus
|
||||
listen: ':9811'
|
||||
listen: ':9091'
|
||||
`)
|
||||
assert.Equal(t, ":9811", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
|
||||
assert.Equal(t, ":9091", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
|
||||
}
|
||||
|
||||
func TestSyslogLoggingOutletFacility(t *testing.T) {
|
||||
@@ -2,8 +2,16 @@ package config
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestConfigEmptyFails(t *testing.T) {
|
||||
conf, err := testConfig(t, "\n")
|
||||
assert.Nil(t, conf)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestJobsOnlyWorks(t *testing.T) {
|
||||
testValidConfig(t, `
|
||||
jobs:
|
||||
@@ -26,7 +34,7 @@ jobs:
|
||||
keep_sender:
|
||||
- type: not_replicated
|
||||
keep_receiver:
|
||||
- type: last_n
|
||||
- type: last_n
|
||||
count: 1
|
||||
`)
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestSnapshotting(t *testing.T) {
|
||||
tmpl := `
|
||||
jobs:
|
||||
- name: foo
|
||||
type: push
|
||||
connect:
|
||||
type: local
|
||||
listener_name: foo
|
||||
client_identity: bar
|
||||
filesystems: {"<": true}
|
||||
%s
|
||||
pruning:
|
||||
keep_sender:
|
||||
- type: last_n
|
||||
count: 10
|
||||
keep_receiver:
|
||||
- type: last_n
|
||||
count: 10
|
||||
`
|
||||
manual := `
|
||||
snapshotting:
|
||||
type: manual
|
||||
`
|
||||
periodic := `
|
||||
snapshotting:
|
||||
type: periodic
|
||||
prefix: zrepl_
|
||||
interval: 10m
|
||||
`
|
||||
|
||||
hooks := `
|
||||
snapshotting:
|
||||
type: periodic
|
||||
prefix: zrepl_
|
||||
interval: 10m
|
||||
hooks:
|
||||
- type: command
|
||||
path: /tmp/path/to/command
|
||||
- type: command
|
||||
path: /tmp/path/to/command
|
||||
filesystems: { "zroot<": true, "<": false }
|
||||
- type: postgres-checkpoint
|
||||
dsn: "host=localhost port=5432 user=postgres sslmode=disable"
|
||||
filesystems: {
|
||||
"tank/postgres/data11": true
|
||||
}
|
||||
- type: mysql-lock-tables
|
||||
dsn: "root@tcp(localhost)/"
|
||||
filesystems: {
|
||||
"tank/mysql": true
|
||||
}
|
||||
`
|
||||
|
||||
fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
|
||||
var c *Config
|
||||
|
||||
t.Run("manual", func(t *testing.T) {
|
||||
c = testValidConfig(t, fillSnapshotting(manual))
|
||||
snm := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingManual)
|
||||
assert.Equal(t, "manual", snm.Type)
|
||||
})
|
||||
|
||||
t.Run("periodic", func(t *testing.T) {
|
||||
c = testValidConfig(t, fillSnapshotting(periodic))
|
||||
snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
|
||||
assert.Equal(t, "periodic", snp.Type)
|
||||
assert.Equal(t, 10*time.Minute, snp.Interval)
|
||||
assert.Equal(t, "zrepl_", snp.Prefix)
|
||||
})
|
||||
|
||||
t.Run("hooks", func(t *testing.T) {
|
||||
c = testValidConfig(t, fillSnapshotting(hooks))
|
||||
hs := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic).Hooks
|
||||
assert.Equal(t, hs[0].Ret.(*HookCommand).Filesystems["<"], true)
|
||||
assert.Equal(t, hs[1].Ret.(*HookCommand).Filesystems["zroot<"], true)
|
||||
assert.Equal(t, hs[2].Ret.(*HookPostgresCheckpoint).Filesystems["tank/postgres/data11"], true)
|
||||
assert.Equal(t, hs[3].Ret.(*HookMySQLLockTables).Filesystems["tank/mysql"], true)
|
||||
})
|
||||
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"text/template"
|
||||
|
||||
"github.com/kr/pretty"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
|
||||
paths, err := filepath.Glob("./samples/*")
|
||||
if err != nil {
|
||||
t.Errorf("glob failed: %+v", err)
|
||||
}
|
||||
|
||||
for _, p := range paths {
|
||||
|
||||
if path.Ext(p) != ".yml" {
|
||||
t.Logf("skipping file %s", p)
|
||||
continue
|
||||
}
|
||||
|
||||
t.Run(p, func(t *testing.T) {
|
||||
c, err := ParseConfig(p)
|
||||
if err != nil {
|
||||
t.Errorf("error parsing %s:\n%+v", p, err)
|
||||
}
|
||||
|
||||
t.Logf("file: %s", p)
|
||||
t.Log(pretty.Sprint(c))
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// template must be a template/text template with a single '{{ . }}' as placehodler for val
|
||||
//nolint[:deadcode,unused]
|
||||
func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
|
||||
tmp, err := template.New("master").Parse(tmpl)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
err = tmp.Execute(&buf, val)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return testValidConfig(t, buf.String())
|
||||
}
|
||||
|
||||
func testValidConfig(t *testing.T, input string) *Config {
|
||||
t.Helper()
|
||||
conf, err := testConfig(t, input)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, conf)
|
||||
return conf
|
||||
}
|
||||
|
||||
func testConfig(t *testing.T, input string) (*Config, error) {
|
||||
t.Helper()
|
||||
return ParseConfigBytes([]byte(input))
|
||||
}
|
||||
|
||||
func trimSpaceEachLineAndPad(s, pad string) string {
|
||||
var out strings.Builder
|
||||
scan := bufio.NewScanner(strings.NewReader(s))
|
||||
for scan.Scan() {
|
||||
fmt.Fprintf(&out, "%s%s\n", pad, bytes.TrimSpace(scan.Bytes()))
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func TestTrimSpaceEachLineAndPad(t *testing.T) {
|
||||
foo := `
|
||||
foo
|
||||
bar baz
|
||||
`
|
||||
assert.Equal(t, " \n foo\n bar baz\n \n", trimSpaceEachLineAndPad(foo, " "))
|
||||
}
|
||||
@@ -37,7 +37,7 @@ func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (
|
||||
return err
|
||||
}
|
||||
|
||||
intervals, err := ParseRetentionIntervalSpec(in)
|
||||
intervals, err := parseRetentionGridIntervalsString(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -64,7 +64,7 @@ func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval,
|
||||
return nil, fmt.Errorf("contains factor <= 0")
|
||||
}
|
||||
|
||||
duration, err := parsePositiveDuration(comps[2])
|
||||
duration, err := parsePostitiveDuration(comps[2])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -102,7 +102,7 @@ func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval,
|
||||
|
||||
}
|
||||
|
||||
func ParseRetentionIntervalSpec(s string) (intervals []RetentionInterval, err error) {
|
||||
func parseRetentionGridIntervalsString(s string) (intervals []RetentionInterval, err error) {
|
||||
|
||||
ges := strings.Split(s, "|")
|
||||
intervals = make([]RetentionInterval, 0, 7*len(ges))
|
||||
@@ -10,8 +10,6 @@ jobs:
|
||||
address: "backup-server.foo.bar:8888"
|
||||
snapshotting:
|
||||
type: manual
|
||||
send:
|
||||
encrypted: false
|
||||
pruning:
|
||||
keep_sender:
|
||||
- type: not_replicated
|
||||
@@ -5,11 +5,7 @@ jobs:
|
||||
type: tcp
|
||||
listen: "0.0.0.0:8888"
|
||||
clients: {
|
||||
"192.168.122.123" : "mysql01",
|
||||
"192.168.122.42" : "mx01",
|
||||
"2001:0db8:85a3::8a2e:0370:7334": "gateway",
|
||||
"10.23.42.0/24": "cluster-*",
|
||||
"fde4:8dba:82e1::/64": "san-*",
|
||||
"192.168.122.123" : "client1"
|
||||
}
|
||||
filesystems: {
|
||||
"<": true,
|
||||
@@ -8,20 +8,17 @@ import (
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
||||
"github.com/zrepl/zrepl/internal/daemon/nethelpers"
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/logger"
|
||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
||||
"github.com/zrepl/zrepl/internal/version"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/internal/zfs/zfscmd"
|
||||
"github.com/zrepl/zrepl/daemon/job"
|
||||
"github.com/zrepl/zrepl/daemon/nethelpers"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/util/envconst"
|
||||
"github.com/zrepl/zrepl/version"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type controlJob struct {
|
||||
@@ -47,8 +44,6 @@ func (j *controlJob) Status() *job.Status { return &job.Status{Type: job.TypeInt
|
||||
|
||||
func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
|
||||
|
||||
func (j *controlJob) SenderConfig() *endpoint.SenderConfig { return nil }
|
||||
|
||||
var promControl struct {
|
||||
requestBegin *prometheus.CounterVec
|
||||
requestFinished *prometheus.HistogramVec
|
||||
@@ -66,7 +61,7 @@ func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) {
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "control",
|
||||
Name: "request_finished",
|
||||
Help: "time it took a request to finish",
|
||||
Help: "time it took a request to finih",
|
||||
Buckets: []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3, 10e-3, 100e-3, 200e-3, 400e-3, 800e-3, 1, 10, 20},
|
||||
}, []string{"endpoint"})
|
||||
registerer.MustRegister(promControl.requestBegin)
|
||||
@@ -119,16 +114,7 @@ func (j *controlJob) Run(ctx context.Context) {
|
||||
mux.Handle(ControlJobEndpointStatus,
|
||||
// don't log requests to status endpoint, too spammy
|
||||
jsonResponder{log, func() (interface{}, error) {
|
||||
jobs := j.jobs.status()
|
||||
globalZFS := zfscmd.GetReport()
|
||||
envconstReport := envconst.GetReport()
|
||||
s := Status{
|
||||
Jobs: jobs,
|
||||
Global: GlobalStatus{
|
||||
ZFSCmds: globalZFS,
|
||||
Envconst: envconstReport,
|
||||
OsEnviron: os.Environ(),
|
||||
}}
|
||||
s := j.jobs.status()
|
||||
return s, nil
|
||||
}})
|
||||
|
||||
@@ -158,8 +144,8 @@ func (j *controlJob) Run(ctx context.Context) {
|
||||
server := http.Server{
|
||||
Handler: mux,
|
||||
// control socket is local, 1s timeout should be more than sufficient, even on a loaded system
|
||||
WriteTimeout: envconst.Duration("ZREPL_DAEMON_CONTROL_SERVER_WRITE_TIMEOUT", 1*time.Second),
|
||||
ReadTimeout: envconst.Duration("ZREPL_DAEMON_CONTROL_SERVER_READ_TIMEOUT", 1*time.Second),
|
||||
WriteTimeout: 1 * time.Second,
|
||||
ReadTimeout: 1 * time.Second,
|
||||
}
|
||||
|
||||
outer:
|
||||
@@ -261,7 +247,7 @@ func (j jsonRequestResponder) ServeHTTP(w http.ResponseWriter, r *http.Request)
|
||||
var buf bytes.Buffer
|
||||
encodeErr := json.NewEncoder(&buf).Encode(res)
|
||||
if encodeErr != nil {
|
||||
j.log.WithError(producerErr).Error("control handler json marshal error")
|
||||
j.log.WithError(producerErr).Error("control handler json marhsal error")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, err := io.WriteString(w, encodeErr.Error())
|
||||
logIoErr(err)
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
@@ -12,22 +13,18 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
||||
"github.com/zrepl/zrepl/internal/daemon/job/reset"
|
||||
"github.com/zrepl/zrepl/internal/daemon/job/wakeup"
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
||||
"github.com/zrepl/zrepl/internal/logger"
|
||||
"github.com/zrepl/zrepl/internal/version"
|
||||
"github.com/zrepl/zrepl/internal/zfs/zfscmd"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/job"
|
||||
"github.com/zrepl/zrepl/daemon/job/reset"
|
||||
"github.com/zrepl/zrepl/daemon/job/wakeup"
|
||||
"github.com/zrepl/zrepl/daemon/logging"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/version"
|
||||
)
|
||||
|
||||
func Run(ctx context.Context, conf *config.Config) error {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
func Run(conf *config.Config) error {
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
defer cancel()
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
@@ -41,9 +38,8 @@ func Run(ctx context.Context, conf *config.Config) error {
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "cannot build logging from config")
|
||||
}
|
||||
outlets.Add(newPrometheusLogOutlet(), logger.Debug)
|
||||
|
||||
confJobs, err := job.JobsFromConfig(conf, config.ParseFlagsNone)
|
||||
confJobs, err := job.JobsFromConfig(conf)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "cannot build jobs from config")
|
||||
}
|
||||
@@ -51,23 +47,14 @@ func Run(ctx context.Context, conf *config.Config) error {
|
||||
log := logger.NewLogger(outlets, 1*time.Second)
|
||||
log.Info(version.NewZreplVersionInformation().String())
|
||||
|
||||
ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
|
||||
trace.RegisterCallback(trace.Callback{
|
||||
OnBegin: func(ctx context.Context) { logging.GetLogger(ctx, logging.SubsysTraceData).Debug("begin span") },
|
||||
OnEnd: func(ctx context.Context, spanInfo trace.SpanInfo) {
|
||||
logging.
|
||||
GetLogger(ctx, logging.SubsysTraceData).
|
||||
WithField("duration_s", spanInfo.EndedAt().Sub(spanInfo.StartedAt()).Seconds()).
|
||||
Debug("finished span " + spanInfo.TaskAndSpanStack(trace.SpanStackKindAnnotation))
|
||||
},
|
||||
})
|
||||
|
||||
for _, job := range confJobs {
|
||||
if config.IsInternalJobName(job.Name()) {
|
||||
if IsInternalJobName(job.Name()) {
|
||||
panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME
|
||||
}
|
||||
}
|
||||
|
||||
ctx = job.WithLogger(ctx, log)
|
||||
|
||||
jobs := newJobs()
|
||||
|
||||
// start control socket
|
||||
@@ -89,17 +76,11 @@ func Run(ctx context.Context, conf *config.Config) error {
|
||||
return errors.Errorf("unknown monitoring job #%d (type %T)", i, v)
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "cannot build monitoring job #%d", i)
|
||||
return errors.Wrapf(err, "cannot build monitorin gjob #%d", i)
|
||||
}
|
||||
jobs.start(ctx, job, true)
|
||||
}
|
||||
|
||||
// register global (=non job-local) metrics
|
||||
version.PrometheusRegister(prometheus.DefaultRegisterer)
|
||||
zfscmd.RegisterMetrics(prometheus.DefaultRegisterer)
|
||||
trace.RegisterMetrics(prometheus.DefaultRegisterer)
|
||||
endpoint.RegisterMetrics(prometheus.DefaultRegisterer)
|
||||
|
||||
log.Info("starting daemon")
|
||||
|
||||
// start regular jobs
|
||||
@@ -113,8 +94,6 @@ func Run(ctx context.Context, conf *config.Config) error {
|
||||
case <-ctx.Done():
|
||||
log.WithError(ctx.Err()).Info("context finished")
|
||||
}
|
||||
log.Info("waiting for jobs to finish")
|
||||
<-jobs.wait()
|
||||
log.Info("daemon exiting")
|
||||
return nil
|
||||
}
|
||||
@@ -137,26 +116,18 @@ func newJobs() *jobs {
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
logJobField string = "job"
|
||||
)
|
||||
|
||||
func (s *jobs) wait() <-chan struct{} {
|
||||
ch := make(chan struct{})
|
||||
go func() {
|
||||
s.wg.Wait()
|
||||
close(ch)
|
||||
}()
|
||||
return ch
|
||||
}
|
||||
|
||||
type Status struct {
|
||||
Jobs map[string]*job.Status
|
||||
Global GlobalStatus
|
||||
}
|
||||
|
||||
type GlobalStatus struct {
|
||||
ZFSCmds *zfscmd.Report
|
||||
Envconst *envconst.Report
|
||||
OsEnviron []string
|
||||
}
|
||||
|
||||
func (s *jobs) status() map[string]*job.Status {
|
||||
s.m.RLock()
|
||||
defer s.m.RUnlock()
|
||||
@@ -210,19 +181,22 @@ const (
|
||||
jobNameControl = "_control"
|
||||
)
|
||||
|
||||
func IsInternalJobName(s string) bool {
|
||||
return strings.HasPrefix(s, "_")
|
||||
}
|
||||
|
||||
func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
|
||||
s.m.Lock()
|
||||
defer s.m.Unlock()
|
||||
|
||||
ctx = logging.WithInjectedField(ctx, logging.JobField, j.Name())
|
||||
|
||||
jobLog := job.GetLogger(ctx).
|
||||
WithField(logJobField, j.Name()).
|
||||
WithOutlet(newPrometheusLogOutlet(j.Name()), logger.Debug)
|
||||
jobName := j.Name()
|
||||
|
||||
// package `config` enforces these with clean errors, these are just assertions
|
||||
if !internal && config.IsInternalJobName(jobName) {
|
||||
if !internal && IsInternalJobName(jobName) {
|
||||
panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName))
|
||||
}
|
||||
if internal && !config.IsInternalJobName(jobName) {
|
||||
if internal && !IsInternalJobName(jobName) {
|
||||
panic(fmt.Sprintf("internal job does not use internal job name %s", jobName))
|
||||
}
|
||||
if _, ok := s.jobs[jobName]; ok {
|
||||
@@ -232,7 +206,7 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
|
||||
j.RegisterMetrics(prometheus.DefaultRegisterer)
|
||||
|
||||
s.jobs[jobName] = j
|
||||
ctx = zfscmd.WithJobID(ctx, j.Name())
|
||||
ctx = job.WithLogger(ctx, jobLog)
|
||||
ctx, wakeup := wakeup.Context(ctx)
|
||||
ctx, resetFunc := reset.Context(ctx)
|
||||
s.wakeups[jobName] = wakeup
|
||||
@@ -241,8 +215,8 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
|
||||
s.wg.Add(1)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
job.GetLogger(ctx).Info("starting job")
|
||||
defer job.GetLogger(ctx).Info("job exited")
|
||||
jobLog.Info("starting job")
|
||||
defer jobLog.Info("job exited")
|
||||
j.Run(ctx)
|
||||
}()
|
||||
}
|
||||
@@ -6,8 +6,8 @@ import (
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type DatasetMapFilter struct {
|
||||
@@ -160,16 +160,8 @@ func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
func (m DatasetMapFilter) UserSpecifiedDatasets() (datasets zfs.UserSpecifiedDatasetsSet) {
|
||||
datasets = make(zfs.UserSpecifiedDatasetsSet)
|
||||
for i := range m.entries {
|
||||
datasets[m.entries[i].path.ToString()] = true
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Construct a new filter-only DatasetMapFilter from a mapping
|
||||
// The new filter allows exactly those paths that were not forbidden by the mapping.
|
||||
// The new filter allows excactly those paths that were not forbidden by the mapping.
|
||||
func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
|
||||
|
||||
if m.filterMode {
|
||||
@@ -1,11 +1,11 @@
|
||||
package filters
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestDatasetMapFilter(t *testing.T) {
|
||||
|
||||
type testCase struct {
|
||||
@@ -0,0 +1,41 @@
|
||||
package filters
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type AnyFSVFilter struct{}
|
||||
|
||||
func NewAnyFSVFilter() AnyFSVFilter {
|
||||
return AnyFSVFilter{}
|
||||
}
|
||||
|
||||
var _ zfs.FilesystemVersionFilter = AnyFSVFilter{}
|
||||
|
||||
func (AnyFSVFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
type PrefixFilter struct {
|
||||
prefix string
|
||||
fstype zfs.VersionType
|
||||
fstypeSet bool // optionals anyone?
|
||||
}
|
||||
|
||||
var _ zfs.FilesystemVersionFilter = &PrefixFilter{}
|
||||
|
||||
func NewPrefixFilter(prefix string) *PrefixFilter {
|
||||
return &PrefixFilter{prefix: prefix}
|
||||
}
|
||||
|
||||
func NewTypedPrefixFilter(prefix string, versionType zfs.VersionType) *PrefixFilter {
|
||||
return &PrefixFilter{prefix, versionType, true}
|
||||
}
|
||||
|
||||
func (f *PrefixFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) {
|
||||
fstypeMatches := (!f.fstypeSet || t == f.fstype)
|
||||
prefixMatches := strings.HasPrefix(name, f.prefix)
|
||||
return fstypeMatches && prefixMatches, nil
|
||||
}
|
||||
@@ -3,8 +3,8 @@ package hooks
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type List []Hook
|
||||
@@ -5,7 +5,7 @@
|
||||
//
|
||||
// This package also provides all supported hook type implementations and abstractions around them.
|
||||
//
|
||||
// # Use For Other Kinds Of ExpectStepReports
|
||||
// Use For Other Kinds Of ExpectStepReports
|
||||
//
|
||||
// This package REQUIRES REFACTORING before it can be used for other activities than snapshots, e.g. pre- and post-replication:
|
||||
//
|
||||
@@ -15,7 +15,7 @@
|
||||
// The hook implementations should move out of this package.
|
||||
// However, there is a lot of tight coupling which to untangle isn't worth it ATM.
|
||||
//
|
||||
// # How This Package Is Used By Package Snapper
|
||||
// How This Package Is Used By Package Snapper
|
||||
//
|
||||
// Deserialize a config.List using ListFromConfig().
|
||||
// Then it MUST filter the list to only contain hooks for a particular filesystem using
|
||||
@@ -30,4 +30,5 @@
|
||||
// Command hooks make it available in the environment variable ZREPL_DRYRUN.
|
||||
//
|
||||
// Plan.Report() can be called while Plan.Run() is executing to give an overview of plan execution progress (future use in "zrepl status").
|
||||
//
|
||||
package hooks
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
// Re-export type here so that
|
||||
@@ -6,17 +6,29 @@ import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
||||
"github.com/zrepl/zrepl/internal/logger"
|
||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/util/envconst"
|
||||
)
|
||||
|
||||
type contextKey int
|
||||
|
||||
const (
|
||||
contextKeyLog contextKey = 0
|
||||
)
|
||||
|
||||
type Logger = logger.Logger
|
||||
|
||||
func WithLogger(ctx context.Context, log Logger) context.Context {
|
||||
return context.WithValue(ctx, contextKeyLog, log)
|
||||
}
|
||||
|
||||
func GetLogger(ctx context.Context) Logger { return getLogger(ctx) }
|
||||
|
||||
func getLogger(ctx context.Context) Logger {
|
||||
return logging.GetLogger(ctx, logging.SubsysHooks)
|
||||
if log, ok := ctx.Value(contextKeyLog).(Logger); ok {
|
||||
return log
|
||||
}
|
||||
return logger.NewNullLogger()
|
||||
}
|
||||
|
||||
const MAX_HOOK_LOG_SIZE_DEFAULT int = 1 << 20
|
||||
@@ -4,8 +4,8 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type HookJobCallback func(ctx context.Context) error
|
||||
@@ -12,11 +12,11 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
||||
"github.com/zrepl/zrepl/internal/logger"
|
||||
"github.com/zrepl/zrepl/internal/util/circlog"
|
||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/util/circlog"
|
||||
"github.com/zrepl/zrepl/util/envconst"
|
||||
)
|
||||
|
||||
type HookEnvVar string
|
||||
@@ -93,14 +93,7 @@ func (r *CommandHookReport) String() string {
|
||||
cmdLine.WriteString(fmt.Sprintf("%s'%s'", sep, a))
|
||||
}
|
||||
|
||||
var msg string
|
||||
if r.Err == nil {
|
||||
msg = "command hook"
|
||||
} else {
|
||||
msg = fmt.Sprintf("command hook failed with %q", r.Err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s: \"%s\"", msg, cmdLine.String()) // no %q to make copy-pastable
|
||||
return fmt.Sprintf("command hook invocation: \"%s\"", cmdLine.String()) // no %q to make copy-pastable
|
||||
}
|
||||
func (r *CommandHookReport) Error() string {
|
||||
if r.Err == nil {
|
||||
+11
-14
@@ -12,27 +12,24 @@ import (
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
// Hook to implement the following recommmendation from MySQL docs
|
||||
// https://dev.mysql.com/doc/mysql-backup-excerpt/5.7/en/backup-methods.html
|
||||
//
|
||||
// Making Backups Using a File System Snapshot:
|
||||
// Making Backups Using a File System Snapshot:
|
||||
//
|
||||
// If you are using a Veritas file system, you can make a backup like this:
|
||||
// If you are using a Veritas file system, you can make a backup like this:
|
||||
//
|
||||
// From a client program, execute FLUSH TABLES WITH READ LOCK.
|
||||
// From another shell, execute mount vxfs snapshot.
|
||||
// From the first client, execute UNLOCK TABLES.
|
||||
// Copy files from the snapshot.
|
||||
// Unmount the snapshot.
|
||||
// From a client program, execute FLUSH TABLES WITH READ LOCK.
|
||||
// From another shell, execute mount vxfs snapshot.
|
||||
// From the first client, execute UNLOCK TABLES.
|
||||
// Copy files from the snapshot.
|
||||
// Unmount the snapshot.
|
||||
//
|
||||
// Similar snapshot capabilities may be available in other file systems, such as LVM or ZFS.
|
||||
//
|
||||
|
||||
// Similar snapshot capabilities may be available in other file systems, such as LVM or ZFS.
|
||||
type MySQLLockTables struct {
|
||||
errIsFatal bool
|
||||
connector sqldriver.Connector
|
||||
+3
-3
@@ -10,9 +10,9 @@ import (
|
||||
"github.com/lib/pq"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type PgChkptHook struct {
|
||||
@@ -11,13 +11,10 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/hooks"
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
||||
"github.com/zrepl/zrepl/internal/logger"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/hooks"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type comparisonAssertionFunc func(require.TestingT, interface{}, interface{}, ...interface{})
|
||||
@@ -72,9 +69,6 @@ func curry(f comparisonAssertionFunc, expected interface{}, right bool) (ret val
|
||||
}
|
||||
|
||||
func TestHooks(t *testing.T) {
|
||||
ctx, end := trace.WithTaskFromStack(context.Background())
|
||||
defer end()
|
||||
|
||||
testFSName := "testpool/testdataset"
|
||||
testSnapshotName := "testsnap"
|
||||
|
||||
@@ -161,7 +155,7 @@ jobs:
|
||||
ExpectedEdge: hooks.Pre,
|
||||
ExpectStatus: hooks.StepErr,
|
||||
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
|
||||
ErrorTest: regexpTest("^command hook failed.*exit status 1$"),
|
||||
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
|
||||
},
|
||||
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
|
||||
expectStep{
|
||||
@@ -185,7 +179,7 @@ jobs:
|
||||
ExpectedEdge: hooks.Pre,
|
||||
ExpectStatus: hooks.StepErr,
|
||||
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
|
||||
ErrorTest: regexpTest("^command hook failed.*exit status 1$"),
|
||||
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
|
||||
},
|
||||
expectStep{
|
||||
ExpectedEdge: hooks.Pre,
|
||||
@@ -234,7 +228,7 @@ jobs:
|
||||
ExpectedEdge: hooks.Post,
|
||||
ExpectStatus: hooks.StepErr,
|
||||
OutputTest: containsTest(fmt.Sprintf("TEST ERROR post_testing %s@%s", testFSName, testSnapshotName)),
|
||||
ErrorTest: regexpTest("^command hook failed.*exit status 1$"),
|
||||
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -267,7 +261,7 @@ jobs:
|
||||
ExpectedEdge: hooks.Pre,
|
||||
ExpectStatus: hooks.StepErr,
|
||||
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
|
||||
ErrorTest: regexpTest("^command hook failed.*exit status 1$"),
|
||||
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
|
||||
},
|
||||
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
|
||||
expectStep{
|
||||
@@ -295,7 +289,7 @@ jobs:
|
||||
ExpectedEdge: hooks.Pre,
|
||||
ExpectStatus: hooks.StepErr,
|
||||
OutputTest: containsTest(fmt.Sprintf("TEST ERROR pre_testing %s@%s", testFSName, testSnapshotName)),
|
||||
ErrorTest: regexpTest("^command hook failed.*exit status 1$"),
|
||||
ErrorTest: regexpTest("^command hook invocation.*exit status 1$"),
|
||||
},
|
||||
expectStep{ExpectedEdge: hooks.Callback, ExpectStatus: hooks.StepOk},
|
||||
expectStep{
|
||||
@@ -424,8 +418,9 @@ jobs:
|
||||
|
||||
cbReached = false
|
||||
|
||||
ctx := context.Background()
|
||||
if testing.Verbose() && !tt.SuppressOutput {
|
||||
ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(log))
|
||||
ctx = hooks.WithLogger(ctx, log)
|
||||
}
|
||||
plan.Run(ctx, false)
|
||||
report := plan.Report()
|
||||
@@ -1,5 +1,6 @@
|
||||
// Code generated by "enumer -type=StepStatus -trimprefix=Step"; DO NOT EDIT.
|
||||
|
||||
//
|
||||
package hooks
|
||||
|
||||
import (
|
||||
@@ -0,0 +1,434 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/daemon/job/reset"
|
||||
"github.com/zrepl/zrepl/daemon/job/wakeup"
|
||||
"github.com/zrepl/zrepl/daemon/logging"
|
||||
"github.com/zrepl/zrepl/daemon/pruner"
|
||||
"github.com/zrepl/zrepl/daemon/snapper"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/replication"
|
||||
"github.com/zrepl/zrepl/replication/driver"
|
||||
"github.com/zrepl/zrepl/replication/logic"
|
||||
"github.com/zrepl/zrepl/replication/report"
|
||||
"github.com/zrepl/zrepl/rpc"
|
||||
"github.com/zrepl/zrepl/transport"
|
||||
"github.com/zrepl/zrepl/transport/fromconfig"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type ActiveSide struct {
|
||||
mode activeMode
|
||||
name string
|
||||
connecter transport.Connecter
|
||||
|
||||
prunerFactory *pruner.PrunerFactory
|
||||
|
||||
promRepStateSecs *prometheus.HistogramVec // labels: state
|
||||
promPruneSecs *prometheus.HistogramVec // labels: prune_side
|
||||
promBytesReplicated *prometheus.CounterVec // labels: filesystem
|
||||
|
||||
tasksMtx sync.Mutex
|
||||
tasks activeSideTasks
|
||||
}
|
||||
|
||||
//go:generate enumer -type=ActiveSideState
|
||||
type ActiveSideState int
|
||||
|
||||
const (
|
||||
ActiveSideReplicating ActiveSideState = 1 << iota
|
||||
ActiveSidePruneSender
|
||||
ActiveSidePruneReceiver
|
||||
ActiveSideDone // also errors
|
||||
)
|
||||
|
||||
type activeSideTasks struct {
|
||||
state ActiveSideState
|
||||
|
||||
// valid for state ActiveSideReplicating, ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
|
||||
replicationReport driver.ReportFunc
|
||||
replicationCancel context.CancelFunc
|
||||
|
||||
// valid for state ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
|
||||
prunerSender, prunerReceiver *pruner.Pruner
|
||||
|
||||
// valid for state ActiveSidePruneReceiver, ActiveSideDone
|
||||
prunerSenderCancel, prunerReceiverCancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (a *ActiveSide) updateTasks(u func(*activeSideTasks)) activeSideTasks {
|
||||
a.tasksMtx.Lock()
|
||||
defer a.tasksMtx.Unlock()
|
||||
copy := a.tasks
|
||||
if u == nil {
|
||||
return copy
|
||||
}
|
||||
u(©)
|
||||
a.tasks = copy
|
||||
return copy
|
||||
}
|
||||
|
||||
type activeMode interface {
|
||||
ConnectEndpoints(rpcLoggers rpc.Loggers, connecter transport.Connecter)
|
||||
DisconnectEndpoints()
|
||||
SenderReceiver() (logic.Sender, logic.Receiver)
|
||||
Type() Type
|
||||
RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{})
|
||||
SnapperReport() *snapper.Report
|
||||
ResetConnectBackoff()
|
||||
}
|
||||
|
||||
type modePush struct {
|
||||
setupMtx sync.Mutex
|
||||
sender *endpoint.Sender
|
||||
receiver *rpc.Client
|
||||
fsfilter endpoint.FSFilter
|
||||
snapper *snapper.PeriodicOrManual
|
||||
}
|
||||
|
||||
func (m *modePush) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Connecter) {
|
||||
m.setupMtx.Lock()
|
||||
defer m.setupMtx.Unlock()
|
||||
if m.receiver != nil || m.sender != nil {
|
||||
panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
|
||||
}
|
||||
m.sender = endpoint.NewSender(m.fsfilter)
|
||||
m.receiver = rpc.NewClient(connecter, loggers)
|
||||
}
|
||||
|
||||
func (m *modePush) DisconnectEndpoints() {
|
||||
m.setupMtx.Lock()
|
||||
defer m.setupMtx.Unlock()
|
||||
m.receiver.Close()
|
||||
m.sender = nil
|
||||
m.receiver = nil
|
||||
}
|
||||
|
||||
func (m *modePush) SenderReceiver() (logic.Sender, logic.Receiver) {
|
||||
m.setupMtx.Lock()
|
||||
defer m.setupMtx.Unlock()
|
||||
return m.sender, m.receiver
|
||||
}
|
||||
|
||||
func (m *modePush) Type() Type { return TypePush }
|
||||
|
||||
func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
|
||||
m.snapper.Run(ctx, wakeUpCommon)
|
||||
}
|
||||
|
||||
func (m *modePush) SnapperReport() *snapper.Report {
|
||||
return m.snapper.Report()
|
||||
}
|
||||
|
||||
func (m *modePush) ResetConnectBackoff() {
|
||||
m.setupMtx.Lock()
|
||||
defer m.setupMtx.Unlock()
|
||||
if m.receiver != nil {
|
||||
m.receiver.ResetConnectBackoff()
|
||||
}
|
||||
}
|
||||
|
||||
func modePushFromConfig(g *config.Global, in *config.PushJob) (*modePush, error) {
|
||||
m := &modePush{}
|
||||
fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannnot build filesystem filter")
|
||||
}
|
||||
m.fsfilter = fsf
|
||||
|
||||
if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build snapper")
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
type modePull struct {
|
||||
setupMtx sync.Mutex
|
||||
receiver *endpoint.Receiver
|
||||
sender *rpc.Client
|
||||
rootFS *zfs.DatasetPath
|
||||
interval config.PositiveDurationOrManual
|
||||
}
|
||||
|
||||
func (m *modePull) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Connecter) {
|
||||
m.setupMtx.Lock()
|
||||
defer m.setupMtx.Unlock()
|
||||
if m.receiver != nil || m.sender != nil {
|
||||
panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
|
||||
}
|
||||
m.receiver = endpoint.NewReceiver(m.rootFS, false)
|
||||
m.sender = rpc.NewClient(connecter, loggers)
|
||||
}
|
||||
|
||||
func (m *modePull) DisconnectEndpoints() {
|
||||
m.setupMtx.Lock()
|
||||
defer m.setupMtx.Unlock()
|
||||
m.sender.Close()
|
||||
m.sender = nil
|
||||
m.receiver = nil
|
||||
}
|
||||
|
||||
func (m *modePull) SenderReceiver() (logic.Sender, logic.Receiver) {
|
||||
m.setupMtx.Lock()
|
||||
defer m.setupMtx.Unlock()
|
||||
return m.sender, m.receiver
|
||||
}
|
||||
|
||||
func (*modePull) Type() Type { return TypePull }
|
||||
|
||||
func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
|
||||
if m.interval.Manual {
|
||||
GetLogger(ctx).Info("manual pull configured, periodic pull disabled")
|
||||
// "waiting for wakeups" is printed in common ActiveSide.do
|
||||
return
|
||||
}
|
||||
t := time.NewTicker(m.interval.Interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
select {
|
||||
case wakeUpCommon <- struct{}{}:
|
||||
default:
|
||||
GetLogger(ctx).
|
||||
WithField("pull_interval", m.interval).
|
||||
Warn("pull job took longer than pull interval")
|
||||
wakeUpCommon <- struct{}{} // block anyways, to queue up the wakeup
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *modePull) SnapperReport() *snapper.Report {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *modePull) ResetConnectBackoff() {
|
||||
m.setupMtx.Lock()
|
||||
defer m.setupMtx.Unlock()
|
||||
if m.sender != nil {
|
||||
m.sender.ResetConnectBackoff()
|
||||
}
|
||||
}
|
||||
|
||||
func modePullFromConfig(g *config.Global, in *config.PullJob) (m *modePull, err error) {
|
||||
m = &modePull{}
|
||||
m.interval = in.Interval
|
||||
|
||||
m.rootFS, err = zfs.NewDatasetPath(in.RootFS)
|
||||
if err != nil {
|
||||
return nil, errors.New("RootFS is not a valid zfs filesystem path")
|
||||
}
|
||||
if m.rootFS.Length() <= 0 {
|
||||
return nil, errors.New("RootFS must not be empty") // duplicates error check of receiver
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func activeSide(g *config.Global, in *config.ActiveJob, mode activeMode) (j *ActiveSide, err error) {
|
||||
|
||||
j = &ActiveSide{mode: mode}
|
||||
j.name = in.Name
|
||||
j.promRepStateSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "replication",
|
||||
Name: "state_time",
|
||||
Help: "seconds spent during replication",
|
||||
ConstLabels: prometheus.Labels{"zrepl_job": j.name},
|
||||
}, []string{"state"})
|
||||
j.promBytesReplicated = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "replication",
|
||||
Name: "bytes_replicated",
|
||||
Help: "number of bytes replicated from sender to receiver per filesystem",
|
||||
ConstLabels: prometheus.Labels{"zrepl_job": j.name},
|
||||
}, []string{"filesystem"})
|
||||
|
||||
j.connecter, err = fromconfig.ConnecterFromConfig(g, in.Connect)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build client")
|
||||
}
|
||||
|
||||
j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "pruning",
|
||||
Name: "time",
|
||||
Help: "seconds spent in pruner",
|
||||
ConstLabels: prometheus.Labels{"zrepl_job": j.name},
|
||||
}, []string{"prune_side"})
|
||||
j.prunerFactory, err = pruner.NewPrunerFactory(in.Pruning, j.promPruneSecs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return j, nil
|
||||
}
|
||||
|
||||
func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) {
|
||||
registerer.MustRegister(j.promRepStateSecs)
|
||||
registerer.MustRegister(j.promPruneSecs)
|
||||
registerer.MustRegister(j.promBytesReplicated)
|
||||
}
|
||||
|
||||
func (j *ActiveSide) Name() string { return j.name }
|
||||
|
||||
type ActiveSideStatus struct {
|
||||
Replication *report.Report
|
||||
PruningSender, PruningReceiver *pruner.Report
|
||||
Snapshotting *snapper.Report
|
||||
}
|
||||
|
||||
func (j *ActiveSide) Status() *Status {
|
||||
tasks := j.updateTasks(nil)
|
||||
|
||||
s := &ActiveSideStatus{}
|
||||
t := j.mode.Type()
|
||||
if tasks.replicationReport != nil {
|
||||
s.Replication = tasks.replicationReport()
|
||||
}
|
||||
if tasks.prunerSender != nil {
|
||||
s.PruningSender = tasks.prunerSender.Report()
|
||||
}
|
||||
if tasks.prunerReceiver != nil {
|
||||
s.PruningReceiver = tasks.prunerReceiver.Report()
|
||||
}
|
||||
s.Snapshotting = j.mode.SnapperReport()
|
||||
return &Status{Type: t, JobSpecific: s}
|
||||
}
|
||||
|
||||
func (j *ActiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
|
||||
pull, ok := j.mode.(*modePull)
|
||||
if !ok {
|
||||
_ = j.mode.(*modePush) // make sure we didn't introduce a new job type
|
||||
return nil, false
|
||||
}
|
||||
return pull.rootFS.Copy(), true
|
||||
}
|
||||
|
||||
func (j *ActiveSide) Run(ctx context.Context) {
|
||||
log := GetLogger(ctx)
|
||||
ctx = logging.WithSubsystemLoggers(ctx, log)
|
||||
|
||||
defer log.Info("job exiting")
|
||||
|
||||
periodicDone := make(chan struct{})
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
go j.mode.RunPeriodic(ctx, periodicDone)
|
||||
|
||||
invocationCount := 0
|
||||
outer:
|
||||
for {
|
||||
log.Info("wait for wakeups")
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.WithError(ctx.Err()).Info("context")
|
||||
break outer
|
||||
|
||||
case <-wakeup.Wait(ctx):
|
||||
j.mode.ResetConnectBackoff()
|
||||
case <-periodicDone:
|
||||
}
|
||||
invocationCount++
|
||||
invLog := log.WithField("invocation", invocationCount)
|
||||
j.do(WithLogger(ctx, invLog))
|
||||
}
|
||||
}
|
||||
|
||||
func (j *ActiveSide) do(ctx context.Context) {
|
||||
|
||||
log := GetLogger(ctx)
|
||||
ctx = logging.WithSubsystemLoggers(ctx, log)
|
||||
loggers := rpc.GetLoggersOrPanic(ctx) // filled by WithSubsystemLoggers
|
||||
j.mode.ConnectEndpoints(loggers, j.connecter)
|
||||
defer j.mode.DisconnectEndpoints()
|
||||
|
||||
// allow cancellation of an invocation (this function)
|
||||
ctx, cancelThisRun := context.WithCancel(ctx)
|
||||
defer cancelThisRun()
|
||||
go func() {
|
||||
select {
|
||||
case <-reset.Wait(ctx):
|
||||
log.Info("reset received, cancelling current invocation")
|
||||
cancelThisRun()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
|
||||
sender, receiver := j.mode.SenderReceiver()
|
||||
|
||||
{
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
ctx, repCancel := context.WithCancel(ctx)
|
||||
var repWait driver.WaitFunc
|
||||
j.updateTasks(func(tasks *activeSideTasks) {
|
||||
// reset it
|
||||
*tasks = activeSideTasks{}
|
||||
tasks.replicationCancel = repCancel
|
||||
tasks.replicationReport, repWait = replication.Do(
|
||||
ctx, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver),
|
||||
)
|
||||
tasks.state = ActiveSideReplicating
|
||||
})
|
||||
log.Info("start replication")
|
||||
repWait(true) // wait blocking
|
||||
repCancel() // always cancel to free up context resources
|
||||
}
|
||||
|
||||
{
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
ctx, senderCancel := context.WithCancel(ctx)
|
||||
tasks := j.updateTasks(func(tasks *activeSideTasks) {
|
||||
tasks.prunerSender = j.prunerFactory.BuildSenderPruner(ctx, sender, sender)
|
||||
tasks.prunerSenderCancel = senderCancel
|
||||
tasks.state = ActiveSidePruneSender
|
||||
})
|
||||
log.Info("start pruning sender")
|
||||
tasks.prunerSender.Prune()
|
||||
log.Info("finished pruning sender")
|
||||
senderCancel()
|
||||
}
|
||||
{
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
ctx, receiverCancel := context.WithCancel(ctx)
|
||||
tasks := j.updateTasks(func(tasks *activeSideTasks) {
|
||||
tasks.prunerReceiver = j.prunerFactory.BuildReceiverPruner(ctx, receiver, sender)
|
||||
tasks.prunerReceiverCancel = receiverCancel
|
||||
tasks.state = ActiveSidePruneReceiver
|
||||
})
|
||||
log.Info("start pruning receiver")
|
||||
tasks.prunerReceiver.Prune()
|
||||
log.Info("finished pruning receiver")
|
||||
receiverCancel()
|
||||
}
|
||||
|
||||
j.updateTasks(func(tasks *activeSideTasks) {
|
||||
tasks.state = ActiveSideDone
|
||||
})
|
||||
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
// Code generated by "enumer -type=ActiveSideState"; DO NOT EDIT.
|
||||
|
||||
//
|
||||
package job
|
||||
|
||||
import (
|
||||
@@ -7,14 +7,13 @@ import (
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/util/bandwidthlimit"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
)
|
||||
|
||||
func JobsFromConfig(c *config.Config, parseFlags config.ParseFlags) ([]Job, error) {
|
||||
func JobsFromConfig(c *config.Config) ([]Job, error) {
|
||||
js := make([]Job, len(c.Jobs))
|
||||
for i := range c.Jobs {
|
||||
j, err := buildJob(c.Global, c.Jobs[i], parseFlags)
|
||||
j, err := buildJob(c.Global, c.Jobs[i])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -24,22 +23,45 @@ func JobsFromConfig(c *config.Config, parseFlags config.ParseFlags) ([]Job, erro
|
||||
js[i] = j
|
||||
}
|
||||
|
||||
// receiving-side root filesystems must not overlap
|
||||
{
|
||||
rfss := make([]string, 0, len(js))
|
||||
for _, j := range js {
|
||||
jrfs, ok := j.OwnedDatasetSubtreeRoot()
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
rfss = append(rfss, jrfs.ToString())
|
||||
}
|
||||
if err := validateReceivingSidesDoNotOverlap(rfss); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return js, nil
|
||||
}
|
||||
|
||||
func buildJob(c *config.Global, in config.JobEnum, parseFlags config.ParseFlags) (j Job, err error) {
|
||||
func buildJob(c *config.Global, in config.JobEnum) (j Job, err error) {
|
||||
cannotBuildJob := func(e error, name string) (Job, error) {
|
||||
return nil, errors.Wrapf(e, "cannot build job %q", name)
|
||||
}
|
||||
// FIXME prettify this
|
||||
switch v := in.Ret.(type) {
|
||||
case *config.SinkJob:
|
||||
j, err = passiveSideFromConfig(c, &v.PassiveJob, v, parseFlags)
|
||||
m, err := modeSinkFromConfig(c, v)
|
||||
if err != nil {
|
||||
return cannotBuildJob(err, v.Name)
|
||||
}
|
||||
j, err = passiveSideFromConfig(c, &v.PassiveJob, m)
|
||||
if err != nil {
|
||||
return cannotBuildJob(err, v.Name)
|
||||
}
|
||||
case *config.SourceJob:
|
||||
j, err = passiveSideFromConfig(c, &v.PassiveJob, v, parseFlags)
|
||||
m, err := modeSourceFromConfig(c, v)
|
||||
if err != nil {
|
||||
return cannotBuildJob(err, v.Name)
|
||||
}
|
||||
j, err = passiveSideFromConfig(c, &v.PassiveJob, m)
|
||||
if err != nil {
|
||||
return cannotBuildJob(err, v.Name)
|
||||
}
|
||||
@@ -49,12 +71,20 @@ func buildJob(c *config.Global, in config.JobEnum, parseFlags config.ParseFlags)
|
||||
return cannotBuildJob(err, v.Name)
|
||||
}
|
||||
case *config.PushJob:
|
||||
j, err = activeSide(c, &v.ActiveJob, v, parseFlags)
|
||||
m, err := modePushFromConfig(c, v)
|
||||
if err != nil {
|
||||
return cannotBuildJob(err, v.Name)
|
||||
}
|
||||
j, err = activeSide(c, &v.ActiveJob, m)
|
||||
if err != nil {
|
||||
return cannotBuildJob(err, v.Name)
|
||||
}
|
||||
case *config.PullJob:
|
||||
j, err = activeSide(c, &v.ActiveJob, v, parseFlags)
|
||||
m, err := modePullFromConfig(c, v)
|
||||
if err != nil {
|
||||
return cannotBuildJob(err, v.Name)
|
||||
}
|
||||
j, err = activeSide(c, &v.ActiveJob, m)
|
||||
if err != nil {
|
||||
return cannotBuildJob(err, v.Name)
|
||||
}
|
||||
@@ -74,11 +104,6 @@ func validateReceivingSidesDoNotOverlap(receivingRootFSs []string) error {
|
||||
sort.Slice(rfss, func(i, j int) bool {
|
||||
return strings.Compare(rfss[i], rfss[j]) == -1
|
||||
})
|
||||
// add tailing slash because of hierarchy-simulation
|
||||
// rootfs/ is not root of rootfs2/
|
||||
for i := 0; i < len(rfss); i++ {
|
||||
rfss[i] += "/"
|
||||
}
|
||||
// idea:
|
||||
// no path in rfss must be prefix of another
|
||||
//
|
||||
@@ -93,13 +118,3 @@ func validateReceivingSidesDoNotOverlap(receivingRootFSs []string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildBandwidthLimitConfig(in *config.BandwidthLimit) (c bandwidthlimit.Config, _ error) {
|
||||
if in.Max.ToBytes() > 0 && int64(in.Max.ToBytes()) == 0 {
|
||||
return c, fmt.Errorf("bandwidth limit `max` is too small, must at least specify one byte")
|
||||
}
|
||||
return bandwidthlimit.Config{
|
||||
Max: int64(in.Max.ToBytes()),
|
||||
BucketCapacity: int64(in.BucketCapacity.ToBytes()),
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
|
||||
type testCase struct {
|
||||
err bool
|
||||
input []string
|
||||
}
|
||||
tcs := []testCase{
|
||||
{false, nil},
|
||||
{false, []string{}},
|
||||
{false, []string{""}}, // not our job to determine valid paths
|
||||
{false, []string{"a"}},
|
||||
{false, []string{"some/path"}},
|
||||
{false, []string{"zroot/sink1", "zroot/sink2", "zroot/sink3"}},
|
||||
{true, []string{"zroot/b", "zroot/b"}},
|
||||
{true, []string{"zroot/foo", "zroot/foo/bar", "zroot/baz"}},
|
||||
{false, []string{"a/x", "b/x"}},
|
||||
{false, []string{"a", "b"}},
|
||||
{true, []string{"a", "a"}},
|
||||
{true, []string{"a/x/y", "a/x"}},
|
||||
{true, []string{"a/x", "a/x/y"}},
|
||||
{true, []string{"a/x", "b/x", "a/x/y"}},
|
||||
{true, []string{"a", "a/b", "a/c", "a/b"}},
|
||||
{true, []string{"a/b", "a/c", "a/b", "a/d", "a/c"}},
|
||||
}
|
||||
|
||||
for _, tc := range tcs {
|
||||
t.Logf("input: %v", tc.input)
|
||||
err := validateReceivingSidesDoNotOverlap(tc.input)
|
||||
if tc.err {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,16 +7,27 @@ import (
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/logger"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type Logger = logger.Logger
|
||||
|
||||
type contextKey int
|
||||
|
||||
const (
|
||||
contextKeyLog contextKey = iota
|
||||
)
|
||||
|
||||
func GetLogger(ctx context.Context) Logger {
|
||||
return logging.GetLogger(ctx, logging.SubsysJob)
|
||||
if l, ok := ctx.Value(contextKeyLog).(Logger); ok {
|
||||
return l
|
||||
}
|
||||
return logger.NewNullLogger()
|
||||
}
|
||||
|
||||
func WithLogger(ctx context.Context, l Logger) context.Context {
|
||||
return context.WithValue(ctx, contextKeyLog, l)
|
||||
}
|
||||
|
||||
type Job interface {
|
||||
@@ -27,7 +38,6 @@ type Job interface {
|
||||
// Jobs that return a subtree of the dataset hierarchy
|
||||
// must return the root of that subtree as rfs and ok = true
|
||||
OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool)
|
||||
SenderConfig() *endpoint.SenderConfig
|
||||
}
|
||||
|
||||
type Type string
|
||||
@@ -0,0 +1,158 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/daemon/logging"
|
||||
"github.com/zrepl/zrepl/daemon/snapper"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/rpc"
|
||||
"github.com/zrepl/zrepl/transport"
|
||||
"github.com/zrepl/zrepl/transport/fromconfig"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type PassiveSide struct {
|
||||
mode passiveMode
|
||||
name string
|
||||
listen transport.AuthenticatedListenerFactory
|
||||
}
|
||||
|
||||
type passiveMode interface {
|
||||
Handler() rpc.Handler
|
||||
RunPeriodic(ctx context.Context)
|
||||
SnapperReport() *snapper.Report // may be nil
|
||||
Type() Type
|
||||
}
|
||||
|
||||
type modeSink struct {
|
||||
rootDataset *zfs.DatasetPath
|
||||
}
|
||||
|
||||
func (m *modeSink) Type() Type { return TypeSink }
|
||||
|
||||
func (m *modeSink) Handler() rpc.Handler {
|
||||
return endpoint.NewReceiver(m.rootDataset, true)
|
||||
}
|
||||
|
||||
func (m *modeSink) RunPeriodic(_ context.Context) {}
|
||||
func (m *modeSink) SnapperReport() *snapper.Report { return nil }
|
||||
|
||||
func modeSinkFromConfig(g *config.Global, in *config.SinkJob) (m *modeSink, err error) {
|
||||
m = &modeSink{}
|
||||
m.rootDataset, err = zfs.NewDatasetPath(in.RootFS)
|
||||
if err != nil {
|
||||
return nil, errors.New("root dataset is not a valid zfs filesystem path")
|
||||
}
|
||||
if m.rootDataset.Length() <= 0 {
|
||||
return nil, errors.New("root dataset must not be empty") // duplicates error check of receiver
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
type modeSource struct {
|
||||
fsfilter zfs.DatasetFilter
|
||||
snapper *snapper.PeriodicOrManual
|
||||
}
|
||||
|
||||
func modeSourceFromConfig(g *config.Global, in *config.SourceJob) (m *modeSource, err error) {
|
||||
// FIXME exact dedup of modePush
|
||||
m = &modeSource{}
|
||||
fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannnot build filesystem filter")
|
||||
}
|
||||
m.fsfilter = fsf
|
||||
|
||||
if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build snapper")
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *modeSource) Type() Type { return TypeSource }
|
||||
|
||||
func (m *modeSource) Handler() rpc.Handler {
|
||||
return endpoint.NewSender(m.fsfilter)
|
||||
}
|
||||
|
||||
func (m *modeSource) RunPeriodic(ctx context.Context) {
|
||||
m.snapper.Run(ctx, nil)
|
||||
}
|
||||
|
||||
func (m *modeSource) SnapperReport() *snapper.Report {
|
||||
return m.snapper.Report()
|
||||
}
|
||||
|
||||
func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, mode passiveMode) (s *PassiveSide, err error) {
|
||||
|
||||
s = &PassiveSide{mode: mode, name: in.Name}
|
||||
if s.listen, err = fromconfig.ListenerFactoryFromConfig(g, in.Serve); err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build listener factory")
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (j *PassiveSide) Name() string { return j.name }
|
||||
|
||||
type PassiveStatus struct {
|
||||
Snapper *snapper.Report
|
||||
}
|
||||
|
||||
func (s *PassiveSide) Status() *Status {
|
||||
st := &PassiveStatus{
|
||||
Snapper: s.mode.SnapperReport(),
|
||||
}
|
||||
return &Status{Type: s.mode.Type(), JobSpecific: st}
|
||||
}
|
||||
|
||||
func (j *PassiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
|
||||
sink, ok := j.mode.(*modeSink)
|
||||
if !ok {
|
||||
_ = j.mode.(*modeSource) // make sure we didn't introduce a new job type
|
||||
return nil, false
|
||||
}
|
||||
return sink.rootDataset.Copy(), true
|
||||
}
|
||||
|
||||
func (*PassiveSide) RegisterMetrics(registerer prometheus.Registerer) {}
|
||||
|
||||
func (j *PassiveSide) Run(ctx context.Context) {
|
||||
|
||||
log := GetLogger(ctx)
|
||||
defer log.Info("job exiting")
|
||||
ctx = logging.WithSubsystemLoggers(ctx, log)
|
||||
{
|
||||
ctx, cancel := context.WithCancel(ctx) // shadowing
|
||||
defer cancel()
|
||||
go j.mode.RunPeriodic(ctx)
|
||||
}
|
||||
|
||||
handler := j.mode.Handler()
|
||||
if handler == nil {
|
||||
panic(fmt.Sprintf("implementation error: j.mode.Handler() returned nil: %#v", j))
|
||||
}
|
||||
|
||||
ctxInterceptor := func(handlerCtx context.Context) context.Context {
|
||||
return logging.WithSubsystemLoggers(handlerCtx, log)
|
||||
}
|
||||
|
||||
rpcLoggers := rpc.GetLoggersOrPanic(ctx) // WithSubsystemLoggers above
|
||||
server := rpc.NewServer(handler, rpcLoggers, ctxInterceptor)
|
||||
|
||||
listener, err := j.listen()
|
||||
if err != nil {
|
||||
log.WithError(err).Error("cannot listen")
|
||||
return
|
||||
}
|
||||
|
||||
server.Serve(ctx, listener)
|
||||
}
|
||||
@@ -4,39 +4,34 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
||||
"github.com/zrepl/zrepl/internal/util/bandwidthlimit"
|
||||
"github.com/zrepl/zrepl/internal/util/nodefault"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
||||
"github.com/zrepl/zrepl/internal/daemon/job/wakeup"
|
||||
"github.com/zrepl/zrepl/internal/daemon/pruner"
|
||||
"github.com/zrepl/zrepl/internal/daemon/snapper"
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/replication/logic/pdu"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/daemon/job/wakeup"
|
||||
"github.com/zrepl/zrepl/daemon/logging"
|
||||
"github.com/zrepl/zrepl/daemon/pruner"
|
||||
"github.com/zrepl/zrepl/daemon/snapper"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/replication/logic/pdu"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type SnapJob struct {
|
||||
name endpoint.JobID
|
||||
name string
|
||||
fsfilter zfs.DatasetFilter
|
||||
snapper snapper.Snapper
|
||||
snapper *snapper.PeriodicOrManual
|
||||
|
||||
prunerFactory *pruner.LocalPrunerFactory
|
||||
|
||||
promPruneSecs *prometheus.HistogramVec // labels: prune_side
|
||||
|
||||
prunerMtx sync.Mutex
|
||||
pruner *pruner.Pruner
|
||||
pruner *pruner.Pruner
|
||||
}
|
||||
|
||||
func (j *SnapJob) Name() string { return j.name.String() }
|
||||
func (j *SnapJob) Name() string { return j.name }
|
||||
|
||||
func (j *SnapJob) Type() Type { return TypeSnap }
|
||||
|
||||
@@ -44,23 +39,20 @@ func snapJobFromConfig(g *config.Global, in *config.SnapJob) (j *SnapJob, err er
|
||||
j = &SnapJob{}
|
||||
fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build filesystem filter")
|
||||
return nil, errors.Wrap(err, "cannnot build filesystem filter")
|
||||
}
|
||||
j.fsfilter = fsf
|
||||
|
||||
if j.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build snapper")
|
||||
}
|
||||
j.name, err = endpoint.MakeJobID(in.Name)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "invalid job name")
|
||||
}
|
||||
j.name = in.Name
|
||||
j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "zrepl",
|
||||
Subsystem: "pruning",
|
||||
Name: "time",
|
||||
Help: "seconds spent in pruner",
|
||||
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
|
||||
ConstLabels: prometheus.Labels{"zrepl_job": j.name},
|
||||
}, []string{"prune_side"})
|
||||
j.prunerFactory, err = pruner.NewLocalPrunerFactory(in.Pruning, j.promPruneSecs)
|
||||
if err != nil {
|
||||
@@ -81,13 +73,10 @@ type SnapJobStatus struct {
|
||||
func (j *SnapJob) Status() *Status {
|
||||
s := &SnapJobStatus{}
|
||||
t := j.Type()
|
||||
j.prunerMtx.Lock()
|
||||
if j.pruner != nil {
|
||||
s.Pruning = j.pruner.Report()
|
||||
}
|
||||
j.prunerMtx.Unlock()
|
||||
r := j.snapper.Report()
|
||||
s.Snapshotting = &r
|
||||
s.Snapshotting = j.snapper.Report()
|
||||
return &Status{Type: t, JobSpecific: s}
|
||||
}
|
||||
|
||||
@@ -95,21 +84,16 @@ func (j *SnapJob) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (j *SnapJob) SenderConfig() *endpoint.SenderConfig { return nil }
|
||||
|
||||
func (j *SnapJob) Run(ctx context.Context) {
|
||||
ctx, endTask := trace.WithTaskAndSpan(ctx, "snap-job", j.Name())
|
||||
defer endTask()
|
||||
log := GetLogger(ctx)
|
||||
ctx = logging.WithSubsystemLoggers(ctx, log)
|
||||
|
||||
defer log.Info("job exiting")
|
||||
|
||||
periodicDone := make(chan struct{})
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
periodicCtx, endTask := trace.WithTask(ctx, "snapshotting")
|
||||
defer endTask()
|
||||
go j.snapper.Run(periodicCtx, periodicDone)
|
||||
go j.snapper.Run(ctx, periodicDone)
|
||||
|
||||
invocationCount := 0
|
||||
outer:
|
||||
@@ -124,10 +108,8 @@ outer:
|
||||
case <-periodicDone:
|
||||
}
|
||||
invocationCount++
|
||||
|
||||
invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", invocationCount))
|
||||
j.doPrune(invocationCtx)
|
||||
endSpan()
|
||||
invLog := log.WithField("invocation", invocationCount)
|
||||
j.doPrune(WithLogger(ctx, invLog))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -138,7 +120,7 @@ outer:
|
||||
// TODO:
|
||||
// This is a work-around for the current package daemon/pruner
|
||||
// and package pruning.Snapshot limitation: they require the
|
||||
// `Replicated` getter method be present, but obviously,
|
||||
// `Replicated` getter method be present, but obviously,
|
||||
// a local job like SnapJob can't deliver on that.
|
||||
// But the pruner.Pruner gives up on an FS if no replication
|
||||
// cursor is present, which is why this pruner returns the
|
||||
@@ -148,9 +130,12 @@ type alwaysUpToDateReplicationCursorHistory struct {
|
||||
target pruner.Target
|
||||
}
|
||||
|
||||
var _ pruner.Sender = (*alwaysUpToDateReplicationCursorHistory)(nil)
|
||||
var _ pruner.History = (*alwaysUpToDateReplicationCursorHistory)(nil)
|
||||
|
||||
func (h alwaysUpToDateReplicationCursorHistory) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
|
||||
if req.GetGet() == nil {
|
||||
return nil, fmt.Errorf("unsupported ReplicationCursor request: SnapJob only supports GETting a (faked) cursor")
|
||||
}
|
||||
fsvReq := &pdu.ListFilesystemVersionsReq{
|
||||
Filesystem: req.GetFilesystem(),
|
||||
}
|
||||
@@ -175,21 +160,10 @@ func (h alwaysUpToDateReplicationCursorHistory) ListFilesystems(ctx context.Cont
|
||||
}
|
||||
|
||||
func (j *SnapJob) doPrune(ctx context.Context) {
|
||||
ctx, endSpan := trace.WithSpan(ctx, "snap-job-do-prune")
|
||||
defer endSpan()
|
||||
log := GetLogger(ctx)
|
||||
sender := endpoint.NewSender(endpoint.SenderConfig{
|
||||
JobID: j.name,
|
||||
FSF: j.fsfilter,
|
||||
// FIXME the following config fields are irrelevant for SnapJob
|
||||
// because the endpoint is only used as pruner.Target.
|
||||
// However, the implementation requires them to be set.
|
||||
Encrypt: &nodefault.Bool{B: true},
|
||||
BandwidthLimit: bandwidthlimit.NoLimitConfig(),
|
||||
})
|
||||
j.prunerMtx.Lock()
|
||||
ctx = logging.WithSubsystemLoggers(ctx, log)
|
||||
sender := endpoint.NewSender(j.fsfilter)
|
||||
j.pruner = j.prunerFactory.BuildLocalPruner(ctx, sender, alwaysUpToDateReplicationCursorHistory{sender})
|
||||
j.prunerMtx.Unlock()
|
||||
log.Info("start pruning")
|
||||
j.pruner.Prune()
|
||||
log.Info("finished pruning")
|
||||
@@ -10,10 +10,18 @@ import (
|
||||
"github.com/mattn/go-isatty"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
||||
"github.com/zrepl/zrepl/internal/logger"
|
||||
"github.com/zrepl/zrepl/internal/tlsconf"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/hooks"
|
||||
"github.com/zrepl/zrepl/daemon/pruner"
|
||||
"github.com/zrepl/zrepl/daemon/snapper"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/replication/driver"
|
||||
"github.com/zrepl/zrepl/replication/logic"
|
||||
"github.com/zrepl/zrepl/rpc"
|
||||
"github.com/zrepl/zrepl/rpc/transportmux"
|
||||
"github.com/zrepl/zrepl/tlsconf"
|
||||
"github.com/zrepl/zrepl/transport"
|
||||
)
|
||||
|
||||
func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error) {
|
||||
@@ -61,10 +69,8 @@ func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error)
|
||||
type Subsystem string
|
||||
|
||||
const (
|
||||
SubsysMeta Subsystem = "meta"
|
||||
SubsysJob Subsystem = "job"
|
||||
SubsysReplication Subsystem = "repl"
|
||||
SubsysEndpoint Subsystem = "endpoint"
|
||||
SubsyEndpoint Subsystem = "endpoint"
|
||||
SubsysPruning Subsystem = "pruning"
|
||||
SubsysSnapshot Subsystem = "snapshot"
|
||||
SubsysHooks Subsystem = "hook"
|
||||
@@ -73,104 +79,29 @@ const (
|
||||
SubsysRPC Subsystem = "rpc"
|
||||
SubsysRPCControl Subsystem = "rpc.ctrl"
|
||||
SubsysRPCData Subsystem = "rpc.data"
|
||||
SubsysZFSCmd Subsystem = "zfs.cmd"
|
||||
SubsysTraceData Subsystem = "trace.data"
|
||||
SubsysPlatformtest Subsystem = "platformtest"
|
||||
)
|
||||
|
||||
var AllSubsystems = []Subsystem{
|
||||
SubsysMeta,
|
||||
SubsysJob,
|
||||
SubsysReplication,
|
||||
SubsysEndpoint,
|
||||
SubsysPruning,
|
||||
SubsysSnapshot,
|
||||
SubsysHooks,
|
||||
SubsysTransport,
|
||||
SubsysTransportMux,
|
||||
SubsysRPC,
|
||||
SubsysRPCControl,
|
||||
SubsysRPCData,
|
||||
SubsysZFSCmd,
|
||||
SubsysTraceData,
|
||||
SubsysPlatformtest,
|
||||
func WithSubsystemLoggers(ctx context.Context, log logger.Logger) context.Context {
|
||||
ctx = logic.WithLogger(ctx, log.WithField(SubsysField, SubsysReplication))
|
||||
ctx = driver.WithLogger(ctx, log.WithField(SubsysField, SubsysReplication))
|
||||
ctx = endpoint.WithLogger(ctx, log.WithField(SubsysField, SubsyEndpoint))
|
||||
ctx = pruner.WithLogger(ctx, log.WithField(SubsysField, SubsysPruning))
|
||||
ctx = snapper.WithLogger(ctx, log.WithField(SubsysField, SubsysSnapshot))
|
||||
ctx = hooks.WithLogger(ctx, log.WithField(SubsysField, SubsysHooks))
|
||||
ctx = transport.WithLogger(ctx, log.WithField(SubsysField, SubsysTransport))
|
||||
ctx = transportmux.WithLogger(ctx, log.WithField(SubsysField, SubsysTransportMux))
|
||||
ctx = rpc.WithLoggers(ctx,
|
||||
rpc.Loggers{
|
||||
General: log.WithField(SubsysField, SubsysRPC),
|
||||
Control: log.WithField(SubsysField, SubsysRPCControl),
|
||||
Data: log.WithField(SubsysField, SubsysRPCData),
|
||||
},
|
||||
)
|
||||
return ctx
|
||||
}
|
||||
|
||||
type injectedField struct {
|
||||
field string
|
||||
value interface{}
|
||||
parent *injectedField
|
||||
}
|
||||
|
||||
func WithInjectedField(ctx context.Context, field string, value interface{}) context.Context {
|
||||
var parent *injectedField
|
||||
parentI := ctx.Value(contextKeyInjectedField)
|
||||
if parentI != nil {
|
||||
parent = parentI.(*injectedField)
|
||||
}
|
||||
// TODO sanity-check `field` now
|
||||
this := &injectedField{field, value, parent}
|
||||
return context.WithValue(ctx, contextKeyInjectedField, this)
|
||||
}
|
||||
|
||||
func iterInjectedFields(ctx context.Context, cb func(field string, value interface{})) {
|
||||
injI := ctx.Value(contextKeyInjectedField)
|
||||
if injI == nil {
|
||||
return
|
||||
}
|
||||
inj := injI.(*injectedField)
|
||||
for ; inj != nil; inj = inj.parent {
|
||||
cb(inj.field, inj.value)
|
||||
}
|
||||
}
|
||||
|
||||
type SubsystemLoggers map[Subsystem]logger.Logger
|
||||
|
||||
func SubsystemLoggersWithUniversalLogger(l logger.Logger) SubsystemLoggers {
|
||||
loggers := make(SubsystemLoggers)
|
||||
for _, s := range AllSubsystems {
|
||||
loggers[s] = l
|
||||
}
|
||||
return loggers
|
||||
}
|
||||
|
||||
func WithLoggers(ctx context.Context, loggers SubsystemLoggers) context.Context {
|
||||
return context.WithValue(ctx, contextKeyLoggers, loggers)
|
||||
}
|
||||
|
||||
func GetLoggers(ctx context.Context) SubsystemLoggers {
|
||||
loggers, ok := ctx.Value(contextKeyLoggers).(SubsystemLoggers)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return loggers
|
||||
}
|
||||
|
||||
func GetLogger(ctx context.Context, subsys Subsystem) logger.Logger {
|
||||
return getLoggerImpl(ctx, subsys, true)
|
||||
}
|
||||
|
||||
func getLoggerImpl(ctx context.Context, subsys Subsystem, panicIfEnded bool) logger.Logger {
|
||||
loggers, ok := ctx.Value(contextKeyLoggers).(SubsystemLoggers)
|
||||
if !ok || loggers == nil {
|
||||
return logger.NewNullLogger()
|
||||
}
|
||||
l, ok := loggers[subsys]
|
||||
if !ok {
|
||||
return logger.NewNullLogger()
|
||||
}
|
||||
|
||||
l = l.WithField(SubsysField, subsys)
|
||||
|
||||
l = l.WithField(SpanField, trace.GetSpanStackOrDefault(ctx, *trace.StackKindId, "NOSPAN"))
|
||||
|
||||
fields := make(logger.Fields)
|
||||
iterInjectedFields(ctx, func(field string, value interface{}) {
|
||||
fields[field] = value
|
||||
})
|
||||
l = l.WithFields(fields)
|
||||
|
||||
return l
|
||||
func LogSubsystem(log logger.Logger, subsys Subsystem) logger.Logger {
|
||||
return log.ReplaceField(SubsysField, subsys)
|
||||
}
|
||||
|
||||
func parseLogFormat(i interface{}) (f EntryFormatter, err error) {
|
||||
+5
-6
@@ -10,7 +10,7 @@ import (
|
||||
"github.com/go-logfmt/logfmt"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/logger"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -22,7 +22,6 @@ const (
|
||||
const (
|
||||
JobField string = "job"
|
||||
SubsysField string = "subsystem"
|
||||
SpanField string = "span"
|
||||
)
|
||||
|
||||
type MetadataFlags int64
|
||||
@@ -86,7 +85,7 @@ func (f *HumanFormatter) Format(e *logger.Entry) (out []byte, err error) {
|
||||
fmt.Fprintf(&line, "[%s]", col.Sprint(e.Level.Short()))
|
||||
}
|
||||
|
||||
prefixFields := []string{JobField, SubsysField, SpanField}
|
||||
prefixFields := []string{JobField, SubsysField}
|
||||
prefixed := make(map[string]bool, len(prefixFields)+2)
|
||||
for _, field := range prefixFields {
|
||||
val, ok := e.Fields[field]
|
||||
@@ -175,8 +174,8 @@ func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) {
|
||||
}
|
||||
|
||||
// at least try and put job and task in front
|
||||
prefixed := make(map[string]bool, 3)
|
||||
prefix := []string{JobField, SubsysField, SpanField}
|
||||
prefixed := make(map[string]bool, 2)
|
||||
prefix := []string{JobField, SubsysField}
|
||||
for _, pf := range prefix {
|
||||
v, ok := e.Fields[pf]
|
||||
if !ok {
|
||||
@@ -212,7 +211,7 @@ func logfmtTryEncodeKeyval(enc *logfmt.Encoder, field, value interface{}) error
|
||||
case logfmt.ErrUnsupportedValueType:
|
||||
err := enc.EncodeKeyval(field, fmt.Sprintf("<%T>", value))
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "cannot encode unsupported value type Go type")
|
||||
return errors.Wrap(err, "cannot encode unsuuported value type Go type")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/logger"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
)
|
||||
|
||||
type EntryFormatter interface {
|
||||
@@ -63,7 +63,7 @@ func NewTCPOutlet(formatter EntryFormatter, network, address string, tlsConfig *
|
||||
return
|
||||
}
|
||||
|
||||
entryChan := make(chan *bytes.Buffer, 1) // allow one message in flight while previous is in io.Copy()
|
||||
entryChan := make(chan *bytes.Buffer, 1) // allow one message in flight while previos is in io.Copy()
|
||||
|
||||
o := &TCPOutlet{
|
||||
formatter: formatter,
|
||||
@@ -0,0 +1,16 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
)
|
||||
|
||||
type Logger = logger.Logger
|
||||
|
||||
var DaemonCmd = &cli.Subcommand{
|
||||
Use: "daemon",
|
||||
Short: "run the zrepl daemon",
|
||||
Run: func(subcommand *cli.Subcommand, args []string) error {
|
||||
return Run(subcommand.Config())
|
||||
},
|
||||
}
|
||||
@@ -8,11 +8,7 @@ import (
|
||||
"net"
|
||||
"net/http/pprof"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"golang.org/x/net/websocket"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
||||
"github.com/zrepl/zrepl/daemon/job"
|
||||
)
|
||||
|
||||
type pprofServer struct {
|
||||
@@ -68,8 +64,6 @@ outer:
|
||||
mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
|
||||
mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
|
||||
mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
|
||||
mux.Handle("/metrics", promhttp.Handler())
|
||||
mux.Handle("/debug/zrepl/activity-trace", websocket.Handler(trace.ChrometraceClientWebsocketHandler))
|
||||
go func() {
|
||||
err := http.Serve(s.listener, mux)
|
||||
if ctx.Err() != nil {
|
||||
@@ -8,26 +8,22 @@ import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/logger"
|
||||
"github.com/zrepl/zrepl/internal/rpc/dataconn/frameconn"
|
||||
"github.com/zrepl/zrepl/internal/util/tcpsock"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/job"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/rpc/dataconn/frameconn"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type prometheusJob struct {
|
||||
listen string
|
||||
freeBind bool
|
||||
listen string
|
||||
}
|
||||
|
||||
func newPrometheusJobFromConfig(in *config.PrometheusMonitoring) (*prometheusJob, error) {
|
||||
if _, _, err := net.SplitHostPort(in.Listen); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &prometheusJob{in.Listen, in.ListenFreeBind}, nil
|
||||
return &prometheusJob{in.Listen}, nil
|
||||
}
|
||||
|
||||
var prom struct {
|
||||
@@ -50,8 +46,6 @@ func (j *prometheusJob) Status() *job.Status { return &job.Status{Type: job.Type
|
||||
|
||||
func (j *prometheusJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
|
||||
|
||||
func (j *prometheusJob) SenderConfig() *endpoint.SenderConfig { return nil }
|
||||
|
||||
func (j *prometheusJob) RegisterMetrics(registerer prometheus.Registerer) {}
|
||||
|
||||
func (j *prometheusJob) Run(ctx context.Context) {
|
||||
@@ -66,7 +60,7 @@ func (j *prometheusJob) Run(ctx context.Context) {
|
||||
|
||||
log := job.GetLogger(ctx)
|
||||
|
||||
l, err := tcpsock.Listen(j.listen, j.freeBind)
|
||||
l, err := net.Listen("tcp", j.listen)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("cannot listen")
|
||||
return
|
||||
@@ -87,19 +81,16 @@ func (j *prometheusJob) Run(ctx context.Context) {
|
||||
}
|
||||
|
||||
type prometheusJobOutlet struct {
|
||||
jobName string
|
||||
}
|
||||
|
||||
var _ logger.Outlet = prometheusJobOutlet{}
|
||||
|
||||
func newPrometheusLogOutlet() prometheusJobOutlet {
|
||||
return prometheusJobOutlet{}
|
||||
func newPrometheusLogOutlet(jobName string) prometheusJobOutlet {
|
||||
return prometheusJobOutlet{jobName}
|
||||
}
|
||||
|
||||
func (o prometheusJobOutlet) WriteEntry(entry logger.Entry) error {
|
||||
jobFieldVal, ok := entry.Fields[logging.JobField].(string)
|
||||
if !ok {
|
||||
jobFieldVal = "_nojobid"
|
||||
}
|
||||
prom.taskLogEntries.WithLabelValues(jobFieldVal, entry.Level.String()).Inc()
|
||||
prom.taskLogEntries.WithLabelValues(o.jobName, entry.Level.String()).Inc()
|
||||
return nil
|
||||
}
|
||||
@@ -11,29 +11,20 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
||||
"github.com/zrepl/zrepl/internal/logger"
|
||||
"github.com/zrepl/zrepl/internal/pruning"
|
||||
"github.com/zrepl/zrepl/internal/replication/logic/pdu"
|
||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/pruning"
|
||||
"github.com/zrepl/zrepl/replication/logic/pdu"
|
||||
"github.com/zrepl/zrepl/util/envconst"
|
||||
)
|
||||
|
||||
// The sender in the replication setup.
|
||||
// The pruner uses the Sender to determine which of the Target's filesystems need to be pruned.
|
||||
// Also, it asks the Sender about the replication cursor of each filesystem
|
||||
// to enable the 'not_replicated' pruning rule.
|
||||
//
|
||||
// Try to keep it compatible with github.com/zrepl/zrepl/endpoint.Endpoint
|
||||
type Sender interface {
|
||||
// Try to keep it compatible with gitub.com/zrepl/zrepl/endpoint.Endpoint
|
||||
type History interface {
|
||||
ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error)
|
||||
ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
|
||||
}
|
||||
|
||||
// The pruning target, i.e., on which snapshots are destroyed.
|
||||
// This can be a replication sender or receiver.
|
||||
//
|
||||
// Try to keep it compatible with github.com/zrepl/zrepl/endpoint.Endpoint
|
||||
// Try to keep it compatible with gitub.com/zrepl/zrepl/endpoint.Endpoint
|
||||
type Target interface {
|
||||
ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
|
||||
ListFilesystemVersions(ctx context.Context, req *pdu.ListFilesystemVersionsReq) (*pdu.ListFilesystemVersionsRes, error)
|
||||
@@ -44,19 +35,23 @@ type Logger = logger.Logger
|
||||
|
||||
type contextKey int
|
||||
|
||||
const (
|
||||
contextKeyPruneSide contextKey = 1 + iota
|
||||
)
|
||||
const contextKeyLogger contextKey = 0
|
||||
|
||||
func WithLogger(ctx context.Context, log Logger) context.Context {
|
||||
return context.WithValue(ctx, contextKeyLogger, log)
|
||||
}
|
||||
|
||||
func GetLogger(ctx context.Context) Logger {
|
||||
pruneSide := ctx.Value(contextKeyPruneSide).(string)
|
||||
return logging.GetLogger(ctx, logging.SubsysPruning).WithField("prune_side", pruneSide)
|
||||
if l, ok := ctx.Value(contextKeyLogger).(Logger); ok {
|
||||
return l
|
||||
}
|
||||
return logger.NewNullLogger()
|
||||
}
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
target Target
|
||||
sender Sender
|
||||
receiver History
|
||||
rules []pruning.KeepRule
|
||||
retryWait time.Duration
|
||||
considerSnapAtCursorReplicated bool
|
||||
@@ -140,12 +135,12 @@ func NewPrunerFactory(in config.PruningSenderReceiver, promPruneSecs *prometheus
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, sender Sender) *Pruner {
|
||||
func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, receiver History) *Pruner {
|
||||
p := &Pruner{
|
||||
args: args{
|
||||
context.WithValue(ctx, contextKeyPruneSide, "sender"),
|
||||
WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "sender")),
|
||||
target,
|
||||
sender,
|
||||
receiver,
|
||||
f.senderRules,
|
||||
f.retryWait,
|
||||
f.considerSnapAtCursorReplicated,
|
||||
@@ -156,12 +151,12 @@ func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, se
|
||||
return p
|
||||
}
|
||||
|
||||
func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target, sender Sender) *Pruner {
|
||||
func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target, receiver History) *Pruner {
|
||||
p := &Pruner{
|
||||
args: args{
|
||||
context.WithValue(ctx, contextKeyPruneSide, "receiver"),
|
||||
WithLogger(ctx, GetLogger(ctx).WithField("prune_side", "receiver")),
|
||||
target,
|
||||
sender,
|
||||
receiver,
|
||||
f.receiverRules,
|
||||
f.retryWait,
|
||||
false, // senseless here anyways
|
||||
@@ -172,12 +167,12 @@ func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target,
|
||||
return p
|
||||
}
|
||||
|
||||
func (f *LocalPrunerFactory) BuildLocalPruner(ctx context.Context, target Target, history Sender) *Pruner {
|
||||
func (f *LocalPrunerFactory) BuildLocalPruner(ctx context.Context, target Target, receiver History) *Pruner {
|
||||
p := &Pruner{
|
||||
args: args{
|
||||
context.WithValue(ctx, contextKeyPruneSide, "local"),
|
||||
ctx,
|
||||
target,
|
||||
history,
|
||||
receiver,
|
||||
f.keepRules,
|
||||
f.retryWait,
|
||||
false, // considerSnapAtCursorReplicated is not relevant for local pruning
|
||||
@@ -199,16 +194,6 @@ const (
|
||||
Done
|
||||
)
|
||||
|
||||
// Returns true in case the State is a terminal state(PlanErr, ExecErr, Done)
|
||||
func (s State) IsTerminal() bool {
|
||||
switch s {
|
||||
case PlanErr, ExecErr, Done:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type updater func(func(*Pruner))
|
||||
|
||||
func (p *Pruner) Prune() {
|
||||
@@ -361,9 +346,9 @@ func (s snapshot) Date() time.Time { return s.date }
|
||||
|
||||
func doOneAttempt(a *args, u updater) {
|
||||
|
||||
ctx, target, sender := a.ctx, a.target, a.sender
|
||||
ctx, target, receiver := a.ctx, a.target, a.receiver
|
||||
|
||||
sfssres, err := sender.ListFilesystems(ctx, &pdu.ListFilesystemReq{})
|
||||
sfssres, err := receiver.ListFilesystems(ctx, &pdu.ListFilesystemReq{})
|
||||
if err != nil {
|
||||
u(func(p *Pruner) {
|
||||
p.state = PlanErr
|
||||
@@ -427,8 +412,11 @@ tfss_loop:
|
||||
|
||||
rcReq := &pdu.ReplicationCursorReq{
|
||||
Filesystem: tfs.Path,
|
||||
Op: &pdu.ReplicationCursorReq_Get{
|
||||
Get: &pdu.ReplicationCursorReq_GetOp{},
|
||||
},
|
||||
}
|
||||
rc, err := sender.ReplicationCursor(ctx, rcReq)
|
||||
rc, err := receiver.ReplicationCursor(ctx, rcReq)
|
||||
if err != nil {
|
||||
pfsPlanErrAndLog(err, "cannot get replication cursor bookmark")
|
||||
continue tfss_loop
|
||||
@@ -474,7 +462,7 @@ tfss_loop:
|
||||
})
|
||||
}
|
||||
if preCursor {
|
||||
pfsPlanErrAndLog(fmt.Errorf("prune target has no snapshot that corresponds to sender replication cursor bookmark"), "")
|
||||
pfsPlanErrAndLog(fmt.Errorf("replication cursor not found in prune target filesystem versions"), "")
|
||||
continue tfss_loop
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// Code generated by "enumer -type=State"; DO NOT EDIT.
|
||||
|
||||
//
|
||||
package pruner
|
||||
|
||||
import (
|
||||
@@ -0,0 +1,465 @@
|
||||
package snapper
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/daemon/hooks"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
//go:generate stringer -type=SnapState
|
||||
type SnapState uint
|
||||
|
||||
const (
|
||||
SnapPending SnapState = 1 << iota
|
||||
SnapStarted
|
||||
SnapDone
|
||||
SnapError
|
||||
)
|
||||
|
||||
// All fields protected by Snapper.mtx
|
||||
type snapProgress struct {
|
||||
state SnapState
|
||||
|
||||
// SnapStarted, SnapDone, SnapError
|
||||
name string
|
||||
startAt time.Time
|
||||
hookPlan *hooks.Plan
|
||||
|
||||
// SnapDone
|
||||
doneAt time.Time
|
||||
|
||||
// SnapErr TODO disambiguate state
|
||||
runResults hooks.PlanReport
|
||||
}
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
log Logger
|
||||
prefix string
|
||||
interval time.Duration
|
||||
fsf *filters.DatasetMapFilter
|
||||
snapshotsTaken chan<- struct{}
|
||||
hooks *hooks.List
|
||||
dryRun bool
|
||||
}
|
||||
|
||||
type Snapper struct {
|
||||
args args
|
||||
|
||||
mtx sync.Mutex
|
||||
state State
|
||||
|
||||
// set in state Plan, used in Waiting
|
||||
lastInvocation time.Time
|
||||
|
||||
// valid for state Snapshotting
|
||||
plan map[*zfs.DatasetPath]*snapProgress
|
||||
|
||||
// valid for state SyncUp and Waiting
|
||||
sleepUntil time.Time
|
||||
|
||||
// valid for state Err
|
||||
err error
|
||||
}
|
||||
|
||||
//go:generate stringer -type=State
|
||||
type State uint
|
||||
|
||||
const (
|
||||
SyncUp State = 1 << iota
|
||||
SyncUpErrWait
|
||||
Planning
|
||||
Snapshotting
|
||||
Waiting
|
||||
ErrorWait
|
||||
Stopped
|
||||
)
|
||||
|
||||
func (s State) sf() state {
|
||||
m := map[State]state{
|
||||
SyncUp: syncUp,
|
||||
SyncUpErrWait: wait,
|
||||
Planning: plan,
|
||||
Snapshotting: snapshot,
|
||||
Waiting: wait,
|
||||
ErrorWait: wait,
|
||||
Stopped: nil,
|
||||
}
|
||||
return m[s]
|
||||
}
|
||||
|
||||
type updater func(u func(*Snapper)) State
|
||||
type state func(a args, u updater) state
|
||||
|
||||
type contextKey int
|
||||
|
||||
const (
|
||||
contextKeyLog contextKey = 0
|
||||
)
|
||||
|
||||
type Logger = logger.Logger
|
||||
|
||||
func WithLogger(ctx context.Context, log Logger) context.Context {
|
||||
return context.WithValue(ctx, contextKeyLog, log)
|
||||
}
|
||||
|
||||
func getLogger(ctx context.Context) Logger {
|
||||
if log, ok := ctx.Value(contextKeyLog).(Logger); ok {
|
||||
return log
|
||||
}
|
||||
return logger.NewNullLogger()
|
||||
}
|
||||
|
||||
func PeriodicFromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in *config.SnapshottingPeriodic) (*Snapper, error) {
|
||||
if in.Prefix == "" {
|
||||
return nil, errors.New("prefix must not be empty")
|
||||
}
|
||||
if in.Interval <= 0 {
|
||||
return nil, errors.New("interval must be positive")
|
||||
}
|
||||
|
||||
hookList, err := hooks.ListFromConfig(&in.Hooks)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "hook config error")
|
||||
}
|
||||
|
||||
args := args{
|
||||
prefix: in.Prefix,
|
||||
interval: in.Interval,
|
||||
fsf: fsf,
|
||||
hooks: hookList,
|
||||
// ctx and log is set in Run()
|
||||
}
|
||||
|
||||
return &Snapper{state: SyncUp, args: args}, nil
|
||||
}
|
||||
|
||||
func (s *Snapper) Run(ctx context.Context, snapshotsTaken chan<- struct{}) {
|
||||
|
||||
getLogger(ctx).Debug("start")
|
||||
defer getLogger(ctx).Debug("stop")
|
||||
|
||||
s.args.snapshotsTaken = snapshotsTaken
|
||||
s.args.ctx = ctx
|
||||
s.args.log = getLogger(ctx)
|
||||
s.args.dryRun = false // for future expansion
|
||||
|
||||
u := func(u func(*Snapper)) State {
|
||||
s.mtx.Lock()
|
||||
defer s.mtx.Unlock()
|
||||
if u != nil {
|
||||
u(s)
|
||||
}
|
||||
return s.state
|
||||
}
|
||||
|
||||
var st state = syncUp
|
||||
|
||||
for st != nil {
|
||||
pre := u(nil)
|
||||
st = st(s.args, u)
|
||||
post := u(nil)
|
||||
getLogger(ctx).
|
||||
WithField("transition", fmt.Sprintf("%s=>%s", pre, post)).
|
||||
Debug("state transition")
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func onErr(err error, u updater) state {
|
||||
return u(func(s *Snapper) {
|
||||
s.err = err
|
||||
preState := s.state
|
||||
switch s.state {
|
||||
case SyncUp:
|
||||
s.state = SyncUpErrWait
|
||||
case Planning:
|
||||
fallthrough
|
||||
case Snapshotting:
|
||||
s.state = ErrorWait
|
||||
}
|
||||
s.args.log.WithError(err).WithField("pre_state", preState).WithField("post_state", s.state).Error("snapshotting error")
|
||||
}).sf()
|
||||
}
|
||||
|
||||
func onMainCtxDone(ctx context.Context, u updater) state {
|
||||
return u(func(s *Snapper) {
|
||||
s.err = ctx.Err()
|
||||
s.state = Stopped
|
||||
}).sf()
|
||||
}
|
||||
|
||||
func syncUp(a args, u updater) state {
|
||||
u(func(snapper *Snapper) {
|
||||
snapper.lastInvocation = time.Now()
|
||||
})
|
||||
fss, err := listFSes(a.ctx, a.fsf)
|
||||
if err != nil {
|
||||
return onErr(err, u)
|
||||
}
|
||||
syncPoint, err := findSyncPoint(a.log, fss, a.prefix, a.interval)
|
||||
if err != nil {
|
||||
return onErr(err, u)
|
||||
}
|
||||
u(func(s *Snapper) {
|
||||
s.sleepUntil = syncPoint
|
||||
})
|
||||
t := time.NewTimer(time.Until(syncPoint))
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-t.C:
|
||||
return u(func(s *Snapper) {
|
||||
s.state = Planning
|
||||
}).sf()
|
||||
case <-a.ctx.Done():
|
||||
return onMainCtxDone(a.ctx, u)
|
||||
}
|
||||
}
|
||||
|
||||
func plan(a args, u updater) state {
|
||||
u(func(snapper *Snapper) {
|
||||
snapper.lastInvocation = time.Now()
|
||||
})
|
||||
fss, err := listFSes(a.ctx, a.fsf)
|
||||
if err != nil {
|
||||
return onErr(err, u)
|
||||
}
|
||||
|
||||
plan := make(map[*zfs.DatasetPath]*snapProgress, len(fss))
|
||||
for _, fs := range fss {
|
||||
plan[fs] = &snapProgress{state: SnapPending}
|
||||
}
|
||||
return u(func(s *Snapper) {
|
||||
s.state = Snapshotting
|
||||
s.plan = plan
|
||||
s.err = nil
|
||||
}).sf()
|
||||
}
|
||||
|
||||
func snapshot(a args, u updater) state {
|
||||
|
||||
var plan map[*zfs.DatasetPath]*snapProgress
|
||||
u(func(snapper *Snapper) {
|
||||
plan = snapper.plan
|
||||
})
|
||||
|
||||
hookMatchCount := make(map[hooks.Hook]int, len(*a.hooks))
|
||||
for _, h := range *a.hooks {
|
||||
hookMatchCount[h] = 0
|
||||
}
|
||||
|
||||
anyFsHadErr := false
|
||||
// TODO channel programs -> allow a little jitter?
|
||||
for fs, progress := range plan {
|
||||
suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
|
||||
snapname := fmt.Sprintf("%s%s", a.prefix, suffix)
|
||||
|
||||
l := a.log.
|
||||
WithField("fs", fs.ToString()).
|
||||
WithField("snap", snapname)
|
||||
|
||||
hookEnvExtra := hooks.Env{
|
||||
hooks.EnvFS: fs.ToString(),
|
||||
hooks.EnvSnapshot: snapname,
|
||||
}
|
||||
|
||||
jobCallback := hooks.NewCallbackHookForFilesystem("snapshot", fs, func(_ context.Context) (err error) {
|
||||
l.Debug("create snapshot")
|
||||
err = zfs.ZFSSnapshot(fs, snapname, false) // TODO propagagte context to ZFSSnapshot
|
||||
if err != nil {
|
||||
l.WithError(err).Error("cannot create snapshot")
|
||||
}
|
||||
return
|
||||
})
|
||||
|
||||
fsHadErr := false
|
||||
var planReport hooks.PlanReport
|
||||
var plan *hooks.Plan
|
||||
{
|
||||
filteredHooks, err := a.hooks.CopyFilteredForFilesystem(fs)
|
||||
if err != nil {
|
||||
l.WithError(err).Error("unexpected filter error")
|
||||
fsHadErr = true
|
||||
goto updateFSState
|
||||
}
|
||||
// account for running hooks
|
||||
for _, h := range filteredHooks {
|
||||
hookMatchCount[h] = hookMatchCount[h] + 1
|
||||
}
|
||||
|
||||
var planErr error
|
||||
plan, planErr = hooks.NewPlan(&filteredHooks, hooks.PhaseSnapshot, jobCallback, hookEnvExtra)
|
||||
if planErr != nil {
|
||||
fsHadErr = true
|
||||
l.WithError(planErr).Error("cannot create job hook plan")
|
||||
goto updateFSState
|
||||
}
|
||||
}
|
||||
u(func(snapper *Snapper) {
|
||||
progress.name = snapname
|
||||
progress.startAt = time.Now()
|
||||
progress.hookPlan = plan
|
||||
progress.state = SnapStarted
|
||||
})
|
||||
{
|
||||
l := hooks.GetLogger(a.ctx).WithField("fs", fs.ToString()).WithField("snap", snapname)
|
||||
l.WithField("report", plan.Report().String()).Debug("begin run job plan")
|
||||
plan.Run(hooks.WithLogger(a.ctx, l), a.dryRun)
|
||||
planReport = plan.Report()
|
||||
fsHadErr = planReport.HadError() // not just fatal errors
|
||||
if fsHadErr {
|
||||
l.WithField("report", planReport.String()).Error("end run job plan with error")
|
||||
} else {
|
||||
l.WithField("report", planReport.String()).Info("end run job plan successful")
|
||||
}
|
||||
}
|
||||
|
||||
updateFSState:
|
||||
anyFsHadErr = anyFsHadErr || fsHadErr
|
||||
u(func(snapper *Snapper) {
|
||||
progress.doneAt = time.Now()
|
||||
progress.state = SnapDone
|
||||
if fsHadErr {
|
||||
progress.state = SnapError
|
||||
}
|
||||
progress.runResults = planReport
|
||||
})
|
||||
}
|
||||
|
||||
select {
|
||||
case a.snapshotsTaken <- struct{}{}:
|
||||
default:
|
||||
if a.snapshotsTaken != nil {
|
||||
a.log.Warn("callback channel is full, discarding snapshot update event")
|
||||
}
|
||||
}
|
||||
|
||||
for h, mc := range hookMatchCount {
|
||||
if mc == 0 {
|
||||
hookIdx := -1
|
||||
for idx, ah := range *a.hooks {
|
||||
if ah == h {
|
||||
hookIdx = idx
|
||||
break
|
||||
}
|
||||
}
|
||||
a.log.WithField("hook", h.String()).WithField("hook_number", hookIdx+1).Warn("hook did not match any snapshotted filesystems")
|
||||
}
|
||||
}
|
||||
|
||||
return u(func(snapper *Snapper) {
|
||||
if anyFsHadErr {
|
||||
snapper.state = ErrorWait
|
||||
snapper.err = errors.New("one or more snapshots could not be created, check logs for details")
|
||||
} else {
|
||||
snapper.state = Waiting
|
||||
snapper.err = nil
|
||||
}
|
||||
}).sf()
|
||||
}
|
||||
|
||||
func wait(a args, u updater) state {
|
||||
var sleepUntil time.Time
|
||||
u(func(snapper *Snapper) {
|
||||
lastTick := snapper.lastInvocation
|
||||
snapper.sleepUntil = lastTick.Add(a.interval)
|
||||
sleepUntil = snapper.sleepUntil
|
||||
log := a.log.WithField("sleep_until", sleepUntil).WithField("duration", a.interval)
|
||||
logFunc := log.Debug
|
||||
if snapper.state == ErrorWait || snapper.state == SyncUpErrWait {
|
||||
logFunc = log.Error
|
||||
}
|
||||
logFunc("enter wait-state after error")
|
||||
})
|
||||
|
||||
t := time.NewTimer(time.Until(sleepUntil))
|
||||
defer t.Stop()
|
||||
|
||||
select {
|
||||
case <-t.C:
|
||||
return u(func(snapper *Snapper) {
|
||||
snapper.state = Planning
|
||||
}).sf()
|
||||
case <-a.ctx.Done():
|
||||
return onMainCtxDone(a.ctx, u)
|
||||
}
|
||||
}
|
||||
|
||||
func listFSes(ctx context.Context, mf *filters.DatasetMapFilter) (fss []*zfs.DatasetPath, err error) {
|
||||
return zfs.ZFSListMapping(ctx, mf)
|
||||
}
|
||||
|
||||
func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval time.Duration) (syncPoint time.Time, err error) {
|
||||
type snapTime struct {
|
||||
ds *zfs.DatasetPath
|
||||
time time.Time
|
||||
}
|
||||
|
||||
if len(fss) == 0 {
|
||||
return time.Now(), nil
|
||||
}
|
||||
|
||||
snaptimes := make([]snapTime, 0, len(fss))
|
||||
|
||||
now := time.Now()
|
||||
|
||||
log.Debug("examine filesystem state")
|
||||
for _, d := range fss {
|
||||
|
||||
l := log.WithField("fs", d.ToString())
|
||||
|
||||
fsvs, err := zfs.ZFSListFilesystemVersions(d, filters.NewTypedPrefixFilter(prefix, zfs.Snapshot))
|
||||
if err != nil {
|
||||
l.WithError(err).Error("cannot list filesystem versions")
|
||||
continue
|
||||
}
|
||||
if len(fsvs) <= 0 {
|
||||
l.WithField("prefix", prefix).Debug("no filesystem versions with prefix")
|
||||
continue
|
||||
}
|
||||
|
||||
// Sort versions by creation
|
||||
sort.SliceStable(fsvs, func(i, j int) bool {
|
||||
return fsvs[i].CreateTXG < fsvs[j].CreateTXG
|
||||
})
|
||||
|
||||
latest := fsvs[len(fsvs)-1]
|
||||
l.WithField("creation", latest.Creation).
|
||||
Debug("found latest snapshot")
|
||||
|
||||
since := now.Sub(latest.Creation)
|
||||
if since < 0 {
|
||||
l.WithField("snapshot", latest.Name).
|
||||
WithField("creation", latest.Creation).
|
||||
Error("snapshot is from the future")
|
||||
continue
|
||||
}
|
||||
next := now
|
||||
if since < interval {
|
||||
next = latest.Creation.Add(interval)
|
||||
}
|
||||
snaptimes = append(snaptimes, snapTime{d, next})
|
||||
}
|
||||
|
||||
if len(snaptimes) == 0 {
|
||||
snaptimes = append(snaptimes, snapTime{nil, now})
|
||||
}
|
||||
|
||||
sort.Slice(snaptimes, func(i, j int) bool {
|
||||
return snaptimes[i].time.Before(snaptimes[j].time)
|
||||
})
|
||||
|
||||
return snaptimes[0].time, nil
|
||||
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package snapper
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
)
|
||||
|
||||
// FIXME: properly abstract snapshotting:
|
||||
// - split up things that trigger snapshotting from the mechanism
|
||||
// - timer-based trigger (periodic)
|
||||
// - call from control socket (manual)
|
||||
// - mixed modes?
|
||||
// - support a `zrepl snapshot JOBNAME` subcommand for config.SnapshottingManual
|
||||
type PeriodicOrManual struct {
|
||||
s *Snapper
|
||||
}
|
||||
|
||||
func (s *PeriodicOrManual) Run(ctx context.Context, wakeUpCommon chan<- struct{}) {
|
||||
if s.s != nil {
|
||||
s.s.Run(ctx, wakeUpCommon)
|
||||
}
|
||||
}
|
||||
|
||||
// Returns nil if manual
|
||||
func (s *PeriodicOrManual) Report() *Report {
|
||||
if s.s != nil {
|
||||
return s.s.Report()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func FromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in config.SnapshottingEnum) (*PeriodicOrManual, error) {
|
||||
switch v := in.Ret.(type) {
|
||||
case *config.SnapshottingPeriodic:
|
||||
snapper, err := PeriodicFromConfig(g, fsf, v)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &PeriodicOrManual{snapper}, nil
|
||||
case *config.SnapshottingManual:
|
||||
return &PeriodicOrManual{}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown snapshotting type %T", v)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package snapper
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/zrepl/zrepl/daemon/hooks"
|
||||
)
|
||||
|
||||
type Report struct {
|
||||
State State
|
||||
// valid in state SyncUp and Waiting
|
||||
SleepUntil time.Time
|
||||
// valid in state Err
|
||||
Error string
|
||||
// valid in state Snapshotting
|
||||
Progress []*ReportFilesystem
|
||||
}
|
||||
|
||||
type ReportFilesystem struct {
|
||||
Path string
|
||||
State SnapState
|
||||
|
||||
// Valid in SnapStarted and later
|
||||
SnapName string
|
||||
StartAt time.Time
|
||||
Hooks string
|
||||
HooksHadError bool
|
||||
|
||||
// Valid in SnapDone | SnapError
|
||||
DoneAt time.Time
|
||||
}
|
||||
|
||||
func errOrEmptyString(e error) string {
|
||||
if e != nil {
|
||||
return e.Error()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *Snapper) Report() *Report {
|
||||
s.mtx.Lock()
|
||||
defer s.mtx.Unlock()
|
||||
|
||||
pReps := make([]*ReportFilesystem, 0, len(s.plan))
|
||||
for fs, p := range s.plan {
|
||||
var hooksStr string
|
||||
var hooksHadError bool
|
||||
if p.hookPlan != nil {
|
||||
hr := p.hookPlan.Report()
|
||||
// FIXME: technically this belongs into client
|
||||
// but we can't serialize hooks.Step ATM
|
||||
rightPad := func(str string, length int, pad string) string {
|
||||
if len(str) > length {
|
||||
return str[:length]
|
||||
}
|
||||
return str + strings.Repeat(pad, length-len(str))
|
||||
}
|
||||
hooksHadError = hr.HadError()
|
||||
rows := make([][]string, len(hr))
|
||||
const numCols = 4
|
||||
lens := make([]int, numCols)
|
||||
for i, e := range hr {
|
||||
rows[i] = make([]string, numCols)
|
||||
rows[i][0] = fmt.Sprintf("%d", i+1)
|
||||
rows[i][1] = e.Status.String()
|
||||
runTime := "..."
|
||||
if e.Status != hooks.StepPending {
|
||||
runTime = e.End.Sub(e.Begin).Round(time.Millisecond).String()
|
||||
}
|
||||
rows[i][2] = runTime
|
||||
rows[i][3] = ""
|
||||
if e.Report != nil {
|
||||
rows[i][3] = e.Report.String()
|
||||
}
|
||||
for j, col := range lens {
|
||||
if len(rows[i][j]) > col {
|
||||
lens[j] = len(rows[i][j])
|
||||
}
|
||||
}
|
||||
}
|
||||
rowsFlat := make([]string, len(hr))
|
||||
for i, r := range rows {
|
||||
colsPadded := make([]string, len(r))
|
||||
for j, c := range r[:len(r)-1] {
|
||||
colsPadded[j] = rightPad(c, lens[j], " ")
|
||||
}
|
||||
colsPadded[len(r)-1] = r[len(r)-1]
|
||||
rowsFlat[i] = strings.Join(colsPadded, " ")
|
||||
}
|
||||
hooksStr = strings.Join(rowsFlat, "\n")
|
||||
}
|
||||
pReps = append(pReps, &ReportFilesystem{
|
||||
Path: fs.ToString(),
|
||||
State: p.state,
|
||||
SnapName: p.name,
|
||||
StartAt: p.startAt,
|
||||
DoneAt: p.doneAt,
|
||||
Hooks: hooksStr,
|
||||
HooksHadError: hooksHadError,
|
||||
})
|
||||
}
|
||||
|
||||
sort.Slice(pReps, func(i, j int) bool {
|
||||
return strings.Compare(pReps[i].Path, pReps[j].Path) == -1
|
||||
})
|
||||
|
||||
r := &Report{
|
||||
State: s.state,
|
||||
SleepUntil: s.sleepUntil,
|
||||
Error: errOrEmptyString(s.err),
|
||||
Progress: pReps,
|
||||
}
|
||||
|
||||
return r
|
||||
}
|
||||
+595
@@ -0,0 +1,595 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"$$hashKey": "object:3351",
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"gnetId": null,
|
||||
"graphTooltip": 1,
|
||||
"id": 7,
|
||||
"iteration": 1552746418826,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"content": "# zrepl Prometheus Metrics\n\nzrepl exposes Prometheus metrics and ships with this Grafana dashboard.\nThe exported metrics are suitable for health checks:\n\n* The log should generally be warning & error-free\n * The `Log Messages that require attention` graph visualizes log message counts indicating problems.\n* The number of goroutines should not grow unboundedly over time.\n * During replication, the number of goroutines can be way higher than during idle time.\n * If the goroutine count grows with each replication, there is clearly a goroutine leak. Please open a bug report.\n* The sys memory consumption should not grow unboundedly over time.\n * Note that the Go runtime pre-allocates some of its heap from the OS.\n * zrepl actually uses much less memory than allocated from the OS.\n * Since Go 1.11, Go pre-allocates more aggressively.\n* Monitor that some data is replicated, although that metric does not guarantee that replication was successful.\n\n**In general, note that the exported metrics are not stable unless declared otherwise.**",
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 35,
|
||||
"mode": "markdown",
|
||||
"title": "Panel Title",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": null,
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
},
|
||||
"id": 15,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": true,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"$$hashKey": "object:3436",
|
||||
"expr": "up{job='$prom_job_name'}",
|
||||
"format": "time_series",
|
||||
"intervalFactor": 1,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "zrepl Instances Up",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": "5",
|
||||
"min": "0",
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": null,
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 9
|
||||
},
|
||||
"id": 22,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "increase(zrepl_daemon_log_entries{job='$prom_job_name',level=~'warn|error'}[$__interval])",
|
||||
"format": "time_series",
|
||||
"intervalFactor": 1,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Log Messages that require attention",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": "0",
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": null,
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 18
|
||||
},
|
||||
"id": 33,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [
|
||||
{
|
||||
"alias": "/replicated bytes in last.*/",
|
||||
"yaxis": 2
|
||||
}
|
||||
],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(zrepl_replication_bytes_replicated{job='$prom_job_name'}[$__interval])) by (zrepl_job)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "replication data rate zrepl_job={{zrepl_job}}",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum(increase(zrepl_replication_bytes_replicated{job='$prom_job_name'}[10m])) by (zrepl_job)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"interval": "",
|
||||
"intervalFactor": 1,
|
||||
"legendFormat": "replicated bytes in last 10min zrepl_job={{zrepl_job}}",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Replication Data Rate and Volume(integrates last 10min)",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "Bps",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "bytes",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": null,
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 18
|
||||
},
|
||||
"id": 23,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(increase(zrepl_daemon_log_entries{job='$prom_job_name',zrepl_job=~\"^[^_].*\"}[$__interval])) by (instance,zrepl_job)",
|
||||
"format": "time_series",
|
||||
"intervalFactor": 1,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Log Activity (without internal jobs)",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": "0",
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": null,
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 27
|
||||
},
|
||||
"id": 17,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"$$hashKey": "object:3535",
|
||||
"expr": "go_memstats_sys_bytes{job='$prom_job_name'}",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 1,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Memory Allocated by the Go runtime from the OS (should not grow unboundedly)",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "bytes",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": "0",
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": null,
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 27
|
||||
},
|
||||
"id": 19,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "go_goroutines{job='$prom_job_name'}",
|
||||
"format": "time_series",
|
||||
"intervalFactor": 1,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "number of goroutines (should not grow unboundedly)",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": "0",
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"refresh": "1m",
|
||||
"schemaVersion": 16,
|
||||
"style": "dark",
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"allValue": null,
|
||||
"current": {
|
||||
"text": "zrepl",
|
||||
"value": "zrepl"
|
||||
},
|
||||
"datasource": "prometheus",
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Prometheus Job Name",
|
||||
"multi": false,
|
||||
"name": "prom_job_name",
|
||||
"options": [],
|
||||
"query": "label_values(up, job)",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"tagValuesQuery": "",
|
||||
"tags": [],
|
||||
"tagsQuery": "",
|
||||
"type": "query",
|
||||
"useTags": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-2d",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
"refresh_intervals": [
|
||||
"5s",
|
||||
"10s",
|
||||
"30s",
|
||||
"1m",
|
||||
"5m",
|
||||
"15m",
|
||||
"30m",
|
||||
"1h",
|
||||
"2h",
|
||||
"1d"
|
||||
],
|
||||
"time_options": [
|
||||
"5m",
|
||||
"15m",
|
||||
"1h",
|
||||
"6h",
|
||||
"12h",
|
||||
"24h",
|
||||
"2d",
|
||||
"7d",
|
||||
"30d"
|
||||
]
|
||||
},
|
||||
"timezone": "",
|
||||
"title": "zrepl 0.1",
|
||||
"uid": "xTljn4qmk",
|
||||
"version": 6
|
||||
}
|
||||
-1647
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user