Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Christian Schwarz
|
bdd0c9a199 | ||
|
Christian Schwarz
|
498036194d |
+222
-229
@@ -1,8 +1,4 @@
|
|||||||
version: 2.1
|
version: 2.1
|
||||||
orbs:
|
|
||||||
# NB: this is not the Go version, but the Orb version
|
|
||||||
# https://circleci.com/developer/orbs/orb/circleci/go#usage-go-modules-cache
|
|
||||||
go: circleci/go@1.11.0
|
|
||||||
|
|
||||||
commands:
|
commands:
|
||||||
setup-home-local-bin:
|
setup-home-local-bin:
|
||||||
@@ -16,149 +12,132 @@ commands:
|
|||||||
echo "$line" >> $BASH_ENV
|
echo "$line" >> $BASH_ENV
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# NOTE: UV version is defined in .uv-version file at repository root
|
invoke-lazy-sh:
|
||||||
install-docdep:
|
|
||||||
steps:
|
|
||||||
- run:
|
|
||||||
name: Read UV version from .uv-version
|
|
||||||
command: |
|
|
||||||
UV_VERSION=$(cat .uv-version)
|
|
||||||
echo "export UV_VERSION=$UV_VERSION" >> $BASH_ENV
|
|
||||||
# Python is managed by uv - it will automatically download the version
|
|
||||||
# specified in docs/.python-version when needed
|
|
||||||
- run:
|
|
||||||
name: Install uv
|
|
||||||
command: curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh
|
|
||||||
- run:
|
|
||||||
name: Add uv to PATH and set cache dir
|
|
||||||
command: |
|
|
||||||
echo 'export PATH="$HOME/.local/bin:$PATH"' >> $BASH_ENV
|
|
||||||
echo 'export UV_CACHE_DIR="$HOME/.cache/uv"' >> $BASH_ENV
|
|
||||||
- restore_cache:
|
|
||||||
name: Restore uv cache
|
|
||||||
keys:
|
|
||||||
- uv-cache-v1-${UV_VERSION}-{{ checksum "docs/uv.lock" }}
|
|
||||||
- uv-cache-v1-${UV_VERSION}-
|
|
||||||
|
|
||||||
save-uv-cache:
|
|
||||||
steps:
|
|
||||||
- run:
|
|
||||||
name: Prune uv cache for CI
|
|
||||||
command: uv cache prune --ci
|
|
||||||
- save_cache:
|
|
||||||
name: Save uv cache
|
|
||||||
key: uv-cache-v1-${UV_VERSION}-{{ checksum "docs/uv.lock" }}
|
|
||||||
paths:
|
|
||||||
- ~/.cache/uv
|
|
||||||
|
|
||||||
install-zfs-from-source:
|
|
||||||
parameters:
|
parameters:
|
||||||
zfs_release:
|
subcommand:
|
||||||
type: string
|
type: string
|
||||||
steps:
|
steps:
|
||||||
- run:
|
- run:
|
||||||
name: Record kernel version for cache key
|
environment:
|
||||||
command: uname -r > /tmp/kernel-version
|
TERM: xterm
|
||||||
|
command: ./lazy.sh <<parameters.subcommand>>
|
||||||
|
|
||||||
|
apt-update-and-install-common-deps:
|
||||||
|
steps:
|
||||||
|
- run: sudo apt update && sudo apt install gawk make
|
||||||
|
|
||||||
|
restore-cache-gomod:
|
||||||
|
steps:
|
||||||
- restore_cache:
|
- restore_cache:
|
||||||
name: Restore ZFS native debs cache
|
key: go-mod-v4-{{ checksum "go.sum" }}
|
||||||
keys:
|
save-cache-gomod:
|
||||||
- zfs-debs-v2-<<parameters.zfs_release>>-{{ checksum "/tmp/kernel-version" }}
|
steps:
|
||||||
|
- save_cache:
|
||||||
|
key: go-mod-v4-{{ checksum "go.sum" }}
|
||||||
|
paths:
|
||||||
|
- "/go/pkg/mod"
|
||||||
|
|
||||||
|
install-godep:
|
||||||
|
steps:
|
||||||
|
- apt-update-and-install-common-deps
|
||||||
|
- invoke-lazy-sh:
|
||||||
|
subcommand: godep
|
||||||
|
|
||||||
|
install-docdep:
|
||||||
|
steps:
|
||||||
|
- apt-update-and-install-common-deps
|
||||||
|
- run: sudo apt install python3 python3-pip libgirepository1.0-dev
|
||||||
|
- invoke-lazy-sh:
|
||||||
|
subcommand: docdep
|
||||||
|
|
||||||
|
download-and-install-minio-client:
|
||||||
|
steps:
|
||||||
|
- setup-home-local-bin
|
||||||
|
- restore_cache:
|
||||||
|
key: minio-client-v2
|
||||||
- run:
|
- run:
|
||||||
# https://openzfs.github.io/openzfs-docs/Developer%20Resources/Building%20ZFS.html
|
shell: /bin/bash -eo pipefail
|
||||||
name: Build ZFS <<parameters.zfs_release>> native debs (if not cached)
|
|
||||||
no_output_timeout: 20m
|
|
||||||
command: |
|
command: |
|
||||||
# CircleCI machine images have pyenv Python 3.13 shadowing the system
|
if which mc; then exit 0; fi
|
||||||
# Python 3.12. The apt python3-* packages (setuptools, cffi, etc.) only
|
sudo curl -sSL https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2020-08-20T00-23-01Z \
|
||||||
# install for the system Python, and ZFS's dpkg-buildpackage needs them
|
-o "$HOME/.local/bin/mc"
|
||||||
# for --enable-pyzfs. Use the system Python so apt packages are visible.
|
sudo chmod +x "$HOME/.local/bin/mc"
|
||||||
export PYENV_VERSION=system
|
- save_cache:
|
||||||
if [ -d /tmp/zfs-debs ]; then
|
key: minio-client-v2
|
||||||
echo "ZFS debs cache hit, skipping build"
|
paths:
|
||||||
|
- "$HOME/.local/bin/mc"
|
||||||
|
|
||||||
|
upload-minio:
|
||||||
|
parameters:
|
||||||
|
src:
|
||||||
|
type: string
|
||||||
|
dst:
|
||||||
|
type: string
|
||||||
|
steps:
|
||||||
|
- run:
|
||||||
|
shell: /bin/bash -eo pipefail
|
||||||
|
when: always
|
||||||
|
command: |
|
||||||
|
if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
|
||||||
|
echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
sudo apt-get update
|
set -u # from now on
|
||||||
sudo apt-get install -y \
|
|
||||||
alien autoconf automake build-essential debhelper-compat dh-autoreconf \
|
|
||||||
dh-dkms dh-python dkms fakeroot gawk git libaio-dev libattr1-dev \
|
|
||||||
libblkid-dev libcurl4-openssl-dev libelf-dev libffi-dev libpam0g-dev \
|
|
||||||
libssl-dev libtirpc-dev libtool libudev-dev linux-headers-$(uname -r) lsb-release \
|
|
||||||
parallel po-debconf python3 python3-all-dev python3-cffi python3-dev \
|
|
||||||
python3-packaging python3-setuptools python3-sphinx uuid-dev zlib1g-dev
|
|
||||||
mkdir -p /tmp/zfs-build
|
|
||||||
cd /tmp/zfs-build
|
|
||||||
git clone --depth 1 --branch <<parameters.zfs_release>> https://github.com/openzfs/zfs.git zfs-src
|
|
||||||
cd zfs-src
|
|
||||||
sh autogen.sh
|
|
||||||
./configure
|
|
||||||
make native-deb
|
|
||||||
mkdir -p /tmp/zfs-debs
|
|
||||||
find /tmp/zfs-build -name '*.deb' -exec mv -t /tmp/zfs-debs/ {} +
|
|
||||||
- save_cache:
|
|
||||||
name: Save ZFS native debs cache
|
|
||||||
key: zfs-debs-v2-<<parameters.zfs_release>>-{{ checksum "/tmp/kernel-version" }}
|
|
||||||
paths:
|
|
||||||
- /tmp/zfs-debs
|
|
||||||
- run:
|
|
||||||
name: Install ZFS <<parameters.zfs_release>>
|
|
||||||
command: |
|
|
||||||
ls /tmp/zfs-debs/
|
|
||||||
# Only install the debs we need. Exclude dracut (conflicts with
|
|
||||||
# initramfs-tools), dkms (we have prebuilt modules), and packages
|
|
||||||
# we don't need (pyzfs, test, doc).
|
|
||||||
sudo apt-get install -y $(find /tmp/zfs-debs -name '*.deb' \
|
|
||||||
! -name '*dracut*' \
|
|
||||||
! -name '*dkms*' \
|
|
||||||
! -name '*initramfs*' \
|
|
||||||
! -name '*pyzfs*' \
|
|
||||||
! -name '*doc*' \
|
|
||||||
! -name '*test*' \
|
|
||||||
-print)
|
|
||||||
sudo modprobe zfs
|
|
||||||
- run:
|
|
||||||
name: Verify ZFS version
|
|
||||||
command: |
|
|
||||||
sudo zfs version
|
|
||||||
sudo zpool version
|
|
||||||
|
|
||||||
docs-publish-sh:
|
mc config host add --api s3v4 zrepl-minio https://minio.cschwarz.com ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
|
||||||
|
|
||||||
|
# keep in sync with set-github-minio-status
|
||||||
|
jobprefix=zrepl-ci-artifacts/${CIRCLE_SHA1}-pipeline-<<pipeline.number>>/${CIRCLE_JOB}
|
||||||
|
|
||||||
|
# Upload artifacts
|
||||||
|
mkdir -p ./artifacts
|
||||||
|
mc cp -r <<parameters.src>> "zrepl-minio/$jobprefix/<<parameters.dst>>"
|
||||||
|
|
||||||
|
set-github-minio-status:
|
||||||
parameters:
|
parameters:
|
||||||
push:
|
context:
|
||||||
type: boolean
|
type: string
|
||||||
|
description:
|
||||||
|
type: string
|
||||||
|
minio-dst:
|
||||||
|
type: string
|
||||||
steps:
|
steps:
|
||||||
- checkout
|
|
||||||
- run:
|
- run:
|
||||||
|
shell: /bin/bash -eo pipefail
|
||||||
command: |
|
command: |
|
||||||
git config --global user.email "zreplbot@cschwarz.com"
|
if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
|
||||||
git config --global user.name "zrepl-github-io-ci"
|
echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
set -u # from now on
|
||||||
|
|
||||||
# Configure git to use the GitHub token for HTTPS authentication.
|
# keep in sync with with upload-minio command
|
||||||
# The token is stored in the 'zrepl-github-io-deploy' context.
|
jobprefix=zrepl-ci-artifacts/${CIRCLE_SHA1}-pipeline-<<pipeline.number>>/${CIRCLE_JOB}
|
||||||
- when:
|
# Push Artifact Link to GitHub
|
||||||
condition: << parameters.push >>
|
REPO="zrepl/zrepl"
|
||||||
steps:
|
COMMIT="${CIRCLE_SHA1}"
|
||||||
- run:
|
JOB_NAME="${CIRCLE_JOB}"
|
||||||
name: Configure git to use GitHub token for push
|
CONTEXT="<<parameters.context>>"
|
||||||
# GITHUB_PAGES_TOKEN is from the 'zrepl-github-io-deploy' context.
|
DESCRIPTION="<<parameters.description>>"
|
||||||
# CircleCI's secret masking automatically redacts context variables in logs.
|
TARGETURL=https://minio.cschwarz.com/minio/"$jobprefix"/"<<parameters.minio-dst>>"
|
||||||
command: |
|
curl "https://api.github.com/repos/$REPO/statuses/$COMMIT" \
|
||||||
# Unset CircleCI's SSH URL rewriting that checkout step configured
|
-H "Content-Type: application/json" \
|
||||||
git config --global --unset-all url."ssh://git@github.com".insteadOf || true
|
-H "Authorization: token $GITHUB_COMMIT_STATUS_TOKEN" \
|
||||||
# Set up credential helper with GitHub token
|
-X POST \
|
||||||
git config --global credential.helper store
|
-d '{"context":"'"$CONTEXT"'", "state": "success", "description":"'"$DESCRIPTION"'", "target_url":"'"$TARGETURL"'"}'
|
||||||
echo "https://x-access-token:${GITHUB_PAGES_TOKEN}@github.com" > ~/.git-credentials
|
|
||||||
chmod 600 ~/.git-credentials
|
|
||||||
|
|
||||||
# caller must install-docdep
|
|
||||||
- when:
|
trigger-pipeline:
|
||||||
condition: << parameters.push >>
|
parameters:
|
||||||
steps:
|
body_no_shell_subst:
|
||||||
- run: bash -x docs/publish.sh -c -a -P
|
type: string
|
||||||
- when:
|
steps:
|
||||||
condition:
|
- run: |
|
||||||
not: << parameters.push >>
|
curl -X POST https://circleci.com/api/v2/project/github/zrepl/zrepl/pipeline \
|
||||||
steps:
|
-H 'Content-Type: application/json' \
|
||||||
- run: bash -x docs/publish.sh -c -a
|
-H 'Accept: application/json' \
|
||||||
|
-H "Circle-Token: $ZREPL_BOT_CIRCLE_TOKEN" \
|
||||||
|
--data '<<parameters.body_no_shell_subst>>'
|
||||||
|
|
||||||
parameters:
|
parameters:
|
||||||
do_ci:
|
do_ci:
|
||||||
@@ -169,29 +148,31 @@ parameters:
|
|||||||
type: boolean
|
type: boolean
|
||||||
default: false
|
default: false
|
||||||
|
|
||||||
|
release_docker_baseimage_tag:
|
||||||
|
type: string
|
||||||
|
default: "1.19"
|
||||||
|
|
||||||
workflows:
|
workflows:
|
||||||
version: 2
|
version: 2
|
||||||
|
|
||||||
ci:
|
ci:
|
||||||
when: << pipeline.parameters.do_ci >>
|
when: << pipeline.parameters.do_ci >>
|
||||||
jobs:
|
jobs:
|
||||||
- run-docs-publish-sh:
|
- quickcheck-docs
|
||||||
name: quickcheck-docs
|
- quickcheck-go: &quickcheck-go-smoketest
|
||||||
push: false
|
name: quickcheck-go-amd64-linux-1.19
|
||||||
- quickcheck-go:
|
goversion: &latest-go-release "1.19"
|
||||||
name: quickcheck-go-amd64-linux-1.25.7
|
|
||||||
goversion: &latest-go-release "1.25.7"
|
|
||||||
goos: linux
|
goos: linux
|
||||||
goarch: amd64
|
goarch: amd64
|
||||||
- test-go:
|
- test-go-on-latest-go-release:
|
||||||
goversion: *latest-go-release
|
goversion: *latest-go-release
|
||||||
- quickcheck-go:
|
- quickcheck-go:
|
||||||
requires:
|
requires:
|
||||||
- quickcheck-go-amd64-linux-1.25.7 #quickcheck-go-smoketest.name
|
- quickcheck-go-amd64-linux-1.19 #quickcheck-go-smoketest.name
|
||||||
matrix:
|
matrix: &quickcheck-go-matrix
|
||||||
alias: quickcheck-go-matrix
|
alias: quickcheck-go-matrix
|
||||||
parameters:
|
parameters:
|
||||||
goversion: [*latest-go-release, "1.24.13"]
|
goversion: [*latest-go-release, "1.12"]
|
||||||
goos: ["linux", "freebsd"]
|
goos: ["linux", "freebsd"]
|
||||||
goarch: ["amd64", "arm64"]
|
goarch: ["amd64", "arm64"]
|
||||||
exclude:
|
exclude:
|
||||||
@@ -199,16 +180,10 @@ workflows:
|
|||||||
- goversion: *latest-go-release
|
- goversion: *latest-go-release
|
||||||
goos: linux
|
goos: linux
|
||||||
goarch: amd64
|
goarch: amd64
|
||||||
- platformtest:
|
# not supported by Go 1.12
|
||||||
matrix:
|
- goversion: "1.12"
|
||||||
parameters:
|
goos: freebsd
|
||||||
goversion: [*latest-go-release]
|
goarch: arm64
|
||||||
goos: ["linux"]
|
|
||||||
goarch: ["amd64"]
|
|
||||||
zfs_release: ["zfs-2.2.9", "zfs-2.3.5", "zfs-2.4.0"]
|
|
||||||
requires:
|
|
||||||
- test-go
|
|
||||||
- quickcheck-go-<< matrix.goarch >>-<< matrix.goos >>-<< matrix.goversion >>
|
|
||||||
|
|
||||||
release:
|
release:
|
||||||
when: << pipeline.parameters.do_release >>
|
when: << pipeline.parameters.do_release >>
|
||||||
@@ -226,31 +201,44 @@ workflows:
|
|||||||
- release-deb
|
- release-deb
|
||||||
- release-rpm
|
- release-rpm
|
||||||
|
|
||||||
publish-zrepl.github.io:
|
periodic:
|
||||||
jobs:
|
triggers:
|
||||||
- run-docs-publish-sh:
|
- schedule:
|
||||||
name: publish-zrepl.github.io
|
cron: "00 17 * * *"
|
||||||
push: true
|
|
||||||
context:
|
|
||||||
- zrepl-github-io-deploy
|
|
||||||
filters:
|
filters:
|
||||||
branches:
|
branches:
|
||||||
only:
|
only:
|
||||||
- master
|
- master
|
||||||
|
- stable
|
||||||
|
- problame/circleci-build
|
||||||
|
jobs:
|
||||||
|
- periodic-full-pipeline-run
|
||||||
|
|
||||||
|
zrepl.github.io:
|
||||||
|
jobs:
|
||||||
|
- publish-zrepl-github-io:
|
||||||
|
filters:
|
||||||
|
branches:
|
||||||
|
only:
|
||||||
|
- stable
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
run-docs-publish-sh:
|
quickcheck-docs:
|
||||||
parameters:
|
|
||||||
push:
|
|
||||||
type: boolean
|
|
||||||
docker:
|
docker:
|
||||||
- image: cimg/base:current
|
- image: cimg/base:2020.08
|
||||||
steps:
|
steps:
|
||||||
- checkout
|
- checkout
|
||||||
- install-docdep
|
- install-docdep
|
||||||
- docs-publish-sh:
|
- run: make docs
|
||||||
push: << parameters.push >>
|
|
||||||
- save-uv-cache
|
- download-and-install-minio-client
|
||||||
|
- upload-minio:
|
||||||
|
src: artifacts
|
||||||
|
dst: ""
|
||||||
|
- set-github-minio-status:
|
||||||
|
context: artifacts/${CIRCLE_JOB}
|
||||||
|
description: artifacts of CI job ${CIRCLE_JOB}
|
||||||
|
minio-dst: ""
|
||||||
|
|
||||||
quickcheck-go:
|
quickcheck-go:
|
||||||
parameters:
|
parameters:
|
||||||
@@ -261,99 +249,62 @@ jobs:
|
|||||||
goarch:
|
goarch:
|
||||||
type: string
|
type: string
|
||||||
docker:
|
docker:
|
||||||
- image: &cimg_with_modern_go cimg/go:1.25
|
- image: cimg/go:<<parameters.goversion>>
|
||||||
environment:
|
environment:
|
||||||
GOOS: <<parameters.goos>>
|
GOOS: <<parameters.goos>>
|
||||||
GOARCH: <<parameters.goarch>>
|
GOARCH: <<parameters.goarch>>
|
||||||
GOTOOLCHAIN: "go<<parameters.goversion>>"
|
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- checkout
|
- checkout
|
||||||
|
|
||||||
- go/load-cache:
|
- restore-cache-gomod
|
||||||
key: quickcheck-<<parameters.goversion>>
|
|
||||||
- run: make build/install
|
|
||||||
- run: go mod download
|
- run: go mod download
|
||||||
- run: cd build && go mod download
|
- run: cd build && go mod download
|
||||||
- go/save-cache:
|
- save-cache-gomod
|
||||||
key: quickcheck-<<parameters.goversion>>
|
|
||||||
|
|
||||||
# ensure all code has been generated
|
- install-godep
|
||||||
- run: make generate
|
- run: make formatcheck
|
||||||
- run: |
|
- run: make generate-platform-test-list
|
||||||
if output=$(git status --porcelain) && [ -z "$output" ]; then
|
|
||||||
echo "Working directory clean"
|
|
||||||
else
|
|
||||||
echo "Uncommitted changes"
|
|
||||||
echo ""
|
|
||||||
echo "$output"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# other checks
|
|
||||||
- run: make zrepl-bin test-platform-bin
|
- run: make zrepl-bin test-platform-bin
|
||||||
- run: make vet
|
- run: make vet
|
||||||
- run: make lint
|
- run: make lint
|
||||||
|
|
||||||
|
- download-and-install-minio-client
|
||||||
|
- run: rm -f artifacts/generate-platform-test-list
|
||||||
- store_artifacts:
|
- store_artifacts:
|
||||||
path: artifacts
|
path: artifacts
|
||||||
- persist_to_workspace:
|
- upload-minio:
|
||||||
root: .
|
src: artifacts
|
||||||
paths: [.]
|
dst: ""
|
||||||
|
- set-github-minio-status:
|
||||||
|
context: artifacts/${CIRCLE_JOB}
|
||||||
|
description: artifacts of CI job ${CIRCLE_JOB}
|
||||||
|
minio-dst: ""
|
||||||
|
|
||||||
platformtest:
|
test-go-on-latest-go-release:
|
||||||
parameters:
|
|
||||||
goversion:
|
|
||||||
type: string
|
|
||||||
goos:
|
|
||||||
type: string
|
|
||||||
goarch:
|
|
||||||
type: string
|
|
||||||
zfs_release:
|
|
||||||
type: string
|
|
||||||
machine:
|
|
||||||
# pinned (not :current) to keep ZFS build cache valid across runs
|
|
||||||
image: ubuntu-2404:2025.09.1
|
|
||||||
resource_class: medium
|
|
||||||
environment:
|
|
||||||
GOOS: <<parameters.goos>>
|
|
||||||
GOARCH: <<parameters.goarch>>
|
|
||||||
steps:
|
|
||||||
- attach_workspace:
|
|
||||||
at: .
|
|
||||||
- install-zfs-from-source:
|
|
||||||
zfs_release: <<parameters.zfs_release>>
|
|
||||||
- run: sudo make test-platform GOOS="$GOOS" GOARCH="$GOARCH"
|
|
||||||
|
|
||||||
test-go:
|
|
||||||
parameters:
|
parameters:
|
||||||
goversion:
|
goversion:
|
||||||
type: string
|
type: string
|
||||||
docker:
|
docker:
|
||||||
- image: *cimg_with_modern_go
|
- image: cimg/go:<<parameters.goversion>>
|
||||||
environment:
|
|
||||||
GOTOOLCHAIN: "go<<parameters.goversion>>"
|
|
||||||
steps:
|
steps:
|
||||||
- checkout
|
- checkout
|
||||||
- go/load-cache:
|
- restore-cache-gomod
|
||||||
key: make-test-go
|
|
||||||
- run: make test-go
|
- run: make test-go
|
||||||
- go/save-cache:
|
# don't save-cache-gomod here, test-go doesn't pull all the dependencies
|
||||||
key: make-test-go
|
|
||||||
|
|
||||||
release-build:
|
release-build:
|
||||||
machine:
|
machine:
|
||||||
image: &release-vm-image "ubuntu-2404:current"
|
image: ubuntu-2004:202201-02
|
||||||
resource_class: large
|
|
||||||
steps:
|
steps:
|
||||||
- checkout
|
- checkout
|
||||||
- run: make release-docker
|
- run: make release-docker RELEASE_DOCKER_BASEIMAGE_TAG=<<pipeline.parameters.release_docker_baseimage_tag>>
|
||||||
- persist_to_workspace:
|
- persist_to_workspace:
|
||||||
root: .
|
root: .
|
||||||
paths: [.]
|
paths: [.]
|
||||||
release-deb:
|
release-deb:
|
||||||
machine:
|
machine:
|
||||||
image: *release-vm-image
|
image: ubuntu-2004:202201-02
|
||||||
steps:
|
steps:
|
||||||
- attach_workspace:
|
- attach_workspace:
|
||||||
at: .
|
at: .
|
||||||
@@ -365,7 +316,7 @@ jobs:
|
|||||||
|
|
||||||
release-rpm:
|
release-rpm:
|
||||||
machine:
|
machine:
|
||||||
image: *release-vm-image
|
image: ubuntu-2004:202201-02
|
||||||
steps:
|
steps:
|
||||||
- attach_workspace:
|
- attach_workspace:
|
||||||
at: .
|
at: .
|
||||||
@@ -377,10 +328,52 @@ jobs:
|
|||||||
|
|
||||||
release-upload:
|
release-upload:
|
||||||
docker:
|
docker:
|
||||||
- image: cimg/base:2024.09
|
- image: cimg/base:2020.08
|
||||||
steps:
|
steps:
|
||||||
- attach_workspace:
|
- attach_workspace:
|
||||||
at: .
|
at: .
|
||||||
- run: make wrapup-and-checksum
|
|
||||||
- store_artifacts:
|
- store_artifacts:
|
||||||
path: artifacts/release
|
path: artifacts
|
||||||
|
- download-and-install-minio-client
|
||||||
|
- upload-minio:
|
||||||
|
src: artifacts
|
||||||
|
dst: ""
|
||||||
|
- set-github-minio-status:
|
||||||
|
context: artifacts/release
|
||||||
|
description: CI-generated release artifacts
|
||||||
|
minio-dst: ""
|
||||||
|
|
||||||
|
periodic-full-pipeline-run:
|
||||||
|
docker:
|
||||||
|
- image: cimg/base:2020.08
|
||||||
|
steps:
|
||||||
|
- trigger-pipeline:
|
||||||
|
body_no_shell_subst: '{"branch":"<<pipeline.git.branch>>", "parameters": { "do_ci": true, "do_release": true }}'
|
||||||
|
|
||||||
|
publish-zrepl-github-io:
|
||||||
|
docker:
|
||||||
|
- image: cimg/python:3.7
|
||||||
|
steps:
|
||||||
|
- checkout
|
||||||
|
- invoke-lazy-sh:
|
||||||
|
subcommand: docdep
|
||||||
|
- run:
|
||||||
|
command: |
|
||||||
|
git config --global user.email "zreplbot@cschwarz.com"
|
||||||
|
git config --global user.name "zrepl-github-io-ci"
|
||||||
|
|
||||||
|
# https://circleci.com/docs/2.0/add-ssh-key/#adding-multiple-keys-with-blank-hostnames
|
||||||
|
- run: ssh-add -D
|
||||||
|
# the default circleci ssh config only additional ssh keys for Host !github.com
|
||||||
|
- run:
|
||||||
|
command: |
|
||||||
|
cat > ~/.ssh/config \<<EOF
|
||||||
|
Host *
|
||||||
|
IdentityFile /home/circleci/.ssh/id_rsa_458e62c517f6c480e40452126ce47421
|
||||||
|
EOF
|
||||||
|
- add_ssh_keys:
|
||||||
|
fingerprints:
|
||||||
|
# deploy key for zrepl.github.io
|
||||||
|
- "45:8e:62:c5:17:f6:c4:80:e4:04:52:12:6c:e4:74:21"
|
||||||
|
|
||||||
|
- run: bash -x docs/publish.sh -c -a
|
||||||
|
|||||||
@@ -1,92 +0,0 @@
|
|||||||
import argparse
|
|
||||||
from pathlib import Path
|
|
||||||
import re
|
|
||||||
|
|
||||||
import requests
|
|
||||||
|
|
||||||
import time
|
|
||||||
|
|
||||||
import os
|
|
||||||
import yaml
|
|
||||||
import argparse
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
|
|
||||||
circle_token = os.environ.get('CIRCLE_TOKEN')
|
|
||||||
if not circle_token:
|
|
||||||
cli_yml = Path.home() / ".circleci" / "cli.yml"
|
|
||||||
if cli_yml.exists():
|
|
||||||
with open(cli_yml) as f:
|
|
||||||
data = yaml.safe_load(f)
|
|
||||||
if data:
|
|
||||||
circle_token = data.get("token")
|
|
||||||
if not circle_token:
|
|
||||||
raise ValueError('CIRCLE_TOKEN not set and no token found in ~/.circleci/cli.yml')
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description='Download artifacts from CircleCI')
|
|
||||||
parser.add_argument('build_num', type=str, help='Build number')
|
|
||||||
parser.add_argument('dst', type=Path, help='Destination directory')
|
|
||||||
parser.add_argument('--prefix', type=str, default='', help='Filter for prefix')
|
|
||||||
parser.add_argument('--match', type=str, default='.*', help='Only include paths matching the given regex')
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
res = requests.get(
|
|
||||||
f"https://circleci.com/api/v1.1/project/github/zrepl/zrepl/{args.build_num}/artifacts",
|
|
||||||
headers={
|
|
||||||
"Circle-Token": circle_token,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
res.raise_for_status()
|
|
||||||
|
|
||||||
# https://circleci.com/docs/api/v1/index.html#artifacts-of-a-job
|
|
||||||
# [ {
|
|
||||||
# "path" : "raw-test-output/go-test-report.xml",
|
|
||||||
# "pretty_path" : "raw-test-output/go-test-report.xml",
|
|
||||||
# "node_index" : 0,
|
|
||||||
# "url" : "https://24-88881093-gh.circle-artifacts.com/0/raw-test-output/go-test-report.xml"
|
|
||||||
# }, {
|
|
||||||
# "path" : "raw-test-output/go-test.out",
|
|
||||||
# "pretty_path" : "raw-test-output/go-test.out",
|
|
||||||
# "node_index" : 0,
|
|
||||||
# "url" : "https://24-88881093-gh.circle-artifacts.com/0/raw-test-output/go-test.out"
|
|
||||||
# } ]
|
|
||||||
res = res.json()
|
|
||||||
|
|
||||||
for artifact in res:
|
|
||||||
if not artifact["pretty_path"].startswith(args.prefix):
|
|
||||||
continue
|
|
||||||
if not re.match(args.match, artifact["pretty_path"]):
|
|
||||||
continue
|
|
||||||
stripped = artifact["pretty_path"][len(args.prefix):]
|
|
||||||
print(f"Downloading {artifact['pretty_path']} to {args.dst / stripped}")
|
|
||||||
artifact_rel = Path(stripped)
|
|
||||||
artifact_dst = args.dst / artifact_rel
|
|
||||||
artifact_dst.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
res = requests.get(
|
|
||||||
artifact["url"],
|
|
||||||
headers={
|
|
||||||
"Circle-Token": circle_token,
|
|
||||||
},
|
|
||||||
stream=True,
|
|
||||||
)
|
|
||||||
res.raise_for_status()
|
|
||||||
|
|
||||||
total_size = int(res.headers.get("Content-Length", 0))
|
|
||||||
block_size = 128 * 1024
|
|
||||||
with open(artifact_dst, "wb") as f:
|
|
||||||
progress = 0
|
|
||||||
start_time = time.time()
|
|
||||||
for chunk in res.iter_content(chunk_size=block_size):
|
|
||||||
f.write(chunk)
|
|
||||||
progress += len(chunk)
|
|
||||||
percent = progress / total_size * 100
|
|
||||||
elapsed_time = time.time() - start_time
|
|
||||||
if elapsed_time >= 5:
|
|
||||||
print(f"Downloaded {progress}/{total_size} bytes ({percent:.2f}%)", end="\r")
|
|
||||||
start_time = time.time()
|
|
||||||
print(f"Downloaded {progress}/{total_size} bytes ({percent:.2f}%)")
|
|
||||||
print("Download complete!")
|
|
||||||
|
|
||||||
print("All files downloaded")
|
|
||||||
+10
@@ -0,0 +1,10 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
COMMIT="$1"
|
||||||
|
GO_VERSION="$2"
|
||||||
|
|
||||||
|
curl -v -X POST https://api.github.com/repos/zrepl/debian-binary-packaging/dispatches \
|
||||||
|
-H 'Accept: application/vnd.github.v3+json' \
|
||||||
|
-H "Authorization: token $GITHUB_ACCESS_TOKEN" \
|
||||||
|
--data '{"event_type": "push", "client_payload": { "zrepl_main_repo_commit": "'"$COMMIT"'", "go_version": "'"$GO_VERSION"'" }}'
|
||||||
@@ -1,95 +0,0 @@
|
|||||||
---
|
|
||||||
description: Generate GitHub release notes from docs/changelog.rst and create a draft release with artifacts
|
|
||||||
argument-hint: <version-tag, e.g. v0.7.0>
|
|
||||||
model: sonnet
|
|
||||||
allowed-tools: Read, Write
|
|
||||||
---
|
|
||||||
|
|
||||||
# Task
|
|
||||||
|
|
||||||
Create a draft GitHub release for zrepl version $ARGUMENTS and upload release artifacts.
|
|
||||||
|
|
||||||
## Inputs
|
|
||||||
|
|
||||||
Here is the current changelog RST source:
|
|
||||||
|
|
||||||
!`cat docs/changelog.rst`
|
|
||||||
|
|
||||||
## Instructions
|
|
||||||
|
|
||||||
### Phase 1: Pre-flight Sanity Checks
|
|
||||||
|
|
||||||
Before creating the release, verify that the previous build steps have been completed.
|
|
||||||
Run these checks using bash commands:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
test -d artifacts/release && echo "✓ artifacts/release directory exists" || echo "✗ Missing artifacts/release directory - run: make download-circleci-release JOB_NUM=<num>"
|
|
||||||
test -f artifacts/release/sha512sum.txt && echo "✓ sha512sum.txt exists" || echo "✗ Missing sha512sum.txt - run: make download-circleci-release JOB_NUM=<num>"
|
|
||||||
test -f artifacts/release/sha512sum.txt.asc && echo "✓ sha512sum.txt.asc exists (signed)" || echo "✗ Missing signature - run: make verify-and-sign"
|
|
||||||
ls artifacts/release/zrepl-* >/dev/null 2>&1 && echo "✓ Release artifacts present" || echo "✗ No release artifacts found"
|
|
||||||
```
|
|
||||||
|
|
||||||
If ANY check fails (shows ✗), **STOP** and print the error messages. Do not proceed with release creation.
|
|
||||||
|
|
||||||
### Phase 2: Generate Release Notes
|
|
||||||
|
|
||||||
2. Extract the changelog section for version $ARGUMENTS from the RST source above.
|
|
||||||
3. Extract all GitHub usernames mentioned in that changelog section (look for @username patterns or other contributor attributions).
|
|
||||||
4. Write a short "Highlights" section (2-4 bullets) summarizing the most impactful user-facing changes in plain language. Do NOT include commit/issue links.
|
|
||||||
5. Write a "Contributors" thank you line for all GitHub users found in step 3. Format as: "Thanks to @user1, @user2, and @user3 for their contributions to this release!"
|
|
||||||
6. Write a "Breaking Changes" section. Convert RST formatting to plain Markdown text (no commit or issue links):
|
|
||||||
- `:commit:\`abc123\`` → just remove it or describe what it changed
|
|
||||||
- `:issue:\`123\`` → just remove it or describe the issue in plain text
|
|
||||||
- `:ref:\`display text <anchor>\`` → just use `display text` as plain text
|
|
||||||
- `:repomasterlink:\`path\`` → just mention the path without a link
|
|
||||||
- RST inline code ` ``code`` ` → markdown `` `code` ``
|
|
||||||
- RST links `` `text <url>`_ `` → just use `text` without the link
|
|
||||||
If there are no breaking changes, write "No breaking changes. vX.Y.Z-1 is interoperable with vX.Y.Z." (fill in actual versions).
|
|
||||||
7. Assemble the full release notes using the template below. **IMPORTANT**: Do NOT include a detailed changelog list. Only include what is specified in the template.
|
|
||||||
8. Write the result to `artifacts/release-notes.md`.
|
|
||||||
9. Validate all links in the release notes by checking HTTP status codes with curl:
|
|
||||||
- Extract all URLs from the markdown file
|
|
||||||
- Check each URL with `curl -sI -w "%{http_code}" -o /dev/null <url>`
|
|
||||||
- Report any broken links (non-200 status codes)
|
|
||||||
- If any links are broken, stop and notify the user before creating the release
|
|
||||||
|
|
||||||
### Phase 3: Create Draft Release and Upload Artifacts
|
|
||||||
|
|
||||||
10. Run: `gh release create $ARGUMENTS --title "$ARGUMENTS" --notes-file artifacts/release-notes.md --draft`
|
|
||||||
11. Upload all artifacts: `gh release upload $ARGUMENTS artifacts/release/*`
|
|
||||||
12. Verify upload succeeded: `gh release view $ARGUMENTS --json assets --jq '.assets | length'`
|
|
||||||
13. Print the release URL and confirm that artifacts were uploaded successfully.
|
|
||||||
|
|
||||||
## Template
|
|
||||||
|
|
||||||
```
|
|
||||||
The full changelog is [on the docs site](https://zrepl.github.io/changelog.html).
|
|
||||||
|
|
||||||
## Highlights
|
|
||||||
|
|
||||||
{2-4 bullet plain-language summary of the most impactful changes}
|
|
||||||
|
|
||||||
{Thank all GitHub users mentioned in the changelog, e.g., "Thanks to @user1, @user2, and @user3 for their contributions to this release!"}
|
|
||||||
|
|
||||||
## Breaking Changes
|
|
||||||
|
|
||||||
{breaking changes, or "No breaking changes."}
|
|
||||||
|
|
||||||
## New Users
|
|
||||||
|
|
||||||
We provide [quick-start guides](https://zrepl.github.io/quickstart.html) for different usage scenarios.
|
|
||||||
We also recommend studying the [overview section of the configuration chapter](https://zrepl.github.io/configuration/overview.html).
|
|
||||||
|
|
||||||
## Testing & Upgrading
|
|
||||||
|
|
||||||
* Read the [Changelog](https://zrepl.github.io/changelog.html)
|
|
||||||
* [Run the platform tests](https://zrepl.github.io/usage.html#platform-tests) on a test system.
|
|
||||||
* Download & deploy the `zrepl` binary / distro package.
|
|
||||||
|
|
||||||
## Donations
|
|
||||||
|
|
||||||
zrepl is a spare-time project primarily developed by [Christian Schwarz](https://cschwarz.com).
|
|
||||||
Express your support through a donation to keep maintenance and feature development going.
|
|
||||||
|
|
||||||
[](https://patreon.com/zrepl) [](https://github.com/sponsors/problame) [](https://liberapay.com/zrepl/donate) [](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
|
|
||||||
```
|
|
||||||
@@ -1,149 +0,0 @@
|
|||||||
---
|
|
||||||
description: Update supporters list from git history and changelog contributors
|
|
||||||
argument-hint: <time-range-or-release>
|
|
||||||
model: sonnet
|
|
||||||
---
|
|
||||||
|
|
||||||
# Task
|
|
||||||
|
|
||||||
Automatically update `docs/supporters.rst` by extracting GitHub contributors from:
|
|
||||||
1. Git commit history (code contributors)
|
|
||||||
2. `docs/changelog.rst` (all mentioned GitHub users)
|
|
||||||
|
|
||||||
## Arguments
|
|
||||||
|
|
||||||
Specify a time range or release range to filter contributors:
|
|
||||||
|
|
||||||
**Time range examples:**
|
|
||||||
- `--since="2024-01-01"` - Contributors since a specific date
|
|
||||||
- `--since="1 year ago"` - Contributors in the last year
|
|
||||||
- `--since="6 months ago"` - Contributors in the last 6 months
|
|
||||||
|
|
||||||
**Release range examples:**
|
|
||||||
- `v0.6.0..HEAD` - Contributors between v0.6.0 and current HEAD
|
|
||||||
- `v0.6.0..v0.7.0` - Contributors between two releases
|
|
||||||
- `--all` - All contributors (default if no argument provided)
|
|
||||||
|
|
||||||
## Instructions
|
|
||||||
|
|
||||||
### Step 1: Extract Contributors from Git History
|
|
||||||
|
|
||||||
Parse the arguments to determine the git log range:
|
|
||||||
- If `--since="..."` is provided, use: `git log --since="..." --format='%aN <%aE>' | sort -u`
|
|
||||||
- If a commit range like `v0.6.0..HEAD` is provided, use: `git log v0.6.0..HEAD --format='%aN <%aE>' | sort -u`
|
|
||||||
- If `--all` or no argument, use: `git log --format='%aN <%aE>' | sort -u`
|
|
||||||
|
|
||||||
Parse the output to identify:
|
|
||||||
- GitHub users (look for @users.noreply.github.com emails or known GitHub patterns)
|
|
||||||
- Real names of code contributors
|
|
||||||
- Filter out the main maintainer (Christian Schwarz / problame) as they're already credited
|
|
||||||
|
|
||||||
### Step 2: Extract GitHub Users from Changelog
|
|
||||||
|
|
||||||
Read `docs/changelog.rst` and extract all GitHub usernames mentioned:
|
|
||||||
- Pattern: `` `@username <https://github.com/username>`_ ``
|
|
||||||
- Pattern: `@username` in text
|
|
||||||
- Collect all unique usernames
|
|
||||||
|
|
||||||
### Step 3: Categorize Contributors
|
|
||||||
|
|
||||||
For each contributor, determine their tier:
|
|
||||||
|
|
||||||
**Code contributors** (`|supporter-code|`):
|
|
||||||
- Anyone who appears in git history as a commit author
|
|
||||||
- Prioritize those with substantial contributions (multiple commits)
|
|
||||||
|
|
||||||
**Other contributors** (keep existing `|supporter-gold|` and `|supporter-std|`):
|
|
||||||
- Keep all existing gold/std supporters as-is (monetary supporters)
|
|
||||||
- Only update code contributors
|
|
||||||
|
|
||||||
### Step 4: Read Current Supporters File
|
|
||||||
|
|
||||||
Read `docs/supporters.rst` and parse existing entries to:
|
|
||||||
- Preserve all `|supporter-gold|` entries (monetary supporters - don't touch these)
|
|
||||||
- Preserve all `|supporter-std|` entries (monetary supporters - don't touch these)
|
|
||||||
- Preserve ALL existing `|supporter-code|` entries in their current order
|
|
||||||
- Extract list of existing code contributor names/GitHub usernames to avoid duplicates
|
|
||||||
|
|
||||||
### Step 5: Identify Contributors to Promote/Add
|
|
||||||
|
|
||||||
**IMPORTANT: Find contributors in the range, and either ADD them (if new) or MOVE them up (if recurring).**
|
|
||||||
|
|
||||||
For each GitHub user found in changelog or git history for the specified range:
|
|
||||||
1. Check if they're already listed as `|supporter-code|` in the existing file
|
|
||||||
2. Categorize them:
|
|
||||||
- **NEW**: Not currently in the file → will be added
|
|
||||||
- **RECURRING**: Already in the file with new contributions in this range → will be moved to top
|
|
||||||
- **UNCHANGED**: Not in this range → stays in original position
|
|
||||||
3. For both NEW and RECURRING contributors:
|
|
||||||
- Get their latest contribution date from the range: `git log <range> --author="email" --format="%ai" -1`
|
|
||||||
- Format as: `* |supporter-code| `Name <https://github.com/username>`_`
|
|
||||||
- Use their real name if available from git history, otherwise use GitHub username
|
|
||||||
4. Sort NEW + RECURRING contributors together by their contribution date (newest first)
|
|
||||||
5. Keep list of UNCHANGED contributors in their original order
|
|
||||||
|
|
||||||
**The final order will be: [NEW + RECURRING from range, sorted newest first] + [UNCHANGED in original order]**
|
|
||||||
|
|
||||||
### Step 6: Generate Updated File
|
|
||||||
|
|
||||||
Create the updated supporters list:
|
|
||||||
- Keep all `|supporter-gold|` entries unchanged and in their exact positions
|
|
||||||
- Keep all `|supporter-std|` entries unchanged and in their exact positions
|
|
||||||
- For `|supporter-code|` entries, reorganize as follows:
|
|
||||||
1. **Top section**: Contributors from the analyzed range (NEW + RECURRING), sorted newest first
|
|
||||||
2. **Bottom section**: UNCHANGED contributors (not in this range), in their original order
|
|
||||||
- Update the spacer comment showing the range that was just processed:
|
|
||||||
```rst
|
|
||||||
..
|
|
||||||
↓ claude --permission-mode default /update-supporters <actual-range-used>
|
|
||||||
```
|
|
||||||
Replace `<actual-range-used>` with the actual argument passed (e.g., `v0.6.1..v0.7.0`)
|
|
||||||
This comment should appear immediately before the first code contributor entry
|
|
||||||
If there's an existing comment in this format, replace it; otherwise add it
|
|
||||||
- Maintain all other file structure and RST formatting, including the "Before /update-supporters" marker comment
|
|
||||||
|
|
||||||
### Step 7: Apply Changes
|
|
||||||
|
|
||||||
- Use Edit tool to update the `docs/supporters.rst` file
|
|
||||||
- Display summary of changes made:
|
|
||||||
- Time/release range analyzed
|
|
||||||
- Number of contributors found
|
|
||||||
- New code contributors added
|
|
||||||
- Final counts by tier
|
|
||||||
|
|
||||||
## Implementation Notes
|
|
||||||
|
|
||||||
- **DO NOT modify, remove, or re-sort** any `|supporter-gold|` or `|supporter-std|` entries
|
|
||||||
- **For code contributors**: ADD new ones and MOVE UP recurring ones (those with contributions in the analyzed range)
|
|
||||||
- When extracting from git history, focus on substantial contributors (filter out single-commit or trivial changes if needed)
|
|
||||||
- **CRITICAL:** Contributors from the analyzed range go to the top
|
|
||||||
- Use `git log <range> --author="<email>" --format="%ai" -1` to get latest contribution date in the range
|
|
||||||
- Both NEW and RECURRING contributors from this range are sorted together (newest first) and placed at the TOP
|
|
||||||
- UNCHANGED contributors (not in this range) stay in their original relative order below
|
|
||||||
- This creates a chronological history: most recent release's contributors appear first
|
|
||||||
- Add/update the spacer comment showing the processed range
|
|
||||||
- Handle edge cases:
|
|
||||||
- Contributors mentioned in changelog but not in git history (external contributors, reporters)
|
|
||||||
- Contributors in git history but not mentioned in changelog (decide whether to include)
|
|
||||||
|
|
||||||
## Example Output
|
|
||||||
|
|
||||||
```
|
|
||||||
Analyzing contributors for range: v0.6.0..v0.7.0
|
|
||||||
- Found 8 code contributors in git history
|
|
||||||
- Found 12 GitHub users mentioned in changelog.rst (v0.7.0 section)
|
|
||||||
- Current supporters.rst has 8 gold, 12 std, 10 code contributors
|
|
||||||
|
|
||||||
New contributors:
|
|
||||||
+ Jane Doe (@janedoe)
|
|
||||||
+ John Smith (@jsmith)
|
|
||||||
|
|
||||||
Promoted (recurring) contributors:
|
|
||||||
↑ Alice Developer (@alice) - new contributions in v0.6.0..v0.7.0
|
|
||||||
|
|
||||||
Updated docs/supporters.rst:
|
|
||||||
- Gold supporters: 8 (unchanged)
|
|
||||||
- Std supporters: 12 (unchanged)
|
|
||||||
- Code contributors: 10 → 12
|
|
||||||
- Promoted to top: 3 contributors (2 new + 1 recurring)
|
|
||||||
```
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
version: 2
|
|
||||||
updates:
|
|
||||||
# Docs use Python only for static site generation.
|
|
||||||
- package-ecosystem: "pip"
|
|
||||||
directory: /docs
|
|
||||||
schedule:
|
|
||||||
interval: "weekly"
|
|
||||||
ignore:
|
|
||||||
- dependency-name: "*"
|
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
# Build
|
# Build
|
||||||
artifacts/
|
artifacts/
|
||||||
build/install.tmp
|
|
||||||
|
|
||||||
# Golang
|
# Golang
|
||||||
vendor/
|
vendor/
|
||||||
|
|||||||
+14
-40
@@ -1,42 +1,16 @@
|
|||||||
version: "2"
|
|
||||||
linters:
|
linters:
|
||||||
enable:
|
enable:
|
||||||
- revive
|
- goimports
|
||||||
settings:
|
|
||||||
revive:
|
issues:
|
||||||
rules:
|
exclude-rules:
|
||||||
- name: time-equal
|
- path: _test\.go
|
||||||
exclusions:
|
linters:
|
||||||
generated: lax
|
- errcheck
|
||||||
presets:
|
# Disable staticcheck 'Empty body in an if or else branch' as it's useful
|
||||||
- comments
|
# to put a comment into an empty else-clause that explains why whatever
|
||||||
- common-false-positives
|
# is done in the if-caluse is not necessary if the condition is false.
|
||||||
- legacy
|
- linters:
|
||||||
- std-error-handling
|
- staticcheck
|
||||||
rules:
|
text: "SA9003:"
|
||||||
- linters:
|
|
||||||
- errcheck
|
|
||||||
path: _test\.go
|
|
||||||
- linters:
|
|
||||||
- staticcheck
|
|
||||||
text: 'SA9003:'
|
|
||||||
- linters:
|
|
||||||
- staticcheck
|
|
||||||
text: '(QF1001|QF1011|ST1012|QF1008|ST1005|ST1023|QF1003|ST1006|ST1001|QF1004):'
|
|
||||||
paths:
|
|
||||||
- third_party$
|
|
||||||
- builtin$
|
|
||||||
- examples$
|
|
||||||
formatters:
|
|
||||||
enable:
|
|
||||||
- goimports
|
|
||||||
settings:
|
|
||||||
goimports:
|
|
||||||
local-prefixes:
|
|
||||||
- github.com/zrepl/zrepl
|
|
||||||
exclusions:
|
|
||||||
generated: lax
|
|
||||||
paths:
|
|
||||||
- third_party$
|
|
||||||
- builtin$
|
|
||||||
- examples$
|
|
||||||
|
|||||||
@@ -1 +0,0 @@
|
|||||||
0.9.30
|
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
.PHONY: generate build test vet cover release docs docs-clean clean format lint platformtest
|
.PHONY: generate build test vet cover release docs docs-clean clean format lint platformtest
|
||||||
.PHONY: release release-noarch
|
.PHONY: release bins-all release-noarch
|
||||||
.DEFAULT_GOAL := zrepl-bin
|
.DEFAULT_GOAL := zrepl-bin
|
||||||
|
|
||||||
ARTIFACTDIR := artifacts
|
ARTIFACTDIR := artifacts
|
||||||
@@ -14,25 +14,22 @@ ifndef _ZREPL_VERSION
|
|||||||
endif
|
endif
|
||||||
endif
|
endif
|
||||||
|
|
||||||
ZREPL_PACKAGE_RELEASE := 1
|
|
||||||
|
|
||||||
GO := go
|
GO := go
|
||||||
GOOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOOS"')
|
GOOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOOS"')
|
||||||
GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"')
|
GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"')
|
||||||
GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"')
|
GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"')
|
||||||
GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"')
|
GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"')
|
||||||
GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
|
GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
|
||||||
GO_ENV_VARS := CGO_ENABLED=0
|
GO_ENV_VARS := GO111MODULE=on
|
||||||
GO_LDFLAGS := "-X github.com/zrepl/zrepl/internal/version.zreplVersion=$(_ZREPL_VERSION)"
|
GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
|
||||||
GO_MOD_READONLY := -mod=readonly
|
GO_MOD_READONLY := -mod=readonly
|
||||||
GO_EXTRA_BUILDFLAGS :=
|
GO_EXTRA_BUILDFLAGS :=
|
||||||
GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
|
GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
|
||||||
GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
|
GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
|
||||||
|
GOLANGCI_LINT := golangci-lint
|
||||||
GOCOVMERGE := gocovmerge
|
GOCOVMERGE := gocovmerge
|
||||||
RELEASE_GOVERSION ?= go1.25.7
|
RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.19
|
||||||
STRIPPED_GOVERSION := $(subst go,,$(RELEASE_GOVERSION))
|
RELEASE_DOCKER_BASEIMAGE ?= golang:$(RELEASE_DOCKER_BASEIMAGE_TAG)
|
||||||
RELEASE_DOCKER_BASEIMAGE ?= golang:$(STRIPPED_GOVERSION)
|
|
||||||
RELEASE_DOCKER_CACHEMOUNT :=
|
|
||||||
|
|
||||||
ifneq ($(GOARM),)
|
ifneq ($(GOARM),)
|
||||||
ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM)
|
ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM)
|
||||||
@@ -40,49 +37,34 @@ else
|
|||||||
ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)
|
ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)
|
||||||
endif
|
endif
|
||||||
|
|
||||||
|
.PHONY: printvars
|
||||||
|
printvars:
|
||||||
|
@echo GOOS=$(GOOS)
|
||||||
|
@echo GOARCH=$(GOARCH)
|
||||||
|
@echo GOARM=$(GOARM)
|
||||||
|
|
||||||
ifneq ($(RELEASE_DOCKER_CACHEMOUNT),)
|
|
||||||
_RELEASE_DOCKER_CACHEMOUNT := -v $(RELEASE_DOCKER_CACHEMOUNT)/mod:/go/pkg/mod -v $(RELEASE_DOCKER_CACHEMOUNT)/xdg-cache:/.cache/go-build
|
|
||||||
.PHONY: release-docker-mkcachemount
|
|
||||||
release-docker-mkcachemount:
|
|
||||||
mkdir -p $(RELEASE_DOCKER_CACHEMOUNT)
|
|
||||||
mkdir -p $(RELEASE_DOCKER_CACHEMOUNT)/mod
|
|
||||||
mkdir -p $(RELEASE_DOCKER_CACHEMOUNT)/xdg-cache
|
|
||||||
else
|
|
||||||
_RELEASE_DOCKER_CACHEMOUNT :=
|
|
||||||
.PHONY: release-docker-mkcachemount
|
|
||||||
release-docker-mkcachemount:
|
|
||||||
# nothing to do
|
|
||||||
endif
|
|
||||||
|
|
||||||
##################### PRODUCING A RELEASE #############
|
##################### PRODUCING A RELEASE #############
|
||||||
.PHONY: release wrapup-and-checksum check-git-clean verify-and-sign clean ensure-release-toolchain
|
.PHONY: release wrapup-and-checksum check-git-clean sign clean
|
||||||
|
|
||||||
ensure-release-toolchain:
|
release: clean
|
||||||
# ensure the toolchain is actually the one we expect
|
# no cross-platform support for target test
|
||||||
test $(RELEASE_GOVERSION) = "$$($(GO_ENV_VARS) $(GO) env GOVERSION)"
|
$(MAKE) test-go
|
||||||
|
$(MAKE) bins-all
|
||||||
release: ensure-release-toolchain
|
|
||||||
$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="vet"
|
|
||||||
$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="lint"
|
|
||||||
$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="zrepl-bin"
|
|
||||||
$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="test-platform-bin"
|
|
||||||
$(MAKE) noarch
|
$(MAKE) noarch
|
||||||
|
$(MAKE) wrapup-and-checksum
|
||||||
|
$(MAKE) check-git-clean
|
||||||
|
ifeq (SIGN, 1)
|
||||||
|
$(MAKE) sign
|
||||||
|
endif
|
||||||
|
@echo "ZREPL RELEASE ARTIFACTS AVAILABLE IN artifacts/release"
|
||||||
|
|
||||||
release-docker: $(ARTIFACTDIR) release-docker-mkcachemount
|
release-docker: $(ARTIFACTDIR)
|
||||||
sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build/build.Dockerfile > $(ARTIFACTDIR)/build.Dockerfile
|
sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > artifacts/release-docker.Dockerfile
|
||||||
docker build -t zrepl_release --pull \
|
docker build -t zrepl_release --pull -f artifacts/release-docker.Dockerfile .
|
||||||
--build-arg BUILD_UID=$$(id -u) \
|
docker run --rm -i -v $(CURDIR):/src -u $$(id -u):$$(id -g) \
|
||||||
--build-arg BUILD_GID=$$(id -g) \
|
|
||||||
-f $(ARTIFACTDIR)/build.Dockerfile .
|
|
||||||
docker run --rm -i$$(test -t 0 && echo t) \
|
|
||||||
$(_RELEASE_DOCKER_CACHEMOUNT) \
|
|
||||||
-v $(CURDIR):/src \
|
|
||||||
zrepl_release \
|
zrepl_release \
|
||||||
make release \
|
make release GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
|
||||||
GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
|
|
||||||
ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE) \
|
|
||||||
RELEASE_GOVERSION=$(RELEASE_GOVERSION)
|
|
||||||
|
|
||||||
debs-docker:
|
debs-docker:
|
||||||
$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=deb
|
$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=deb
|
||||||
@@ -99,14 +81,9 @@ rpm: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion
|
|||||||
$(eval _ZREPL_RPM_TOPDIR_ABS := $(CURDIR)/$(ARTIFACTDIR)/rpmbuild)
|
$(eval _ZREPL_RPM_TOPDIR_ABS := $(CURDIR)/$(ARTIFACTDIR)/rpmbuild)
|
||||||
rm -rf "$(_ZREPL_RPM_TOPDIR_ABS)"
|
rm -rf "$(_ZREPL_RPM_TOPDIR_ABS)"
|
||||||
mkdir "$(_ZREPL_RPM_TOPDIR_ABS)"
|
mkdir "$(_ZREPL_RPM_TOPDIR_ABS)"
|
||||||
for d in BUILD BUILDROOT RPMS SOURCES SPECS SRPMS; do \
|
mkdir -p "$(_ZREPL_RPM_TOPDIR_ABS)"/{SPECS,RPMS,BUILD,BUILDROOT}
|
||||||
mkdir -p "$(_ZREPL_RPM_TOPDIR_ABS)/$$d"; \
|
sed "s/^Version:.*/Version: $(_ZREPL_RPM_VERSION)/g" \
|
||||||
done
|
packaging/rpm/zrepl.spec > $(_ZREPL_RPM_TOPDIR_ABS)/SPECS/zrepl.spec
|
||||||
sed \
|
|
||||||
-e "s/^Version:.*/Version: $(_ZREPL_RPM_VERSION)/g" \
|
|
||||||
-e "s/^Release:.*/Release: $(ZREPL_PACKAGE_RELEASE)/g" \
|
|
||||||
packaging/rpm/zrepl.spec \
|
|
||||||
> $(_ZREPL_RPM_TOPDIR_ABS)/SPECS/zrepl.spec
|
|
||||||
|
|
||||||
# see /usr/lib/rpm/platform
|
# see /usr/lib/rpm/platform
|
||||||
ifeq ($(GOARCH),amd64)
|
ifeq ($(GOARCH),amd64)
|
||||||
@@ -133,16 +110,13 @@ rpm-docker:
|
|||||||
docker build -t zrepl_rpm_pkg --pull -f packaging/rpm/Dockerfile .
|
docker build -t zrepl_rpm_pkg --pull -f packaging/rpm/Dockerfile .
|
||||||
docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
|
docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
|
||||||
zrepl_rpm_pkg \
|
zrepl_rpm_pkg \
|
||||||
make rpm \
|
make rpm GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
|
||||||
GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
|
|
||||||
ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE)
|
|
||||||
|
|
||||||
|
|
||||||
deb: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
|
deb: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
|
||||||
|
|
||||||
cp packaging/deb/debian/changelog.template packaging/deb/debian/changelog
|
cp packaging/deb/debian/changelog.template packaging/deb/debian/changelog
|
||||||
sed -i 's/DATE_DASH_R_OUTPUT/$(shell date -R)/' packaging/deb/debian/changelog
|
sed -i 's/DATE_DASH_R_OUTPUT/$(shell date -R)/' packaging/deb/debian/changelog
|
||||||
VERSION="$(subst -,.,$(_ZREPL_VERSION))-$(ZREPL_PACKAGE_RELEASE)"; \
|
VERSION="$(subst -,.,$(_ZREPL_VERSION))"; \
|
||||||
export VERSION="$${VERSION#v}"; \
|
export VERSION="$${VERSION#v}"; \
|
||||||
sed -i 's/VERSION/'"$$VERSION"'/' packaging/deb/debian/changelog
|
sed -i 's/VERSION/'"$$VERSION"'/' packaging/deb/debian/changelog
|
||||||
|
|
||||||
@@ -160,36 +134,28 @@ endif
|
|||||||
|
|
||||||
deb-docker:
|
deb-docker:
|
||||||
docker build -t zrepl_debian_pkg --pull -f packaging/deb/Dockerfile .
|
docker build -t zrepl_debian_pkg --pull -f packaging/deb/Dockerfile .
|
||||||
# Use a small open file limit to make fakeroot work. If we don't
|
|
||||||
# specify it, docker daemon will use its file limit. I don't know
|
|
||||||
# what changed (Docker, its systemd service, its Go version). But I
|
|
||||||
# observed fakeroot iterating close(i) up to i > 1000000, which costs
|
|
||||||
# a good amount of CPU time and makes the build slow.
|
|
||||||
docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
|
docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
|
||||||
--ulimit nofile=1024:1024 \
|
|
||||||
zrepl_debian_pkg \
|
zrepl_debian_pkg \
|
||||||
make deb \
|
make deb GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
|
||||||
GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
|
|
||||||
ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE)
|
|
||||||
|
|
||||||
# expects `release`, `deb` & `rpm` targets to have run before
|
# expects `release` target to have run before
|
||||||
NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
|
NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
|
||||||
wrapup-and-checksum:
|
wrapup-and-checksum:
|
||||||
rm -f $(NOARCH_TARBALL)
|
rm -f $(NOARCH_TARBALL)
|
||||||
tar --mtime='1970-01-01' --sort=name \
|
tar --mtime='1970-01-01' --sort=name \
|
||||||
--transform 's/$(ARTIFACTDIR)/zrepl-$(_ZREPL_VERSION)-noarch/' \
|
--transform 's/$(ARTIFACTDIR)/zrepl-$(_ZREPL_VERSION)-noarch/' \
|
||||||
--transform 's#dist#zrepl-$(_ZREPL_VERSION)-noarch/dist#' \
|
--transform 's#dist#zrepl-$(_ZREPL_VERSION)-noarch/dist#' \
|
||||||
--transform 's#internal/config/samples#zrepl-$(_ZREPL_VERSION)-noarch/config#' \
|
--transform 's#config/samples#zrepl-$(_ZREPL_VERSION)-noarch/config#' \
|
||||||
-acf $(NOARCH_TARBALL) \
|
-acf $(NOARCH_TARBALL) \
|
||||||
$(ARTIFACTDIR)/docs/html \
|
$(ARTIFACTDIR)/docs/html \
|
||||||
$(ARTIFACTDIR)/bash_completion \
|
$(ARTIFACTDIR)/bash_completion \
|
||||||
$(ARTIFACTDIR)/_zrepl.zsh_completion \
|
$(ARTIFACTDIR)/_zrepl.zsh_completion \
|
||||||
$(ARTIFACTDIR)/go_env.txt \
|
$(ARTIFACTDIR)/go_env.txt \
|
||||||
dist \
|
dist \
|
||||||
internal/config/samples
|
config/samples
|
||||||
rm -rf "$(ARTIFACTDIR)/release"
|
rm -rf "$(ARTIFACTDIR)/release"
|
||||||
mkdir -p "$(ARTIFACTDIR)/release"
|
mkdir -p "$(ARTIFACTDIR)/release"
|
||||||
cp -l $(ARTIFACTDIR)/zrepl* \
|
cp -l $(ARTIFACTDIR)/zrepl-* \
|
||||||
$(ARTIFACTDIR)/platformtest-* \
|
$(ARTIFACTDIR)/platformtest-* \
|
||||||
"$(ARTIFACTDIR)/release"
|
"$(ARTIFACTDIR)/release"
|
||||||
cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
|
cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
|
||||||
@@ -209,46 +175,42 @@ check-git-clean:
|
|||||||
|
|
||||||
tag-release:
|
tag-release:
|
||||||
test -n "$(ZREPL_TAG_VERSION)" || exit 1
|
test -n "$(ZREPL_TAG_VERSION)" || exit 1
|
||||||
git tag -u '328A6627FA98061D!' -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)"
|
git tag -u E27CA5FC -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)"
|
||||||
|
|
||||||
verify-and-sign:
|
sign:
|
||||||
cd "$(ARTIFACTDIR)/release" && sha512sum -c sha512sum.txt
|
gpg -u "89BC 5D89 C845 568B F578 B306 CDBD 8EC8 E27C A5FC" \
|
||||||
gpg -u '328A6627FA98061D!' \
|
|
||||||
--armor \
|
--armor \
|
||||||
--detach-sign $(ARTIFACTDIR)/release/sha512sum.txt
|
--detach-sign $(ARTIFACTDIR)/release/sha512sum.txt
|
||||||
|
|
||||||
clean: docs-clean
|
clean: docs-clean
|
||||||
rm -rf "$(ARTIFACTDIR)"
|
rm -rf "$(ARTIFACTDIR)"
|
||||||
|
|
||||||
download-circleci-release:
|
##################### BINARIES #####################
|
||||||
rm -rf "$(ARTIFACTDIR)"
|
.PHONY: bins-all lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin generate-platform-test-list
|
||||||
mkdir -p "$(ARTIFACTDIR)/release"
|
|
||||||
python3 .circleci/download_artifacts.py --prefix 'artifacts/release/' "$(JOB_NUM)" "$(ARTIFACTDIR)/release"
|
|
||||||
|
|
||||||
##################### MULTI-ARCH HELPERS #####################
|
BINS_ALL_TARGETS := zrepl-bin test-platform-bin vet lint
|
||||||
|
GO_SUPPORTS_ILLUMOS := $(shell $(GO) version | gawk -F '.' '/^go version /{split($$0, comps, " "); split(comps[3], v, "."); if (v[1] == "go1" && v[2] >= 13) { print "illumos"; } else { print "noillumos"; }}')
|
||||||
|
bins-all:
|
||||||
|
$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd GOARCH=amd64
|
||||||
|
$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd GOARCH=386
|
||||||
|
$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd GOARCH=arm GOARM=7
|
||||||
|
$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd GOARCH=arm64
|
||||||
|
$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux GOARCH=amd64
|
||||||
|
$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux GOARCH=arm64
|
||||||
|
$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux GOARCH=arm GOARM=7
|
||||||
|
$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux GOARCH=386
|
||||||
|
$(MAKE) $(BINS_ALL_TARGETS) GOOS=darwin GOARCH=amd64
|
||||||
|
$(MAKE) $(BINS_ALL_TARGETS) GOOS=solaris GOARCH=amd64
|
||||||
|
ifeq ($(GO_SUPPORTS_ILLUMOS), illumos)
|
||||||
|
$(MAKE) $(BINS_ALL_TARGETS) GOOS=illumos GOARCH=amd64
|
||||||
|
else ifeq ($(GO_SUPPORTS_ILLUMOS), noillumos)
|
||||||
|
@echo "SKIPPING ILLUMOS BUILD BECAUSE GO VERSION DOESN'T SUPPORT IT"
|
||||||
|
else
|
||||||
|
@echo "CANNOT DETERMINE WHETHER GO VERSION SUPPORTS GOOS=illumos"; exit 1
|
||||||
|
endif
|
||||||
|
|
||||||
_run_make_foreach_target_tuple:
|
lint:
|
||||||
if [ "$(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG)" = "" ]; then \
|
$(GO_ENV_VARS) $(GOLANGCI_LINT) run ./...
|
||||||
echo "RUN_MAKE_FOREACH_TARGET_TUPLE_ARG must be set"; \
|
|
||||||
exit 1; \
|
|
||||||
fi
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd GOARCH=amd64
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd GOARCH=386
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd GOARCH=arm GOARM=7
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd GOARCH=arm64
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux GOARCH=amd64
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux GOARCH=arm64
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux GOARCH=arm GOARM=7
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux GOARCH=386
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=darwin GOARCH=amd64
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=solaris GOARCH=amd64
|
|
||||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=illumos GOARCH=amd64
|
|
||||||
|
|
||||||
##################### REGULAR TARGETS #####################
|
|
||||||
.PHONY: lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin
|
|
||||||
|
|
||||||
lint: build/install
|
|
||||||
$(GO_ENV_VARS) build/install/golangci_lint/golangci-lint run ./...
|
|
||||||
|
|
||||||
vet:
|
vet:
|
||||||
$(GO_ENV_VARS) $(GO) vet $(GO_BUILDFLAGS) ./...
|
$(GO_ENV_VARS) $(GO) vet $(GO_BUILDFLAGS) ./...
|
||||||
@@ -269,6 +231,9 @@ endif
|
|||||||
zrepl-bin:
|
zrepl-bin:
|
||||||
$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(ZREPL_TARGET_TUPLE)"
|
$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(ZREPL_TARGET_TUPLE)"
|
||||||
|
|
||||||
|
generate-platform-test-list:
|
||||||
|
$(GO_BUILD) -o $(ARTIFACTDIR)/generate-platform-test-list ./platformtest/tests/gen
|
||||||
|
|
||||||
COVER_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-cover-$(ZREPL_TARGET_TUPLE)
|
COVER_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-cover-$(ZREPL_TARGET_TUPLE)
|
||||||
cover-platform-bin:
|
cover-platform-bin:
|
||||||
$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
|
$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
|
||||||
@@ -285,7 +250,7 @@ cover-platform:
|
|||||||
|
|
||||||
TEST_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)
|
TEST_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)
|
||||||
test-platform-bin:
|
test-platform-bin:
|
||||||
$(GO_BUILD) -o "$(TEST_PLATFORM_BIN_PATH)" ./internal/platformtest/harness
|
$(GO_BUILD) -o "$(TEST_PLATFORM_BIN_PATH)" ./platformtest/harness
|
||||||
test-platform:
|
test-platform:
|
||||||
export _TEST_PLATFORM_CMD="\"$(TEST_PLATFORM_BIN_PATH)\""; \
|
export _TEST_PLATFORM_CMD="\"$(TEST_PLATFORM_BIN_PATH)\""; \
|
||||||
$(MAKE) _test-or-cover-platform-impl
|
$(MAKE) _test-or-cover-platform-impl
|
||||||
@@ -302,7 +267,7 @@ ifndef _TEST_PLATFORM_CMD
|
|||||||
endif
|
endif
|
||||||
rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
|
rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
|
||||||
rm -f "$(ARTIFACTDIR)/platformtest.cover"
|
rm -f "$(ARTIFACTDIR)/platformtest.cover"
|
||||||
internal/platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
|
platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
|
||||||
$(_TEST_PLATFORM_CMD) \
|
$(_TEST_PLATFORM_CMD) \
|
||||||
-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
|
-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
|
||||||
-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
|
-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
|
||||||
@@ -324,39 +289,23 @@ cover-full:
|
|||||||
|
|
||||||
##################### DEV TARGETS #####################
|
##################### DEV TARGETS #####################
|
||||||
# not part of the build, must do that manually
|
# not part of the build, must do that manually
|
||||||
.PHONY: generate format
|
.PHONY: generate formatcheck format
|
||||||
|
|
||||||
build/install:
|
generate: generate-platform-test-list
|
||||||
rm -rf build/install.tmp
|
protoc -I=replication/logic/pdu --go_out=replication/logic/pdu --go-grpc_out=replication/logic/pdu replication/logic/pdu/pdu.proto
|
||||||
mkdir build/install.tmp
|
protoc -I=rpc/grpcclientidentity/example --go_out=rpc/grpcclientidentity/example/pdu --go-grpc_out=rpc/grpcclientidentity/example/pdu rpc/grpcclientidentity/example/grpcauth.proto
|
||||||
|
$(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./...
|
||||||
-echo "installing protoc"
|
|
||||||
mkdir build/install.tmp/protoc
|
|
||||||
bash -x build/get_protoc.bash build/install.tmp/protoc
|
|
||||||
|
|
||||||
-echo "installing golangci-lint"
|
|
||||||
mkdir -p build/install.tmp/golangci_lint
|
|
||||||
GOHOSTARCH=$(GOHOSTARCH) \
|
|
||||||
build/get_golangci_lint.bash build/install.tmp/golangci_lint
|
|
||||||
|
|
||||||
-echo "installing go tools"
|
|
||||||
build/go_install_tools.bash build/install.tmp/gobin
|
|
||||||
|
|
||||||
mv build/install.tmp build/install
|
|
||||||
|
|
||||||
|
|
||||||
generate: build/install
|
|
||||||
# TODO: would be nice to run with a pure path here
|
|
||||||
PATH="$(CURDIR)/build/install/gobin:$(CURDIR)/build/install/protoc/bin:$$PATH" && \
|
|
||||||
build/install/protoc/bin/protoc -I=internal/replication/logic/pdu --go_out=internal/replication/logic/pdu --go-grpc_out=internal/replication/logic/pdu internal/replication/logic/pdu/pdu.proto && \
|
|
||||||
build/install/protoc/bin/protoc -I=internal/rpc/grpcclientidentity/example --go_out=internal/rpc/grpcclientidentity/example/pdu --go-grpc_out=internal/rpc/grpcclientidentity/example/pdu internal/rpc/grpcclientidentity/example/grpcauth.proto && \
|
|
||||||
$(GO) generate $(GO_BUILDFLAGS) -x ./... && \
|
|
||||||
true
|
|
||||||
|
|
||||||
GOIMPORTS := goimports -srcdir . -local 'github.com/zrepl/zrepl'
|
GOIMPORTS := goimports -srcdir . -local 'github.com/zrepl/zrepl'
|
||||||
|
FINDSRCFILES := find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go'
|
||||||
|
|
||||||
format: build/install
|
formatcheck:
|
||||||
@ build/install/gobin/goimports -w -local 'github.com/zrepl/zrepl' $$(find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go')
|
@# goimports doesn't have a knob to exit with non-zero status code if formatting is needed
|
||||||
|
@# see https://go-review.googlesource.com/c/tools/+/237378
|
||||||
|
@ affectedfiles=$$($(GOIMPORTS) -l $(shell $(FINDSRCFILES)) | tee /dev/stderr | wc -l); test "$$affectedfiles" = 0
|
||||||
|
|
||||||
|
format:
|
||||||
|
@ $(GOIMPORTS) -w -d $(shell $(FINDSRCFILES))
|
||||||
|
|
||||||
##################### NOARCH #####################
|
##################### NOARCH #####################
|
||||||
.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
|
.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
|
||||||
@@ -384,9 +333,12 @@ $(ARTIFACTDIR)/go_env.txt:
|
|||||||
|
|
||||||
docs: $(ARTIFACTDIR)/docs
|
docs: $(ARTIFACTDIR)/docs
|
||||||
# https://www.sphinx-doc.org/en/master/man/sphinx-build.html
|
# https://www.sphinx-doc.org/en/master/man/sphinx-build.html
|
||||||
cd docs && uv sync --frozen
|
$(MAKE) -C docs \
|
||||||
cd docs && uv run sphinx-build -W --keep-going -n . ../artifacts/docs/html
|
html \
|
||||||
|
BUILDDIR=../artifacts/docs \
|
||||||
|
SPHINXOPTS="-W --keep-going -n"
|
||||||
|
|
||||||
docs-clean:
|
docs-clean:
|
||||||
rm -rf artifacts/docs
|
$(MAKE) -C docs \
|
||||||
rm -rf docs/.venv
|
clean \
|
||||||
|
BUILDDIR=../artifacts/docs
|
||||||
|
|||||||
@@ -26,40 +26,24 @@ zrepl is a one-stop ZFS backup & replication solution.
|
|||||||
If so, think of an expressive configuration example.
|
If so, think of an expressive configuration example.
|
||||||
2. Think of at least one use case that generalizes from your concrete application.
|
2. Think of at least one use case that generalizes from your concrete application.
|
||||||
3. Open an issue on GitHub with example conf & use case attached.
|
3. Open an issue on GitHub with example conf & use case attached.
|
||||||
4. **Optional**: [Contact Christian Schwarz](https://cschwarz.com) for contract work.
|
4. **Optional**: [Post a bounty](https://www.bountysource.com/teams/zrepl) on the issue, or [contact Christian Schwarz](https://cschwarz.com) for contract work.
|
||||||
|
|
||||||
The above does not apply if you already implemented everything.
|
The above does not apply if you already implemented everything.
|
||||||
Check out the *Coding Workflow* section below for details.
|
Check out the *Coding Workflow* section below for details.
|
||||||
|
|
||||||
## Development
|
## Building, Releasing, Downstream-Packaging
|
||||||
|
|
||||||
|
This section provides an overview of the zrepl build & release process.
|
||||||
|
Check out `docs/installation/compile-from-source.rst` for build-from-source instructions.
|
||||||
|
|
||||||
|
### Overview
|
||||||
|
|
||||||
zrepl is written in [Go](https://golang.org) and uses [Go modules](https://github.com/golang/go/wiki/Modules) to manage dependencies.
|
zrepl is written in [Go](https://golang.org) and uses [Go modules](https://github.com/golang/go/wiki/Modules) to manage dependencies.
|
||||||
The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
|
The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
|
||||||
|
|
||||||
### Building
|
Install **build dependencies** using `./lazy.sh devsetup`.
|
||||||
|
`lazy.sh` uses `python3-pip` to fetch the build dependencies for the docs - you might want to use a [venv](https://docs.python.org/3/library/venv.html).
|
||||||
#### Go Code
|
If you just want to install the Go dependencies, run `./lazy.sh godep`.
|
||||||
|
|
||||||
Dependencies:
|
|
||||||
|
|
||||||
* Go
|
|
||||||
* GNU Make
|
|
||||||
* Git
|
|
||||||
* wget (`make generate`)
|
|
||||||
* unzip (`make generate`)
|
|
||||||
|
|
||||||
Some Go code is **generated**, and generated code is committed to the source tree.
|
|
||||||
Therefore, building does not require having code generation tools set up.
|
|
||||||
When making changes that require code to be (re-)generated, run `make generate`.
|
|
||||||
I downloads and installs pinned versions of the code generation tools into `./build/install`.
|
|
||||||
There is a CI check that ensures Git state is clean, i.e., code generation has been done by a PR and is deterministic.
|
|
||||||
|
|
||||||
#### Docs
|
|
||||||
|
|
||||||
Install [uv](https://docs.astral.sh/uv/getting-started/installation/), then run `make docs`.
|
|
||||||
uv automatically manages Python and dependencies.
|
|
||||||
|
|
||||||
### Testing
|
|
||||||
|
|
||||||
The **test suite** is split into pure **Go tests** (`make test-go`) and **platform tests** that interact with ZFS and thus generally **require root privileges** (`sudo make test-platform`).
|
The **test suite** is split into pure **Go tests** (`make test-go`) and **platform tests** that interact with ZFS and thus generally **require root privileges** (`sudo make test-platform`).
|
||||||
Platform tests run on their own pool with the name `zreplplatformtest`, which is created using the file vdev in `/tmp`.
|
Platform tests run on their own pool with the name `zreplplatformtest`, which is created using the file vdev in `/tmp`.
|
||||||
@@ -67,156 +51,51 @@ Platform tests run on their own pool with the name `zreplplatformtest`, which is
|
|||||||
For a full **code coverage** profile, run `make test-go COVER=1 && sudo make test-platform && make cover-merge`.
|
For a full **code coverage** profile, run `make test-go COVER=1 && sudo make test-platform && make cover-merge`.
|
||||||
An HTML report can be generated using `make cover-html`.
|
An HTML report can be generated using `make cover-html`.
|
||||||
|
|
||||||
### Circle CI
|
**Code generation** is triggered by `make generate`. Generated code is committed to the source tree.
|
||||||
|
|
||||||
We use CircleCI for automated build & test pre- and post-merge.
|
### Build & Release Process
|
||||||
|
|
||||||
|
**The `Makefile` is catering to the needs of developers & CI, not distro packagers**.
|
||||||
|
It provides phony targets for
|
||||||
|
* local development (building, running tests, etc)
|
||||||
|
* building a release in Docker (used by the CI & release management)
|
||||||
|
* building .deb and .rpm packages out of the release artifacts.
|
||||||
|
|
||||||
|
**Build tooling & dependencies** are documented as code in `lazy.sh`.
|
||||||
|
Go dependencies are then fetched by the go command and pip dependencies are pinned through a `requirements.txt`.
|
||||||
|
|
||||||
|
**We use CircleCI for continuous integration**.
|
||||||
There are two workflows:
|
There are two workflows:
|
||||||
|
|
||||||
* `ci` runs for every commit / branch / tag pushed to GitHub.
|
* `ci` runs for every commit / branch / tag pushed to GitHub.
|
||||||
It is supposed to run very fast (<5min and provides quick feedback to developers).
|
It is supposed to run very fast (<5min and provides quick feedback to developers).
|
||||||
It runs formatting checks, lints and tests on the most important OSes / architectures.
|
It runs formatting checks, lints and tests on the most important OSes / architectures.
|
||||||
|
Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
|
||||||
|
|
||||||
* `release` runs
|
* `release` runs
|
||||||
* on manual triggers through the CircleCI API (in order to produce a release)
|
* on manual triggers through the CircleCI API (in order to produce a release)
|
||||||
* periodically on `master`
|
* periodically on `master`
|
||||||
|
Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
|
||||||
|
|
||||||
Artifacts are stored in CircleCI.
|
**Releases** are issued via Git tags + GitHub Releases feature.
|
||||||
|
|
||||||
### Releasing
|
|
||||||
|
|
||||||
All zrepl releases are git-tagged and then published as a GitHub Release.
|
|
||||||
There is a git tag for each zrepl release, usually `vMAJOR.MINOR.0`.
|
|
||||||
We don't move git tags once the release has been published.
|
|
||||||
|
|
||||||
The procedure to issue a release is as follows:
|
The procedure to issue a release is as follows:
|
||||||
|
* Issue the source release:
|
||||||
* Prepare the release (as a PR to `master`):
|
* Git tag the release on the `master` branch.
|
||||||
* Finalize `docs/changelog.rst` for the release.
|
|
||||||
* Update the supporters list based on contributors for this release:
|
|
||||||
```bash
|
|
||||||
claude --permission-mode default '/update-supporters v0.6.1..v0.7.0'
|
|
||||||
```
|
|
||||||
Replace version tags with the appropriate range for your release.
|
|
||||||
* Merge the PR. Docs are auto-published to zrepl.github.io on merge.
|
|
||||||
* Tag the release:
|
|
||||||
* Git tag the release on the `master` branch (e.g., `vMAJOR.MINOR.0`).
|
|
||||||
```
|
|
||||||
make tag-release ZREPL_TAG_VERSION=v0.7.0
|
|
||||||
```
|
|
||||||
* Push the tag.
|
* Push the tag.
|
||||||
* Build and publish:
|
* Run `./docs/publish.sh` to re-build & push zrepl.github.io.
|
||||||
* Run the `release` pipeline against the `master` branch (trigger via CircleCI UI).
|
* Issue the official binary release:
|
||||||
This URL: https://app.circleci.com/pipelines/github/zrepl/zrepl?branch=master.
|
* Run the `release` pipeline (triggered via CircleCI API)
|
||||||
Example pipeline: https://app.circleci.com/pipelines/github/zrepl/zrepl/8547
|
* Download the artifacts to the release manager's machine.
|
||||||
* Download artifacts using this handy makefile target.
|
* Create a GitHub release, edit the changelog, upload all the release artifacts, including .rpm and .deb files.
|
||||||
Note: `JOB_NUM` must be the **job number** from the `release-upload` job, **not the pipeline number**.
|
* Issue the GitHub release.
|
||||||
Find it via: pipeline → `release` workflow → `release-upload` job number.
|
* Add the .rpm and .deb files to the official zrepl repos, publish those.
|
||||||
Example URL: https://app.circleci.com/pipelines/github/zrepl/zrepl/8547/workflows/65feb2c9-15d7-46ab-a551-46d62a5769b0/jobs/66079/steps
|
|
||||||
|
|
||||||
```
|
**Official binary releases are not re-built when Go receives an update. If the Go update is critical to zrepl (e.g. a Go security update that affects zrepl), we'd issue a new source release**.
|
||||||
make download-circleci-release JOB_NUM=66079
|
The rationale for this is that whereas distros provide a mechanism for this (`$zrepl_source_release-$distro_package_revision`), GitHub Releases doesn't which means we'd need to update the existing GitHub release's assets, which nobody would notice (no RSS feed updates, etc.).
|
||||||
```
|
Downstream packagers can read the changelog to determine whether they want to push that minor release into their distro or simply skip it.
|
||||||
* Verify checksums and sign the checksums file:
|
|
||||||
```
|
|
||||||
make verify-and-sign
|
|
||||||
```
|
|
||||||
* Create GitHub draft release and upload artifacts:
|
|
||||||
```bash
|
|
||||||
claude --permission-mode default '/draft-release v0.7.0'
|
|
||||||
```
|
|
||||||
This command will verify that artifacts are ready, create the draft release, and upload all artifacts.
|
|
||||||
* Review the draft release on GitHub, then publish,
|
|
||||||
with the box checked to create a GitHub "Discussion" for the release.
|
|
||||||
* Immediately after hitting publish.
|
|
||||||
* Update the release on GitHub to link to the discussion.
|
|
||||||
* Update `docs/changelog.rst` to link to the GitHub release.
|
|
||||||
* Update `docs/_templates/versions.html` to link to the GitHub release.
|
|
||||||
* Announce release in the zrepl Matrix channel & other socials as applicable.
|
|
||||||
Link to the GitHub release as the entrypoint.
|
|
||||||
* After a couple of days
|
|
||||||
* Add the .rpm and .deb files to the official zrepl repos.
|
|
||||||
* Code for management of these repos: https://github.com/zrepl/package-repo-ops (private repo at this time)
|
|
||||||
|
|
||||||
#### Patch releases, Go toolchain updates, APT/RPM Package rebuilds
|
### Additional Notes to Distro Package Maintainers
|
||||||
|
|
||||||
`vMAJOR.MINOR.0` is typically a tagged commit on `master`, because development velocity isn't high
|
|
||||||
and thus release branches for stabilization aren't necessary.
|
|
||||||
|
|
||||||
Occasionally though there is a need for patch changes to a release, e.g.
|
|
||||||
- security issue in a dependency
|
|
||||||
- Go toolchain update (e.g. security issue in standard library)
|
|
||||||
|
|
||||||
The procedure for this is the following
|
|
||||||
- create a branch off the release tag we need to patch, named `releases/MAJOR.MINOR.X`
|
|
||||||
- that branch will never be merged into `master`, it'll be a dead-end for this specific patch
|
|
||||||
- make changes in that branch
|
|
||||||
- make the final commit that bumps version numbers
|
|
||||||
- create the git tag
|
|
||||||
- follow general procedure for publishing the release (previous sectino
|
|
||||||
|
|
||||||
For Go toolchain updates and package rebuilds with no source code changes, leverage the APT/RPM package revision field.
|
|
||||||
Control via the `ZREPL_PACKAGE_RELEASE` Makefile variable, see `origin/releases/0.6.1-2` for an example.
|
|
||||||
|
|
||||||
|
|
||||||
### Updating Dependencies
|
|
||||||
|
|
||||||
- Update the `go` directive and `toolchain` directive in `go.mod`
|
|
||||||
- `go` is the minimum supported version
|
|
||||||
- `toolchain` is the preferred toolchain version if `GOTOOLCHAIN` is not specified
|
|
||||||
|
|
||||||
Run `go mod tidy` to ensure consistency.
|
|
||||||
|
|
||||||
Update Go module dependencies:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Update all other dependencies
|
|
||||||
go get -u -t ./...
|
|
||||||
go mod tidy
|
|
||||||
|
|
||||||
# Above might fail if there are version selection conflicts.
|
|
||||||
# Figure out what's going on by updating packages from error messages first.
|
|
||||||
# Example:
|
|
||||||
go get -u google.golang.org/genproto google.golang.org/grpc google.golang.org/protobuf
|
|
||||||
```
|
|
||||||
|
|
||||||
Update codegen & lint tools
|
|
||||||
- `protoc` => `build/get_protoc.bash`
|
|
||||||
- GH releases publish sha256 sums
|
|
||||||
- `golangci-lint` => `build/get_golangci_lint.bash`
|
|
||||||
- bump versions in `build/tools.go`
|
|
||||||
- we use the tools.go trick:
|
|
||||||
- `go get -tags tools -u example.com/tool ; go mod tidy`
|
|
||||||
- review whether we're ready to switch to `go tool`: https://github.com/zrepl/zrepl/pull/909
|
|
||||||
|
|
||||||
Now run `make generate`.
|
|
||||||
|
|
||||||
Run `make lint` and `make vet`.
|
|
||||||
|
|
||||||
Update the CI configuration `.circleci/config.yml`:
|
|
||||||
- Update Go version references (we reference the minimum and max supported version)
|
|
||||||
- Set `Makefile` `RELEASE_GOVERSION` to the new Go version
|
|
||||||
- Update the pinned ZFS release tags in the `platformtest` matrix (`zfs_release` parameter) to the latest patch releases of each OpenZFS branch (currently 2.2, 2.3, 2.4)
|
|
||||||
- Update the pinned Ubuntu machine image for `platformtest` (e.g. `ubuntu-2404:2025.09.1`) — this is pinned rather than `current` so the ZFS build cache stays valid across runs
|
|
||||||
|
|
||||||
Update docs build tooling:
|
|
||||||
- Update `uv` version in `.circleci/config.yml` (search for `astral.sh/uv/` and cache keys containing the version)
|
|
||||||
- Check if there's now a CircleCI orb for uv that we could use
|
|
||||||
- Update Python version in `docs/.python-version`
|
|
||||||
|
|
||||||
Update docs dependencies (Sphinx, sphinx-rtd-theme):
|
|
||||||
- Check current versions in `docs/pyproject.toml`
|
|
||||||
- Review upstream changelogs for breaking changes
|
|
||||||
- Update version constraints in `pyproject.toml` and the `uv` lockfile (see [uv docs on dependencies](https://docs.astral.sh/uv/concepts/projects/dependencies/)):
|
|
||||||
- Test locally with `make docs`
|
|
||||||
|
|
||||||
Kick a full CI pipeline run (`do_ci=true` and `do_release=true`).
|
|
||||||
|
|
||||||
Merge PR with merge commit.
|
|
||||||
|
|
||||||
## Notes to Distro Package Maintainers
|
|
||||||
|
|
||||||
* The `Makefile` in this project is not suitable for builds in distros.
|
|
||||||
* Run the platform tests (Docs -> Usage -> Platform Tests) **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
|
* Run the platform tests (Docs -> Usage -> Platform Tests) **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
|
||||||
* Ship a default config that adheres to your distro's `hier` and logging system.
|
* Ship a default config that adheres to your distro's `hier` and logging system.
|
||||||
* Ship a service manager file and _please_ try to upstream it to this repository.
|
* Ship a service manager file and _please_ try to upstream it to this repository.
|
||||||
@@ -226,3 +105,31 @@ Merge PR with merge commit.
|
|||||||
This is how `zrepl version` knows what version number to show.
|
This is how `zrepl version` knows what version number to show.
|
||||||
Your build system should set the `ldFlags` flags appropriately and add a prefix or suffix that indicates that the given zrepl binary is a distro build, not an official one.
|
Your build system should set the `ldFlags` flags appropriately and add a prefix or suffix that indicates that the given zrepl binary is a distro build, not an official one.
|
||||||
* Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
|
* Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
|
||||||
|
|
||||||
|
|
||||||
|
## Contributing Code
|
||||||
|
|
||||||
|
* Open an issue when starting to hack on a new feature
|
||||||
|
* Commits should reference the issue they are related to
|
||||||
|
* Docs improvements not documenting new features do not require an issue.
|
||||||
|
|
||||||
|
### Breaking Changes
|
||||||
|
|
||||||
|
Backward-incompatible changes must be documented in the git commit message and are listed in `docs/changelog.rst`.
|
||||||
|
|
||||||
|
### Glossary & Naming Inconsistencies
|
||||||
|
|
||||||
|
In ZFS, *dataset* refers to the objects *filesystem*, *ZVOL* and *snapshot*. <br />
|
||||||
|
However, we need a word for *filesystem* & *ZVOL* but not a snapshot, bookmark, etc.
|
||||||
|
|
||||||
|
Toward the user, the following terminology is used:
|
||||||
|
|
||||||
|
* **filesystem**: a ZFS filesystem or a ZVOL
|
||||||
|
* **filesystem version**: a ZFS snapshot or a bookmark
|
||||||
|
|
||||||
|
Sadly, the zrepl implementation is inconsistent in its use of these words:
|
||||||
|
variables and types are often named *dataset* when they in fact refer to a *filesystem*.
|
||||||
|
|
||||||
|
There will not be a big refactoring (an attempt was made, but it's destroying too much history without much gain).
|
||||||
|
|
||||||
|
However, new contributions & patches should fix naming without further notice in the commit message.
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
FROM !SUBSTITUTED_BY_MAKEFILE
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y \
|
||||||
|
python3-pip \
|
||||||
|
unzip \
|
||||||
|
gawk
|
||||||
|
|
||||||
|
ADD build.installprotoc.bash ./
|
||||||
|
RUN bash build.installprotoc.bash
|
||||||
|
|
||||||
|
ADD lazy.sh /tmp/lazy.sh
|
||||||
|
ADD docs/requirements.txt /tmp/requirements.txt
|
||||||
|
ENV ZREPL_LAZY_DOCS_REQPATH=/tmp/requirements.txt
|
||||||
|
RUN /tmp/lazy.sh docdep
|
||||||
|
|
||||||
|
# prepare volume mount of git checkout to /zrepl
|
||||||
|
RUN mkdir -p /src/github.com/zrepl/zrepl
|
||||||
|
RUN mkdir -p /.cache && chmod -R 0777 /.cache
|
||||||
|
|
||||||
|
# $GOPATH is /go
|
||||||
|
# Go 1.12 doesn't use modules within GOPATH, but 1.13 and later do
|
||||||
|
# => store source outside of GOPATH
|
||||||
|
WORKDIR /src
|
||||||
|
|
||||||
|
# Install build tools (e.g. protobuf generator, stringer) into $GOPATH/bin
|
||||||
|
ADD build/ /tmp/build
|
||||||
|
RUN /tmp/lazy.sh godep
|
||||||
|
|
||||||
|
RUN chmod -R 0777 /go
|
||||||
|
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
set -x
|
||||||
|
|
||||||
|
MACH=$(uname -m)
|
||||||
|
MACH="${MACH/aarch64/aarch_64}"
|
||||||
|
|
||||||
|
VERSION=3.6.1
|
||||||
|
FILENAME=protoc-"$VERSION"-linux-"$MACH".zip
|
||||||
|
|
||||||
|
if [ -e "$FILENAME" ]; then
|
||||||
|
echo "$FILENAME" already exists 1>&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
wget https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
|
||||||
|
|
||||||
|
stat "$FILENAME"
|
||||||
|
|
||||||
|
sha256sum -c --ignore-missing <<EOF
|
||||||
|
6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807 protoc-3.6.1-linux-x86_64.zip
|
||||||
|
af8e5aaaf39ddec62ec8dd2be1b8d9602c6da66564883a16393ade5f71170922 protoc-3.6.1-linux-aarch_64.zip
|
||||||
|
EOF
|
||||||
|
|
||||||
|
unzip -d /usr "$FILENAME"
|
||||||
@@ -1 +0,0 @@
|
|||||||
install/
|
|
||||||
@@ -1,30 +0,0 @@
|
|||||||
FROM !SUBSTITUTED_BY_MAKEFILE
|
|
||||||
|
|
||||||
ARG BUILD_UID=1000
|
|
||||||
ARG BUILD_GID=1000
|
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y \
|
|
||||||
python3 \
|
|
||||||
unzip \
|
|
||||||
gawk \
|
|
||||||
curl
|
|
||||||
|
|
||||||
# Create build user with the host's UID/GID before installing uv
|
|
||||||
RUN groupadd -g ${BUILD_GID} zrepl_build && \
|
|
||||||
useradd -u ${BUILD_UID} -g ${BUILD_GID} -m zrepl_build
|
|
||||||
|
|
||||||
# Go toolchain uses xdg-cache
|
|
||||||
RUN mkdir -p /.cache && chmod -R 0777 /.cache
|
|
||||||
|
|
||||||
# Install uv as the build user - version from .uv-version file
|
|
||||||
ADD .uv-version /tmp/.uv-version
|
|
||||||
USER zrepl_build
|
|
||||||
RUN UV_VERSION=$(cat /tmp/.uv-version) && \
|
|
||||||
curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh
|
|
||||||
|
|
||||||
# Go devtools are managed by Makefile
|
|
||||||
|
|
||||||
WORKDIR /src
|
|
||||||
ENV PATH="/home/zrepl_build/.local/bin:$PATH" \
|
|
||||||
GOCACHE="/.cache/go-build"
|
|
||||||
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
set -x
|
|
||||||
|
|
||||||
# golangci-lint recommends against using `go get` and friends to install it
|
|
||||||
|
|
||||||
cd "$1"
|
|
||||||
|
|
||||||
VERSION=2.8.0
|
|
||||||
FILENAME=golangci-lint-"$VERSION"-linux-"$GOHOSTARCH".tar.gz
|
|
||||||
|
|
||||||
if [ -e "$FILENAME" ]; then
|
|
||||||
echo "$FILENAME" already exists 1>&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
wget --continue https://github.com/golangci/golangci-lint/releases/download/v"$VERSION"/"$FILENAME"
|
|
||||||
|
|
||||||
stat "$FILENAME"
|
|
||||||
|
|
||||||
# Select the correct checksum for the downloaded architecture
|
|
||||||
case "$GOHOSTARCH" in
|
|
||||||
arm64) EXPECTED_SHA256="2a58388db8af5ab9330791cea0ebdd4100723cd05ad7185d92febaaee272ec9a" ;;
|
|
||||||
amd64) EXPECTED_SHA256="7048bc6b25c9515ed092c83f9fa8709ca97937ead52d9ff317a143299ee97a50" ;;
|
|
||||||
*) echo "Unknown architecture: $GOHOSTARCH" >&2; exit 1 ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
# Verify checksum explicitly - fails if hash doesn't match
|
|
||||||
echo "$EXPECTED_SHA256 $FILENAME" | sha256sum -c -
|
|
||||||
|
|
||||||
tar -x --strip-components=1 -f "$FILENAME"
|
|
||||||
|
|
||||||
stat ./golangci-lint
|
|
||||||
@@ -1,32 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
set -x
|
|
||||||
|
|
||||||
cd "$1"
|
|
||||||
|
|
||||||
MACH=$(uname -m)
|
|
||||||
MACH="${MACH/aarch64/aarch_64}"
|
|
||||||
|
|
||||||
VERSION=33.5
|
|
||||||
FILENAME=protoc-"$VERSION"-linux-"$MACH".zip
|
|
||||||
|
|
||||||
if [ -e "$FILENAME" ]; then
|
|
||||||
echo "$FILENAME" already exists 1>&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
wget --continue https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
|
|
||||||
|
|
||||||
stat "$FILENAME"
|
|
||||||
|
|
||||||
# Select the correct checksum for the downloaded architecture
|
|
||||||
case "$MACH" in
|
|
||||||
aarch_64) EXPECTED_SHA256="2b0fcf9b2c32cbadccc0eb7a88b841fffecd4a06fc80acdba2b5be45e815c38a" ;;
|
|
||||||
x86_64) EXPECTED_SHA256="24e58fb231d50306ee28491f33a170301e99540f7e29ca461e0e80fd1239f8d1" ;;
|
|
||||||
*) echo "Unknown architecture: $MACH" >&2; exit 1 ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
# Verify checksum explicitly - fails if hash doesn't match
|
|
||||||
echo "$EXPECTED_SHA256 $FILENAME" | sha256sum -c -
|
|
||||||
|
|
||||||
unzip -d . "$FILENAME"
|
|
||||||
+7
-18
@@ -1,24 +1,13 @@
|
|||||||
module github.com/zrepl/zrepl/build
|
module github.com/zrepl/zrepl/build
|
||||||
|
|
||||||
go 1.24.13
|
go 1.12
|
||||||
|
|
||||||
toolchain go1.25.7
|
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/alvaroloes/enumer v1.1.2
|
github.com/alvaroloes/enumer v1.1.1
|
||||||
|
github.com/golangci/golangci-lint v1.50.1
|
||||||
|
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||||
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad
|
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad
|
||||||
golang.org/x/tools v0.41.0
|
golang.org/x/tools v0.2.0
|
||||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.1
|
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 // indirect
|
||||||
google.golang.org/protobuf v1.36.11
|
google.golang.org/protobuf v1.28.0
|
||||||
)
|
|
||||||
|
|
||||||
require (
|
|
||||||
github.com/pascaldekloe/name v1.0.1 // indirect
|
|
||||||
golang.org/x/mod v0.32.0 // indirect
|
|
||||||
golang.org/x/sync v0.19.0 // indirect
|
|
||||||
golang.org/x/sys v0.41.0 // indirect
|
|
||||||
golang.org/x/telemetry v0.0.0-20260205145544-86a5c4bf3c8d // indirect
|
|
||||||
golang.org/x/text v0.31.0 // indirect
|
|
||||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c // indirect
|
|
||||||
google.golang.org/grpc v1.75.0 // indirect
|
|
||||||
)
|
)
|
||||||
|
|||||||
+1817
-28
File diff suppressed because it is too large
Load Diff
@@ -1,6 +0,0 @@
|
|||||||
export GO111MODULE=on # otherwise, a checkout of this repo in GOPATH will disable modules on Go 1.12 and earlier
|
|
||||||
source <(go env)
|
|
||||||
# build tools for the host platform
|
|
||||||
export GOOS="$GOHOSTOS"
|
|
||||||
export GOARCH="$GOHOSTARCH"
|
|
||||||
# TODO GOARM=$GOHOSTARM?
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
set -euo pipefail
|
|
||||||
set -x
|
|
||||||
|
|
||||||
OUTDIR="$(readlink -f "$1")"
|
|
||||||
if [ -e "$OUTDIR" ]; then
|
|
||||||
echo "$OUTDIR" already exists 1>&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# go install command below will install tools to $GOBIN
|
|
||||||
export GOBIN="$OUTDIR"
|
|
||||||
|
|
||||||
cd "$(dirname "$0")"
|
|
||||||
|
|
||||||
source ./go_install_host_tool.source
|
|
||||||
|
|
||||||
cat tools.go | grep _ | awk -F'"' '{print $2}' | tee | xargs -tI '{}' go install '{}'
|
|
||||||
@@ -6,6 +6,7 @@ package main
|
|||||||
// the lines are parsed by lazy.sh, do not edit
|
// the lines are parsed by lazy.sh, do not edit
|
||||||
import (
|
import (
|
||||||
_ "github.com/alvaroloes/enumer"
|
_ "github.com/alvaroloes/enumer"
|
||||||
|
_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
|
||||||
_ "github.com/wadey/gocovmerge"
|
_ "github.com/wadey/gocovmerge"
|
||||||
_ "golang.org/x/tools/cmd/goimports"
|
_ "golang.org/x/tools/cmd/goimports"
|
||||||
_ "golang.org/x/tools/cmd/stringer"
|
_ "golang.org/x/tools/cmd/stringer"
|
||||||
|
|||||||
@@ -8,9 +8,9 @@ import (
|
|||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
var rootArgs struct {
|
var rootArgs struct {
|
||||||
@@ -11,11 +11,11 @@ import (
|
|||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
"github.com/zrepl/yaml-config"
|
"github.com/zrepl/yaml-config"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/cli"
|
"github.com/zrepl/zrepl/cli"
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
"github.com/zrepl/zrepl/daemon/job"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
"github.com/zrepl/zrepl/daemon/logging"
|
||||||
"github.com/zrepl/zrepl/internal/logger"
|
"github.com/zrepl/zrepl/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
var configcheckArgs struct {
|
var configcheckArgs struct {
|
||||||
@@ -9,12 +9,12 @@ import (
|
|||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
"github.com/zrepl/zrepl/daemon/job"
|
||||||
"github.com/zrepl/zrepl/internal/endpoint"
|
"github.com/zrepl/zrepl/endpoint"
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/cli"
|
"github.com/zrepl/zrepl/cli"
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
package client
|
package client
|
||||||
|
|
||||||
import "github.com/zrepl/zrepl/internal/cli"
|
import "github.com/zrepl/zrepl/cli"
|
||||||
|
|
||||||
var PprofCmd = &cli.Subcommand{
|
var PprofCmd = &cli.Subcommand{
|
||||||
Use: "pprof",
|
Use: "pprof",
|
||||||
@@ -8,7 +8,7 @@ import (
|
|||||||
|
|
||||||
"golang.org/x/net/websocket"
|
"golang.org/x/net/websocket"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/cli"
|
"github.com/zrepl/zrepl/cli"
|
||||||
)
|
)
|
||||||
|
|
||||||
var pprofActivityTraceCmd = &cli.Subcommand{
|
var pprofActivityTraceCmd = &cli.Subcommand{
|
||||||
@@ -6,9 +6,9 @@ import (
|
|||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/cli"
|
"github.com/zrepl/zrepl/cli"
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon"
|
"github.com/zrepl/zrepl/daemon"
|
||||||
)
|
)
|
||||||
|
|
||||||
var pprofListenCmd struct {
|
var pprofListenCmd struct {
|
||||||
@@ -5,9 +5,9 @@ import (
|
|||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/cli"
|
"github.com/zrepl/zrepl/cli"
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon"
|
"github.com/zrepl/zrepl/daemon"
|
||||||
)
|
)
|
||||||
|
|
||||||
var SignalCmd = &cli.Subcommand{
|
var SignalCmd = &cli.Subcommand{
|
||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon"
|
"github.com/zrepl/zrepl/daemon"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Client struct {
|
type Client struct {
|
||||||
@@ -9,11 +9,11 @@ import (
|
|||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/cli"
|
"github.com/zrepl/zrepl/cli"
|
||||||
"github.com/zrepl/zrepl/internal/client/status/client"
|
"github.com/zrepl/zrepl/client/status/client"
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon"
|
"github.com/zrepl/zrepl/daemon"
|
||||||
"github.com/zrepl/zrepl/internal/util/choices"
|
"github.com/zrepl/zrepl/util/choices"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Client interface {
|
type Client interface {
|
||||||
@@ -5,11 +5,11 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/gdamore/tcell/v2"
|
"github.com/gdamore/tcell"
|
||||||
"github.com/mattn/go-isatty"
|
"github.com/mattn/go-isatty"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/client/status/viewmodel"
|
"github.com/zrepl/zrepl/client/status/viewmodel"
|
||||||
)
|
)
|
||||||
|
|
||||||
func dump(c Client, job string) error {
|
func dump(c Client, job string) error {
|
||||||
@@ -8,9 +8,9 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/gdamore/tcell/v2"
|
"github.com/gdamore/tcell/v2"
|
||||||
"github.com/rivo/tview"
|
tview "gitlab.com/tslocum/cview"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/client/status/viewmodel"
|
"github.com/zrepl/zrepl/client/status/viewmodel"
|
||||||
)
|
)
|
||||||
|
|
||||||
func interactive(c Client, flag statusFlags) error {
|
func interactive(c Client, flag statusFlags) error {
|
||||||
@@ -29,6 +29,7 @@ func interactive(c Client, flag statusFlags) error {
|
|||||||
jobMenuRoot.SetSelectable(true)
|
jobMenuRoot.SetSelectable(true)
|
||||||
jobMenu.SetRoot(jobMenuRoot)
|
jobMenu.SetRoot(jobMenuRoot)
|
||||||
jobMenu.SetCurrentNode(jobMenuRoot)
|
jobMenu.SetCurrentNode(jobMenuRoot)
|
||||||
|
jobMenu.SetSelectedTextColor(tcell.ColorGreen)
|
||||||
jobTextDetail := tview.NewTextView()
|
jobTextDetail := tview.NewTextView()
|
||||||
jobTextDetail.SetWrap(false)
|
jobTextDetail.SetWrap(false)
|
||||||
|
|
||||||
@@ -109,8 +110,10 @@ func interactive(c Client, flag statusFlags) error {
|
|||||||
}
|
}
|
||||||
app.SetRoot(toolbarSplit, true)
|
app.SetRoot(toolbarSplit, true)
|
||||||
app.SetFocus(preModalFocus)
|
app.SetFocus(preModalFocus)
|
||||||
|
app.Draw()
|
||||||
})
|
})
|
||||||
app.SetRoot(m, true)
|
app.SetRoot(m, true)
|
||||||
|
app.Draw()
|
||||||
}
|
}
|
||||||
|
|
||||||
app.SetRoot(toolbarSplit, true)
|
app.SetRoot(toolbarSplit, true)
|
||||||
@@ -167,14 +170,12 @@ func interactive(c Client, flag statusFlags) error {
|
|||||||
redrawJobsList = true
|
redrawJobsList = true
|
||||||
}
|
}
|
||||||
if redrawJobsList {
|
if redrawJobsList {
|
||||||
selectedTextStyle := tcell.StyleDefault.Bold(true)
|
|
||||||
selectedJobN = nil
|
selectedJobN = nil
|
||||||
children := make([]*tview.TreeNode, len(jobs))
|
children := make([]*tview.TreeNode, len(jobs))
|
||||||
for i := range jobs {
|
for i := range jobs {
|
||||||
jobN := tview.NewTreeNode(jobs[i].JobTreeTitle())
|
jobN := tview.NewTreeNode(jobs[i].JobTreeTitle())
|
||||||
jobN.SetReference(jobs[i])
|
jobN.SetReference(jobs[i])
|
||||||
jobN.SetSelectable(true)
|
jobN.SetSelectable(true)
|
||||||
jobN.SetSelectedTextStyle(selectedTextStyle)
|
|
||||||
children[i] = jobN
|
children[i] = jobN
|
||||||
jobN.SetSelectedFunc(func() {
|
jobN.SetSelectedFunc(func() {
|
||||||
viewmodelupdate(func(p *viewmodel.Params) {
|
viewmodelupdate(func(p *viewmodel.Params) {
|
||||||
@@ -186,7 +187,6 @@ func interactive(c Client, flag statusFlags) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
jobMenuRoot.SetChildren(children)
|
jobMenuRoot.SetChildren(children)
|
||||||
jobMenuRoot.SetSelectedTextStyle(selectedTextStyle)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if selectedJobN != nil && jobMenu.GetCurrentNode() != selectedJobN {
|
if selectedJobN != nil && jobMenu.GetCurrentNode() != selectedJobN {
|
||||||
@@ -207,6 +207,9 @@ func interactive(c Client, flag statusFlags) error {
|
|||||||
bottombar.ResizeItem(bottombarDateView, len(bottombardatestring), 0)
|
bottombar.ResizeItem(bottombarDateView, len(bottombardatestring), 0)
|
||||||
|
|
||||||
bottomBarStatus.SetText(m.BottomBarStatus())
|
bottomBarStatus.SetText(m.BottomBarStatus())
|
||||||
|
|
||||||
|
app.Draw()
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
@@ -249,7 +252,6 @@ func interactive(c Client, flag statusFlags) error {
|
|||||||
|
|
||||||
app.SetInputCapture(func(e *tcell.EventKey) *tcell.EventKey {
|
app.SetInputCapture(func(e *tcell.EventKey) *tcell.EventKey {
|
||||||
if e.Key() == tcell.KeyTab {
|
if e.Key() == tcell.KeyTab {
|
||||||
// TODO: only if there's no modal showing (long-time bug in zrepl status)
|
|
||||||
tabbableCycle()
|
tabbableCycle()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -282,8 +284,9 @@ func interactive(c Client, flag statusFlags) error {
|
|||||||
signals := []string{"wakeup", "reset"}
|
signals := []string{"wakeup", "reset"}
|
||||||
clientFuncs := []func(job string) error{c.SignalWakeup, c.SignalReset}
|
clientFuncs := []func(job string) error{c.SignalWakeup, c.SignalReset}
|
||||||
sigMod := tview.NewModal()
|
sigMod := tview.NewModal()
|
||||||
|
sigMod.SetBackgroundColor(tcell.ColorDefault)
|
||||||
sigMod.SetBorder(true)
|
sigMod.SetBorder(true)
|
||||||
sigMod.SetButtonActivatedStyle(tcell.StyleDefault.Bold(true).Reverse(true))
|
sigMod.GetForm().SetButtonTextColorFocused(tcell.ColorGreen)
|
||||||
sigMod.AddButtons(signals)
|
sigMod.AddButtons(signals)
|
||||||
sigMod.SetText(fmt.Sprintf("Send a signal to job %q", job.Name()))
|
sigMod.SetText(fmt.Sprintf("Send a signal to job %q", job.Name()))
|
||||||
showModal(sigMod, func(idx int, _ string) {
|
showModal(sigMod, func(idx int, _ string) {
|
||||||
@@ -10,9 +10,9 @@ import (
|
|||||||
"github.com/gdamore/tcell/v2"
|
"github.com/gdamore/tcell/v2"
|
||||||
"github.com/mattn/go-isatty"
|
"github.com/mattn/go-isatty"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/rivo/tview"
|
tview "gitlab.com/tslocum/cview"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/client/status/viewmodel"
|
"github.com/zrepl/zrepl/client/status/viewmodel"
|
||||||
)
|
)
|
||||||
|
|
||||||
func legacy(c Client, flag statusFlags) error {
|
func legacy(c Client, flag statusFlags) error {
|
||||||
@@ -28,7 +28,7 @@ func legacy(c Client, flag statusFlags) error {
|
|||||||
textView := tview.NewTextView()
|
textView := tview.NewTextView()
|
||||||
textView.SetWrap(true)
|
textView.SetWrap(true)
|
||||||
textView.SetScrollable(true) // so that it allows us to set scroll position
|
textView.SetScrollable(true) // so that it allows us to set scroll position
|
||||||
// textView.SetScrollBarVisibility(tview.ScrollBarNever)
|
textView.SetScrollBarVisibility(tview.ScrollBarNever)
|
||||||
|
|
||||||
app.SetRoot(textView, true)
|
app.SetRoot(textView, true)
|
||||||
|
|
||||||
@@ -10,12 +10,12 @@ import (
|
|||||||
"github.com/go-playground/validator/v10"
|
"github.com/go-playground/validator/v10"
|
||||||
yaml "github.com/zrepl/yaml-config"
|
yaml "github.com/zrepl/yaml-config"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/client/status/viewmodel/stringbuilder"
|
"github.com/zrepl/zrepl/client/status/viewmodel/stringbuilder"
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/daemon"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
"github.com/zrepl/zrepl/daemon/job"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/pruner"
|
"github.com/zrepl/zrepl/daemon/pruner"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/snapper"
|
"github.com/zrepl/zrepl/daemon/snapper"
|
||||||
"github.com/zrepl/zrepl/internal/replication/report"
|
"github.com/zrepl/zrepl/replication/report"
|
||||||
)
|
)
|
||||||
|
|
||||||
type M struct {
|
type M struct {
|
||||||
@@ -85,7 +85,7 @@ func (m *M) Update(p Params) {
|
|||||||
// filter out internal jobs
|
// filter out internal jobs
|
||||||
var jobsList []*Job
|
var jobsList []*Job
|
||||||
for _, j := range m.jobsList {
|
for _, j := range m.jobsList {
|
||||||
if config.IsInternalJobName(j.name) {
|
if daemon.IsInternalJobName(j.name) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
jobsList = append(jobsList, j)
|
jobsList = append(jobsList, j)
|
||||||
@@ -374,15 +374,13 @@ func renderReplicationReport(t *stringbuilder.B, rep *report.Report, history *by
|
|||||||
eta = time.Duration((float64(expected)-float64(replicated))/float64(rate)) * time.Second
|
eta = time.Duration((float64(expected)-float64(replicated))/float64(rate)) * time.Second
|
||||||
}
|
}
|
||||||
|
|
||||||
if !latest.State.IsTerminal() {
|
t.Write("Progress: ")
|
||||||
t.Write("Progress: ")
|
t.DrawBar(50, replicated, expected, changeCount)
|
||||||
t.DrawBar(50, replicated, expected, changeCount)
|
t.Write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected), ByteCountBinary(rate)))
|
||||||
t.Write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected), ByteCountBinary(rate)))
|
if eta != 0 {
|
||||||
if eta != 0 {
|
t.Write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
|
||||||
t.Write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
|
|
||||||
}
|
|
||||||
t.Newline()
|
|
||||||
}
|
}
|
||||||
|
t.Newline()
|
||||||
if containsInvalidSizeEstimates {
|
if containsInvalidSizeEstimates {
|
||||||
t.Write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
|
t.Write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
|
||||||
t.Newline()
|
t.Newline()
|
||||||
@@ -495,17 +493,15 @@ func renderPrunerReport(t *stringbuilder.B, r *pruner.Report, fsfilter FilterFun
|
|||||||
}
|
}
|
||||||
|
|
||||||
// global progress bar
|
// global progress bar
|
||||||
if !state.IsTerminal() {
|
progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
|
||||||
progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
|
t.Write("Progress: ")
|
||||||
t.Write("Progress: ")
|
t.Write("[")
|
||||||
t.Write("[")
|
t.Write(stringbuilder.Times("=", progress))
|
||||||
t.Write(stringbuilder.Times("=", progress))
|
t.Write(">")
|
||||||
t.Write(">")
|
t.Write(stringbuilder.Times("-", 80-progress))
|
||||||
t.Write(stringbuilder.Times("-", 80-progress))
|
t.Write("]")
|
||||||
t.Write("]")
|
t.Printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
|
||||||
t.Printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
|
t.Newline()
|
||||||
t.Newline()
|
|
||||||
}
|
|
||||||
|
|
||||||
sort.SliceStable(all, func(i, j int) bool {
|
sort.SliceStable(all, func(i, j int) bool {
|
||||||
return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
|
return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
|
||||||
@@ -5,8 +5,8 @@ import (
|
|||||||
|
|
||||||
"github.com/problame/go-netssh"
|
"github.com/problame/go-netssh"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/cli"
|
"github.com/zrepl/zrepl/cli"
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
|
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
@@ -9,10 +9,10 @@ import (
|
|||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/cli"
|
"github.com/zrepl/zrepl/cli"
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
"github.com/zrepl/zrepl/daemon/filters"
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
var TestCmd = &cli.Subcommand{
|
var TestCmd = &cli.Subcommand{
|
||||||
@@ -7,10 +7,10 @@ import (
|
|||||||
|
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/cli"
|
"github.com/zrepl/zrepl/cli"
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon"
|
"github.com/zrepl/zrepl/daemon"
|
||||||
"github.com/zrepl/zrepl/internal/version"
|
"github.com/zrepl/zrepl/version"
|
||||||
)
|
)
|
||||||
|
|
||||||
var versionArgs struct {
|
var versionArgs struct {
|
||||||
@@ -7,10 +7,10 @@ import (
|
|||||||
|
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/cli"
|
"github.com/zrepl/zrepl/cli"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
"github.com/zrepl/zrepl/daemon/filters"
|
||||||
"github.com/zrepl/zrepl/internal/endpoint"
|
"github.com/zrepl/zrepl/endpoint"
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
package client
|
package client
|
||||||
|
|
||||||
import "github.com/zrepl/zrepl/internal/cli"
|
import "github.com/zrepl/zrepl/cli"
|
||||||
|
|
||||||
var zabsCmdCreate = &cli.Subcommand{
|
var zabsCmdCreate = &cli.Subcommand{
|
||||||
Use: "create",
|
Use: "create",
|
||||||
+3
-3
@@ -7,9 +7,9 @@ import (
|
|||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/cli"
|
"github.com/zrepl/zrepl/cli"
|
||||||
"github.com/zrepl/zrepl/internal/endpoint"
|
"github.com/zrepl/zrepl/endpoint"
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
var zabsCreateStepHoldFlags struct {
|
var zabsCreateStepHoldFlags struct {
|
||||||
@@ -11,9 +11,9 @@ import (
|
|||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/cli"
|
"github.com/zrepl/zrepl/cli"
|
||||||
"github.com/zrepl/zrepl/internal/endpoint"
|
"github.com/zrepl/zrepl/endpoint"
|
||||||
"github.com/zrepl/zrepl/internal/util/chainlock"
|
"github.com/zrepl/zrepl/util/chainlock"
|
||||||
)
|
)
|
||||||
|
|
||||||
var zabsListFlags struct {
|
var zabsListFlags struct {
|
||||||
@@ -10,8 +10,8 @@ import (
|
|||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/spf13/pflag"
|
"github.com/spf13/pflag"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/cli"
|
"github.com/zrepl/zrepl/cli"
|
||||||
"github.com/zrepl/zrepl/internal/endpoint"
|
"github.com/zrepl/zrepl/endpoint"
|
||||||
)
|
)
|
||||||
|
|
||||||
// shared between release-all and release-step
|
// shared between release-all and release-step
|
||||||
@@ -2,19 +2,20 @@ package config
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
"log/syslog"
|
"log/syslog"
|
||||||
"os"
|
"os"
|
||||||
pathpkg "path"
|
"reflect"
|
||||||
"path/filepath"
|
"regexp"
|
||||||
"strings"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/robfig/cron/v3"
|
"github.com/robfig/cron/v3"
|
||||||
"github.com/zrepl/yaml-config"
|
"github.com/zrepl/yaml-config"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/util/datasizeunit"
|
"github.com/zrepl/zrepl/util/datasizeunit"
|
||||||
zfsprop "github.com/zrepl/zrepl/internal/zfs/property"
|
zfsprop "github.com/zrepl/zrepl/zfs/property"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ParseFlags uint
|
type ParseFlags uint
|
||||||
@@ -25,9 +26,8 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Jobs []JobEnum `yaml:"jobs,optional"`
|
Jobs []JobEnum `yaml:"jobs"`
|
||||||
Global *Global `yaml:"global,optional,fromdefaults"`
|
Global *Global `yaml:"global,optional,fromdefaults"`
|
||||||
Include []string `yaml:"include,optional"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Config) Job(name string) (*JobEnum, error) {
|
func (c *Config) Job(name string) (*JobEnum, error) {
|
||||||
@@ -67,6 +67,7 @@ type ActiveJob struct {
|
|||||||
Name string `yaml:"name"`
|
Name string `yaml:"name"`
|
||||||
Connect ConnectEnum `yaml:"connect"`
|
Connect ConnectEnum `yaml:"connect"`
|
||||||
Pruning PruningSenderReceiver `yaml:"pruning"`
|
Pruning PruningSenderReceiver `yaml:"pruning"`
|
||||||
|
Debug JobDebugSettings `yaml:"debug,optional"`
|
||||||
Replication *Replication `yaml:"replication,optional,fromdefaults"`
|
Replication *Replication `yaml:"replication,optional,fromdefaults"`
|
||||||
ConflictResolution *ConflictResolution `yaml:"conflict_resolution,optional,fromdefaults"`
|
ConflictResolution *ConflictResolution `yaml:"conflict_resolution,optional,fromdefaults"`
|
||||||
}
|
}
|
||||||
@@ -76,15 +77,17 @@ type ConflictResolution struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type PassiveJob struct {
|
type PassiveJob struct {
|
||||||
Type string `yaml:"type"`
|
Type string `yaml:"type"`
|
||||||
Name string `yaml:"name"`
|
Name string `yaml:"name"`
|
||||||
Serve ServeEnum `yaml:"serve"`
|
Serve ServeEnum `yaml:"serve"`
|
||||||
|
Debug JobDebugSettings `yaml:"debug,optional"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type SnapJob struct {
|
type SnapJob struct {
|
||||||
Type string `yaml:"type"`
|
Type string `yaml:"type"`
|
||||||
Name string `yaml:"name"`
|
Name string `yaml:"name"`
|
||||||
Pruning PruningLocal `yaml:"pruning"`
|
Pruning PruningLocal `yaml:"pruning"`
|
||||||
|
Debug JobDebugSettings `yaml:"debug,optional"`
|
||||||
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
|
Snapshotting SnapshottingEnum `yaml:"snapshotting"`
|
||||||
Filesystems FilesystemsFilter `yaml:"filesystems"`
|
Filesystems FilesystemsFilter `yaml:"filesystems"`
|
||||||
}
|
}
|
||||||
@@ -187,10 +190,13 @@ func (i *PositiveDurationOrManual) UnmarshalYAML(u func(interface{}, bool) error
|
|||||||
return fmt.Errorf("value must not be empty")
|
return fmt.Errorf("value must not be empty")
|
||||||
default:
|
default:
|
||||||
i.Manual = false
|
i.Manual = false
|
||||||
i.Interval, err = parsePositiveDuration(s)
|
i.Interval, err = time.ParseDuration(s)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if i.Interval <= 0 {
|
||||||
|
return fmt.Errorf("value must be a positive duration, got %q", s)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -222,16 +228,10 @@ type SnapshottingEnum struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type SnapshottingPeriodic struct {
|
type SnapshottingPeriodic struct {
|
||||||
Type string `yaml:"type"`
|
Type string `yaml:"type"`
|
||||||
Prefix string `yaml:"prefix"`
|
Prefix string `yaml:"prefix"`
|
||||||
Interval *PositiveDuration `yaml:"interval"`
|
Interval time.Duration `yaml:"interval,positive"`
|
||||||
Hooks HookList `yaml:"hooks,optional"`
|
Hooks HookList `yaml:"hooks,optional"`
|
||||||
TimestampFormattingSpec `yaml:",inline"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type TimestampFormattingSpec struct {
|
|
||||||
TimestampFormat string `yaml:"timestamp_format,optional,default=dense"`
|
|
||||||
TimestampLocation string `yaml:"timestamp_location,optional,default=UTC"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type CronSpec struct {
|
type CronSpec struct {
|
||||||
@@ -260,11 +260,10 @@ func (s *CronSpec) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool)
|
|||||||
}
|
}
|
||||||
|
|
||||||
type SnapshottingCron struct {
|
type SnapshottingCron struct {
|
||||||
Type string `yaml:"type"`
|
Type string `yaml:"type"`
|
||||||
Prefix string `yaml:"prefix"`
|
Prefix string `yaml:"prefix"`
|
||||||
Cron CronSpec `yaml:"cron"`
|
Cron CronSpec `yaml:"cron"`
|
||||||
Hooks HookList `yaml:"hooks,optional"`
|
Hooks HookList `yaml:"hooks,optional"`
|
||||||
TimestampFormattingSpec `yaml:",inline"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type SnapshottingManual struct {
|
type SnapshottingManual struct {
|
||||||
@@ -306,6 +305,18 @@ type Global struct {
|
|||||||
Serve *GlobalServe `yaml:"serve,optional,fromdefaults"`
|
Serve *GlobalServe `yaml:"serve,optional,fromdefaults"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func Default(i interface{}) {
|
||||||
|
v := reflect.ValueOf(i)
|
||||||
|
if v.Kind() != reflect.Ptr {
|
||||||
|
panic(v)
|
||||||
|
}
|
||||||
|
y := `{}`
|
||||||
|
err := yaml.Unmarshal([]byte(y), v.Interface())
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type ConnectEnum struct {
|
type ConnectEnum struct {
|
||||||
Ret interface{}
|
Ret interface{}
|
||||||
}
|
}
|
||||||
@@ -472,6 +483,14 @@ type GlobalStdinServer struct {
|
|||||||
SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
|
SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type JobDebugSettings struct {
|
||||||
|
Conn *struct {
|
||||||
|
ReadDump string `yaml:"read_dump"`
|
||||||
|
WriteDump string `yaml:"write_dump"`
|
||||||
|
} `yaml:"conn,optional"`
|
||||||
|
RPCLog bool `yaml:"rpc_log,optional,default=false"`
|
||||||
|
}
|
||||||
|
|
||||||
type HookList []HookEnum
|
type HookList []HookEnum
|
||||||
|
|
||||||
type HookEnum struct {
|
type HookEnum struct {
|
||||||
@@ -659,9 +678,8 @@ var ConfigFileDefaultLocations = []string{
|
|||||||
"/usr/local/etc/zrepl/zrepl.yml",
|
"/usr/local/etc/zrepl/zrepl.yml",
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseConfig(path string) (rootConfig *Config, err error) {
|
func ParseConfig(path string) (i *Config, err error) {
|
||||||
|
|
||||||
// Parse main configuration file
|
|
||||||
if path == "" {
|
if path == "" {
|
||||||
// Try default locations
|
// Try default locations
|
||||||
for _, l := range ConfigFileDefaultLocations {
|
for _, l := range ConfigFileDefaultLocations {
|
||||||
@@ -680,94 +698,11 @@ func ParseConfig(path string) (rootConfig *Config, err error) {
|
|||||||
|
|
||||||
var bytes []byte
|
var bytes []byte
|
||||||
|
|
||||||
if bytes, err = os.ReadFile(path); err != nil {
|
if bytes, err = ioutil.ReadFile(path); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
rootConfig, err = ParseConfigBytes(bytes)
|
return ParseConfigBytes(bytes)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = expandConfigInclude(path, rootConfig)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = validateJobNames(rootConfig); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return rootConfig, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func IsInternalJobName(s string) bool {
|
|
||||||
return strings.HasPrefix(s, "_")
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateJobNames(config *Config) error {
|
|
||||||
seen := make(map[string]struct{})
|
|
||||||
for _, job := range config.Jobs {
|
|
||||||
name := job.Name()
|
|
||||||
if IsInternalJobName(name) {
|
|
||||||
return errors.Errorf("job name %q is reserved for internal use (starts with _)", name)
|
|
||||||
}
|
|
||||||
if _, ok := seen[name]; ok {
|
|
||||||
return errors.Errorf("duplicate job name %q", name)
|
|
||||||
}
|
|
||||||
seen[name] = struct{}{}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func expandConfigInclude(configPath string, config *Config) (err error) {
|
|
||||||
var includeConfigPaths []string
|
|
||||||
for _, path := range config.Include {
|
|
||||||
if !pathpkg.IsAbs(configPath) {
|
|
||||||
path = pathpkg.Join(pathpkg.Dir(configPath), path)
|
|
||||||
}
|
|
||||||
|
|
||||||
stat, statErr := os.Stat(path)
|
|
||||||
if statErr != nil {
|
|
||||||
return errors.Wrapf(statErr, "stat path %q", path)
|
|
||||||
}
|
|
||||||
|
|
||||||
if stat.Mode().IsDir() {
|
|
||||||
directoryPaths, err := filepath.Glob(path + "/*.yml")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
includeConfigPaths = append(includeConfigPaths, directoryPaths...)
|
|
||||||
} else if stat.Mode().IsRegular() {
|
|
||||||
if extention := filepath.Ext(path); extention != ".yml" {
|
|
||||||
return fmt.Errorf("include config files must end with `.yml`: %s", path)
|
|
||||||
}
|
|
||||||
includeConfigPaths = append(includeConfigPaths, path)
|
|
||||||
} else {
|
|
||||||
return fmt.Errorf("not a file or directory: %s", path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, path := range includeConfigPaths {
|
|
||||||
var bytes []byte
|
|
||||||
if bytes, err = os.ReadFile(path); err != nil {
|
|
||||||
return errors.Wrapf(err, "read file: %q", path)
|
|
||||||
}
|
|
||||||
|
|
||||||
includedConfig, err := ParseConfigBytes(bytes)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(includedConfig.Include) > 0 {
|
|
||||||
return errors.Errorf("included configuration files must not include other files: %s", path)
|
|
||||||
}
|
|
||||||
|
|
||||||
config.Jobs = append(config.Jobs, includedConfig.Jobs...)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseConfigBytes(bytes []byte) (*Config, error) {
|
func ParseConfigBytes(bytes []byte) (*Config, error) {
|
||||||
@@ -775,16 +710,46 @@ func ParseConfigBytes(bytes []byte) (*Config, error) {
|
|||||||
if err := yaml.UnmarshalStrict(bytes, &c); err != nil {
|
if err := yaml.UnmarshalStrict(bytes, &c); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if c != nil {
|
|
||||||
return c, nil
|
|
||||||
}
|
|
||||||
// There was no yaml document in the file, deserialize from default.
|
|
||||||
// => See TestFromdefaultsEmptyDoc in yaml-config package.
|
|
||||||
if err := yaml.UnmarshalStrict([]byte("{}"), &c); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if c == nil {
|
if c == nil {
|
||||||
panic("the fallback to deserialize from `{}` should work")
|
return nil, fmt.Errorf("config is empty or only consists of comments")
|
||||||
}
|
}
|
||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
|
||||||
|
|
||||||
|
func parsePositiveDuration(e string) (d time.Duration, err error) {
|
||||||
|
comps := durationStringRegex.FindStringSubmatch(e)
|
||||||
|
if len(comps) != 3 {
|
||||||
|
err = fmt.Errorf("does not match regex: %s %#v", e, comps)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if durationFactor <= 0 {
|
||||||
|
return 0, errors.New("duration must be positive integer")
|
||||||
|
}
|
||||||
|
|
||||||
|
var durationUnit time.Duration
|
||||||
|
switch comps[2] {
|
||||||
|
case "s":
|
||||||
|
durationUnit = time.Second
|
||||||
|
case "m":
|
||||||
|
durationUnit = time.Minute
|
||||||
|
case "h":
|
||||||
|
durationUnit = time.Hour
|
||||||
|
case "d":
|
||||||
|
durationUnit = 24 * time.Hour
|
||||||
|
case "w":
|
||||||
|
durationUnit = 24 * 7 * time.Hour
|
||||||
|
default:
|
||||||
|
err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
d = time.Duration(durationFactor) * durationUnit
|
||||||
|
return
|
||||||
|
}
|
||||||
@@ -2,8 +2,16 @@ package config
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestConfigEmptyFails(t *testing.T) {
|
||||||
|
conf, err := testConfig(t, "\n")
|
||||||
|
assert.Nil(t, conf)
|
||||||
|
assert.Error(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
func TestJobsOnlyWorks(t *testing.T) {
|
func TestJobsOnlyWorks(t *testing.T) {
|
||||||
testValidConfig(t, `
|
testValidConfig(t, `
|
||||||
jobs:
|
jobs:
|
||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
zfsprop "github.com/zrepl/zrepl/internal/zfs/property"
|
zfsprop "github.com/zrepl/zrepl/zfs/property"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRecvOptions(t *testing.T) {
|
func TestRecvOptions(t *testing.T) {
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSnapshotting(t *testing.T) {
|
||||||
|
tmpl := `
|
||||||
|
jobs:
|
||||||
|
- name: foo
|
||||||
|
type: push
|
||||||
|
connect:
|
||||||
|
type: local
|
||||||
|
listener_name: foo
|
||||||
|
client_identity: bar
|
||||||
|
filesystems: {"<": true}
|
||||||
|
%s
|
||||||
|
pruning:
|
||||||
|
keep_sender:
|
||||||
|
- type: last_n
|
||||||
|
count: 10
|
||||||
|
keep_receiver:
|
||||||
|
- type: last_n
|
||||||
|
count: 10
|
||||||
|
`
|
||||||
|
manual := `
|
||||||
|
snapshotting:
|
||||||
|
type: manual
|
||||||
|
`
|
||||||
|
periodic := `
|
||||||
|
snapshotting:
|
||||||
|
type: periodic
|
||||||
|
prefix: zrepl_
|
||||||
|
interval: 10m
|
||||||
|
`
|
||||||
|
|
||||||
|
hooks := `
|
||||||
|
snapshotting:
|
||||||
|
type: periodic
|
||||||
|
prefix: zrepl_
|
||||||
|
interval: 10m
|
||||||
|
hooks:
|
||||||
|
- type: command
|
||||||
|
path: /tmp/path/to/command
|
||||||
|
- type: command
|
||||||
|
path: /tmp/path/to/command
|
||||||
|
filesystems: { "zroot<": true, "<": false }
|
||||||
|
- type: postgres-checkpoint
|
||||||
|
dsn: "host=localhost port=5432 user=postgres sslmode=disable"
|
||||||
|
filesystems: {
|
||||||
|
"tank/postgres/data11": true
|
||||||
|
}
|
||||||
|
- type: mysql-lock-tables
|
||||||
|
dsn: "root@tcp(localhost)/"
|
||||||
|
filesystems: {
|
||||||
|
"tank/mysql": true
|
||||||
|
}
|
||||||
|
`
|
||||||
|
|
||||||
|
fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
|
||||||
|
var c *Config
|
||||||
|
|
||||||
|
t.Run("manual", func(t *testing.T) {
|
||||||
|
c = testValidConfig(t, fillSnapshotting(manual))
|
||||||
|
snm := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingManual)
|
||||||
|
assert.Equal(t, "manual", snm.Type)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("periodic", func(t *testing.T) {
|
||||||
|
c = testValidConfig(t, fillSnapshotting(periodic))
|
||||||
|
snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
|
||||||
|
assert.Equal(t, "periodic", snp.Type)
|
||||||
|
assert.Equal(t, 10*time.Minute, snp.Interval)
|
||||||
|
assert.Equal(t, "zrepl_", snp.Prefix)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("hooks", func(t *testing.T) {
|
||||||
|
c = testValidConfig(t, fillSnapshotting(hooks))
|
||||||
|
hs := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic).Hooks
|
||||||
|
assert.Equal(t, hs[0].Ret.(*HookCommand).Filesystems["<"], true)
|
||||||
|
assert.Equal(t, hs[1].Ret.(*HookCommand).Filesystems["zroot<"], true)
|
||||||
|
assert.Equal(t, hs[2].Ret.(*HookPostgresCheckpoint).Filesystems["tank/postgres/data11"], true)
|
||||||
|
assert.Equal(t, hs[3].Ret.(*HookMySQLLockTables).Filesystems["tank/mysql"], true)
|
||||||
|
})
|
||||||
|
|
||||||
|
}
|
||||||
@@ -22,8 +22,6 @@ func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
|
|||||||
t.Errorf("glob failed: %+v", err)
|
t.Errorf("glob failed: %+v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
paths = append(paths, "../../packaging/systemd-default-zrepl.yml")
|
|
||||||
|
|
||||||
for _, p := range paths {
|
for _, p := range paths {
|
||||||
|
|
||||||
if path.Ext(p) != ".yml" {
|
if path.Ext(p) != ".yml" {
|
||||||
@@ -45,23 +43,9 @@ func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestInvalidSampleConfigsFailToParse(t *testing.T) {
|
|
||||||
paths, err := filepath.Glob("./samples/invalid/*/zrepl.yml")
|
|
||||||
require.NoError(t, err, "glob failed")
|
|
||||||
require.NotEmpty(t, paths, "no invalid sample configs found")
|
|
||||||
|
|
||||||
for _, p := range paths {
|
|
||||||
t.Run(p, func(t *testing.T) {
|
|
||||||
_, err := ParseConfig(p)
|
|
||||||
require.Error(t, err, "expected config %s to fail parsing", p)
|
|
||||||
t.Logf("config %s failed as expected: %v", p, err)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// template must be a template/text template with a single '{{ . }}' as placeholder for val
|
// template must be a template/text template with a single '{{ . }}' as placeholder for val
|
||||||
//
|
//
|
||||||
//nolint:unused
|
//nolint:deadcode,unused
|
||||||
func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
|
func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
|
||||||
tmp, err := template.New("master").Parse(tmpl)
|
tmp, err := template.New("master").Parse(tmpl)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -154,18 +138,3 @@ func TestCronSpec(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEmptyConfig(t *testing.T) {
|
|
||||||
cases := []string{
|
|
||||||
"",
|
|
||||||
"\n",
|
|
||||||
"---",
|
|
||||||
"---\n",
|
|
||||||
}
|
|
||||||
for _, input := range cases {
|
|
||||||
config := testValidConfig(t, input)
|
|
||||||
require.NotNil(t, config)
|
|
||||||
require.NotNil(t, config.Global)
|
|
||||||
require.Empty(t, config.Jobs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -14,14 +14,14 @@ import (
|
|||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
"github.com/zrepl/zrepl/daemon/job"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/nethelpers"
|
"github.com/zrepl/zrepl/daemon/nethelpers"
|
||||||
"github.com/zrepl/zrepl/internal/endpoint"
|
"github.com/zrepl/zrepl/endpoint"
|
||||||
"github.com/zrepl/zrepl/internal/logger"
|
"github.com/zrepl/zrepl/logger"
|
||||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
"github.com/zrepl/zrepl/util/envconst"
|
||||||
"github.com/zrepl/zrepl/internal/version"
|
"github.com/zrepl/zrepl/version"
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
"github.com/zrepl/zrepl/internal/zfs/zfscmd"
|
"github.com/zrepl/zrepl/zfs/zfscmd"
|
||||||
)
|
)
|
||||||
|
|
||||||
type controlJob struct {
|
type controlJob struct {
|
||||||
@@ -3,8 +3,10 @@ package daemon
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"math/rand"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
@@ -12,18 +14,18 @@ import (
|
|||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||||
"github.com/zrepl/zrepl/internal/endpoint"
|
"github.com/zrepl/zrepl/endpoint"
|
||||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
"github.com/zrepl/zrepl/util/envconst"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
"github.com/zrepl/zrepl/daemon/job"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/job/reset"
|
"github.com/zrepl/zrepl/daemon/job/reset"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/job/wakeup"
|
"github.com/zrepl/zrepl/daemon/job/wakeup"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
"github.com/zrepl/zrepl/daemon/logging"
|
||||||
"github.com/zrepl/zrepl/internal/logger"
|
"github.com/zrepl/zrepl/logger"
|
||||||
"github.com/zrepl/zrepl/internal/version"
|
"github.com/zrepl/zrepl/version"
|
||||||
"github.com/zrepl/zrepl/internal/zfs/zfscmd"
|
"github.com/zrepl/zrepl/zfs/zfscmd"
|
||||||
)
|
)
|
||||||
|
|
||||||
func Run(ctx context.Context, conf *config.Config) error {
|
func Run(ctx context.Context, conf *config.Config) error {
|
||||||
@@ -37,6 +39,12 @@ func Run(ctx context.Context, conf *config.Config) error {
|
|||||||
cancel()
|
cancel()
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
// The math/rand package is used presently for generating trace IDs, we
|
||||||
|
// seed it with the current time and pid so that the IDs are mostly
|
||||||
|
// unique.
|
||||||
|
rand.Seed(time.Now().UnixNano())
|
||||||
|
rand.Seed(int64(os.Getpid()))
|
||||||
|
|
||||||
outlets, err := logging.OutletsFromConfig(*conf.Global.Logging)
|
outlets, err := logging.OutletsFromConfig(*conf.Global.Logging)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, "cannot build logging from config")
|
return errors.Wrap(err, "cannot build logging from config")
|
||||||
@@ -63,7 +71,7 @@ func Run(ctx context.Context, conf *config.Config) error {
|
|||||||
})
|
})
|
||||||
|
|
||||||
for _, job := range confJobs {
|
for _, job := range confJobs {
|
||||||
if config.IsInternalJobName(job.Name()) {
|
if IsInternalJobName(job.Name()) {
|
||||||
panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME
|
panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -210,6 +218,10 @@ const (
|
|||||||
jobNameControl = "_control"
|
jobNameControl = "_control"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func IsInternalJobName(s string) bool {
|
||||||
|
return strings.HasPrefix(s, "_")
|
||||||
|
}
|
||||||
|
|
||||||
func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
|
func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
|
||||||
s.m.Lock()
|
s.m.Lock()
|
||||||
defer s.m.Unlock()
|
defer s.m.Unlock()
|
||||||
@@ -217,12 +229,10 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
|
|||||||
ctx = logging.WithInjectedField(ctx, logging.JobField, j.Name())
|
ctx = logging.WithInjectedField(ctx, logging.JobField, j.Name())
|
||||||
|
|
||||||
jobName := j.Name()
|
jobName := j.Name()
|
||||||
|
if !internal && IsInternalJobName(jobName) {
|
||||||
// package `config` enforces these with clean errors, these are just assertions
|
|
||||||
if !internal && config.IsInternalJobName(jobName) {
|
|
||||||
panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName))
|
panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName))
|
||||||
}
|
}
|
||||||
if internal && !config.IsInternalJobName(jobName) {
|
if internal && !IsInternalJobName(jobName) {
|
||||||
panic(fmt.Sprintf("internal job does not use internal job name %s", jobName))
|
panic(fmt.Sprintf("internal job does not use internal job name %s", jobName))
|
||||||
}
|
}
|
||||||
if _, ok := s.jobs[jobName]; ok {
|
if _, ok := s.jobs[jobName]; ok {
|
||||||
@@ -6,8 +6,8 @@ import (
|
|||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/endpoint"
|
"github.com/zrepl/zrepl/endpoint"
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
type DatasetMapFilter struct {
|
type DatasetMapFilter struct {
|
||||||
@@ -160,14 +160,6 @@ func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m DatasetMapFilter) UserSpecifiedDatasets() (datasets zfs.UserSpecifiedDatasetsSet) {
|
|
||||||
datasets = make(zfs.UserSpecifiedDatasetsSet)
|
|
||||||
for i := range m.entries {
|
|
||||||
datasets[m.entries[i].path.ToString()] = true
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Construct a new filter-only DatasetMapFilter from a mapping
|
// Construct a new filter-only DatasetMapFilter from a mapping
|
||||||
// The new filter allows exactly those paths that were not forbidden by the mapping.
|
// The new filter allows exactly those paths that were not forbidden by the mapping.
|
||||||
func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
|
func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
|
||||||
@@ -3,7 +3,7 @@ package filters
|
|||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDatasetMapFilter(t *testing.T) {
|
func TestDatasetMapFilter(t *testing.T) {
|
||||||
@@ -3,8 +3,8 @@ package hooks
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
type List []Hook
|
type List []Hook
|
||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Re-export type here so that
|
// Re-export type here so that
|
||||||
@@ -6,9 +6,9 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
"github.com/zrepl/zrepl/daemon/logging"
|
||||||
"github.com/zrepl/zrepl/internal/logger"
|
"github.com/zrepl/zrepl/logger"
|
||||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
"github.com/zrepl/zrepl/util/envconst"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Logger = logger.Logger
|
type Logger = logger.Logger
|
||||||
@@ -4,8 +4,8 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
"github.com/zrepl/zrepl/daemon/filters"
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
type HookJobCallback func(ctx context.Context) error
|
type HookJobCallback func(ctx context.Context) error
|
||||||
@@ -12,11 +12,11 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
"github.com/zrepl/zrepl/daemon/filters"
|
||||||
"github.com/zrepl/zrepl/internal/logger"
|
"github.com/zrepl/zrepl/logger"
|
||||||
"github.com/zrepl/zrepl/internal/util/circlog"
|
"github.com/zrepl/zrepl/util/circlog"
|
||||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
"github.com/zrepl/zrepl/util/envconst"
|
||||||
)
|
)
|
||||||
|
|
||||||
type HookEnvVar string
|
type HookEnvVar string
|
||||||
+3
-3
@@ -12,9 +12,9 @@ import (
|
|||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
"github.com/zrepl/zrepl/daemon/filters"
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Hook to implement the following recommmendation from MySQL docs
|
// Hook to implement the following recommmendation from MySQL docs
|
||||||
+3
-3
@@ -10,9 +10,9 @@ import (
|
|||||||
"github.com/lib/pq"
|
"github.com/lib/pq"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
"github.com/zrepl/zrepl/daemon/filters"
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
type PgChkptHook struct {
|
type PgChkptHook struct {
|
||||||
@@ -11,13 +11,13 @@ import (
|
|||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/hooks"
|
"github.com/zrepl/zrepl/daemon/hooks"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
"github.com/zrepl/zrepl/daemon/logging"
|
||||||
"github.com/zrepl/zrepl/internal/logger"
|
"github.com/zrepl/zrepl/logger"
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
type comparisonAssertionFunc func(require.TestingT, interface{}, interface{}, ...interface{})
|
type comparisonAssertionFunc func(require.TestingT, interface{}, interface{}, ...interface{})
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
// Code generated by "enumer -type=StepStatus -trimprefix=Step"; DO NOT EDIT.
|
// Code generated by "enumer -type=StepStatus -trimprefix=Step"; DO NOT EDIT.
|
||||||
|
|
||||||
|
//
|
||||||
package hooks
|
package hooks
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -8,24 +8,25 @@ import (
|
|||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
"github.com/prometheus/common/log"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
"github.com/zrepl/zrepl/util/envconst"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/config"
|
"github.com/zrepl/zrepl/config"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/job/reset"
|
"github.com/zrepl/zrepl/daemon/job/reset"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/job/wakeup"
|
"github.com/zrepl/zrepl/daemon/job/wakeup"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/pruner"
|
"github.com/zrepl/zrepl/daemon/pruner"
|
||||||
"github.com/zrepl/zrepl/internal/daemon/snapper"
|
"github.com/zrepl/zrepl/daemon/snapper"
|
||||||
"github.com/zrepl/zrepl/internal/endpoint"
|
"github.com/zrepl/zrepl/endpoint"
|
||||||
"github.com/zrepl/zrepl/internal/replication"
|
"github.com/zrepl/zrepl/replication"
|
||||||
"github.com/zrepl/zrepl/internal/replication/driver"
|
"github.com/zrepl/zrepl/replication/driver"
|
||||||
"github.com/zrepl/zrepl/internal/replication/logic"
|
"github.com/zrepl/zrepl/replication/logic"
|
||||||
"github.com/zrepl/zrepl/internal/replication/report"
|
"github.com/zrepl/zrepl/replication/report"
|
||||||
"github.com/zrepl/zrepl/internal/rpc"
|
"github.com/zrepl/zrepl/rpc"
|
||||||
"github.com/zrepl/zrepl/internal/transport"
|
"github.com/zrepl/zrepl/transport"
|
||||||
"github.com/zrepl/zrepl/internal/transport/fromconfig"
|
"github.com/zrepl/zrepl/transport/fromconfig"
|
||||||
"github.com/zrepl/zrepl/internal/zfs"
|
"github.com/zrepl/zrepl/zfs"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ActiveSide struct {
|
type ActiveSide struct {
|
||||||
@@ -481,7 +482,7 @@ func (j *ActiveSide) do(ctx context.Context) {
|
|||||||
go func() {
|
go func() {
|
||||||
select {
|
select {
|
||||||
case <-reset.Wait(ctx):
|
case <-reset.Wait(ctx):
|
||||||
GetLogger(ctx).Info("reset received, cancelling current invocation")
|
log.Info("reset received, cancelling current invocation")
|
||||||
cancelThisRun()
|
cancelThisRun()
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
}
|
}
|
||||||
@@ -6,8 +6,8 @@ import (
|
|||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
"github.com/zrepl/zrepl/internal/endpoint"
|
"github.com/zrepl/zrepl/endpoint"
|
||||||
"github.com/zrepl/zrepl/internal/transport"
|
"github.com/zrepl/zrepl/transport"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestFakeActiveSideDirectMethodInvocationClientIdentityDoesNotPassValidityTest(t *testing.T) {
|
func TestFakeActiveSideDirectMethodInvocationClientIdentityDoesNotPassValidityTest(t *testing.T) {
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user