Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Christian Schwarz
|
beecb4b93d | ||
|
Christian Schwarz
|
c8afaf83ab | ||
|
Christian Schwarz
|
b0caa2d151 |
+89
-101
@@ -1,9 +1,8 @@
|
||||
version: 2.1
|
||||
orbs:
|
||||
# NB: this is not the Go version, but the Orb version
|
||||
# NB: 1.7.2 is not the Go version, but the Orb version
|
||||
# https://circleci.com/developer/orbs/orb/circleci/go#usage-go-modules-cache
|
||||
go: circleci/go@1.11.0
|
||||
|
||||
go: circleci/go@1.7.2
|
||||
commands:
|
||||
setup-home-local-bin:
|
||||
steps:
|
||||
@@ -16,40 +15,37 @@ commands:
|
||||
echo "$line" >> $BASH_ENV
|
||||
fi
|
||||
|
||||
# NOTE: UV version is defined in .uv-version file at repository root
|
||||
invoke-lazy-sh:
|
||||
parameters:
|
||||
subcommand:
|
||||
type: string
|
||||
steps:
|
||||
- run:
|
||||
environment:
|
||||
TERM: xterm
|
||||
command: ./lazy.sh <<parameters.subcommand>>
|
||||
|
||||
apt-update-and-install-common-deps:
|
||||
steps:
|
||||
- run: sudo apt-get update
|
||||
- run: sudo apt-get install -y gawk make
|
||||
# CircleCI doesn't update its cimg/go images.
|
||||
# So, need to update manually to get up-to-date trust chains.
|
||||
# The need for this was required for cimg/go:1.12, but let's future proof this here and now.
|
||||
- run: sudo apt-get install -y git ca-certificates
|
||||
|
||||
install-godep:
|
||||
steps:
|
||||
- apt-update-and-install-common-deps
|
||||
- invoke-lazy-sh:
|
||||
subcommand: godep
|
||||
|
||||
install-docdep:
|
||||
steps:
|
||||
- run:
|
||||
name: Read UV version from .uv-version
|
||||
command: |
|
||||
UV_VERSION=$(cat .uv-version)
|
||||
echo "export UV_VERSION=$UV_VERSION" >> $BASH_ENV
|
||||
# Python is managed by uv - it will automatically download the version
|
||||
# specified in docs/.python-version when needed
|
||||
- run:
|
||||
name: Install uv
|
||||
command: curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh
|
||||
- run:
|
||||
name: Add uv to PATH and set cache dir
|
||||
command: |
|
||||
echo 'export PATH="$HOME/.local/bin:$PATH"' >> $BASH_ENV
|
||||
echo 'export UV_CACHE_DIR="$HOME/.cache/uv"' >> $BASH_ENV
|
||||
- restore_cache:
|
||||
name: Restore uv cache
|
||||
keys:
|
||||
- uv-cache-v1-${UV_VERSION}-{{ checksum "docs/uv.lock" }}
|
||||
- uv-cache-v1-${UV_VERSION}-
|
||||
|
||||
save-uv-cache:
|
||||
steps:
|
||||
- run:
|
||||
name: Prune uv cache for CI
|
||||
command: uv cache prune --ci
|
||||
- save_cache:
|
||||
name: Save uv cache
|
||||
key: uv-cache-v1-${UV_VERSION}-{{ checksum "docs/uv.lock" }}
|
||||
paths:
|
||||
- ~/.cache/uv
|
||||
- apt-update-and-install-common-deps
|
||||
- run: sudo apt install python3 python3-pip libgirepository1.0-dev
|
||||
- invoke-lazy-sh:
|
||||
subcommand: docdep
|
||||
|
||||
docs-publish-sh:
|
||||
parameters:
|
||||
@@ -62,22 +58,26 @@ commands:
|
||||
git config --global user.email "zreplbot@cschwarz.com"
|
||||
git config --global user.name "zrepl-github-io-ci"
|
||||
|
||||
# Configure git to use the GitHub token for HTTPS authentication.
|
||||
# The token is stored in the 'zrepl-github-io-deploy' context.
|
||||
# if we're pushing, we need to add the deploy key
|
||||
# which is stored as "Additional SSH Keys" in the CircleCI project settings.
|
||||
# We can't use the CircleCI-manage deploy key because we're pushing
|
||||
# to a different repo than the one we're building.
|
||||
- when:
|
||||
condition: << parameters.push >>
|
||||
steps:
|
||||
# https://circleci.com/docs/2.0/add-ssh-key/#adding-multiple-keys-with-blank-hostnames
|
||||
- run: ssh-add -D
|
||||
# the default circleci ssh config only additional ssh keys for Host !github.com
|
||||
- run:
|
||||
name: Configure git to use GitHub token for push
|
||||
# GITHUB_PAGES_TOKEN is from the 'zrepl-github-io-deploy' context.
|
||||
# CircleCI's secret masking automatically redacts context variables in logs.
|
||||
command: |
|
||||
# Unset CircleCI's SSH URL rewriting that checkout step configured
|
||||
git config --global --unset-all url."ssh://git@github.com".insteadOf || true
|
||||
# Set up credential helper with GitHub token
|
||||
git config --global credential.helper store
|
||||
echo "https://x-access-token:${GITHUB_PAGES_TOKEN}@github.com" > ~/.git-credentials
|
||||
chmod 600 ~/.git-credentials
|
||||
cat > ~/.ssh/config \<<EOF
|
||||
Host *
|
||||
IdentityFile /home/circleci/.ssh/id_rsa_458e62c517f6c480e40452126ce47421
|
||||
EOF
|
||||
- add_ssh_keys:
|
||||
fingerprints:
|
||||
# deploy key for zrepl.github.io
|
||||
- "45:8e:62:c5:17:f6:c4:80:e4:04:52:12:6c:e4:74:21"
|
||||
|
||||
# caller must install-docdep
|
||||
- when:
|
||||
@@ -99,29 +99,31 @@ parameters:
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
release_docker_baseimage_tag:
|
||||
type: string
|
||||
default: "1.21"
|
||||
|
||||
workflows:
|
||||
version: 2
|
||||
|
||||
ci:
|
||||
when: << pipeline.parameters.do_ci >>
|
||||
jobs:
|
||||
- run-docs-publish-sh:
|
||||
name: quickcheck-docs
|
||||
push: false
|
||||
- quickcheck-go:
|
||||
name: quickcheck-go-amd64-linux-1.25.7
|
||||
goversion: &latest-go-release "1.25.7"
|
||||
- quickcheck-docs
|
||||
- quickcheck-go: &quickcheck-go-smoketest
|
||||
name: quickcheck-go-amd64-linux-1.21
|
||||
goversion: &latest-go-release "1.21"
|
||||
goos: linux
|
||||
goarch: amd64
|
||||
- test-go:
|
||||
- test-go-on-latest-go-release:
|
||||
goversion: *latest-go-release
|
||||
- quickcheck-go:
|
||||
requires:
|
||||
- quickcheck-go-amd64-linux-1.25.7 #quickcheck-go-smoketest.name
|
||||
matrix:
|
||||
- quickcheck-go-amd64-linux-1.21 #quickcheck-go-smoketest.name
|
||||
matrix: &quickcheck-go-matrix
|
||||
alias: quickcheck-go-matrix
|
||||
parameters:
|
||||
goversion: [*latest-go-release, "1.24.13"]
|
||||
goversion: [*latest-go-release, "1.20"]
|
||||
goos: ["linux", "freebsd"]
|
||||
goarch: ["amd64", "arm64"]
|
||||
exclude:
|
||||
@@ -135,9 +137,7 @@ workflows:
|
||||
goversion: [*latest-go-release]
|
||||
goos: ["linux"]
|
||||
goarch: ["amd64"]
|
||||
image: ["ubuntu-2204:current", "ubuntu-2404:current"]
|
||||
requires:
|
||||
- test-go
|
||||
- quickcheck-go-<< matrix.goarch >>-<< matrix.goos >>-<< matrix.goversion >>
|
||||
|
||||
release:
|
||||
@@ -158,29 +158,24 @@ workflows:
|
||||
|
||||
publish-zrepl.github.io:
|
||||
jobs:
|
||||
- run-docs-publish-sh:
|
||||
name: publish-zrepl.github.io
|
||||
push: true
|
||||
context:
|
||||
- zrepl-github-io-deploy
|
||||
- publish-zrepl-github-io:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
- stable
|
||||
|
||||
jobs:
|
||||
run-docs-publish-sh:
|
||||
parameters:
|
||||
push:
|
||||
type: boolean
|
||||
quickcheck-docs:
|
||||
docker:
|
||||
- image: cimg/base:current
|
||||
- image: cimg/base:2023.09
|
||||
steps:
|
||||
- checkout
|
||||
- install-docdep
|
||||
# do the current docs build
|
||||
- run: make docs
|
||||
# does the publish.sh script still work?
|
||||
- docs-publish-sh:
|
||||
push: << parameters.push >>
|
||||
- save-uv-cache
|
||||
push: false
|
||||
|
||||
quickcheck-go:
|
||||
parameters:
|
||||
@@ -191,40 +186,29 @@ jobs:
|
||||
goarch:
|
||||
type: string
|
||||
docker:
|
||||
- image: &cimg_with_modern_go cimg/go:1.25
|
||||
- image: cimg/go:<<parameters.goversion>>
|
||||
environment:
|
||||
GOOS: <<parameters.goos>>
|
||||
GOARCH: <<parameters.goarch>>
|
||||
GOTOOLCHAIN: "go<<parameters.goversion>>"
|
||||
|
||||
steps:
|
||||
- checkout
|
||||
|
||||
- go/load-cache:
|
||||
key: quickcheck-<<parameters.goversion>>
|
||||
- run: make build/install
|
||||
- install-godep
|
||||
- run: go mod download
|
||||
- run: cd build && go mod download
|
||||
- go/save-cache:
|
||||
key: quickcheck-<<parameters.goversion>>
|
||||
|
||||
# ensure all code has been generated
|
||||
- run: make generate
|
||||
- run: |
|
||||
if output=$(git status --porcelain) && [ -z "$output" ]; then
|
||||
echo "Working directory clean"
|
||||
else
|
||||
echo "Uncommitted changes"
|
||||
echo ""
|
||||
echo "$output"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# other checks
|
||||
- run: make formatcheck
|
||||
- run: make generate-platform-test-list
|
||||
- run: make zrepl-bin test-platform-bin
|
||||
- run: make vet
|
||||
- run: make lint
|
||||
|
||||
- run: rm -f artifacts/generate-platform-test-list
|
||||
- store_artifacts:
|
||||
path: artifacts
|
||||
- persist_to_workspace:
|
||||
@@ -239,10 +223,8 @@ jobs:
|
||||
type: string
|
||||
goarch:
|
||||
type: string
|
||||
image:
|
||||
type: string
|
||||
machine:
|
||||
image: <<parameters.image>>
|
||||
image: ubuntu-2204:current
|
||||
resource_class: medium
|
||||
environment:
|
||||
GOOS: <<parameters.goos>>
|
||||
@@ -255,14 +237,12 @@ jobs:
|
||||
- run: sudo zfs version
|
||||
- run: sudo make test-platform GOOS="$GOOS" GOARCH="$GOARCH"
|
||||
|
||||
test-go:
|
||||
test-go-on-latest-go-release:
|
||||
parameters:
|
||||
goversion:
|
||||
type: string
|
||||
docker:
|
||||
- image: *cimg_with_modern_go
|
||||
environment:
|
||||
GOTOOLCHAIN: "go<<parameters.goversion>>"
|
||||
- image: cimg/go:<<parameters.goversion>>
|
||||
steps:
|
||||
- checkout
|
||||
- go/load-cache:
|
||||
@@ -273,17 +253,16 @@ jobs:
|
||||
|
||||
release-build:
|
||||
machine:
|
||||
image: &release-vm-image "ubuntu-2404:current"
|
||||
resource_class: large
|
||||
image: ubuntu-2004:202201-02
|
||||
steps:
|
||||
- checkout
|
||||
- run: make release-docker
|
||||
- run: make release-docker RELEASE_DOCKER_BASEIMAGE_TAG=<<pipeline.parameters.release_docker_baseimage_tag>>
|
||||
- persist_to_workspace:
|
||||
root: .
|
||||
paths: [.]
|
||||
release-deb:
|
||||
machine:
|
||||
image: *release-vm-image
|
||||
image: ubuntu-2004:202201-02
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: .
|
||||
@@ -295,7 +274,7 @@ jobs:
|
||||
|
||||
release-rpm:
|
||||
machine:
|
||||
image: *release-vm-image
|
||||
image: ubuntu-2004:202201-02
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: .
|
||||
@@ -307,10 +286,19 @@ jobs:
|
||||
|
||||
release-upload:
|
||||
docker:
|
||||
- image: cimg/base:2024.09
|
||||
- image: cimg/base:2020.08
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: .
|
||||
- run: make wrapup-and-checksum
|
||||
- store_artifacts:
|
||||
path: artifacts/release
|
||||
path: artifacts
|
||||
|
||||
publish-zrepl-github-io:
|
||||
docker:
|
||||
- image: cimg/base:2023.09
|
||||
steps:
|
||||
- checkout
|
||||
- install-docdep
|
||||
- docs-publish-sh:
|
||||
push: true
|
||||
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
COMMIT="$1"
|
||||
GO_VERSION="$2"
|
||||
|
||||
curl -v -X POST https://api.github.com/repos/zrepl/debian-binary-packaging/dispatches \
|
||||
-H 'Accept: application/vnd.github.v3+json' \
|
||||
-H "Authorization: token $GITHUB_ACCESS_TOKEN" \
|
||||
--data '{"event_type": "push", "client_payload": { "zrepl_main_repo_commit": "'"$COMMIT"'", "go_version": "'"$GO_VERSION"'" }}'
|
||||
@@ -1,9 +0,0 @@
|
||||
version: 2
|
||||
updates:
|
||||
# Docs use Python only for static site generation.
|
||||
- package-ecosystem: "pip"
|
||||
directory: /docs
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
ignore:
|
||||
- dependency-name: "*"
|
||||
@@ -1,6 +1,5 @@
|
||||
# Build
|
||||
artifacts/
|
||||
build/install.tmp
|
||||
|
||||
# Golang
|
||||
vendor/
|
||||
|
||||
+14
-40
@@ -1,42 +1,16 @@
|
||||
version: "2"
|
||||
linters:
|
||||
enable:
|
||||
- revive
|
||||
settings:
|
||||
revive:
|
||||
rules:
|
||||
- name: time-equal
|
||||
exclusions:
|
||||
generated: lax
|
||||
presets:
|
||||
- comments
|
||||
- common-false-positives
|
||||
- legacy
|
||||
- std-error-handling
|
||||
rules:
|
||||
- linters:
|
||||
- errcheck
|
||||
path: _test\.go
|
||||
- linters:
|
||||
- staticcheck
|
||||
text: 'SA9003:'
|
||||
- linters:
|
||||
- staticcheck
|
||||
text: '(QF1001|QF1011|ST1012|QF1008|ST1005|ST1023|QF1003|ST1006|ST1001|QF1004):'
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
formatters:
|
||||
enable:
|
||||
- goimports
|
||||
settings:
|
||||
goimports:
|
||||
local-prefixes:
|
||||
- github.com/zrepl/zrepl
|
||||
exclusions:
|
||||
generated: lax
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
- goimports
|
||||
|
||||
issues:
|
||||
exclude-rules:
|
||||
- path: _test\.go
|
||||
linters:
|
||||
- errcheck
|
||||
# Disable staticcheck 'Empty body in an if or else branch' as it's useful
|
||||
# to put a comment into an empty else-clause that explains why whatever
|
||||
# is done in the if-caluse is not necessary if the condition is false.
|
||||
- linters:
|
||||
- staticcheck
|
||||
text: "SA9003:"
|
||||
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
0.9.30
|
||||
@@ -22,17 +22,16 @@ GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"')
|
||||
GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"')
|
||||
GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"')
|
||||
GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
|
||||
GO_ENV_VARS := CGO_ENABLED=0
|
||||
GO_LDFLAGS := "-X github.com/zrepl/zrepl/internal/version.zreplVersion=$(_ZREPL_VERSION)"
|
||||
GO_ENV_VARS := GO111MODULE=on CGO_ENABLED=0
|
||||
GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
|
||||
GO_MOD_READONLY := -mod=readonly
|
||||
GO_EXTRA_BUILDFLAGS :=
|
||||
GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
|
||||
GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
|
||||
GOLANGCI_LINT := golangci-lint
|
||||
GOCOVMERGE := gocovmerge
|
||||
RELEASE_GOVERSION ?= go1.25.7
|
||||
STRIPPED_GOVERSION := $(subst go,,$(RELEASE_GOVERSION))
|
||||
RELEASE_DOCKER_BASEIMAGE ?= golang:$(STRIPPED_GOVERSION)
|
||||
RELEASE_DOCKER_CACHEMOUNT :=
|
||||
RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.21
|
||||
RELEASE_DOCKER_BASEIMAGE ?= golang:$(RELEASE_DOCKER_BASEIMAGE_TAG)
|
||||
|
||||
ifneq ($(GOARM),)
|
||||
ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM)
|
||||
@@ -40,49 +39,33 @@ else
|
||||
ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)
|
||||
endif
|
||||
|
||||
.PHONY: printvars
|
||||
printvars:
|
||||
@echo GOOS=$(GOOS)
|
||||
@echo GOARCH=$(GOARCH)
|
||||
@echo GOARM=$(GOARM)
|
||||
|
||||
ifneq ($(RELEASE_DOCKER_CACHEMOUNT),)
|
||||
_RELEASE_DOCKER_CACHEMOUNT := -v $(RELEASE_DOCKER_CACHEMOUNT)/mod:/go/pkg/mod -v $(RELEASE_DOCKER_CACHEMOUNT)/xdg-cache:/.cache/go-build
|
||||
.PHONY: release-docker-mkcachemount
|
||||
release-docker-mkcachemount:
|
||||
mkdir -p $(RELEASE_DOCKER_CACHEMOUNT)
|
||||
mkdir -p $(RELEASE_DOCKER_CACHEMOUNT)/mod
|
||||
mkdir -p $(RELEASE_DOCKER_CACHEMOUNT)/xdg-cache
|
||||
else
|
||||
_RELEASE_DOCKER_CACHEMOUNT :=
|
||||
.PHONY: release-docker-mkcachemount
|
||||
release-docker-mkcachemount:
|
||||
# nothing to do
|
||||
endif
|
||||
|
||||
##################### PRODUCING A RELEASE #############
|
||||
.PHONY: release wrapup-and-checksum check-git-clean sign clean ensure-release-toolchain
|
||||
.PHONY: release wrapup-and-checksum check-git-clean sign clean
|
||||
|
||||
ensure-release-toolchain:
|
||||
# ensure the toolchain is actually the one we expect
|
||||
test $(RELEASE_GOVERSION) = "$$($(GO_ENV_VARS) $(GO) env GOVERSION)"
|
||||
|
||||
release: ensure-release-toolchain
|
||||
release: clean
|
||||
# no cross-platform support for target test
|
||||
$(MAKE) test-go
|
||||
$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="vet"
|
||||
$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="lint"
|
||||
$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="zrepl-bin"
|
||||
$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="test-platform-bin"
|
||||
$(MAKE) noarch
|
||||
|
||||
release-docker: $(ARTIFACTDIR) release-docker-mkcachemount
|
||||
sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build/build.Dockerfile > $(ARTIFACTDIR)/build.Dockerfile
|
||||
docker build -t zrepl_release --pull \
|
||||
--build-arg BUILD_UID=$$(id -u) \
|
||||
--build-arg BUILD_GID=$$(id -g) \
|
||||
-f $(ARTIFACTDIR)/build.Dockerfile .
|
||||
docker run --rm -i$$(test -t 0 && echo t) \
|
||||
$(_RELEASE_DOCKER_CACHEMOUNT) \
|
||||
-v $(CURDIR):/src \
|
||||
release-docker: $(ARTIFACTDIR)
|
||||
sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > artifacts/release-docker.Dockerfile
|
||||
docker build -t zrepl_release --pull -f artifacts/release-docker.Dockerfile .
|
||||
docker run --rm -i -v $(CURDIR):/src -u $$(id -u):$$(id -g) \
|
||||
zrepl_release \
|
||||
make release \
|
||||
GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
|
||||
ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE) \
|
||||
RELEASE_GOVERSION=$(RELEASE_GOVERSION)
|
||||
ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE)
|
||||
|
||||
debs-docker:
|
||||
$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=deb
|
||||
@@ -179,14 +162,14 @@ wrapup-and-checksum:
|
||||
tar --mtime='1970-01-01' --sort=name \
|
||||
--transform 's/$(ARTIFACTDIR)/zrepl-$(_ZREPL_VERSION)-noarch/' \
|
||||
--transform 's#dist#zrepl-$(_ZREPL_VERSION)-noarch/dist#' \
|
||||
--transform 's#internal/config/samples#zrepl-$(_ZREPL_VERSION)-noarch/config#' \
|
||||
--transform 's#config/samples#zrepl-$(_ZREPL_VERSION)-noarch/config#' \
|
||||
-acf $(NOARCH_TARBALL) \
|
||||
$(ARTIFACTDIR)/docs/html \
|
||||
$(ARTIFACTDIR)/bash_completion \
|
||||
$(ARTIFACTDIR)/_zrepl.zsh_completion \
|
||||
$(ARTIFACTDIR)/go_env.txt \
|
||||
dist \
|
||||
internal/config/samples
|
||||
config/samples
|
||||
rm -rf "$(ARTIFACTDIR)/release"
|
||||
mkdir -p "$(ARTIFACTDIR)/release"
|
||||
cp -l $(ARTIFACTDIR)/zrepl* \
|
||||
@@ -244,10 +227,10 @@ _run_make_foreach_target_tuple:
|
||||
$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=illumos GOARCH=amd64
|
||||
|
||||
##################### REGULAR TARGETS #####################
|
||||
.PHONY: lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin
|
||||
.PHONY: lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin generate-platform-test-list
|
||||
|
||||
lint: build/install
|
||||
$(GO_ENV_VARS) build/install/golangci_lint/golangci-lint run ./...
|
||||
lint:
|
||||
$(GO_ENV_VARS) $(GOLANGCI_LINT) run ./...
|
||||
|
||||
vet:
|
||||
$(GO_ENV_VARS) $(GO) vet $(GO_BUILDFLAGS) ./...
|
||||
@@ -268,12 +251,15 @@ endif
|
||||
zrepl-bin:
|
||||
$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(ZREPL_TARGET_TUPLE)"
|
||||
|
||||
generate-platform-test-list:
|
||||
$(GO_BUILD) -o $(ARTIFACTDIR)/generate-platform-test-list ./platformtest/tests/gen
|
||||
|
||||
COVER_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-cover-$(ZREPL_TARGET_TUPLE)
|
||||
cover-platform-bin:
|
||||
$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
|
||||
-c -o "$(COVER_PLATFORM_BIN_PATH)" \
|
||||
-covermode=atomic -cover -coverpkg github.com/zrepl/zrepl/... \
|
||||
./internal/platformtest/harness
|
||||
./platformtest/harness
|
||||
cover-platform:
|
||||
# do not track dependency on cover-platform-bin to allow build of binary outside of test VM
|
||||
export _TEST_PLATFORM_CMD="$(COVER_PLATFORM_BIN_PATH) \
|
||||
@@ -284,18 +270,30 @@ cover-platform:
|
||||
|
||||
TEST_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)
|
||||
test-platform-bin:
|
||||
$(GO_BUILD) -o "$(TEST_PLATFORM_BIN_PATH)" ./internal/platformtest/harness
|
||||
$(GO_BUILD) -o "$(TEST_PLATFORM_BIN_PATH)" ./platformtest/harness
|
||||
test-platform:
|
||||
export _TEST_PLATFORM_CMD="\"$(TEST_PLATFORM_BIN_PATH)\""; \
|
||||
$(MAKE) _test-or-cover-platform-impl
|
||||
|
||||
ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
|
||||
ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
|
||||
ZREPL_PLATFORMTEST_MOUNTPOINT := /tmp/zreplplatformtest.pool
|
||||
ZREPL_PLATFORMTEST_ZFS_LOG := /tmp/zreplplatformtest.zfs.log
|
||||
# ZREPL_PLATFORMTEST_STOP_AND_KEEP := -failure.stop-and-keep-pool
|
||||
ZREPL_PLATFORMTEST_ARGS :=
|
||||
_test-or-cover-platform-impl: $(ARTIFACTDIR)
|
||||
ifndef _TEST_PLATFORM_CMD
|
||||
$(error _TEST_PLATFORM_CMD is undefined, caller 'cover-platform' or 'test-platform' should have defined it)
|
||||
endif
|
||||
rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
|
||||
rm -f "$(ARTIFACTDIR)/platformtest.cover"
|
||||
$(_TEST_PLATFORM_CMD) $(ZREPL_PLATFORMTEST_ARGS)
|
||||
platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
|
||||
$(_TEST_PLATFORM_CMD) \
|
||||
-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
|
||||
-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
|
||||
-mountpoint "$(ZREPL_PLATFORMTEST_MOUNTPOINT)" \
|
||||
$(ZREPL_PLATFORMTEST_STOP_AND_KEEP) \
|
||||
$(ZREPL_PLATFORMTEST_ARGS)
|
||||
|
||||
cover-merge: $(ARTIFACTDIR)
|
||||
$(GOCOVMERGE) $(ARTIFACTDIR)/platformtest.cover $(ARTIFACTDIR)/gotest.cover > $(ARTIFACTDIR)/merged.cover
|
||||
@@ -303,6 +301,7 @@ cover-html: cover-merge
|
||||
$(GO) tool cover -html "$(ARTIFACTDIR)/merged.cover" -o "$(ARTIFACTDIR)/merged.cover.html"
|
||||
|
||||
cover-full:
|
||||
test "$$(id -u)" = "0" || echo "MUST RUN AS ROOT" 1>&2
|
||||
$(MAKE) test-go COVER=1
|
||||
$(MAKE) cover-platform-bin
|
||||
$(MAKE) cover-platform
|
||||
@@ -310,39 +309,23 @@ cover-full:
|
||||
|
||||
##################### DEV TARGETS #####################
|
||||
# not part of the build, must do that manually
|
||||
.PHONY: generate format
|
||||
.PHONY: generate formatcheck format
|
||||
|
||||
build/install:
|
||||
rm -rf build/install.tmp
|
||||
mkdir build/install.tmp
|
||||
|
||||
-echo "installing protoc"
|
||||
mkdir build/install.tmp/protoc
|
||||
bash -x build/get_protoc.bash build/install.tmp/protoc
|
||||
|
||||
-echo "installing golangci-lint"
|
||||
mkdir -p build/install.tmp/golangci_lint
|
||||
GOHOSTARCH=$(GOHOSTARCH) \
|
||||
build/get_golangci_lint.bash build/install.tmp/golangci_lint
|
||||
|
||||
-echo "installing go tools"
|
||||
build/go_install_tools.bash build/install.tmp/gobin
|
||||
|
||||
mv build/install.tmp build/install
|
||||
|
||||
|
||||
generate: build/install
|
||||
# TODO: would be nice to run with a pure path here
|
||||
PATH="$(CURDIR)/build/install/gobin:$(CURDIR)/build/install/protoc/bin:$$PATH" && \
|
||||
build/install/protoc/bin/protoc -I=internal/replication/logic/pdu --go_out=internal/replication/logic/pdu --go-grpc_out=internal/replication/logic/pdu internal/replication/logic/pdu/pdu.proto && \
|
||||
build/install/protoc/bin/protoc -I=internal/rpc/grpcclientidentity/example --go_out=internal/rpc/grpcclientidentity/example/pdu --go-grpc_out=internal/rpc/grpcclientidentity/example/pdu internal/rpc/grpcclientidentity/example/grpcauth.proto && \
|
||||
$(GO) generate $(GO_BUILDFLAGS) -x ./... && \
|
||||
true
|
||||
generate: generate-platform-test-list
|
||||
protoc -I=replication/logic/pdu --go_out=replication/logic/pdu --go-grpc_out=replication/logic/pdu replication/logic/pdu/pdu.proto
|
||||
protoc -I=rpc/grpcclientidentity/example --go_out=rpc/grpcclientidentity/example/pdu --go-grpc_out=rpc/grpcclientidentity/example/pdu rpc/grpcclientidentity/example/grpcauth.proto
|
||||
$(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./...
|
||||
|
||||
GOIMPORTS := goimports -srcdir . -local 'github.com/zrepl/zrepl'
|
||||
FINDSRCFILES := find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go'
|
||||
|
||||
format: build/install
|
||||
@ build/install/gobin/goimports -w -local 'github.com/zrepl/zrepl' $$(find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go')
|
||||
formatcheck:
|
||||
@# goimports doesn't have a knob to exit with non-zero status code if formatting is needed
|
||||
@# see https://go-review.googlesource.com/c/tools/+/237378
|
||||
@ affectedfiles=$$($(GOIMPORTS) -l $(shell $(FINDSRCFILES)) | tee /dev/stderr | wc -l); test "$$affectedfiles" = 0
|
||||
|
||||
format:
|
||||
@ $(GOIMPORTS) -w -d $(shell $(FINDSRCFILES))
|
||||
|
||||
##################### NOARCH #####################
|
||||
.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
|
||||
@@ -370,9 +353,12 @@ $(ARTIFACTDIR)/go_env.txt:
|
||||
|
||||
docs: $(ARTIFACTDIR)/docs
|
||||
# https://www.sphinx-doc.org/en/master/man/sphinx-build.html
|
||||
cd docs && uv sync --frozen
|
||||
cd docs && uv run sphinx-build -W --keep-going -n . ../artifacts/docs/html
|
||||
$(MAKE) -C docs \
|
||||
html \
|
||||
BUILDDIR=../artifacts/docs \
|
||||
SPHINXOPTS="-W --keep-going -n"
|
||||
|
||||
docs-clean:
|
||||
rm -rf artifacts/docs
|
||||
rm -rf docs/.venv
|
||||
$(MAKE) -C docs \
|
||||
clean \
|
||||
BUILDDIR=../artifacts/docs
|
||||
|
||||
@@ -26,40 +26,24 @@ zrepl is a one-stop ZFS backup & replication solution.
|
||||
If so, think of an expressive configuration example.
|
||||
2. Think of at least one use case that generalizes from your concrete application.
|
||||
3. Open an issue on GitHub with example conf & use case attached.
|
||||
4. **Optional**: [Contact Christian Schwarz](https://cschwarz.com) for contract work.
|
||||
4. **Optional**: [Post a bounty](https://www.bountysource.com/teams/zrepl) on the issue, or [contact Christian Schwarz](https://cschwarz.com) for contract work.
|
||||
|
||||
The above does not apply if you already implemented everything.
|
||||
Check out the *Coding Workflow* section below for details.
|
||||
|
||||
## Development
|
||||
## Building, Releasing, Downstream-Packaging
|
||||
|
||||
This section provides an overview of the zrepl build & release process.
|
||||
Check out `docs/installation/compile-from-source.rst` for build-from-source instructions.
|
||||
|
||||
### Overview
|
||||
|
||||
zrepl is written in [Go](https://golang.org) and uses [Go modules](https://github.com/golang/go/wiki/Modules) to manage dependencies.
|
||||
The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
|
||||
|
||||
### Building
|
||||
|
||||
#### Go Code
|
||||
|
||||
Dependencies:
|
||||
|
||||
* Go
|
||||
* GNU Make
|
||||
* Git
|
||||
* wget (`make generate`)
|
||||
* unzip (`make generate`)
|
||||
|
||||
Some Go code is **generated**, and generated code is committed to the source tree.
|
||||
Therefore, building does not require having code generation tools set up.
|
||||
When making changes that require code to be (re-)generated, run `make generate`.
|
||||
I downloads and installs pinned versions of the code generation tools into `./build/install`.
|
||||
There is a CI check that ensures Git state is clean, i.e., code generation has been done by a PR and is deterministic.
|
||||
|
||||
#### Docs
|
||||
|
||||
Install [uv](https://docs.astral.sh/uv/getting-started/installation/), then run `make docs`.
|
||||
uv automatically manages Python and dependencies.
|
||||
|
||||
### Testing
|
||||
Install **build dependencies** using `./lazy.sh devsetup`.
|
||||
`lazy.sh` uses `python3-pip` to fetch the build dependencies for the docs - you might want to use a [venv](https://docs.python.org/3/library/venv.html).
|
||||
If you just want to install the Go dependencies, run `./lazy.sh godep`.
|
||||
|
||||
The **test suite** is split into pure **Go tests** (`make test-go`) and **platform tests** that interact with ZFS and thus generally **require root privileges** (`sudo make test-platform`).
|
||||
Platform tests run on their own pool with the name `zreplplatformtest`, which is created using the file vdev in `/tmp`.
|
||||
@@ -67,129 +51,51 @@ Platform tests run on their own pool with the name `zreplplatformtest`, which is
|
||||
For a full **code coverage** profile, run `make test-go COVER=1 && sudo make test-platform && make cover-merge`.
|
||||
An HTML report can be generated using `make cover-html`.
|
||||
|
||||
### Circle CI
|
||||
**Code generation** is triggered by `make generate`. Generated code is committed to the source tree.
|
||||
|
||||
We use CircleCI for automated build & test pre- and post-merge.
|
||||
### Build & Release Process
|
||||
|
||||
**The `Makefile` is catering to the needs of developers & CI, not distro packagers**.
|
||||
It provides phony targets for
|
||||
* local development (building, running tests, etc)
|
||||
* building a release in Docker (used by the CI & release management)
|
||||
* building .deb and .rpm packages out of the release artifacts.
|
||||
|
||||
**Build tooling & dependencies** are documented as code in `lazy.sh`.
|
||||
Go dependencies are then fetched by the go command and pip dependencies are pinned through a `requirements.txt`.
|
||||
|
||||
**We use CircleCI for continuous integration**.
|
||||
There are two workflows:
|
||||
|
||||
* `ci` runs for every commit / branch / tag pushed to GitHub.
|
||||
It is supposed to run very fast (<5min and provides quick feedback to developers).
|
||||
It runs formatting checks, lints and tests on the most important OSes / architectures.
|
||||
Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
|
||||
|
||||
* `release` runs
|
||||
* on manual triggers through the CircleCI API (in order to produce a release)
|
||||
* periodically on `master`
|
||||
Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
|
||||
|
||||
Artifacts are stored in CircleCI.
|
||||
|
||||
### Releasing
|
||||
|
||||
All zrepl releases are git-tagged and then published as a GitHub Release.
|
||||
There is a git tag for each zrepl release, usually `vMAJOR.MINOR.0`.
|
||||
We don't move git tags once the release has been published.
|
||||
|
||||
**Releases** are issued via Git tags + GitHub Releases feature.
|
||||
The procedure to issue a release is as follows:
|
||||
|
||||
* Prepare the release (as a PR to `master`):
|
||||
* Finalize `docs/changelog.rst` for the release.
|
||||
* Merge the PR. Docs are auto-published to zrepl.github.io on merge.
|
||||
* Tag the release:
|
||||
* Git tag the release on the `master` branch (e.g., `vMAJOR.MINOR.0`).
|
||||
* Issue the source release:
|
||||
* Git tag the release on the `master` branch.
|
||||
* Push the tag.
|
||||
* Build and publish:
|
||||
* Run the `release` pipeline (trigger via CircleCI UI).
|
||||
* Download artifacts: `make download-circleci-release BUILD_NUM=<circleci-build-number>`
|
||||
* Create GitHub release and upload artifacts:
|
||||
```bash
|
||||
gh release create vX.Y.Z --title "vX.Y.Z" --notes "See changelog" --draft
|
||||
gh release upload vX.Y.Z artifacts/release/*
|
||||
```
|
||||
* Review the draft release, edit the changelog, then publish.
|
||||
* Add the .rpm and .deb files to the official zrepl repos.
|
||||
* Code for management of these repos: https://github.com/zrepl/package-repo-ops (private repo at this time)
|
||||
* Update docs version list:
|
||||
* Update `docs/_templates/versions.html` with the new release.
|
||||
* Verify the link to `zrepl-noarch.tar` in the GitHub release works.
|
||||
* Merge to `master` (docs auto-publish).
|
||||
* Run `./docs/publish.sh` to re-build & push zrepl.github.io.
|
||||
* Issue the official binary release:
|
||||
* Run the `release` pipeline (triggered via CircleCI API)
|
||||
* Download the artifacts to the release manager's machine.
|
||||
* Create a GitHub release, edit the changelog, upload all the release artifacts, including .rpm and .deb files.
|
||||
* Issue the GitHub release.
|
||||
* Add the .rpm and .deb files to the official zrepl repos, publish those.
|
||||
|
||||
#### Patch releases, Go toolchain updates, APT/RPM Package rebuilds
|
||||
**Official binary releases are not re-built when Go receives an update. If the Go update is critical to zrepl (e.g. a Go security update that affects zrepl), we'd issue a new source release**.
|
||||
The rationale for this is that whereas distros provide a mechanism for this (`$zrepl_source_release-$distro_package_revision`), GitHub Releases doesn't which means we'd need to update the existing GitHub release's assets, which nobody would notice (no RSS feed updates, etc.).
|
||||
Downstream packagers can read the changelog to determine whether they want to push that minor release into their distro or simply skip it.
|
||||
|
||||
`vMAJOR.MINOR.0` is typically a tagged commit on `master`, because development velocity isn't high
|
||||
and thus release branches for stabilization aren't necessary.
|
||||
### Additional Notes to Distro Package Maintainers
|
||||
|
||||
Occasionally though there is a need for patch changes to a release, e.g.
|
||||
- security issue in a dependency
|
||||
- Go toolchain update (e.g. security issue in standard library)
|
||||
|
||||
The procedure for this is the following
|
||||
- create a branch off the release tag we need to patch, named `releases/MAJOR.MINOR.X`
|
||||
- that branch will never be merged into `master`, it'll be a dead-end for this specific patch
|
||||
- make changes in that branch
|
||||
- make the final commit that bumps version numbers
|
||||
- create the git tag
|
||||
- follow general procedure for publishing the release (previous sectino
|
||||
|
||||
For Go toolchain updates and package rebuilds with no source code changes, leverage the APT/RPM package revision field.
|
||||
Control via the `ZREPL_PACKAGE_RELEASE` Makefile variable, see `origin/releases/0.6.1-2` for an example.
|
||||
|
||||
|
||||
### Updating Dependencies
|
||||
|
||||
- Update the `go` directive and `toolchain` directive in `go.mod`
|
||||
- `go` is the minimum supported version
|
||||
- `toolchain` is the preferred toolchain version if `GOTOOLCHAIN` is not specified
|
||||
|
||||
Run `go mod tidy` to ensure consistency.
|
||||
|
||||
Update Go module dependencies:
|
||||
|
||||
```bash
|
||||
# Update all other dependencies
|
||||
go get -u -t ./...
|
||||
go mod tidy
|
||||
|
||||
# Above might fail if there are version selection conflicts.
|
||||
# Figure out what's going on by updating packages from error messages first.
|
||||
# Example:
|
||||
go get -u google.golang.org/genproto google.golang.org/grpc google.golang.org/protobuf
|
||||
```
|
||||
|
||||
Update codegen & lint tools
|
||||
- `protoc` => `build/get_protoc.bash`
|
||||
- GH releases publish sha256 sums
|
||||
- `golangci-lint` => `build/get_golangci_lint.bash`
|
||||
- bump versions in `build/tools.go`
|
||||
- we use the tools.go trick:
|
||||
- `go get -tags tools -u example.com/tool ; go mod tidy`
|
||||
- review whether we're ready to switch to `go tool`: https://github.com/zrepl/zrepl/pull/909
|
||||
|
||||
Now run `make generate`.
|
||||
|
||||
Run `make lint` and `make vet`.
|
||||
|
||||
Update the CI configuration `.circleci/config.yml`:
|
||||
- Update Go version references (we reference the minimum and max supported version)
|
||||
- Set `Makefile` `RELEASE_GOVERSION` to the new Go version
|
||||
|
||||
Update docs build tooling:
|
||||
- Update `uv` version in `.circleci/config.yml` (search for `astral.sh/uv/` and cache keys containing the version)
|
||||
- Check if there's now a CircleCI orb for uv that we could use
|
||||
- Update Python version in `docs/.python-version`
|
||||
|
||||
Update docs dependencies (Sphinx, sphinx-rtd-theme):
|
||||
- Check current versions in `docs/pyproject.toml`
|
||||
- Review upstream changelogs for breaking changes
|
||||
- Update version constraints in `pyproject.toml` and the `uv` lockfile (see [uv docs on dependencies](https://docs.astral.sh/uv/concepts/projects/dependencies/)):
|
||||
- Test locally with `make docs`
|
||||
|
||||
Kick a full CI pipeline run (`do_ci=true` and `do_release=true`).
|
||||
|
||||
Merge PR with merge commit.
|
||||
|
||||
## Notes to Distro Package Maintainers
|
||||
|
||||
* The `Makefile` in this project is not suitable for builds in distros.
|
||||
* Run the platform tests (Docs -> Usage -> Platform Tests) **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
|
||||
* Ship a default config that adheres to your distro's `hier` and logging system.
|
||||
* Ship a service manager file and _please_ try to upstream it to this repository.
|
||||
@@ -199,3 +105,31 @@ Merge PR with merge commit.
|
||||
This is how `zrepl version` knows what version number to show.
|
||||
Your build system should set the `ldFlags` flags appropriately and add a prefix or suffix that indicates that the given zrepl binary is a distro build, not an official one.
|
||||
* Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
|
||||
|
||||
|
||||
## Contributing Code
|
||||
|
||||
* Open an issue when starting to hack on a new feature
|
||||
* Commits should reference the issue they are related to
|
||||
* Docs improvements not documenting new features do not require an issue.
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
Backward-incompatible changes must be documented in the git commit message and are listed in `docs/changelog.rst`.
|
||||
|
||||
### Glossary & Naming Inconsistencies
|
||||
|
||||
In ZFS, *dataset* refers to the objects *filesystem*, *ZVOL* and *snapshot*. <br />
|
||||
However, we need a word for *filesystem* & *ZVOL* but not a snapshot, bookmark, etc.
|
||||
|
||||
Toward the user, the following terminology is used:
|
||||
|
||||
* **filesystem**: a ZFS filesystem or a ZVOL
|
||||
* **filesystem version**: a ZFS snapshot or a bookmark
|
||||
|
||||
Sadly, the zrepl implementation is inconsistent in its use of these words:
|
||||
variables and types are often named *dataset* when they in fact refer to a *filesystem*.
|
||||
|
||||
There will not be a big refactoring (an attempt was made, but it's destroying too much history without much gain).
|
||||
|
||||
However, new contributions & patches should fix naming without further notice in the commit message.
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
FROM !SUBSTITUTED_BY_MAKEFILE
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
python3-pip \
|
||||
python3-venv \
|
||||
unzip \
|
||||
gawk
|
||||
|
||||
ADD build.installprotoc.bash ./
|
||||
RUN bash build.installprotoc.bash
|
||||
|
||||
# setup venv
|
||||
ENV VIRTUAL_ENV=/opt/venv
|
||||
RUN python3 -m venv $VIRTUAL_ENV
|
||||
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
|
||||
|
||||
ADD lazy.sh /tmp/lazy.sh
|
||||
ADD docs/requirements.txt /tmp/requirements.txt
|
||||
ENV ZREPL_LAZY_DOCS_REQPATH=/tmp/requirements.txt
|
||||
RUN /tmp/lazy.sh docdep
|
||||
|
||||
# prepare volume mount of git checkout to /zrepl
|
||||
RUN mkdir -p /src/github.com/zrepl/zrepl
|
||||
RUN mkdir -p /.cache && chmod -R 0777 /.cache
|
||||
|
||||
# $GOPATH is /go
|
||||
# Go 1.12 doesn't use modules within GOPATH, but 1.13 and later do
|
||||
# => store source outside of GOPATH
|
||||
WORKDIR /src
|
||||
|
||||
# Install build tools (e.g. protobuf generator, stringer) into $GOPATH/bin
|
||||
ADD build/ /tmp/build
|
||||
RUN /tmp/lazy.sh godep
|
||||
|
||||
RUN chmod -R 0777 /go
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
set -x
|
||||
|
||||
MACH=$(uname -m)
|
||||
MACH="${MACH/aarch64/aarch_64}"
|
||||
|
||||
VERSION=3.6.1
|
||||
FILENAME=protoc-"$VERSION"-linux-"$MACH".zip
|
||||
|
||||
if [ -e "$FILENAME" ]; then
|
||||
echo "$FILENAME" already exists 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
wget https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
|
||||
|
||||
stat "$FILENAME"
|
||||
|
||||
sha256sum -c --ignore-missing <<EOF
|
||||
6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807 protoc-3.6.1-linux-x86_64.zip
|
||||
af8e5aaaf39ddec62ec8dd2be1b8d9602c6da66564883a16393ade5f71170922 protoc-3.6.1-linux-aarch_64.zip
|
||||
EOF
|
||||
|
||||
unzip -d /usr "$FILENAME"
|
||||
@@ -1 +0,0 @@
|
||||
install/
|
||||
@@ -1,30 +0,0 @@
|
||||
FROM !SUBSTITUTED_BY_MAKEFILE
|
||||
|
||||
ARG BUILD_UID=1000
|
||||
ARG BUILD_GID=1000
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
python3 \
|
||||
unzip \
|
||||
gawk \
|
||||
curl
|
||||
|
||||
# Create build user with the host's UID/GID before installing uv
|
||||
RUN groupadd -g ${BUILD_GID} zrepl_build && \
|
||||
useradd -u ${BUILD_UID} -g ${BUILD_GID} -m zrepl_build
|
||||
|
||||
# Go toolchain uses xdg-cache
|
||||
RUN mkdir -p /.cache && chmod -R 0777 /.cache
|
||||
|
||||
# Install uv as the build user - version from .uv-version file
|
||||
ADD .uv-version /tmp/.uv-version
|
||||
USER zrepl_build
|
||||
RUN UV_VERSION=$(cat /tmp/.uv-version) && \
|
||||
curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh
|
||||
|
||||
# Go devtools are managed by Makefile
|
||||
|
||||
WORKDIR /src
|
||||
ENV PATH="/home/zrepl_build/.local/bin:$PATH" \
|
||||
GOCACHE="/.cache/go-build"
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
set -x
|
||||
|
||||
# golangci-lint recommends against using `go get` and friends to install it
|
||||
|
||||
cd "$1"
|
||||
|
||||
VERSION=2.8.0
|
||||
FILENAME=golangci-lint-"$VERSION"-linux-"$GOHOSTARCH".tar.gz
|
||||
|
||||
if [ -e "$FILENAME" ]; then
|
||||
echo "$FILENAME" already exists 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
wget --continue https://github.com/golangci/golangci-lint/releases/download/v"$VERSION"/"$FILENAME"
|
||||
|
||||
stat "$FILENAME"
|
||||
|
||||
# Select the correct checksum for the downloaded architecture
|
||||
case "$GOHOSTARCH" in
|
||||
arm64) EXPECTED_SHA256="2a58388db8af5ab9330791cea0ebdd4100723cd05ad7185d92febaaee272ec9a" ;;
|
||||
amd64) EXPECTED_SHA256="7048bc6b25c9515ed092c83f9fa8709ca97937ead52d9ff317a143299ee97a50" ;;
|
||||
*) echo "Unknown architecture: $GOHOSTARCH" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
# Verify checksum explicitly - fails if hash doesn't match
|
||||
echo "$EXPECTED_SHA256 $FILENAME" | sha256sum -c -
|
||||
|
||||
tar -x --strip-components=1 -f "$FILENAME"
|
||||
|
||||
stat ./golangci-lint
|
||||
@@ -1,32 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
set -x
|
||||
|
||||
cd "$1"
|
||||
|
||||
MACH=$(uname -m)
|
||||
MACH="${MACH/aarch64/aarch_64}"
|
||||
|
||||
VERSION=33.5
|
||||
FILENAME=protoc-"$VERSION"-linux-"$MACH".zip
|
||||
|
||||
if [ -e "$FILENAME" ]; then
|
||||
echo "$FILENAME" already exists 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
wget --continue https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
|
||||
|
||||
stat "$FILENAME"
|
||||
|
||||
# Select the correct checksum for the downloaded architecture
|
||||
case "$MACH" in
|
||||
aarch_64) EXPECTED_SHA256="2b0fcf9b2c32cbadccc0eb7a88b841fffecd4a06fc80acdba2b5be45e815c38a" ;;
|
||||
x86_64) EXPECTED_SHA256="24e58fb231d50306ee28491f33a170301e99540f7e29ca461e0e80fd1239f8d1" ;;
|
||||
*) echo "Unknown architecture: $MACH" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
# Verify checksum explicitly - fails if hash doesn't match
|
||||
echo "$EXPECTED_SHA256 $FILENAME" | sha256sum -c -
|
||||
|
||||
unzip -d . "$FILENAME"
|
||||
+34
-18
@@ -1,24 +1,40 @@
|
||||
module github.com/zrepl/zrepl/build
|
||||
|
||||
go 1.24.13
|
||||
|
||||
toolchain go1.25.7
|
||||
go 1.12
|
||||
|
||||
require (
|
||||
github.com/alvaroloes/enumer v1.1.2
|
||||
github.com/alvaroloes/enumer v1.1.1
|
||||
github.com/breml/bidichk v0.2.6 // indirect
|
||||
github.com/breml/errchkjson v0.3.5 // indirect
|
||||
github.com/chavacava/garif v0.1.0 // indirect
|
||||
github.com/daixiang0/gci v0.11.1 // indirect
|
||||
github.com/golangci/golangci-lint v1.54.2
|
||||
github.com/golangci/revgrep v0.5.0 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/jgautheron/goconst v1.6.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||
github.com/mgechev/revive v1.3.3 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
|
||||
github.com/polyfloyd/go-errorlint v1.4.5 // indirect
|
||||
github.com/prometheus/client_golang v1.16.0 // indirect
|
||||
github.com/prometheus/common v0.44.0 // indirect
|
||||
github.com/prometheus/procfs v0.11.1 // indirect
|
||||
github.com/rivo/uniseg v0.4.4 // indirect
|
||||
github.com/ryanrolds/sqlclosecheck v0.5.1 // indirect
|
||||
github.com/spf13/viper v1.16.0 // indirect
|
||||
github.com/stretchr/objx v0.5.1 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/tetafro/godot v1.4.15 // indirect
|
||||
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad
|
||||
golang.org/x/tools v0.41.0
|
||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.1
|
||||
google.golang.org/protobuf v1.36.11
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/pascaldekloe/name v1.0.1 // indirect
|
||||
golang.org/x/mod v0.32.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20260205145544-86a5c4bf3c8d // indirect
|
||||
golang.org/x/text v0.31.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c // indirect
|
||||
google.golang.org/grpc v1.75.0 // indirect
|
||||
github.com/xen0n/gosmopolitan v1.2.2 // indirect
|
||||
gitlab.com/bosi/decorder v0.4.1 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.25.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
|
||||
golang.org/x/exp/typeparams v0.0.0-20230905200255-921286631fa9 // indirect
|
||||
golang.org/x/tools v0.13.0
|
||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0
|
||||
google.golang.org/protobuf v1.31.0
|
||||
mvdan.cc/unparam v0.0.0-20230815095028-f7c6fb1088f0 // indirect
|
||||
)
|
||||
|
||||
+2556
-28
File diff suppressed because it is too large
Load Diff
@@ -1,6 +0,0 @@
|
||||
export GO111MODULE=on # otherwise, a checkout of this repo in GOPATH will disable modules on Go 1.12 and earlier
|
||||
source <(go env)
|
||||
# build tools for the host platform
|
||||
export GOOS="$GOHOSTOS"
|
||||
export GOARCH="$GOHOSTARCH"
|
||||
# TODO GOARM=$GOHOSTARM?
|
||||
@@ -1,19 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
set -x
|
||||
|
||||
OUTDIR="$(readlink -f "$1")"
|
||||
if [ -e "$OUTDIR" ]; then
|
||||
echo "$OUTDIR" already exists 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# go install command below will install tools to $GOBIN
|
||||
export GOBIN="$OUTDIR"
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
source ./go_install_host_tool.source
|
||||
|
||||
cat tools.go | grep _ | awk -F'"' '{print $2}' | tee | xargs -tI '{}' go install '{}'
|
||||
@@ -6,6 +6,7 @@ package main
|
||||
// the lines are parsed by lazy.sh, do not edit
|
||||
import (
|
||||
_ "github.com/alvaroloes/enumer"
|
||||
_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
|
||||
_ "github.com/wadey/gocovmerge"
|
||||
_ "golang.org/x/tools/cmd/goimports"
|
||||
_ "golang.org/x/tools/cmd/stringer"
|
||||
|
||||
@@ -8,9 +8,9 @@ import (
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
)
|
||||
|
||||
var rootArgs struct {
|
||||
@@ -11,11 +11,11 @@ import (
|
||||
"github.com/spf13/pflag"
|
||||
"github.com/zrepl/yaml-config"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/cli"
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
||||
"github.com/zrepl/zrepl/internal/logger"
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/job"
|
||||
"github.com/zrepl/zrepl/daemon/logging"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
)
|
||||
|
||||
var configcheckArgs struct {
|
||||
@@ -9,12 +9,12 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/daemon/job"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/cli"
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -1,6 +1,6 @@
|
||||
package client
|
||||
|
||||
import "github.com/zrepl/zrepl/internal/cli"
|
||||
import "github.com/zrepl/zrepl/cli"
|
||||
|
||||
var PprofCmd = &cli.Subcommand{
|
||||
Use: "pprof",
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
|
||||
"golang.org/x/net/websocket"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/cli"
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
)
|
||||
|
||||
var pprofActivityTraceCmd = &cli.Subcommand{
|
||||
@@ -6,9 +6,9 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/cli"
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon"
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon"
|
||||
)
|
||||
|
||||
var pprofListenCmd struct {
|
||||
@@ -5,9 +5,9 @@ import (
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/cli"
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon"
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon"
|
||||
)
|
||||
|
||||
var SignalCmd = &cli.Subcommand{
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon"
|
||||
"github.com/zrepl/zrepl/daemon"
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
@@ -9,11 +9,11 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/cli"
|
||||
"github.com/zrepl/zrepl/internal/client/status/client"
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon"
|
||||
"github.com/zrepl/zrepl/internal/util/choices"
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/client/status/client"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon"
|
||||
"github.com/zrepl/zrepl/util/choices"
|
||||
)
|
||||
|
||||
type Client interface {
|
||||
@@ -5,11 +5,11 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/gdamore/tcell/v2"
|
||||
"github.com/gdamore/tcell"
|
||||
"github.com/mattn/go-isatty"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/client/status/viewmodel"
|
||||
"github.com/zrepl/zrepl/client/status/viewmodel"
|
||||
)
|
||||
|
||||
func dump(c Client, job string) error {
|
||||
@@ -8,9 +8,9 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gdamore/tcell/v2"
|
||||
"github.com/rivo/tview"
|
||||
tview "gitlab.com/tslocum/cview"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/client/status/viewmodel"
|
||||
"github.com/zrepl/zrepl/client/status/viewmodel"
|
||||
)
|
||||
|
||||
func interactive(c Client, flag statusFlags) error {
|
||||
@@ -29,6 +29,7 @@ func interactive(c Client, flag statusFlags) error {
|
||||
jobMenuRoot.SetSelectable(true)
|
||||
jobMenu.SetRoot(jobMenuRoot)
|
||||
jobMenu.SetCurrentNode(jobMenuRoot)
|
||||
jobMenu.SetSelectedTextColor(tcell.ColorGreen)
|
||||
jobTextDetail := tview.NewTextView()
|
||||
jobTextDetail.SetWrap(false)
|
||||
|
||||
@@ -109,8 +110,10 @@ func interactive(c Client, flag statusFlags) error {
|
||||
}
|
||||
app.SetRoot(toolbarSplit, true)
|
||||
app.SetFocus(preModalFocus)
|
||||
app.Draw()
|
||||
})
|
||||
app.SetRoot(m, true)
|
||||
app.Draw()
|
||||
}
|
||||
|
||||
app.SetRoot(toolbarSplit, true)
|
||||
@@ -167,14 +170,12 @@ func interactive(c Client, flag statusFlags) error {
|
||||
redrawJobsList = true
|
||||
}
|
||||
if redrawJobsList {
|
||||
selectedTextStyle := tcell.StyleDefault.Bold(true)
|
||||
selectedJobN = nil
|
||||
children := make([]*tview.TreeNode, len(jobs))
|
||||
for i := range jobs {
|
||||
jobN := tview.NewTreeNode(jobs[i].JobTreeTitle())
|
||||
jobN.SetReference(jobs[i])
|
||||
jobN.SetSelectable(true)
|
||||
jobN.SetSelectedTextStyle(selectedTextStyle)
|
||||
children[i] = jobN
|
||||
jobN.SetSelectedFunc(func() {
|
||||
viewmodelupdate(func(p *viewmodel.Params) {
|
||||
@@ -186,7 +187,6 @@ func interactive(c Client, flag statusFlags) error {
|
||||
}
|
||||
}
|
||||
jobMenuRoot.SetChildren(children)
|
||||
jobMenuRoot.SetSelectedTextStyle(selectedTextStyle)
|
||||
}
|
||||
|
||||
if selectedJobN != nil && jobMenu.GetCurrentNode() != selectedJobN {
|
||||
@@ -207,6 +207,9 @@ func interactive(c Client, flag statusFlags) error {
|
||||
bottombar.ResizeItem(bottombarDateView, len(bottombardatestring), 0)
|
||||
|
||||
bottomBarStatus.SetText(m.BottomBarStatus())
|
||||
|
||||
app.Draw()
|
||||
|
||||
}
|
||||
|
||||
go func() {
|
||||
@@ -249,7 +252,6 @@ func interactive(c Client, flag statusFlags) error {
|
||||
|
||||
app.SetInputCapture(func(e *tcell.EventKey) *tcell.EventKey {
|
||||
if e.Key() == tcell.KeyTab {
|
||||
// TODO: only if there's no modal showing (long-time bug in zrepl status)
|
||||
tabbableCycle()
|
||||
return nil
|
||||
}
|
||||
@@ -282,8 +284,9 @@ func interactive(c Client, flag statusFlags) error {
|
||||
signals := []string{"wakeup", "reset"}
|
||||
clientFuncs := []func(job string) error{c.SignalWakeup, c.SignalReset}
|
||||
sigMod := tview.NewModal()
|
||||
sigMod.SetBackgroundColor(tcell.ColorDefault)
|
||||
sigMod.SetBorder(true)
|
||||
sigMod.SetButtonActivatedStyle(tcell.StyleDefault.Bold(true).Reverse(true))
|
||||
sigMod.GetForm().SetButtonTextColorFocused(tcell.ColorGreen)
|
||||
sigMod.AddButtons(signals)
|
||||
sigMod.SetText(fmt.Sprintf("Send a signal to job %q", job.Name()))
|
||||
showModal(sigMod, func(idx int, _ string) {
|
||||
@@ -10,9 +10,9 @@ import (
|
||||
"github.com/gdamore/tcell/v2"
|
||||
"github.com/mattn/go-isatty"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rivo/tview"
|
||||
tview "gitlab.com/tslocum/cview"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/client/status/viewmodel"
|
||||
"github.com/zrepl/zrepl/client/status/viewmodel"
|
||||
)
|
||||
|
||||
func legacy(c Client, flag statusFlags) error {
|
||||
@@ -28,7 +28,7 @@ func legacy(c Client, flag statusFlags) error {
|
||||
textView := tview.NewTextView()
|
||||
textView.SetWrap(true)
|
||||
textView.SetScrollable(true) // so that it allows us to set scroll position
|
||||
// textView.SetScrollBarVisibility(tview.ScrollBarNever)
|
||||
textView.SetScrollBarVisibility(tview.ScrollBarNever)
|
||||
|
||||
app.SetRoot(textView, true)
|
||||
|
||||
@@ -10,12 +10,12 @@ import (
|
||||
"github.com/go-playground/validator/v10"
|
||||
yaml "github.com/zrepl/yaml-config"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/client/status/viewmodel/stringbuilder"
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
||||
"github.com/zrepl/zrepl/internal/daemon/pruner"
|
||||
"github.com/zrepl/zrepl/internal/daemon/snapper"
|
||||
"github.com/zrepl/zrepl/internal/replication/report"
|
||||
"github.com/zrepl/zrepl/client/status/viewmodel/stringbuilder"
|
||||
"github.com/zrepl/zrepl/daemon"
|
||||
"github.com/zrepl/zrepl/daemon/job"
|
||||
"github.com/zrepl/zrepl/daemon/pruner"
|
||||
"github.com/zrepl/zrepl/daemon/snapper"
|
||||
"github.com/zrepl/zrepl/replication/report"
|
||||
)
|
||||
|
||||
type M struct {
|
||||
@@ -85,7 +85,7 @@ func (m *M) Update(p Params) {
|
||||
// filter out internal jobs
|
||||
var jobsList []*Job
|
||||
for _, j := range m.jobsList {
|
||||
if config.IsInternalJobName(j.name) {
|
||||
if daemon.IsInternalJobName(j.name) {
|
||||
continue
|
||||
}
|
||||
jobsList = append(jobsList, j)
|
||||
@@ -5,8 +5,8 @@ import (
|
||||
|
||||
"github.com/problame/go-netssh"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/cli"
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
|
||||
"context"
|
||||
"errors"
|
||||
@@ -9,10 +9,10 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/cli"
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
var TestCmd = &cli.Subcommand{
|
||||
@@ -7,10 +7,10 @@ import (
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/cli"
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon"
|
||||
"github.com/zrepl/zrepl/internal/version"
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon"
|
||||
"github.com/zrepl/zrepl/version"
|
||||
)
|
||||
|
||||
var versionArgs struct {
|
||||
@@ -7,10 +7,10 @@ import (
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/cli"
|
||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -1,6 +1,6 @@
|
||||
package client
|
||||
|
||||
import "github.com/zrepl/zrepl/internal/cli"
|
||||
import "github.com/zrepl/zrepl/cli"
|
||||
|
||||
var zabsCmdCreate = &cli.Subcommand{
|
||||
Use: "create",
|
||||
+3
-3
@@ -7,9 +7,9 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/cli"
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
var zabsCreateStepHoldFlags struct {
|
||||
@@ -11,9 +11,9 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/cli"
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/util/chainlock"
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/util/chainlock"
|
||||
)
|
||||
|
||||
var zabsListFlags struct {
|
||||
@@ -10,8 +10,8 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/cli"
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/cli"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
)
|
||||
|
||||
// shared between release-all and release-step
|
||||
@@ -2,19 +2,18 @@ package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log/syslog"
|
||||
"os"
|
||||
pathpkg "path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/robfig/cron/v3"
|
||||
"github.com/zrepl/yaml-config"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/util/datasizeunit"
|
||||
zfsprop "github.com/zrepl/zrepl/internal/zfs/property"
|
||||
"github.com/zrepl/zrepl/util/datasizeunit"
|
||||
zfsprop "github.com/zrepl/zrepl/zfs/property"
|
||||
)
|
||||
|
||||
type ParseFlags uint
|
||||
@@ -25,9 +24,8 @@ const (
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Jobs []JobEnum `yaml:"jobs,optional"`
|
||||
Global *Global `yaml:"global,optional,fromdefaults"`
|
||||
Include []string `yaml:"include,optional"`
|
||||
Jobs []JobEnum `yaml:"jobs"`
|
||||
Global *Global `yaml:"global,optional,fromdefaults"`
|
||||
}
|
||||
|
||||
func (c *Config) Job(name string) (*JobEnum, error) {
|
||||
@@ -123,6 +121,7 @@ type BandwidthLimit struct {
|
||||
}
|
||||
|
||||
type Replication struct {
|
||||
Triggers []*ReplicationTriggerEnum
|
||||
Protection *ReplicationOptionsProtection `yaml:"protection,optional,fromdefaults"`
|
||||
Concurrency *ReplicationOptionsConcurrency `yaml:"concurrency,optional,fromdefaults"`
|
||||
}
|
||||
@@ -137,6 +136,32 @@ type ReplicationOptionsConcurrency struct {
|
||||
SizeEstimates int `yaml:"size_estimates,optional,default=4"`
|
||||
}
|
||||
|
||||
type ReplicationTriggerEnum struct {
|
||||
Ret interface{}
|
||||
}
|
||||
|
||||
func (t *ReplicationTriggerEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) {
|
||||
t.Ret, err = enumUnmarshal(u, map[string]interface{}{
|
||||
"manual": &ReplicationTriggerManual{},
|
||||
"periodic": &ReplicationTriggerPeriodic{},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
type ReplicationTriggerManual struct {
|
||||
Type string `yaml:"type"`
|
||||
}
|
||||
|
||||
type ReplicationTriggerPeriodic struct {
|
||||
Type string `yaml:"type"`
|
||||
Interval *PositiveDuration `yaml:"interval"`
|
||||
}
|
||||
|
||||
type ReplicationTriggerCron struct {
|
||||
Type string `yaml:"type"`
|
||||
Cron CronSpec `yaml:"cron"`
|
||||
}
|
||||
|
||||
type PropertyRecvOptions struct {
|
||||
Inherit []zfsprop.Property `yaml:"inherit,optional"`
|
||||
Override map[zfsprop.Property]string `yaml:"override,optional"`
|
||||
@@ -159,7 +184,6 @@ func (j *PushJob) GetSendOptions() *SendOptions { return j.Send }
|
||||
type PullJob struct {
|
||||
ActiveJob `yaml:",inline"`
|
||||
RootFS string `yaml:"root_fs"`
|
||||
Interval PositiveDurationOrManual `yaml:"interval"`
|
||||
Recv *RecvOptions `yaml:"recv,fromdefaults,optional"`
|
||||
}
|
||||
|
||||
@@ -222,16 +246,11 @@ type SnapshottingEnum struct {
|
||||
}
|
||||
|
||||
type SnapshottingPeriodic struct {
|
||||
Type string `yaml:"type"`
|
||||
Prefix string `yaml:"prefix"`
|
||||
Interval *PositiveDuration `yaml:"interval"`
|
||||
Hooks HookList `yaml:"hooks,optional"`
|
||||
TimestampFormattingSpec `yaml:",inline"`
|
||||
}
|
||||
|
||||
type TimestampFormattingSpec struct {
|
||||
TimestampFormat string `yaml:"timestamp_format,optional,default=dense"`
|
||||
TimestampLocation string `yaml:"timestamp_location,optional,default=UTC"`
|
||||
Type string `yaml:"type"`
|
||||
Prefix string `yaml:"prefix"`
|
||||
Interval *PositiveDuration `yaml:"interval"`
|
||||
Hooks HookList `yaml:"hooks,optional"`
|
||||
TimestampFormat string `yaml:"timestamp_format,optional,default=dense"`
|
||||
}
|
||||
|
||||
type CronSpec struct {
|
||||
@@ -260,11 +279,11 @@ func (s *CronSpec) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool)
|
||||
}
|
||||
|
||||
type SnapshottingCron struct {
|
||||
Type string `yaml:"type"`
|
||||
Prefix string `yaml:"prefix"`
|
||||
Cron CronSpec `yaml:"cron"`
|
||||
Hooks HookList `yaml:"hooks,optional"`
|
||||
TimestampFormattingSpec `yaml:",inline"`
|
||||
Type string `yaml:"type"`
|
||||
Prefix string `yaml:"prefix"`
|
||||
Cron CronSpec `yaml:"cron"`
|
||||
Hooks HookList `yaml:"hooks,optional"`
|
||||
TimestampFormat string `yaml:"timestamp_format,optional,default=dense"`
|
||||
}
|
||||
|
||||
type SnapshottingManual struct {
|
||||
@@ -306,6 +325,18 @@ type Global struct {
|
||||
Serve *GlobalServe `yaml:"serve,optional,fromdefaults"`
|
||||
}
|
||||
|
||||
func Default(i interface{}) {
|
||||
v := reflect.ValueOf(i)
|
||||
if v.Kind() != reflect.Ptr {
|
||||
panic(v)
|
||||
}
|
||||
y := `{}`
|
||||
err := yaml.Unmarshal([]byte(y), v.Interface())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
type ConnectEnum struct {
|
||||
Ret interface{}
|
||||
}
|
||||
@@ -659,9 +690,8 @@ var ConfigFileDefaultLocations = []string{
|
||||
"/usr/local/etc/zrepl/zrepl.yml",
|
||||
}
|
||||
|
||||
func ParseConfig(path string) (rootConfig *Config, err error) {
|
||||
func ParseConfig(path string) (i *Config, err error) {
|
||||
|
||||
// Parse main configuration file
|
||||
if path == "" {
|
||||
// Try default locations
|
||||
for _, l := range ConfigFileDefaultLocations {
|
||||
@@ -680,94 +710,11 @@ func ParseConfig(path string) (rootConfig *Config, err error) {
|
||||
|
||||
var bytes []byte
|
||||
|
||||
if bytes, err = os.ReadFile(path); err != nil {
|
||||
if bytes, err = ioutil.ReadFile(path); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
rootConfig, err = ParseConfigBytes(bytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = expandConfigInclude(path, rootConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err = validateJobNames(rootConfig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return rootConfig, err
|
||||
}
|
||||
|
||||
func IsInternalJobName(s string) bool {
|
||||
return strings.HasPrefix(s, "_")
|
||||
}
|
||||
|
||||
func validateJobNames(config *Config) error {
|
||||
seen := make(map[string]struct{})
|
||||
for _, job := range config.Jobs {
|
||||
name := job.Name()
|
||||
if IsInternalJobName(name) {
|
||||
return errors.Errorf("job name %q is reserved for internal use (starts with _)", name)
|
||||
}
|
||||
if _, ok := seen[name]; ok {
|
||||
return errors.Errorf("duplicate job name %q", name)
|
||||
}
|
||||
seen[name] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func expandConfigInclude(configPath string, config *Config) (err error) {
|
||||
var includeConfigPaths []string
|
||||
for _, path := range config.Include {
|
||||
if !pathpkg.IsAbs(configPath) {
|
||||
path = pathpkg.Join(pathpkg.Dir(configPath), path)
|
||||
}
|
||||
|
||||
stat, statErr := os.Stat(path)
|
||||
if statErr != nil {
|
||||
return errors.Wrapf(statErr, "stat path %q", path)
|
||||
}
|
||||
|
||||
if stat.Mode().IsDir() {
|
||||
directoryPaths, err := filepath.Glob(path + "/*.yml")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
includeConfigPaths = append(includeConfigPaths, directoryPaths...)
|
||||
} else if stat.Mode().IsRegular() {
|
||||
if extention := filepath.Ext(path); extention != ".yml" {
|
||||
return fmt.Errorf("include config files must end with `.yml`: %s", path)
|
||||
}
|
||||
includeConfigPaths = append(includeConfigPaths, path)
|
||||
} else {
|
||||
return fmt.Errorf("not a file or directory: %s", path)
|
||||
}
|
||||
}
|
||||
|
||||
for _, path := range includeConfigPaths {
|
||||
var bytes []byte
|
||||
if bytes, err = os.ReadFile(path); err != nil {
|
||||
return errors.Wrapf(err, "read file: %q", path)
|
||||
}
|
||||
|
||||
includedConfig, err := ParseConfigBytes(bytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(includedConfig.Include) > 0 {
|
||||
return errors.Errorf("included configuration files must not include other files: %s", path)
|
||||
}
|
||||
|
||||
config.Jobs = append(config.Jobs, includedConfig.Jobs...)
|
||||
}
|
||||
|
||||
return nil
|
||||
return ParseConfigBytes(bytes)
|
||||
}
|
||||
|
||||
func ParseConfigBytes(bytes []byte) (*Config, error) {
|
||||
@@ -775,16 +722,8 @@ func ParseConfigBytes(bytes []byte) (*Config, error) {
|
||||
if err := yaml.UnmarshalStrict(bytes, &c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c != nil {
|
||||
return c, nil
|
||||
}
|
||||
// There was no yaml document in the file, deserialize from default.
|
||||
// => See TestFromdefaultsEmptyDoc in yaml-config package.
|
||||
if err := yaml.UnmarshalStrict([]byte("{}"), &c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c == nil {
|
||||
panic("the fallback to deserialize from `{}` should work")
|
||||
return nil, fmt.Errorf("config is empty or only consists of comments")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
@@ -2,8 +2,16 @@ package config
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestConfigEmptyFails(t *testing.T) {
|
||||
conf, err := testConfig(t, "\n")
|
||||
assert.Nil(t, conf)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestJobsOnlyWorks(t *testing.T) {
|
||||
testValidConfig(t, `
|
||||
jobs:
|
||||
@@ -26,7 +34,7 @@ jobs:
|
||||
keep_sender:
|
||||
- type: not_replicated
|
||||
keep_receiver:
|
||||
- type: last_n
|
||||
- type: last_n
|
||||
count: 1
|
||||
`)
|
||||
}
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
zfsprop "github.com/zrepl/zrepl/internal/zfs/property"
|
||||
zfsprop "github.com/zrepl/zrepl/zfs/property"
|
||||
)
|
||||
|
||||
func TestRecvOptions(t *testing.T) {
|
||||
@@ -163,8 +163,7 @@ jobs:
|
||||
assert.Equal(t, "periodic", snp.Type)
|
||||
assert.Equal(t, 10*time.Minute, snp.Interval.Duration())
|
||||
assert.Equal(t, "zrepl_", snp.Prefix)
|
||||
assert.Equal(t, snp.TimestampFormat, "dense")
|
||||
assert.Equal(t, snp.TimestampLocation, "UTC")
|
||||
assert.Equal(t, "dense", snp.TimestampFormat) // default was set correctly
|
||||
})
|
||||
|
||||
t.Run("cron", func(t *testing.T) {
|
||||
@@ -172,7 +171,6 @@ jobs:
|
||||
snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingCron)
|
||||
assert.Equal(t, "cron", snp.Type)
|
||||
assert.Equal(t, "zrepl_", snp.Prefix)
|
||||
assert.Equal(t, snp.TimestampFormat, "dense")
|
||||
assert.Equal(t, snp.TimestampLocation, "UTC")
|
||||
assert.Equal(t, "dense", snp.TimestampFormat) // default was set correctly
|
||||
})
|
||||
}
|
||||
@@ -22,8 +22,6 @@ func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
|
||||
t.Errorf("glob failed: %+v", err)
|
||||
}
|
||||
|
||||
paths = append(paths, "../../packaging/systemd-default-zrepl.yml")
|
||||
|
||||
for _, p := range paths {
|
||||
|
||||
if path.Ext(p) != ".yml" {
|
||||
@@ -45,23 +43,9 @@ func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
func TestInvalidSampleConfigsFailToParse(t *testing.T) {
|
||||
paths, err := filepath.Glob("./samples/invalid/*/zrepl.yml")
|
||||
require.NoError(t, err, "glob failed")
|
||||
require.NotEmpty(t, paths, "no invalid sample configs found")
|
||||
|
||||
for _, p := range paths {
|
||||
t.Run(p, func(t *testing.T) {
|
||||
_, err := ParseConfig(p)
|
||||
require.Error(t, err, "expected config %s to fail parsing", p)
|
||||
t.Logf("config %s failed as expected: %v", p, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// template must be a template/text template with a single '{{ . }}' as placeholder for val
|
||||
//
|
||||
//nolint:unused
|
||||
//nolint:deadcode,unused
|
||||
func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
|
||||
tmp, err := template.New("master").Parse(tmpl)
|
||||
if err != nil {
|
||||
@@ -100,7 +84,7 @@ func trimSpaceEachLineAndPad(s, pad string) string {
|
||||
func TestTrimSpaceEachLineAndPad(t *testing.T) {
|
||||
foo := `
|
||||
foo
|
||||
bar baz
|
||||
bar baz
|
||||
`
|
||||
assert.Equal(t, " \n foo\n bar baz\n \n", trimSpaceEachLineAndPad(foo, " "))
|
||||
}
|
||||
@@ -154,18 +138,3 @@ func TestCronSpec(t *testing.T) {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestEmptyConfig(t *testing.T) {
|
||||
cases := []string{
|
||||
"",
|
||||
"\n",
|
||||
"---",
|
||||
"---\n",
|
||||
}
|
||||
for _, input := range cases {
|
||||
config := testValidConfig(t, input)
|
||||
require.NotNil(t, config)
|
||||
require.NotNil(t, config.Global)
|
||||
require.Empty(t, config.Jobs)
|
||||
}
|
||||
}
|
||||
@@ -14,14 +14,14 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
||||
"github.com/zrepl/zrepl/internal/daemon/nethelpers"
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/logger"
|
||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
||||
"github.com/zrepl/zrepl/internal/version"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/internal/zfs/zfscmd"
|
||||
"github.com/zrepl/zrepl/daemon/job"
|
||||
"github.com/zrepl/zrepl/daemon/nethelpers"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/util/envconst"
|
||||
"github.com/zrepl/zrepl/version"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
"github.com/zrepl/zrepl/zfs/zfscmd"
|
||||
)
|
||||
|
||||
type controlJob struct {
|
||||
@@ -3,8 +3,10 @@ package daemon
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
@@ -12,18 +14,18 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/util/envconst"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/job"
|
||||
"github.com/zrepl/zrepl/internal/daemon/job/reset"
|
||||
"github.com/zrepl/zrepl/internal/daemon/job/wakeup"
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
||||
"github.com/zrepl/zrepl/internal/logger"
|
||||
"github.com/zrepl/zrepl/internal/version"
|
||||
"github.com/zrepl/zrepl/internal/zfs/zfscmd"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/job"
|
||||
"github.com/zrepl/zrepl/daemon/job/reset"
|
||||
"github.com/zrepl/zrepl/daemon/job/wakeup"
|
||||
"github.com/zrepl/zrepl/daemon/logging"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/version"
|
||||
"github.com/zrepl/zrepl/zfs/zfscmd"
|
||||
)
|
||||
|
||||
func Run(ctx context.Context, conf *config.Config) error {
|
||||
@@ -37,6 +39,12 @@ func Run(ctx context.Context, conf *config.Config) error {
|
||||
cancel()
|
||||
}()
|
||||
|
||||
// The math/rand package is used presently for generating trace IDs, we
|
||||
// seed it with the current time and pid so that the IDs are mostly
|
||||
// unique.
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
rand.Seed(int64(os.Getpid()))
|
||||
|
||||
outlets, err := logging.OutletsFromConfig(*conf.Global.Logging)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "cannot build logging from config")
|
||||
@@ -63,7 +71,7 @@ func Run(ctx context.Context, conf *config.Config) error {
|
||||
})
|
||||
|
||||
for _, job := range confJobs {
|
||||
if config.IsInternalJobName(job.Name()) {
|
||||
if IsInternalJobName(job.Name()) {
|
||||
panic(fmt.Sprintf("internal job name used for config job '%s'", job.Name())) //FIXME
|
||||
}
|
||||
}
|
||||
@@ -210,6 +218,10 @@ const (
|
||||
jobNameControl = "_control"
|
||||
)
|
||||
|
||||
func IsInternalJobName(s string) bool {
|
||||
return strings.HasPrefix(s, "_")
|
||||
}
|
||||
|
||||
func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
|
||||
s.m.Lock()
|
||||
defer s.m.Unlock()
|
||||
@@ -217,12 +229,10 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
|
||||
ctx = logging.WithInjectedField(ctx, logging.JobField, j.Name())
|
||||
|
||||
jobName := j.Name()
|
||||
|
||||
// package `config` enforces these with clean errors, these are just assertions
|
||||
if !internal && config.IsInternalJobName(jobName) {
|
||||
if !internal && IsInternalJobName(jobName) {
|
||||
panic(fmt.Sprintf("internal job name used for non-internal job %s", jobName))
|
||||
}
|
||||
if internal && !config.IsInternalJobName(jobName) {
|
||||
if internal && !IsInternalJobName(jobName) {
|
||||
panic(fmt.Sprintf("internal job does not use internal job name %s", jobName))
|
||||
}
|
||||
if _, ok := s.jobs[jobName]; ok {
|
||||
@@ -6,8 +6,8 @@ import (
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type DatasetMapFilter struct {
|
||||
@@ -3,7 +3,7 @@ package filters
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
func TestDatasetMapFilter(t *testing.T) {
|
||||
@@ -3,8 +3,8 @@ package hooks
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type List []Hook
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
// Re-export type here so that
|
||||
@@ -6,9 +6,9 @@ import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
||||
"github.com/zrepl/zrepl/internal/logger"
|
||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
||||
"github.com/zrepl/zrepl/daemon/logging"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/util/envconst"
|
||||
)
|
||||
|
||||
type Logger = logger.Logger
|
||||
@@ -4,8 +4,8 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type HookJobCallback func(ctx context.Context) error
|
||||
@@ -12,11 +12,11 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
||||
"github.com/zrepl/zrepl/internal/logger"
|
||||
"github.com/zrepl/zrepl/internal/util/circlog"
|
||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/util/circlog"
|
||||
"github.com/zrepl/zrepl/util/envconst"
|
||||
)
|
||||
|
||||
type HookEnvVar string
|
||||
+3
-3
@@ -12,9 +12,9 @@ import (
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
// Hook to implement the following recommmendation from MySQL docs
|
||||
+3
-3
@@ -10,9 +10,9 @@ import (
|
||||
"github.com/lib/pq"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/filters"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/filters"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type PgChkptHook struct {
|
||||
@@ -11,13 +11,13 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/hooks"
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging"
|
||||
"github.com/zrepl/zrepl/internal/logger"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/hooks"
|
||||
"github.com/zrepl/zrepl/daemon/logging"
|
||||
"github.com/zrepl/zrepl/logger"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type comparisonAssertionFunc func(require.TestingT, interface{}, interface{}, ...interface{})
|
||||
@@ -1,5 +1,6 @@
|
||||
// Code generated by "enumer -type=StepStatus -trimprefix=Step"; DO NOT EDIT.
|
||||
|
||||
//
|
||||
package hooks
|
||||
|
||||
import (
|
||||
@@ -8,24 +8,26 @@ import (
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/common/log"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/daemon/logging/trace"
|
||||
"github.com/zrepl/zrepl/internal/util/envconst"
|
||||
"github.com/zrepl/zrepl/daemon/logging/trace"
|
||||
"github.com/zrepl/zrepl/util/envconst"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/daemon/job/reset"
|
||||
"github.com/zrepl/zrepl/internal/daemon/job/wakeup"
|
||||
"github.com/zrepl/zrepl/internal/daemon/pruner"
|
||||
"github.com/zrepl/zrepl/internal/daemon/snapper"
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/replication"
|
||||
"github.com/zrepl/zrepl/internal/replication/driver"
|
||||
"github.com/zrepl/zrepl/internal/replication/logic"
|
||||
"github.com/zrepl/zrepl/internal/replication/report"
|
||||
"github.com/zrepl/zrepl/internal/rpc"
|
||||
"github.com/zrepl/zrepl/internal/transport"
|
||||
"github.com/zrepl/zrepl/internal/transport/fromconfig"
|
||||
"github.com/zrepl/zrepl/internal/zfs"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/daemon/job/reset"
|
||||
"github.com/zrepl/zrepl/daemon/job/trigger"
|
||||
"github.com/zrepl/zrepl/daemon/job/wakeup"
|
||||
"github.com/zrepl/zrepl/daemon/pruner"
|
||||
"github.com/zrepl/zrepl/daemon/snapper"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/replication"
|
||||
"github.com/zrepl/zrepl/replication/driver"
|
||||
"github.com/zrepl/zrepl/replication/logic"
|
||||
"github.com/zrepl/zrepl/replication/report"
|
||||
"github.com/zrepl/zrepl/rpc"
|
||||
"github.com/zrepl/zrepl/transport"
|
||||
"github.com/zrepl/zrepl/transport/fromconfig"
|
||||
"github.com/zrepl/zrepl/zfs"
|
||||
)
|
||||
|
||||
type ActiveSide struct {
|
||||
@@ -43,6 +45,8 @@ type ActiveSide struct {
|
||||
promReplicationErrors prometheus.Gauge
|
||||
promLastSuccessful prometheus.Gauge
|
||||
|
||||
triggers *trigger.Triggers
|
||||
|
||||
tasksMtx sync.Mutex
|
||||
tasks activeSideTasks
|
||||
}
|
||||
@@ -89,7 +93,7 @@ type activeMode interface {
|
||||
SenderReceiver() (logic.Sender, logic.Receiver)
|
||||
Type() Type
|
||||
PlannerPolicy() logic.PlannerPolicy
|
||||
RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{})
|
||||
RunPeriodic(ctx context.Context, wakeReplication *trigger.Manual)
|
||||
SnapperReport() *snapper.Report
|
||||
ResetConnectBackoff()
|
||||
}
|
||||
@@ -131,8 +135,8 @@ func (m *modePush) Type() Type { return TypePush }
|
||||
|
||||
func (m *modePush) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
|
||||
|
||||
func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
|
||||
m.snapper.Run(ctx, wakeUpCommon)
|
||||
func (m *modePush) RunPeriodic(ctx context.Context, trigger *trigger.Manual) {
|
||||
m.snapper.Run(ctx, trigger)
|
||||
}
|
||||
|
||||
func (m *modePush) SnapperReport() *snapper.Report {
|
||||
@@ -220,7 +224,7 @@ func (*modePull) Type() Type { return TypePull }
|
||||
|
||||
func (m *modePull) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
|
||||
|
||||
func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
|
||||
func (m *modePull) RunPeriodic(ctx context.Context, wakeReplication *trigger.Manual) {
|
||||
if m.interval.Manual {
|
||||
GetLogger(ctx).Info("manual pull configured, periodic pull disabled")
|
||||
// "waiting for wakeups" is printed in common ActiveSide.do
|
||||
@@ -231,14 +235,7 @@ func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
select {
|
||||
case wakeUpCommon <- struct{}{}:
|
||||
default:
|
||||
GetLogger(ctx).
|
||||
WithField("pull_interval", m.interval).
|
||||
Warn("pull job took longer than pull interval")
|
||||
wakeUpCommon <- struct{}{} // block anyways, to queue up the wakeup
|
||||
}
|
||||
wakeReplication.Fire()
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
@@ -369,6 +366,11 @@ func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}, p
|
||||
return nil, errors.Wrap(err, "cannot build replication driver config")
|
||||
}
|
||||
|
||||
j.triggers, err = trigger.FromConfig(in.Replication.Triggers)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannot build triggers")
|
||||
}
|
||||
|
||||
return j, nil
|
||||
}
|
||||
|
||||
@@ -443,12 +445,17 @@ func (j *ActiveSide) Run(ctx context.Context) {
|
||||
|
||||
defer log.Info("job exiting")
|
||||
|
||||
periodicDone := make(chan struct{})
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
periodicCtx, endTask := trace.WithTask(ctx, "periodic")
|
||||
|
||||
periodCtx, endTask := trace.WithTask(ctx, "periodic")
|
||||
defer endTask()
|
||||
go j.mode.RunPeriodic(periodCtx, periodicTrigger)
|
||||
|
||||
wakeupTrigger := wakeup.Trigger(ctx)
|
||||
|
||||
triggered, endTask := j.triggers.Spawn(ctx, []*trigger.Trigger{periodicTrigger, wakeupTrigger})
|
||||
defer endTask()
|
||||
go j.mode.RunPeriodic(periodicCtx, periodicDone)
|
||||
|
||||
invocationCount := 0
|
||||
outer:
|
||||
@@ -458,10 +465,15 @@ outer:
|
||||
case <-ctx.Done():
|
||||
log.WithError(ctx.Err()).Info("context")
|
||||
break outer
|
||||
|
||||
case <-wakeup.Wait(ctx):
|
||||
j.mode.ResetConnectBackoff()
|
||||
case <-periodicDone:
|
||||
case trigger := <-triggered:
|
||||
log :=
|
||||
log.WithField("trigger_id", trigger.ID())
|
||||
log.Info("triggered")
|
||||
switch trigger {
|
||||
case wakeupTrigger:
|
||||
log.Info("trigger is wakeup command, resetting connection backoff")
|
||||
j.mode.ResetConnectBackoff()
|
||||
}
|
||||
}
|
||||
invocationCount++
|
||||
invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", invocationCount))
|
||||
@@ -481,7 +493,7 @@ func (j *ActiveSide) do(ctx context.Context) {
|
||||
go func() {
|
||||
select {
|
||||
case <-reset.Wait(ctx):
|
||||
GetLogger(ctx).Info("reset received, cancelling current invocation")
|
||||
log.Info("reset received, cancelling current invocation")
|
||||
cancelThisRun()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
@@ -6,8 +6,8 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/endpoint"
|
||||
"github.com/zrepl/zrepl/internal/transport"
|
||||
"github.com/zrepl/zrepl/endpoint"
|
||||
"github.com/zrepl/zrepl/transport"
|
||||
)
|
||||
|
||||
func TestFakeActiveSideDirectMethodInvocationClientIdentityDoesNotPassValidityTest(t *testing.T) {
|
||||
@@ -1,5 +1,6 @@
|
||||
// Code generated by "enumer -type=ActiveSideState"; DO NOT EDIT.
|
||||
|
||||
//
|
||||
package job
|
||||
|
||||
import (
|
||||
@@ -7,8 +7,8 @@ import (
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/zrepl/zrepl/internal/config"
|
||||
"github.com/zrepl/zrepl/internal/util/bandwidthlimit"
|
||||
"github.com/zrepl/zrepl/config"
|
||||
"github.com/zrepl/zrepl/util/bandwidthlimit"
|
||||
)
|
||||
|
||||
func JobsFromConfig(c *config.Config, parseFlags config.ParseFlags) ([]Job, error) {
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user