Compare commits

..

3 Commits

Author SHA1 Message Date
Christian Schwarz 83c4bceeb5 WIP 2026-02-13 21:55:25 +00:00
Christian Schwarz 74e23fea43 platformtest: add package doc comment to interposer.go
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-10 20:02:58 +00:00
Christian Schwarz b3f24861c4 platformtest: replace logmockzfs with multi-personality safety interposer
Replace the shell-script-based logmockzfs wrapper with a Go-native
multi-personality binary (busybox-style). The platformtest binary now
acts as a safety interposer for zfs/zpool when invoked via symlinks,
validating that all commands reference the hardcoded test pool before
delegating through sudo.

Key changes:
- New interposer.go: SetupInterposerPath creates a tmpdir with symlinks
  and replaces PATH; RunInterposer validates args and delegates via sudo
- Pool name, image path, and mountpoint are now hardcoded constants
- Remove ZpoolCreateArgs, Zpool struct, and CLI flags for pool config
- Remove logmockzfs/ shell scripts (logzfsenv, zfs wrapper)
- Simplify Makefile: no more logmockzfs invocation or root check
- Add -no-interposer flag for direct execution as root
- Safety: block -a (all) flags without pool reference, validate
  mountpoint paths against traversal, command-aware flag parsing

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-10 19:56:38 +00:00
25 changed files with 483 additions and 656 deletions
+6 -76
View File
@@ -51,76 +51,6 @@ commands:
paths:
- ~/.cache/uv
install-zfs-from-source:
parameters:
zfs_release:
type: string
steps:
- run:
name: Record kernel version for cache key
command: uname -r > /tmp/kernel-version
- restore_cache:
name: Restore ZFS native debs cache
keys:
- zfs-debs-v2-<<parameters.zfs_release>>-{{ checksum "/tmp/kernel-version" }}
- run:
# https://openzfs.github.io/openzfs-docs/Developer%20Resources/Building%20ZFS.html
name: Build ZFS <<parameters.zfs_release>> native debs (if not cached)
no_output_timeout: 20m
command: |
# CircleCI machine images have pyenv Python 3.13 shadowing the system
# Python 3.12. The apt python3-* packages (setuptools, cffi, etc.) only
# install for the system Python, and ZFS's dpkg-buildpackage needs them
# for --enable-pyzfs. Use the system Python so apt packages are visible.
export PYENV_VERSION=system
if [ -d /tmp/zfs-debs ]; then
echo "ZFS debs cache hit, skipping build"
exit 0
fi
sudo apt-get update
sudo apt-get install -y \
alien autoconf automake build-essential debhelper-compat dh-autoreconf \
dh-dkms dh-python dkms fakeroot gawk git libaio-dev libattr1-dev \
libblkid-dev libcurl4-openssl-dev libelf-dev libffi-dev libpam0g-dev \
libssl-dev libtirpc-dev libtool libudev-dev linux-headers-$(uname -r) lsb-release \
parallel po-debconf python3 python3-all-dev python3-cffi python3-dev \
python3-packaging python3-setuptools python3-sphinx uuid-dev zlib1g-dev
mkdir -p /tmp/zfs-build
cd /tmp/zfs-build
git clone --depth 1 --branch <<parameters.zfs_release>> https://github.com/openzfs/zfs.git zfs-src
cd zfs-src
sh autogen.sh
./configure
make native-deb
mkdir -p /tmp/zfs-debs
find /tmp/zfs-build -name '*.deb' -exec mv -t /tmp/zfs-debs/ {} +
- save_cache:
name: Save ZFS native debs cache
key: zfs-debs-v2-<<parameters.zfs_release>>-{{ checksum "/tmp/kernel-version" }}
paths:
- /tmp/zfs-debs
- run:
name: Install ZFS <<parameters.zfs_release>>
command: |
ls /tmp/zfs-debs/
# Only install the debs we need. Exclude dracut (conflicts with
# initramfs-tools), dkms (we have prebuilt modules), and packages
# we don't need (pyzfs, test, doc).
sudo apt-get install -y $(find /tmp/zfs-debs -name '*.deb' \
! -name '*dracut*' \
! -name '*dkms*' \
! -name '*initramfs*' \
! -name '*pyzfs*' \
! -name '*doc*' \
! -name '*test*' \
-print)
sudo modprobe zfs
- run:
name: Verify ZFS version
command: |
sudo zfs version
sudo zpool version
docs-publish-sh:
parameters:
push:
@@ -205,7 +135,7 @@ workflows:
goversion: [*latest-go-release]
goos: ["linux"]
goarch: ["amd64"]
zfs_release: ["zfs-2.2.9", "zfs-2.3.5", "zfs-2.4.0"]
image: ["ubuntu-2204:current", "ubuntu-2404:current"]
requires:
- test-go
- quickcheck-go-<< matrix.goarch >>-<< matrix.goos >>-<< matrix.goversion >>
@@ -309,11 +239,10 @@ jobs:
type: string
goarch:
type: string
zfs_release:
image:
type: string
machine:
# pinned (not :current) to keep ZFS build cache valid across runs
image: ubuntu-2404:2025.09.1
image: <<parameters.image>>
resource_class: medium
environment:
GOOS: <<parameters.goos>>
@@ -321,8 +250,9 @@ jobs:
steps:
- attach_workspace:
at: .
- install-zfs-from-source:
zfs_release: <<parameters.zfs_release>>
- run: sudo apt-get update
- run: sudo apt-get install -y zfsutils-linux
- run: sudo zfs version
- run: sudo make test-platform GOOS="$GOOS" GOARCH="$GOARCH"
test-go:
+1 -9
View File
@@ -7,21 +7,13 @@ import requests
import time
import os
import yaml
import argparse
from pathlib import Path
circle_token = os.environ.get('CIRCLE_TOKEN')
if not circle_token:
cli_yml = Path.home() / ".circleci" / "cli.yml"
if cli_yml.exists():
with open(cli_yml) as f:
data = yaml.safe_load(f)
if data:
circle_token = data.get("token")
if not circle_token:
raise ValueError('CIRCLE_TOKEN not set and no token found in ~/.circleci/cli.yml')
raise ValueError('CIRCLE_TOKEN environment variable not set')
parser = argparse.ArgumentParser(description='Download artifacts from CircleCI')
parser.add_argument('build_num', type=str, help='Build number')
-95
View File
@@ -1,95 +0,0 @@
---
description: Generate GitHub release notes from docs/changelog.rst and create a draft release with artifacts
argument-hint: <version-tag, e.g. v0.7.0>
model: sonnet
allowed-tools: Read, Write
---
# Task
Create a draft GitHub release for zrepl version $ARGUMENTS and upload release artifacts.
## Inputs
Here is the current changelog RST source:
!`cat docs/changelog.rst`
## Instructions
### Phase 1: Pre-flight Sanity Checks
Before creating the release, verify that the previous build steps have been completed.
Run these checks using bash commands:
```bash
test -d artifacts/release && echo "✓ artifacts/release directory exists" || echo "✗ Missing artifacts/release directory - run: make download-circleci-release JOB_NUM=<num>"
test -f artifacts/release/sha512sum.txt && echo "✓ sha512sum.txt exists" || echo "✗ Missing sha512sum.txt - run: make download-circleci-release JOB_NUM=<num>"
test -f artifacts/release/sha512sum.txt.asc && echo "✓ sha512sum.txt.asc exists (signed)" || echo "✗ Missing signature - run: make verify-and-sign"
ls artifacts/release/zrepl-* >/dev/null 2>&1 && echo "✓ Release artifacts present" || echo "✗ No release artifacts found"
```
If ANY check fails (shows ✗), **STOP** and print the error messages. Do not proceed with release creation.
### Phase 2: Generate Release Notes
2. Extract the changelog section for version $ARGUMENTS from the RST source above.
3. Extract all GitHub usernames mentioned in that changelog section (look for @username patterns or other contributor attributions).
4. Write a short "Highlights" section (2-4 bullets) summarizing the most impactful user-facing changes in plain language. Do NOT include commit/issue links.
5. Write a "Contributors" thank you line for all GitHub users found in step 3. Format as: "Thanks to @user1, @user2, and @user3 for their contributions to this release!"
6. Write a "Breaking Changes" section. Convert RST formatting to plain Markdown text (no commit or issue links):
- `:commit:\`abc123\`` → just remove it or describe what it changed
- `:issue:\`123\`` → just remove it or describe the issue in plain text
- `:ref:\`display text <anchor>\`` → just use `display text` as plain text
- `:repomasterlink:\`path\`` → just mention the path without a link
- RST inline code ` ``code`` ` → markdown `` `code` ``
- RST links `` `text <url>`_ `` → just use `text` without the link
If there are no breaking changes, write "No breaking changes. vX.Y.Z-1 is interoperable with vX.Y.Z." (fill in actual versions).
7. Assemble the full release notes using the template below. **IMPORTANT**: Do NOT include a detailed changelog list. Only include what is specified in the template.
8. Write the result to `artifacts/release-notes.md`.
9. Validate all links in the release notes by checking HTTP status codes with curl:
- Extract all URLs from the markdown file
- Check each URL with `curl -sI -w "%{http_code}" -o /dev/null <url>`
- Report any broken links (non-200 status codes)
- If any links are broken, stop and notify the user before creating the release
### Phase 3: Create Draft Release and Upload Artifacts
10. Run: `gh release create $ARGUMENTS --title "$ARGUMENTS" --notes-file artifacts/release-notes.md --draft`
11. Upload all artifacts: `gh release upload $ARGUMENTS artifacts/release/*`
12. Verify upload succeeded: `gh release view $ARGUMENTS --json assets --jq '.assets | length'`
13. Print the release URL and confirm that artifacts were uploaded successfully.
## Template
```
The full changelog is [on the docs site](https://zrepl.github.io/changelog.html).
## Highlights
{2-4 bullet plain-language summary of the most impactful changes}
{Thank all GitHub users mentioned in the changelog, e.g., "Thanks to @user1, @user2, and @user3 for their contributions to this release!"}
## Breaking Changes
{breaking changes, or "No breaking changes."}
## New Users
We provide [quick-start guides](https://zrepl.github.io/quickstart.html) for different usage scenarios.
We also recommend studying the [overview section of the configuration chapter](https://zrepl.github.io/configuration/overview.html).
## Testing & Upgrading
* Read the [Changelog](https://zrepl.github.io/changelog.html)
* [Run the platform tests](https://zrepl.github.io/usage.html#platform-tests) on a test system.
* Download & deploy the `zrepl` binary / distro package.
## Donations
zrepl is a spare-time project primarily developed by [Christian Schwarz](https://cschwarz.com).
Express your support through a donation to keep maintenance and feature development going.
[![Support me on Patreon](https://img.shields.io/badge/dynamic/json?color=yellow&label=Patreon&query=data.attributes.patron_count&suffix=%20patrons&url=https%3A%2F%2Fwww.patreon.com%2Fapi%2Fcampaigns%2F3095079)](https://patreon.com/zrepl) [![Donate via GitHub Sponsors](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&style=flat&color=yellow)](https://github.com/sponsors/problame) [![Donate via Liberapay](https://img.shields.io/liberapay/patrons/zrepl.svg?logo=liberapay)](https://liberapay.com/zrepl/donate) [![Donate via PayPal](https://img.shields.io/badge/donate-paypal-yellow.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
```
-149
View File
@@ -1,149 +0,0 @@
---
description: Update supporters list from git history and changelog contributors
argument-hint: <time-range-or-release>
model: sonnet
---
# Task
Automatically update `docs/supporters.rst` by extracting GitHub contributors from:
1. Git commit history (code contributors)
2. `docs/changelog.rst` (all mentioned GitHub users)
## Arguments
Specify a time range or release range to filter contributors:
**Time range examples:**
- `--since="2024-01-01"` - Contributors since a specific date
- `--since="1 year ago"` - Contributors in the last year
- `--since="6 months ago"` - Contributors in the last 6 months
**Release range examples:**
- `v0.6.0..HEAD` - Contributors between v0.6.0 and current HEAD
- `v0.6.0..v0.7.0` - Contributors between two releases
- `--all` - All contributors (default if no argument provided)
## Instructions
### Step 1: Extract Contributors from Git History
Parse the arguments to determine the git log range:
- If `--since="..."` is provided, use: `git log --since="..." --format='%aN <%aE>' | sort -u`
- If a commit range like `v0.6.0..HEAD` is provided, use: `git log v0.6.0..HEAD --format='%aN <%aE>' | sort -u`
- If `--all` or no argument, use: `git log --format='%aN <%aE>' | sort -u`
Parse the output to identify:
- GitHub users (look for @users.noreply.github.com emails or known GitHub patterns)
- Real names of code contributors
- Filter out the main maintainer (Christian Schwarz / problame) as they're already credited
### Step 2: Extract GitHub Users from Changelog
Read `docs/changelog.rst` and extract all GitHub usernames mentioned:
- Pattern: `` `@username <https://github.com/username>`_ ``
- Pattern: `@username` in text
- Collect all unique usernames
### Step 3: Categorize Contributors
For each contributor, determine their tier:
**Code contributors** (`|supporter-code|`):
- Anyone who appears in git history as a commit author
- Prioritize those with substantial contributions (multiple commits)
**Other contributors** (keep existing `|supporter-gold|` and `|supporter-std|`):
- Keep all existing gold/std supporters as-is (monetary supporters)
- Only update code contributors
### Step 4: Read Current Supporters File
Read `docs/supporters.rst` and parse existing entries to:
- Preserve all `|supporter-gold|` entries (monetary supporters - don't touch these)
- Preserve all `|supporter-std|` entries (monetary supporters - don't touch these)
- Preserve ALL existing `|supporter-code|` entries in their current order
- Extract list of existing code contributor names/GitHub usernames to avoid duplicates
### Step 5: Identify Contributors to Promote/Add
**IMPORTANT: Find contributors in the range, and either ADD them (if new) or MOVE them up (if recurring).**
For each GitHub user found in changelog or git history for the specified range:
1. Check if they're already listed as `|supporter-code|` in the existing file
2. Categorize them:
- **NEW**: Not currently in the file → will be added
- **RECURRING**: Already in the file with new contributions in this range → will be moved to top
- **UNCHANGED**: Not in this range → stays in original position
3. For both NEW and RECURRING contributors:
- Get their latest contribution date from the range: `git log <range> --author="email" --format="%ai" -1`
- Format as: `* |supporter-code| `Name <https://github.com/username>`_`
- Use their real name if available from git history, otherwise use GitHub username
4. Sort NEW + RECURRING contributors together by their contribution date (newest first)
5. Keep list of UNCHANGED contributors in their original order
**The final order will be: [NEW + RECURRING from range, sorted newest first] + [UNCHANGED in original order]**
### Step 6: Generate Updated File
Create the updated supporters list:
- Keep all `|supporter-gold|` entries unchanged and in their exact positions
- Keep all `|supporter-std|` entries unchanged and in their exact positions
- For `|supporter-code|` entries, reorganize as follows:
1. **Top section**: Contributors from the analyzed range (NEW + RECURRING), sorted newest first
2. **Bottom section**: UNCHANGED contributors (not in this range), in their original order
- Update the spacer comment showing the range that was just processed:
```rst
..
↓ claude --permission-mode default /update-supporters <actual-range-used>
```
Replace `<actual-range-used>` with the actual argument passed (e.g., `v0.6.1..v0.7.0`)
This comment should appear immediately before the first code contributor entry
If there's an existing comment in this format, replace it; otherwise add it
- Maintain all other file structure and RST formatting, including the "Before /update-supporters" marker comment
### Step 7: Apply Changes
- Use Edit tool to update the `docs/supporters.rst` file
- Display summary of changes made:
- Time/release range analyzed
- Number of contributors found
- New code contributors added
- Final counts by tier
## Implementation Notes
- **DO NOT modify, remove, or re-sort** any `|supporter-gold|` or `|supporter-std|` entries
- **For code contributors**: ADD new ones and MOVE UP recurring ones (those with contributions in the analyzed range)
- When extracting from git history, focus on substantial contributors (filter out single-commit or trivial changes if needed)
- **CRITICAL:** Contributors from the analyzed range go to the top
- Use `git log <range> --author="<email>" --format="%ai" -1` to get latest contribution date in the range
- Both NEW and RECURRING contributors from this range are sorted together (newest first) and placed at the TOP
- UNCHANGED contributors (not in this range) stay in their original relative order below
- This creates a chronological history: most recent release's contributors appear first
- Add/update the spacer comment showing the processed range
- Handle edge cases:
- Contributors mentioned in changelog but not in git history (external contributors, reporters)
- Contributors in git history but not mentioned in changelog (decide whether to include)
## Example Output
```
Analyzing contributors for range: v0.6.0..v0.7.0
- Found 8 code contributors in git history
- Found 12 GitHub users mentioned in changelog.rst (v0.7.0 section)
- Current supporters.rst has 8 gold, 12 std, 10 code contributors
New contributors:
+ Jane Doe (@janedoe)
+ John Smith (@jsmith)
Promoted (recurring) contributors:
↑ Alice Developer (@alice) - new contributions in v0.6.0..v0.7.0
Updated docs/supporters.rst:
- Gold supporters: 8 (unchanged)
- Std supporters: 12 (unchanged)
- Code contributors: 10 → 12
- Promoted to top: 3 contributors (2 new + 1 recurring)
```
+5 -19
View File
@@ -56,7 +56,7 @@ release-docker-mkcachemount:
endif
##################### PRODUCING A RELEASE #############
.PHONY: release wrapup-and-checksum check-git-clean verify-and-sign clean ensure-release-toolchain
.PHONY: release wrapup-and-checksum check-git-clean sign clean ensure-release-toolchain
ensure-release-toolchain:
# ensure the toolchain is actually the one we expect
@@ -211,8 +211,7 @@ tag-release:
test -n "$(ZREPL_TAG_VERSION)" || exit 1
git tag -u '328A6627FA98061D!' -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)"
verify-and-sign:
cd "$(ARTIFACTDIR)/release" && sha512sum -c sha512sum.txt
sign:
gpg -u '328A6627FA98061D!' \
--armor \
--detach-sign $(ARTIFACTDIR)/release/sha512sum.txt
@@ -223,7 +222,7 @@ clean: docs-clean
download-circleci-release:
rm -rf "$(ARTIFACTDIR)"
mkdir -p "$(ARTIFACTDIR)/release"
python3 .circleci/download_artifacts.py --prefix 'artifacts/release/' "$(JOB_NUM)" "$(ARTIFACTDIR)/release"
python3 .circleci/download_artifacts.py --prefix 'artifacts/release/' "$(BUILD_NUM)" "$(ARTIFACTDIR)/release"
##################### MULTI-ARCH HELPERS #####################
@@ -274,7 +273,7 @@ cover-platform-bin:
$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
-c -o "$(COVER_PLATFORM_BIN_PATH)" \
-covermode=atomic -cover -coverpkg github.com/zrepl/zrepl/... \
./platformtest/harness
./internal/platformtest/harness
cover-platform:
# do not track dependency on cover-platform-bin to allow build of binary outside of test VM
export _TEST_PLATFORM_CMD="$(COVER_PLATFORM_BIN_PATH) \
@@ -290,25 +289,13 @@ test-platform:
export _TEST_PLATFORM_CMD="\"$(TEST_PLATFORM_BIN_PATH)\""; \
$(MAKE) _test-or-cover-platform-impl
ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
ZREPL_PLATFORMTEST_MOUNTPOINT := /tmp/zreplplatformtest.pool
ZREPL_PLATFORMTEST_ZFS_LOG := /tmp/zreplplatformtest.zfs.log
# ZREPL_PLATFORMTEST_STOP_AND_KEEP := -failure.stop-and-keep-pool
ZREPL_PLATFORMTEST_ARGS :=
_test-or-cover-platform-impl: $(ARTIFACTDIR)
ifndef _TEST_PLATFORM_CMD
$(error _TEST_PLATFORM_CMD is undefined, caller 'cover-platform' or 'test-platform' should have defined it)
endif
rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
rm -f "$(ARTIFACTDIR)/platformtest.cover"
internal/platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
$(_TEST_PLATFORM_CMD) \
-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
-mountpoint "$(ZREPL_PLATFORMTEST_MOUNTPOINT)" \
$(ZREPL_PLATFORMTEST_STOP_AND_KEEP) \
$(ZREPL_PLATFORMTEST_ARGS)
$(_TEST_PLATFORM_CMD) $(ZREPL_PLATFORMTEST_ARGS)
cover-merge: $(ARTIFACTDIR)
$(GOCOVMERGE) $(ARTIFACTDIR)/platformtest.cover $(ARTIFACTDIR)/gotest.cover > $(ARTIFACTDIR)/merged.cover
@@ -316,7 +303,6 @@ cover-html: cover-merge
$(GO) tool cover -html "$(ARTIFACTDIR)/merged.cover" -o "$(ARTIFACTDIR)/merged.cover.html"
cover-full:
test "$$(id -u)" = "0" || echo "MUST RUN AS ROOT" 1>&2
$(MAKE) test-go COVER=1
$(MAKE) cover-platform-bin
$(MAKE) cover-platform
+10 -37
View File
@@ -93,50 +93,25 @@ The procedure to issue a release is as follows:
* Prepare the release (as a PR to `master`):
* Finalize `docs/changelog.rst` for the release.
* Update the supporters list based on contributors for this release:
```bash
claude --permission-mode default '/update-supporters v0.6.1..v0.7.0'
```
Replace version tags with the appropriate range for your release.
* Merge the PR. Docs are auto-published to zrepl.github.io on merge.
* Tag the release:
* Git tag the release on the `master` branch (e.g., `vMAJOR.MINOR.0`).
```
make tag-release ZREPL_TAG_VERSION=v0.7.0
```
* Push the tag.
* Build and publish:
* Run the `release` pipeline against the `master` branch (trigger via CircleCI UI).
This URL: https://app.circleci.com/pipelines/github/zrepl/zrepl?branch=master.
Example pipeline: https://app.circleci.com/pipelines/github/zrepl/zrepl/8547
* Download artifacts using this handy makefile target.
Note: `JOB_NUM` must be the **job number** from the `release-upload` job, **not the pipeline number**.
Find it via: pipeline → `release` workflow → `release-upload` job number.
Example URL: https://app.circleci.com/pipelines/github/zrepl/zrepl/8547/workflows/65feb2c9-15d7-46ab-a551-46d62a5769b0/jobs/66079/steps
```
make download-circleci-release JOB_NUM=66079
```
* Verify checksums and sign the checksums file:
```
make verify-and-sign
```
* Create GitHub draft release and upload artifacts:
* Run the `release` pipeline (trigger via CircleCI UI).
* Download artifacts: `make download-circleci-release BUILD_NUM=<circleci-build-number>`
* Create GitHub release and upload artifacts:
```bash
claude --permission-mode default '/draft-release v0.7.0'
gh release create vX.Y.Z --title "vX.Y.Z" --notes "See changelog" --draft
gh release upload vX.Y.Z artifacts/release/*
```
This command will verify that artifacts are ready, create the draft release, and upload all artifacts.
* Review the draft release on GitHub, then publish,
with the box checked to create a GitHub "Discussion" for the release.
* Immediately after hitting publish.
* Update the release on GitHub to link to the discussion.
* Update `docs/changelog.rst` to link to the GitHub release.
* Update `docs/_templates/versions.html` to link to the GitHub release.
* Announce release in the zrepl Matrix channel & other socials as applicable.
Link to the GitHub release as the entrypoint.
* After a couple of days
* Review the draft release, edit the changelog, then publish.
* Add the .rpm and .deb files to the official zrepl repos.
* Code for management of these repos: https://github.com/zrepl/package-repo-ops (private repo at this time)
* Update docs version list:
* Update `docs/_templates/versions.html` with the new release.
* Verify the link to `zrepl-noarch.tar` in the GitHub release works.
* Merge to `master` (docs auto-publish).
#### Patch releases, Go toolchain updates, APT/RPM Package rebuilds
@@ -196,8 +171,6 @@ Run `make lint` and `make vet`.
Update the CI configuration `.circleci/config.yml`:
- Update Go version references (we reference the minimum and max supported version)
- Set `Makefile` `RELEASE_GOVERSION` to the new Go version
- Update the pinned ZFS release tags in the `platformtest` matrix (`zfs_release` parameter) to the latest patch releases of each OpenZFS branch (currently 2.2, 2.3, 2.4)
- Update the pinned Ubuntu machine image for `platformtest` (e.g. `ubuntu-2404:2025.09.1`) — this is pinned rather than `current` so the ZFS build cache stays valid across runs
Update docs build tooling:
- Update `uv` version in `.circleci/config.yml` (search for `astral.sh/uv/` and cache keys containing the version)
-1
View File
@@ -7,7 +7,6 @@
<div class="rst-other-versions">
<dl>
<dt>Releases</dt>
<dd><a href="https://github.com/zrepl/zrepl/releases/tag/v0.7.0">v0.7.0</a></dd>
<dd><a href="https://github.com/zrepl/zrepl/releases/tag/v0.6.1">v0.6.1</a></dd>
<dd><a href="https://github.com/zrepl/zrepl/releases/tag/v0.5.0">v0.5.0</a></dd>
<dd><a href="https://github.com/zrepl/zrepl/releases/tag/v0.4.0">v0.4.0</a></dd>
+43 -36
View File
@@ -13,43 +13,43 @@
Changelog
=========
0.7.0
-----
The changelog summarizes bugfixes that are deemed relevant for users and package maintainers.
Developers should consult the git commit log or GitHub issue tracker.
`GitHub Release <https://github.com/zrepl/zrepl/releases/tag/v0.7.0>`_
Next Release
------------
* |feature| Config file inclusion using ``include`` directive.
This allows distributing zrepl job definitions across multiple YAML files in a ``conf.d`` style directory.
(:commit:`4d6583e`, thanks, `@ZeyadTamimi <https://github.com/zeyadtamimi>`_).
* |feature| Add configurable timezone for snapshot name timestamps. The default remains UTC. (:commit:`b9b9ad1`, thanks, `@mjasnik <https://github.com/mjasnik>`_).
* |feature| Improve Grafana dashboard with additional panels and refinements (:commit:`3f75127`, thanks, `@deajan <https://github.com/deajan>`_).
* |bugfix| Fix replication of placeholder filesystems (:commit:`5615f49`).
* |bugfix| Avoid using the word "error" in info-level log lines about ZFS command execution (:commit:`d3b2295`, thanks, `@bakhtiyarneyman <https://github.com/bakhtiyarneyman>`_).
* |bugfix| Detect duplicate and internal job names at config parse time.
zrepl will now refuse to load a config with duplicate or internally reserved job names, instead of failing later at daemon startup.
(:commit:`860a9be`).
* |bugfix| Fix ``last_n`` keep rule behavior (:commit:`ebc46cf`, thanks, `@dsh2dsh <https://github.com/dsh2dsh>`_).
* |bugfix| Allow different jobs to use the same ``root_fs`` for sinks (:commit:`27012e5`, thanks, `@dsh2dsh <https://github.com/dsh2dsh>`_).
* |docs| Add installation instructions for openSUSE (:commit:`40c4827`, thanks, `@findesgh <https://github.com/findesgh>`_).
* |docs| Improve documentation on ``send_properties`` configuration and NFS/SMB considerations (:commit:`b5d8538`, thanks, `@Malvineous <https://github.com/Malvineous>`_).
* |docs| Warn more prominently about the risks of the ``not_replicated`` keep rule (:commit:`affe00a`, thanks, `@wxiaoguang <https://github.com/wxiaoguang>`_).
* |docs| Improve TLS/EasyRSA setup instructions (:commit:`e524b60`, thanks, `@alorimer <https://github.com/alorimer>`_).
* |docs| Remove dead Bountysource link (:commit:`440b074`, thanks, `@lpulley <https://github.com/lpulley>`_).
* |docs| Fix missing newline in compile-from-source documentation (:commit:`e2fcf9f`, thanks, `@Raupinger <https://github.com/Raupinger>`_).
* |docs| Fix apt repository installation snippet (:commit:`8305367`, thanks, `@fermino <https://github.com/fermino>`_).
* |docs| ``zrepl.github.io``: auto-publish from ``master`` branch, retire ``stable`` branch (:commit:`4f950bb`).
* |maint| ``zrepl status``: switch from the unmaintained ``cview`` fork back to the actively maintained ``tview`` library (:commit:`d7ede3f`).
* |maint| Update to Go 1.25 toolchain with Go 1.24 language level & Go dependencies
* |maint| Debian packages: proper bash completion placement (:commit:`2923009`)
* |maint| RPM packages: AlmaLinux 8 for RPM builds (:commit:`27f4ad1`).
* |maint| Improve build system & CI setup
The plan for the next release is to revisit how zrepl does snapshot management.
High-level goals:
.. NOTE::
| zrepl is a spare-time project primarily developed by `Christian Schwarz <https://cschwarz.com>`_.
| You can support maintenance and feature development through one of the following services:
| |Donate via Patreon| |Donate via GitHub Sponsors| |Donate via Liberapay| |Donate via PayPal|
| Note that PayPal processing fees are relatively high for small donations.
| For SEPA wire transfer and **commercial support**, please `contact Christian directly <https://cschwarz.com>`_.
- Make it easy to decouple snapshot management (snapshotting, pruning) from replication.
- Ability to include/exclude snapshots from replication.
This is useful for aforementioned decoupling, e.g., separate snapshot prefixes for local & remote replication.
Also, it makes explicit that by default, zrepl replicates all snapshots, and that
replication has no concept of "zrepl-created snapshots", which is a common misconception.
- Use of ``zfs snapshot`` comma syntax or channel programs to take snapshots of multiple
datasets atomically.
- Provide an alternative to the ``grid`` pruning policy.
Most likely something based on hourly/daily/weekly/monthly "trains" plus a count.
- Ability to prune at the granularity of the **group** of snapshots created at a given
time, as opposed to the individual snapshots within a dataset.
Maybe this will be addressed by the alternative to the ``grid`` pruning policy,
as it will likely be more predictable.
Those changes will likely come with some breakage in the config.
However, I want to avoid breaking **use cases** that are satisfied by the current design.
There will be beta/RC releases to give users a chance to evaluate.
0.7.0 (unreleased)
------------------
INCOMPLETE LIST OF CHANGES
* |maint| Update to Go 1.25 toolchain with Go 1.24 language level.
* |maint| Fix deprecations exposed by the toolchain update.
* |maint| Long-overdue update of all our dependencies & address deprecations.
* |maint| Publish docs from ``master`` branch, retire ``stable`` branch.
* |maint| The `make release-docker` in CircleCI produces executables that are bit-identical to my personal machine.
0.6.1
-----
@@ -65,6 +65,13 @@ Changelog
* |maint| run platform tests as part of CI.
* |maint| build: upgrade to Go 1.21 and update golangci-lint; minimum Go version for builds is now 1.20
.. NOTE::
| zrepl is a spare-time project primarily developed by `Christian Schwarz <https://cschwarz.com>`_.
| You can support maintenance and feature development through one of the following services:
| |Donate via Patreon| |Donate via GitHub Sponsors| |Donate via Liberapay| |Donate via PayPal|
| Note that PayPal processing fees are relatively high for small donations.
| For SEPA wire transfer and **commercial support**, please `contact Christian directly <https://cschwarz.com>`_.
0.6
---
@@ -251,7 +258,7 @@ Additional changelog:
* |bugfix| systemd + ``ssh+stdinserver``: automatically create ``/var/run/zrepl/stdinserver``
* |bugfix| crash if Prometheus listening socket cannot be opened
* [MAINTAINER NOTICE] ``Makefile`` refactoring, see commit :commit:`080f2c0`
* [MAINTAINER NOTICE] ``Makefile`` refactoring, see :commit:`080f2c0`
0.2
---
@@ -281,7 +288,7 @@ Additional changelog:
0.1.1
-----
* |bugfix| :issue:`162` commit :commit:`d6304f4` : fix I/O timeout errors on variable receive rate
* |bugfix| :issue:`162` :commit:`d6304f4` : fix I/O timeout errors on variable receive rate
* A significant reduction or sudden stall of the receive rate (e.g. recv pool has other I/O to do)
would cause a ``writev I/O timeout`` error after approximately ten seconds.
+1 -1
View File
@@ -182,6 +182,6 @@ extlinks = {
'issue':('https://github.com/zrepl/zrepl/issues/%s', 'issue #%s'),
'repomasterlink':('https://github.com/zrepl/zrepl/blob/master/%s', '%s'),
'sampleconf':('https://github.com/zrepl/zrepl/blob/master/internal/config/samples%s', 'internal/config/samples%s'),
'commit':('https://github.com/zrepl/zrepl/commit/%s', '%s'),
'commit':('https://github.com/zrepl/zrepl/commit/%s', 'commit %s'),
}
-33
View File
@@ -1,33 +0,0 @@
.. include:: global.rst.inc
.. _future:
Future
======
This page contains some notes about future plans for zrepl.
Then again, Christian only has very limited time available for zrepl maintenance these days.
So, don't count on any of this happening in the near future.
One big development that has happened in recent years is this fork of zrepl: https://github.com/dsh2dsh/zrepl
We should figure out whether we can pull in features from there.
The next major step in terms of new feature development for zrepl would be to revise snapshot management:
- Make it easy to decouple snapshot management (snapshotting, pruning) from replication.
- Ability to include/exclude snapshots from replication.
This is useful for aforementioned decoupling, e.g., separate snapshot prefixes for local & remote replication.
Also, it makes explicit that by default, zrepl replicates all snapshots, and that
replication has no concept of "zrepl-created snapshots", which is a common misconception.
- Use of ``zfs snapshot`` comma syntax or channel programs to take snapshots of multiple
datasets atomically.
- Provide an alternative to the ``grid`` pruning policy.
Most likely something based on hourly/daily/weekly/monthly "trains" plus a count.
- Ability to prune at the granularity of the **group** of snapshots created at a given
time, as opposed to the individual snapshots within a dataset.
Maybe this will be addressed by the alternative to the ``grid`` pruning policy,
as it will likely be more predictable.
Those changes will likely come with some breakage in the config.
However, I want to avoid breaking **use cases** that are satisfied by the current design.
There will be beta/RC releases to give users a chance to evaluate.
-1
View File
@@ -136,7 +136,6 @@ Table of Contents
usage
pr
changelog
future
GitHub Repository & Issue Tracker <https://github.com/zrepl/zrepl>
Chat: Matrix <https://matrix.to/#/#zrepl:matrix.org>
supporters
-24
View File
@@ -32,30 +32,6 @@ We would like to thank the following people and organizations for supporting zre
<div class="fa fa-code" style="width: 1em;"></div>
..
The list below is (roughly) sorted by date of latest contribution, newest first.
..
↓ post v0.7.0
* |supporter-std| `drbawb <https://git.sr.ht/~hime>`_
..
↓ claude --permission-mode default /update-supporters v0.6.1..v0.7.0
* |supporter-code| `Bakhtiyar Neyman <https://github.com/bakhtiyarneyman>`_
* |supporter-code| `Zeyad Tamimi <https://github.com/ZeyadTamimi>`_
* |supporter-code| `Orsiris de Jong <https://github.com/deajan>`_
* |supporter-code| `Martin Glatzle <https://github.com/findesgh>`_
* |supporter-code| `Andrew Lorimer <https://github.com/alorimer>`_
* |supporter-code| `Adam Nielsen <https://github.com/Malvineous>`_
* |supporter-code| `wxiaoguang <https://github.com/wxiaoguang>`_
* |supporter-code| `Logan Pulley <https://github.com/lpulley>`_
* |supporter-code| `Florian <https://github.com/Raupinger>`_
* |supporter-code| `Fermín Olaiz <https://github.com/fermino>`_
* |supporter-code| `Denis Shaposhnikov <https://github.com/dsh2dsh>`_
..
↓ Before /update-supporters
* |supporter-gold| `Hostsharing eG die Hosting-Genossenschaft <https://www.hostsharing.net/>`_
* |supporter-std| `Max Christian Pohle <https://coderonline.de>`_
* |supporter-gold| Prominic.NET, Inc.
+9 -13
View File
@@ -101,24 +101,20 @@ Along with the main ``zrepl`` binary, we release the ``platformtest`` binaries.
The zrepl platform tests are an integration test suite that is complementary to the pure Go unit tests.
Any test that needs to interact with ZFS is a platform test.
The platform need to run as root.
The platform tests require ``sudo`` ``NOPASSWD``-access to ``zfs`` and ``zpool``.
For each test, we create a fresh dummy zpool backed by a file-based vdev.
The file path, and a root mountpoint for the dummy zpool, must be specified on the command line:
::
mkdir -p /tmp/zreplplatformtest
./platformtest \
-poolname 'zreplplatformtest' \ # <- name must contain zreplplatformtest
-imagepath /tmp/zreplplatformtest.img \ # <- zrepl will create the file
-mountpoint /tmp/zreplplatformtest # <- must exist
The pool name, image path, and mountpoint are hardcoded in the binary.
The ``platformtest`` binary is a multi-personality binary (similar to busybox).
It acts as a safety interposer for ``zfs`` and ``zpool`` commands, ensuring all operations
reference the hard-coded test pool name ``zreplplatformtest``. It creates symlinks to itself in a temporary
directory and replaces ``PATH`` so that all ``zfs``/``zpool`` invocations go through the interposer.
.. WARNING::
``platformtest`` will unconditionally overwrite the file at `imagepath`
and unconditionally ``zpool destroy $poolname``.
So, don't use a production poolname, and consider running the test in a VM.
``platformtest`` will unconditionally ``zpool destroy`` the test zpool
and unconditionally delete the image file that backs it.
Consider running the test in a VM.
It'll be a lot faster as well because the underlying operations, ``zfs list`` in particular, will be faster.
+1 -1
View File
@@ -34,7 +34,7 @@ require (
golang.org/x/sync v0.19.0
golang.org/x/sys v0.41.0
golang.org/x/tools v0.41.0
google.golang.org/grpc v1.79.3
google.golang.org/grpc v1.78.0
google.golang.org/protobuf v1.36.11
)
+12 -12
View File
@@ -166,16 +166,16 @@ github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd h1:SSo67WLS+99QE
github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd/go.mod h1:JmNwisZzOvW4GfpfLvhZ+gtyKLsIiA+WC+wNKJGJaFg=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48=
go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8=
go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0=
go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs=
go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18=
go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE=
go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8=
go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew=
go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI=
go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA=
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
@@ -235,8 +235,8 @@ gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+25 -18
View File
@@ -26,18 +26,23 @@ var bold = color.New(color.Bold)
var boldRed = color.New(color.Bold, color.FgHiRed)
var boldGreen = color.New(color.Bold, color.FgHiGreen)
const DefaultPoolImageSize = 200 * (1 << 20)
func main() {
// Multi-personality binary: check argv[0] to determine mode.
// When invoked via a symlink named "zfs" or "zpool", act as
// a safety interposer that validates commands and delegates
// to the real binary via sudo.
switch filepath.Base(os.Args[0]) {
case "zfs":
os.Exit(platformtest.RunInterposer("zfs"))
case "zpool":
os.Exit(platformtest.RunInterposer("zpool"))
}
// Test harness mode
var args HarnessArgs
flag.StringVar(&args.CreateArgs.PoolName, "poolname", "", "")
flag.StringVar(&args.CreateArgs.ImagePath, "imagepath", "", "")
flag.Int64Var(&args.CreateArgs.ImageSize, "imagesize", DefaultPoolImageSize, "")
flag.StringVar(&args.CreateArgs.Mountpoint, "mountpoint", "", "")
flag.BoolVar(&args.NoInterposer, "no-interposer", false, "skip safety interposer; symlink zfs/zpool directly to real binaries (requires running as root)")
flag.BoolVar(&args.StopAndKeepPoolOnFail, "failure.stop-and-keep-pool", false, "if a test case fails, stop test execution and keep pool as it was when the test failed")
flag.StringVar(&args.Run, "run", "", "")
flag.StringVar(&args.Run, "run", "", "regex to filter test cases")
flag.DurationVar(&platformtest.ZpoolExportTimeout, "zfs.zpool-export-timeout", platformtest.ZpoolExportTimeout, "")
flag.Parse()
@@ -49,7 +54,7 @@ func main() {
var exitWithErr = fmt.Errorf("exit with error")
type HarnessArgs struct {
CreateArgs platformtest.ZpoolCreateArgs
NoInterposer bool
StopAndKeepPoolOnFail bool
Run string
}
@@ -90,6 +95,14 @@ func (i *invocation) RegisterChild(c *invocation) error {
func HarnessRun(args HarnessArgs) error {
// Set up interposer: create symlinks and replace PATH.
// In direct mode, zfs/zpool symlink to real binaries (no sudo, no validation).
_, cleanup, err := platformtest.SetupInterposerPath("", args.NoInterposer)
if err != nil {
return errors.Wrap(err, "set up interposer PATH")
}
defer cleanup()
runRE := regexp.MustCompile(args.Run)
outlets := logger.NewOutlets()
@@ -104,11 +117,6 @@ func HarnessRun(args HarnessArgs) error {
}
outlets.Add(outlet, level)
logger := logger.NewLogger(outlets, 1*time.Second)
if err := args.CreateArgs.Validate(); err != nil {
logger.Error(err.Error())
panic(err)
}
ctx := context.Background()
defer trace.WithTaskFromStackUpdateCtx(&ctx)()
ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(logger))
@@ -128,14 +136,13 @@ func HarnessRun(args HarnessArgs) error {
bold.Printf("BEGIN TEST CASE %s\n", inv)
pool, err := platformtest.CreateOrReplaceZpool(ctx, ex, args.CreateArgs)
if err != nil {
if err := platformtest.CreateOrReplaceZpool(ctx, ex); err != nil {
panic(errors.Wrap(err, "create test pool"))
}
ctx := &platformtest.Context{
Context: ctx,
RootDataset: filepath.Join(pool.Name(), "rootds"),
RootDataset: filepath.Join(platformtest.PlatformTestPoolName, "rootds"),
QueueSubtest: func(id string, stf func(*platformtest.Context)) {
stinv := newInvocation(stf, id)
err := inv.RegisterChild(stinv)
@@ -158,7 +165,7 @@ func HarnessRun(args HarnessArgs) error {
return exitWithErr
}
if err := pool.Destroy(ctx, ex); err != nil {
if err := platformtest.DestroyZpool(ctx, ex); err != nil {
panic(fmt.Sprintf("error destroying test pool: %s", err))
}
+13 -4
View File
@@ -3,8 +3,11 @@ package main
import (
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"github.com/zrepl/zrepl/internal/platformtest"
)
// Idea taken from
@@ -12,16 +15,22 @@ import (
//
/* How to generate coverage:
go test -c -covermode=atomic -cover -coverpkg github.com/zrepl/zrepl/...
sudo ../logmockzfs/logzfsenv /tmp/zrepl_platform_test.log /usr/bin/zfs \
./harness.test -test.coverprofile=/tmp/harness.out \
-test.v __DEVEL--i-heard-you-like-tests \
-imagepath /tmp/testpool.img -poolname zreplplatformtest
./harness.test -test.coverprofile=/tmp/harness.out \
-test.v __DEVEL--i-heard-you-like-tests
go tool cover -html=/tmp/harness.out -o /tmp/harness.html
*/
// Merge with existing coverage reports using gocovmerge:
// https://github.com/wadey/gocovmerge
func TestMain(t *testing.T) {
// Multi-personality: when invoked as zfs/zpool symlink, run interposer
switch filepath.Base(os.Args[0]) {
case "zfs":
os.Exit(platformtest.RunInterposer("zfs"))
case "zpool":
os.Exit(platformtest.RunInterposer("zpool"))
}
fmt.Println("incoming args: ", os.Args)
var (
+322
View File
@@ -0,0 +1,322 @@
// # Safety Interposer
//
// The platformtest binary is a multi-personality binary (busybox-style).
// When invoked as "zfs" or "zpool" (via argv[0]), it acts as a safety
// interposer that validates commands and delegates to the real binary
// via sudo.
//
// SetupInterposerPath creates a temp directory with symlinks and replaces PATH:
// - zfs/zpool → the platformtest binary itself (interposer mode)
// - zfs.real/zpool.real → actual binaries (for delegation via sudo)
// - sudo/bash → real binaries (so they remain accessible)
//
// When invoked as zfs or zpool, the binary validates that all commands
// reference the test pool "zreplplatformtest" before delegating through
// sudo. The validation rule is simple: the pool name must appear in the
// command args. Special cases:
// - Resume tokens ("zfs send -t <token>"): decoded via the real binary
// to extract and verify the toname field
// - The -a (all) flag is blocked without an explicit pool reference
// - Bare subcommands (feature detection) are allowed
// - Mountpoint paths are validated against traversal on post-create chmod
//
// With -no-interposer, zfs/zpool symlink directly to the real binaries
// (no sudo, no validation). Use this when running as root.
package platformtest
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
)
// SetupInterposerPath creates a temp directory with zfs/zpool symlinks
// and replaces PATH so that all zfs/zpool invocations go through it.
//
// If `explicitDir` is non-empty, the PATH is constructed in that directory
// instead of a temp directory, and the caller is responsible for cleanup.
//
// `direct` controls whether to set up the interposer (false) or to symlink
// zfs/zpool directly to the real binaries (true, no interposer, no sudo).
//
// Returns the directory path and a cleanup function.
func SetupInterposerPath(explicitDir string, direct bool) (string, func(), error) {
// Resolve real binary paths while original PATH is still intact.
lookupNames := []string{"zfs", "zpool", "bash"}
if !direct {
lookupNames = append(lookupNames, "sudo")
}
realBinaries := make(map[string]string)
for _, name := range lookupNames {
p, err := exec.LookPath(name)
if err != nil {
return "", nil, fmt.Errorf("find real %s binary: %w", name, err)
}
realBinaries[name] = p
}
var dir string
var cleanup func()
if explicitDir != "" {
dir = explicitDir
cleanup = func() {} // caller manages lifecycle
} else {
var err error
dir, err = os.MkdirTemp("", "zrepl-platformtest-*")
if err != nil {
return "", nil, fmt.Errorf("create interposer dir: %w", err)
}
cleanup = func() { os.RemoveAll(dir) }
}
if direct {
// Direct mode: symlink zfs/zpool to real binaries (no interposer).
for _, name := range []string{"zfs", "zpool", "bash"} {
link := filepath.Join(dir, name)
os.Remove(link)
if err := os.Symlink(realBinaries[name], link); err != nil {
cleanup()
return "", nil, fmt.Errorf("create symlink %s: %w", name, err)
}
}
} else {
// Interposer mode: symlink zfs/zpool to self, .real to actual binaries.
self, err := os.Executable()
if err != nil {
cleanup()
return "", nil, fmt.Errorf("get executable path: %w", err)
}
for _, name := range []string{"zfs", "zpool"} {
link := filepath.Join(dir, name)
os.Remove(link)
if err := os.Symlink(self, link); err != nil {
cleanup()
return "", nil, fmt.Errorf("create symlink %s: %w", name, err)
}
realLink := filepath.Join(dir, name+".real")
os.Remove(realLink)
if err := os.Symlink(realBinaries[name], realLink); err != nil {
cleanup()
return "", nil, fmt.Errorf("create symlink %s.real: %w", name, err)
}
}
// Symlink other required binaries so they remain accessible
// after PATH replacement.
for _, name := range []string{"sudo", "bash"} {
link := filepath.Join(dir, name)
os.Remove(link)
if err := os.Symlink(realBinaries[name], link); err != nil {
cleanup()
return "", nil, fmt.Errorf("create symlink %s: %w", name, err)
}
}
}
// Replace PATH entirely so ALL zfs/zpool calls go through this directory.
os.Setenv("PATH", dir)
return dir, cleanup, nil
}
// RunInterposer handles the zfs/zpool interposer mode.
// It validates that the command references the test pool,
// then executes the real binary via sudo using the .real
// symlink placed by SetupInterposerPath.
func RunInterposer(command string) int {
args := os.Args[1:]
if err := validatePoolName(command, args, PlatformTestPoolName); err != nil {
fmt.Fprintf(os.Stderr, "SAFETY: %v\n", err)
return 1
}
// The .real symlink in PATH points to the actual binary.
// sudo follows the symlink, so no explicit resolution needed.
realBinary := filepath.Join(os.Getenv("PATH"), command+".real")
sudoArgs := append([]string{realBinary}, args...)
cmd := exec.Command("sudo", sudoArgs...)
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Run()
// After a successful create, chmod the mountpoint so the
// unprivileged test user can operate on it.
if err == nil && isCreateOperation(args) {
if mountpoint := extractCreateMountpoint(command, args); mountpoint != "" {
chmodMountpoint(mountpoint)
}
}
if err != nil {
if exitErr, ok := err.(*exec.ExitError); ok {
return exitErr.ExitCode()
}
fmt.Fprintf(os.Stderr, "interposer: exec error: %v\n", err)
return 1
}
return 0
}
func validatePoolName(command string, args []string, poolName string) error {
if len(args) == 0 {
return nil
}
// Check if any non-flag argument references the pool name.
for _, arg := range args {
if strings.HasPrefix(arg, "-") {
continue
}
if arg == poolName || strings.HasPrefix(arg, poolName+"/") || strings.HasPrefix(arg, poolName+"@") || strings.HasPrefix(arg, poolName+"#") {
return nil
}
}
// Special case: "zfs send -t <token>" (or -nvt, etc.) — the pool
// name is encoded in the resume token, not visible in the args.
// Decode the token via the real binary and verify toname.
if command == "zfs" {
if token := extractResumeToken(args); token != "" {
return validateResumeTokenPool(token, poolName)
}
}
// Block -a (all) flag without pool reference. Commands like
// "zfs unmount -a" or "zpool export -a" affect all pools.
for _, arg := range args {
if !strings.HasPrefix(arg, "-") {
continue
}
if arg == "-a" || (len(arg) > 2 && arg[0] == '-' && arg[1] != '-' &&
strings.ContainsRune(arg[1:], 'a')) {
return fmt.Errorf("%s command rejected: -a flag not allowed without explicit pool reference", command)
}
}
// If only one non-flag arg is present (the subcommand itself),
// this is feature detection (e.g., bare "zfs send", "zfs receive").
nonFlagArgs := 0
for _, arg := range args {
if !strings.HasPrefix(arg, "-") {
nonFlagArgs++
}
}
if nonFlagArgs <= 1 {
return nil
}
return fmt.Errorf("%s command rejected: args %v do not reference pool %q", command, args, poolName)
}
// extractResumeToken returns the resume token from "zfs send -t <token>"
// or combined forms like "zfs send -nvt <token>". Returns "" if not found.
func extractResumeToken(args []string) string {
if len(args) == 0 || args[0] != "send" {
return ""
}
for i, arg := range args {
if !strings.HasPrefix(arg, "-") {
continue
}
// -t standalone, or combined flag containing t (e.g., -nvt)
if arg == "-t" || (len(arg) > 2 && arg[0] == '-' && arg[1] != '-' &&
strings.ContainsRune(arg[1:], 't')) {
if i+1 < len(args) {
return args[i+1]
}
}
}
return ""
}
// validateResumeTokenPool decodes a resume token by calling the real zfs
// binary directly (bypassing the interposer) and checks that the toname
// field references the expected pool.
func validateResumeTokenPool(token, poolName string) error {
realBinary := filepath.Join(os.Getenv("PATH"), "zfs.real")
cmd := exec.Command("sudo", realBinary, "send", "-nvt", token)
output, err := cmd.CombinedOutput()
if err != nil {
// zfs send -nvt may exit non-zero if the dataset no longer exists,
// but it still prints the token contents. Only fail on exec errors.
if _, ok := err.(*exec.ExitError); !ok {
return fmt.Errorf("decode resume token: %w", err)
}
}
for _, line := range strings.Split(string(output), "\n") {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "toname = ") {
toname := strings.TrimPrefix(line, "toname = ")
if toname == poolName || strings.HasPrefix(toname, poolName+"/") || strings.HasPrefix(toname, poolName+"@") {
return nil
}
return fmt.Errorf("resume token rejected: toname %q does not reference pool %q", toname, poolName)
}
}
return fmt.Errorf("resume token rejected: could not extract toname from token")
}
func isCreateOperation(args []string) bool {
return len(args) > 0 && args[0] == "create"
}
func extractCreateMountpoint(command string, args []string) string {
switch command {
case "zpool":
// zpool create ... -O mountpoint=<path> ...
for i, arg := range args {
if arg == "-O" && i+1 < len(args) {
if strings.HasPrefix(args[i+1], "mountpoint=") {
return strings.TrimPrefix(args[i+1], "mountpoint=")
}
}
}
case "zfs":
// zfs create [-p] [-o ...] <dataset>
// Dataset is the last non-flag argument. Its mountpoint is
// derived from the pool mountpoint + dataset path suffix.
dataset := ""
skipNext := false
for _, arg := range args[1:] { // skip "create"
if skipNext {
skipNext = false
continue
}
if arg == "-o" {
skipNext = true
continue
}
if strings.HasPrefix(arg, "-") {
continue
}
dataset = arg
}
if dataset != "" && strings.HasPrefix(dataset, PlatformTestPoolName+"/") {
suffix := strings.TrimPrefix(dataset, PlatformTestPoolName)
return PlatformTestMountpoint + suffix
}
}
return ""
}
func chmodMountpoint(mountpoint string) {
// Validate: only chmod paths under the test mountpoint to prevent
// path traversal (e.g., dataset "pool/../../etc" -> mountpoint "/etc").
cleaned := filepath.Clean(mountpoint)
if cleaned != PlatformTestMountpoint && !strings.HasPrefix(cleaned, PlatformTestMountpoint+"/") {
fmt.Fprintf(os.Stderr, "interposer: refusing to chmod %q (not under %s)\n",
mountpoint, PlatformTestMountpoint)
return
}
// Best-effort: make mountpoint accessible to unprivileged user.
exec.Command("sudo", "chmod", "0777", cleaned).Run() //nolint:errcheck
}
@@ -1,24 +0,0 @@
#!/usr/bin/env bash
set -ue
export ZREPL_MOCK_ZFS_COMMAND_LOG="$1"
shift
export ZREPL_MOCK_ZFS_PATH="$1"
shift
dirname="$(dirname "${BASH_SOURCE[0]}")"
# If we invoke this script from the top zrepl source tree, like so:
# ./platformtest/logmockzfs/logzfsenv ...
# then dirname is relative, i.e., ./platformtest/logmockzfs.
# If we put that relative dir in PATH, then Go >= 1.19 will refuse to
# exec the `./platformtest/logmockzfs/zfs` wrapper script if it finds
# it via PATH lookup. For Example:
# cmd := exec.Command("zfs")
# err := cmd.Run()
# will fail with an error that errors.Is(err, exec.ErrDot), message:
# cannot run executable found relative to current directory
# The solution is to use an abspath.
# Learn more at https://pkg.go.dev/os/exec#hdr-Executables_in_the_current_directory
# and https://go-review.googlesource.com/c/go/+/381374
absdirname="$(readlink -e "$dirname")"
export PATH="$absdirname":"$PATH"
args=("$@")
exec "${args[@]}"
-12
View File
@@ -1,12 +0,0 @@
#!/usr/bin/env bash
set -eu
args=("$@")
if [ -n "$ZREPL_MOCK_ZFS_COMMAND_LOG" ]; then
(
flock -x 200
jq --compact-output -n --argjson args "$(printf '%s\0' "${args[@]}" | jq -Rsc 'split("\u0000")')" '{date: now|strflocaltime("%Y-%m-%dT%H:%M:%S"), command: $args}' >> "$ZREPL_MOCK_ZFS_COMMAND_LOG"
) 200>"$ZREPL_MOCK_ZFS_COMMAND_LOG".lock
fi
exec "$ZREPL_MOCK_ZFS_PATH" "${args[@]}"
+27 -66
View File
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"time"
@@ -14,97 +13,63 @@ import (
"github.com/zrepl/zrepl/internal/zfs"
)
const (
PlatformTestPoolName = "zreplplatformtest"
PlatformTestImagePath = "/tmp/zreplplatformtest.pool.img"
PlatformTestMountpoint = "/tmp/zreplplatformtest.pool"
PlatformTestImageSize = 200 * (1 << 20) // 200 MiB
)
var ZpoolExportTimeout time.Duration = 500 * time.Millisecond
type Zpool struct {
args ZpoolCreateArgs
}
type ZpoolCreateArgs struct {
PoolName string
ImagePath string
ImageSize int64
Mountpoint string
}
func (a ZpoolCreateArgs) Validate() error {
if !filepath.IsAbs(a.ImagePath) {
return errors.Errorf("ImagePath must be absolute, got %q", a.ImagePath)
}
const minImageSize = 1024
if a.ImageSize < minImageSize {
return errors.Errorf("ImageSize must be > %v, got %v", minImageSize, a.ImageSize)
}
if a.Mountpoint == "" || a.Mountpoint[0] != '/' {
return errors.Errorf("Mountpoint must be an absolute path to a directory")
}
if a.PoolName == "" {
return errors.Errorf("PoolName must not be empty")
}
return nil
}
func CreateOrReplaceZpool(ctx context.Context, e Execer, args ZpoolCreateArgs) (*Zpool, error) {
if err := args.Validate(); err != nil {
return nil, errors.Wrap(err, "zpool create args validation error")
}
func CreateOrReplaceZpool(ctx context.Context, e Execer) error {
// export pool if it already exists (idempotence)
if _, err := zfs.ZFSGetRawAnySource(ctx, args.PoolName, []string{"name"}); err != nil {
if _, err := zfs.ZFSGetRawAnySource(ctx, PlatformTestPoolName, []string{"name"}); err != nil {
if _, ok := err.(*zfs.DatasetDoesNotExist); ok {
// we'll create it shortly
} else {
return nil, errors.Wrapf(err, "cannot determine whether test pool %q exists", args.PoolName)
return errors.Wrapf(err, "cannot determine whether test pool %q exists", PlatformTestPoolName)
}
} else {
// exists, export it, OpenFile will destroy it
if err := e.RunExpectSuccessNoOutput(ctx, "zpool", "export", args.PoolName); err != nil {
return nil, errors.Wrapf(err, "cannot destroy test pool %q", args.PoolName)
if err := e.RunExpectSuccessNoOutput(ctx, "zpool", "export", PlatformTestPoolName); err != nil {
return errors.Wrapf(err, "cannot destroy test pool %q", PlatformTestPoolName)
}
}
// clear the mountpoint dir
if err := os.RemoveAll(args.Mountpoint); err != nil {
return nil, errors.Wrapf(err, "remove mountpoint dir %q", args.Mountpoint)
}
if err := os.Mkdir(args.Mountpoint, 0700); err != nil {
return nil, errors.Wrapf(err, "create mountpoint dir %q", args.Mountpoint)
}
// idempotently (re)create the pool image
image, err := os.OpenFile(args.ImagePath, os.O_CREATE|os.O_RDWR, 0600)
image, err := os.OpenFile(PlatformTestImagePath, os.O_CREATE|os.O_RDWR, 0600)
if err != nil {
return nil, errors.Wrap(err, "create image file")
return errors.Wrap(err, "create image file")
}
defer image.Close()
if err := image.Truncate(args.ImageSize); err != nil {
return nil, errors.Wrap(err, "create image: truncate")
if err := image.Truncate(PlatformTestImageSize); err != nil {
return errors.Wrap(err, "create image: truncate")
}
image.Close()
// create the pool
// create the pool (zpool runs via sudo through the interposer,
// which creates the mountpoint and chmods it to 0777)
err = e.RunExpectSuccessNoOutput(ctx, "zpool", "create", "-f",
"-O", fmt.Sprintf("mountpoint=%s", args.Mountpoint),
args.PoolName, args.ImagePath,
"-O", fmt.Sprintf("mountpoint=%s", PlatformTestMountpoint),
PlatformTestPoolName, PlatformTestImagePath,
)
if err != nil {
return nil, errors.Wrap(err, "zpool create")
return errors.Wrap(err, "zpool create")
}
return &Zpool{args}, nil
return nil
}
func (p *Zpool) Name() string { return p.args.PoolName }
func (p *Zpool) Destroy(ctx context.Context, e Execer) error {
func DestroyZpool(ctx context.Context, e Execer) error {
exportDeadline := time.Now().Add(ZpoolExportTimeout)
for {
if time.Now().After(exportDeadline) {
return errors.Errorf("could not zpool export (got 'pool is busy'): %s", p.args.PoolName)
return errors.Errorf("could not zpool export (got 'pool is busy'): %s", PlatformTestPoolName)
}
err := e.RunExpectSuccessNoOutput(ctx, "zpool", "export", p.args.PoolName)
err := e.RunExpectSuccessNoOutput(ctx, "zpool", "export", PlatformTestPoolName)
if err == nil {
break
}
@@ -113,17 +78,13 @@ func (p *Zpool) Destroy(ctx context.Context, e Execer) error {
continue
}
if err != nil {
return errors.Wrapf(err, "export pool %q", p.args.PoolName)
return errors.Wrapf(err, "export pool %q", PlatformTestPoolName)
}
}
if err := os.Remove(p.args.ImagePath); err != nil {
if err := os.Remove(PlatformTestImagePath); err != nil {
return errors.Wrapf(err, "remove pool image")
}
if err := os.RemoveAll(p.args.Mountpoint); err != nil {
return errors.Wrapf(err, "remove mountpoint dir %q", p.args.Mountpoint)
}
return nil
}
+6 -9
View File
@@ -573,9 +573,8 @@ func implReplicationIsResumableFullSend(ctx *platformtest.Context, setup replica
// due to step holds
if setup.expectDatasetIsBusyErrorWhenDestroySnapshotWhilePartiallyReplicated {
ctx.Logf("i=%v", i)
dse, ok := err.(*zfs.DestroySnapshotsError)
require.True(ctx, ok, "expected *zfs.DestroySnapshotsError, got %T: %s", err, err)
require.True(ctx, dse.AllReasonIsHold(), "expected hold reason, got %v", dse.Reason)
require.Error(ctx, err)
require.Contains(ctx, err.Error(), "dataset is busy")
}
}
@@ -757,13 +756,11 @@ func ReplicationIncrementalDestroysStepHoldsIffIncrementalStepHoldsAreDisabledBu
func ReplicationStepCompletedLostBehavior__GuaranteeResumability(ctx *platformtest.Context) {
scenario := replicationStepCompletedLostBehavior_impl(ctx, pdu.ReplicationGuaranteeKind_GuaranteeResumability)
dse1, ok := scenario.deleteSfs1Err.(*zfs.DestroySnapshotsError)
require.True(ctx, ok, "expected *zfs.DestroySnapshotsError, got %T: %s", scenario.deleteSfs1Err, scenario.deleteSfs1Err)
require.True(ctx, dse1.AllReasonIsHold(), "expected hold reason, got %v", dse1.Reason)
require.Error(ctx, scenario.deleteSfs1Err, "protected by holds")
require.Contains(ctx, scenario.deleteSfs1Err.Error(), "dataset is busy")
dse2, ok := scenario.deleteSfs2Err.(*zfs.DestroySnapshotsError)
require.True(ctx, ok, "expected *zfs.DestroySnapshotsError, got %T: %s", scenario.deleteSfs2Err, scenario.deleteSfs2Err)
require.True(ctx, dse2.AllReasonIsHold(), "expected hold reason, got %v", dse2.Reason)
require.Error(ctx, scenario.deleteSfs2Err, "protected by holds")
require.Contains(ctx, scenario.deleteSfs2Err.Error(), "dataset is busy")
require.Nil(ctx, scenario.finalReport.Error())
_ = fsversion(ctx, scenario.rfs, "@3") // @3 ade it to the other side
@@ -31,7 +31,7 @@ func UndestroyableSnapshotParsing(t *platformtest.Context) {
panic(dse.Filesystem)
}
require.Equal(t, []string{"4 5 6"}, dse.Undestroyable)
require.True(t, dse.AllReasonIsHold(), "expected hold reason, got %v", dse.Reason)
require.Equal(t, []string{"dataset is busy"}, dse.Reason)
}
}
-14
View File
@@ -1723,20 +1723,6 @@ type DestroySnapshotsError struct {
Reason []string
}
var destroySnapshotsReasonHeldRegexp = regexp.MustCompile(`^it's being held\. Run 'zfs holds -r .+' to see holders\.$`)
// AllReasonIsHold returns true if every undestroyable snapshot failed because
// it is held. ZFS < 2.4 reports "dataset is busy", ZFS >= 2.4 reports
// "it's being held. Run 'zfs holds -r <snapshot>' to see holders."
func (e *DestroySnapshotsError) AllReasonIsHold() bool {
for _, r := range e.Reason {
if r != "dataset is busy" && !destroySnapshotsReasonHeldRegexp.MatchString(r) {
return false
}
}
return true
}
func (e *DestroySnapshotsError) Error() string {
if len(e.Undestroyable) != len(e.Reason) {
panic(fmt.Sprintf("%v != %v", len(e.Undestroyable), len(e.Reason)))
+1 -1
View File
@@ -69,7 +69,7 @@ cp -a internal/config/samples %{buildroot}%{_datadir}/
%{_bindir}/zrepl
%config %{_unitdir}/zrepl.service
%dir %{_sysconfdir}/zrepl
%config(noreplace) %{_sysconfdir}/zrepl/zrepl.yml
%config %{_sysconfdir}/zrepl/zrepl.yml
%{_datadir}/zsh/site-functions/_zrepl
%{_datadir}/bash-completion/completions/zrepl
%{_datadir}/doc/zrepl