move implementation to internal/ directory (#828)

internal/platformtest/harness/harness.go

@@ -0,0 +1,255 @@
+package main
+
+import (
+	"container/list"
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"time"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/internal/daemon/logging/trace"
+
+	"github.com/zrepl/zrepl/internal/config"
+	"github.com/zrepl/zrepl/internal/daemon/logging"
+	"github.com/zrepl/zrepl/internal/logger"
+	"github.com/zrepl/zrepl/internal/platformtest"
+	"github.com/zrepl/zrepl/internal/platformtest/tests"
+)
+
+var bold = color.New(color.Bold)
+var boldRed = color.New(color.Bold, color.FgHiRed)
+var boldGreen = color.New(color.Bold, color.FgHiGreen)
+
+const DefaultPoolImageSize = 200 * (1 << 20)
+
+func main() {
+
+	var args HarnessArgs
+
+	flag.StringVar(&args.CreateArgs.PoolName, "poolname", "", "")
+	flag.StringVar(&args.CreateArgs.ImagePath, "imagepath", "", "")
+	flag.Int64Var(&args.CreateArgs.ImageSize, "imagesize", DefaultPoolImageSize, "")
+	flag.StringVar(&args.CreateArgs.Mountpoint, "mountpoint", "", "")
+	flag.BoolVar(&args.StopAndKeepPoolOnFail, "failure.stop-and-keep-pool", false, "if a test case fails, stop test execution and keep pool as it was when the test failed")
+	flag.StringVar(&args.Run, "run", "", "")
+	flag.DurationVar(&platformtest.ZpoolExportTimeout, "zfs.zpool-export-timeout", platformtest.ZpoolExportTimeout, "")
+	flag.Parse()
+
+	if err := HarnessRun(args); err != nil {
+		os.Exit(1)
+	}
+}
+
+var exitWithErr = fmt.Errorf("exit with error")
+
+type HarnessArgs struct {
+	CreateArgs            platformtest.ZpoolCreateArgs
+	StopAndKeepPoolOnFail bool
+	Run                   string
+}
+
+type invocation struct {
+	runFunc  tests.Case
+	idstring string
+	result   *testCaseResult
+	children map[string]*invocation
+}
+
+func newInvocation(runFunc tests.Case, id string) *invocation {
+	return &invocation{
+		runFunc:  runFunc,
+		idstring: id,
+		children: make(map[string]*invocation),
+	}
+}
+
+func (i *invocation) String() string {
+	idsuffix := ""
+	if i.idstring != "" {
+		idsuffix = fmt.Sprintf(": %s", i.idstring)
+	}
+	return fmt.Sprintf("%s%s", i.runFunc.String(), idsuffix)
+}
+
+func (i *invocation) RegisterChild(c *invocation) error {
+	if c.idstring == "" {
+		return fmt.Errorf("child must have id string")
+	}
+	if oc := i.children[c.idstring]; oc != nil {
+		return fmt.Errorf("idstring %q is already taken by %s", c.idstring, oc)
+	}
+	i.children[c.idstring] = c
+	return nil
+}
+
+func HarnessRun(args HarnessArgs) error {
+
+	runRE := regexp.MustCompile(args.Run)
+
+	outlets := logger.NewOutlets()
+	outlet, level, err := logging.ParseOutlet(config.LoggingOutletEnum{Ret: &config.StdoutLoggingOutlet{
+		LoggingOutletCommon: config.LoggingOutletCommon{
+			Level:  "debug",
+			Format: "human",
+		},
+	}})
+	if err != nil {
+		panic(err)
+	}
+	outlets.Add(outlet, level)
+	logger := logger.NewLogger(outlets, 1*time.Second)
+
+	if err := args.CreateArgs.Validate(); err != nil {
+		logger.Error(err.Error())
+		panic(err)
+	}
+	ctx := context.Background()
+	defer trace.WithTaskFromStackUpdateCtx(&ctx)()
+	ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(logger))
+	ex := platformtest.NewEx(logger)
+
+	testQueue := list.New()
+	for _, c := range tests.Cases {
+		if runRE.MatchString(c.String()) {
+			testQueue.PushBack(newInvocation(c, ""))
+		}
+	}
+
+	completedTests := list.New()
+
+	for testQueue.Len() > 0 {
+		inv := testQueue.Remove(testQueue.Front()).(*invocation)
+
+		bold.Printf("BEGIN TEST CASE %s\n", inv)
+
+		pool, err := platformtest.CreateOrReplaceZpool(ctx, ex, args.CreateArgs)
+		if err != nil {
+			panic(errors.Wrap(err, "create test pool"))
+		}
+
+		ctx := &platformtest.Context{
+			Context:     ctx,
+			RootDataset: filepath.Join(pool.Name(), "rootds"),
+			QueueSubtest: func(id string, stf func(*platformtest.Context)) {
+				stinv := newInvocation(stf, id)
+				err := inv.RegisterChild(stinv)
+				if err != nil {
+					panic(err)
+				}
+				bold.Printf(" QUEUING SUBTEST %q\n", id)
+				testQueue.PushFront(stinv)
+			},
+		}
+
+		res := runTestCase(ctx, ex, inv.runFunc)
+		inv.result = res
+		if res.failed {
+			fmt.Printf("%+v\n", res.failedStack) // print with stack trace
+		}
+
+		if res.failed && args.StopAndKeepPoolOnFail {
+			boldRed.Printf("STOPPING TEST RUN AT FAILING TEST PER USER REQUEST\n")
+			return exitWithErr
+		}
+
+		if err := pool.Destroy(ctx, ex); err != nil {
+			panic(fmt.Sprintf("error destroying test pool: %s", err))
+		}
+
+		completedTests.PushBack(inv)
+
+		if res.failed {
+			boldRed.Printf("TEST FAILED\n")
+		} else if res.skipped {
+			bold.Printf("TEST SKIPPED\n")
+		} else if res.succeeded {
+			boldGreen.Printf("TEST PASSED\n")
+		} else {
+			panic("unreachable")
+		}
+
+		fmt.Println()
+	}
+
+	var summary struct {
+		succ, fail, skip []*invocation
+	}
+	for completedTests.Len() > 0 {
+		inv := completedTests.Remove(completedTests.Front()).(*invocation)
+		var bucket *[]*invocation
+		if inv.result.failed {
+			bucket = &summary.fail
+		} else if inv.result.skipped {
+			bucket = &summary.skip
+		} else if inv.result.succeeded {
+			bucket = &summary.succ
+		} else {
+			panic("unreachable")
+		}
+		*bucket = append(*bucket, inv)
+	}
+	printBucket := func(bucketName string, c *color.Color, bucket []*invocation) {
+		c.Printf("%s:", bucketName)
+		if len(bucket) == 0 {
+			fmt.Printf(" []\n")
+			return
+		}
+		fmt.Printf("\n")
+		for _, inv := range bucket {
+			fmt.Printf("  %s\n", inv)
+		}
+	}
+	printBucket("PASSING TESTS", boldGreen, summary.succ)
+	printBucket("SKIPPED TESTS", bold, summary.skip)
+	printBucket("FAILED TESTS", boldRed, summary.fail)
+
+	if len(summary.fail) > 0 {
+		return errors.New("at least one test failed")
+	}
+	return nil
+}
+
+type testCaseResult struct {
+	// oneof
+	failed, skipped, succeeded bool
+
+	failedStack error // has stack inside, valid if failed=true
+}
+
+func runTestCase(ctx *platformtest.Context, ex platformtest.Execer, c tests.Case) *testCaseResult {
+
+	// run case
+	var panicked = false
+	var panicValue interface{} = nil
+	var panicStack error
+	func() {
+		defer func() {
+			if item := recover(); item != nil {
+				panicValue = item
+				panicked = true
+				panicStack = errors.Errorf("panic while running test: %v", panicValue)
+			}
+		}()
+		c(ctx)
+	}()
+
+	if panicked {
+		switch panicValue {
+		case platformtest.SkipNowSentinel:
+			return &testCaseResult{skipped: true}
+		case platformtest.FailNowSentinel:
+			return &testCaseResult{failed: true, failedStack: panicStack}
+		default:
+			return &testCaseResult{failed: true, failedStack: panicStack}
+		}
+	} else {
+		return &testCaseResult{succeeded: true}
+	}
+
+}

internal/platformtest/harness/harness_test.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+)
+
+// Idea taken from
+// https://github.com/openSUSE/umoci/blob/v0.2.1/cmd/umoci/main_test.go
+//
+/* How to generate coverage:
+   go test -c -covermode=atomic -cover -coverpkg github.com/zrepl/zrepl/...
+   sudo ../logmockzfs/logzfsenv /tmp/zrepl_platform_test.log /usr/bin/zfs \
+       ./harness.test -test.coverprofile=/tmp/harness.out \
+       -test.v __DEVEL--i-heard-you-like-tests \
+       -imagepath /tmp/testpool.img -poolname zreplplatformtest
+    go tool cover -html=/tmp/harness.out -o /tmp/harness.html
+*/
+// Merge with existing coverage reports using gocovmerge:
+//  https://github.com/wadey/gocovmerge
+
+func TestMain(t *testing.T) {
+	fmt.Println("incoming args: ", os.Args)
+
+	var (
+		args             []string
+		run              bool
+		startCaptureArgs bool
+	)
+
+	for i, arg := range os.Args {
+		switch {
+		case arg == "__DEVEL--i-heard-you-like-tests":
+			run = true
+			startCaptureArgs = true
+		case strings.HasPrefix(arg, "-test"):
+		case strings.HasPrefix(arg, "__DEVEL"):
+		case i == 0:
+			args = append(args, arg)
+		case startCaptureArgs:
+			args = append(args, arg)
+		}
+	}
+	os.Args = args
+	fmt.Println("using args: ", os.Args)
+
+	if run {
+		main()
+	}
+}

internal/platformtest/logmockzfs/logzfsenv

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -ue
+export ZREPL_MOCK_ZFS_COMMAND_LOG="$1"
+shift
+export ZREPL_MOCK_ZFS_PATH="$1"
+shift
+dirname="$(dirname "${BASH_SOURCE[0]}")"
+# If we invoke this script from the top zrepl source tree, like so:
+#    ./platformtest/logmockzfs/logzfsenv ...
+# then dirname is relative, i.e., ./platformtest/logmockzfs.
+# If we put that relative dir in PATH, then Go >= 1.19 will refuse to
+# exec the `./platformtest/logmockzfs/zfs` wrapper script if it finds
+# it via PATH lookup. For Example:
+#   cmd := exec.Command("zfs")
+#   err := cmd.Run()
+# will fail with an error that errors.Is(err, exec.ErrDot), message:
+#   cannot run executable found relative to current directory
+# The solution is to use an abspath.
+# Learn more at https://pkg.go.dev/os/exec#hdr-Executables_in_the_current_directory
+# and https://go-review.googlesource.com/c/go/+/381374
+absdirname="$(readlink -e "$dirname")"
+export PATH="$absdirname":"$PATH"
+args=("$@")
+exec "${args[@]}"

internal/platformtest/logmockzfs/zfs

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -eu
+args=("$@")
+
+if [ -n "$ZREPL_MOCK_ZFS_COMMAND_LOG" ]; then
+    (
+        flock -x 200
+        jq --compact-output -n --argjson args "$(printf '%s\0' "${args[@]}" | jq -Rsc 'split("\u0000")')" '{date: now|strflocaltime("%Y-%m-%dT%H:%M:%S"), command: $args}' >> "$ZREPL_MOCK_ZFS_COMMAND_LOG"
+    ) 200>"$ZREPL_MOCK_ZFS_COMMAND_LOG".lock
+fi
+
+exec "$ZREPL_MOCK_ZFS_PATH" "${args[@]}"

internal/platformtest/platformtest.go

@@ -0,0 +1,46 @@
+package platformtest
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type Context struct {
+	context.Context
+	RootDataset string
+	// Use this callback from a top-level test case to queue the
+	// execution of sub-tests after this test case is complete.
+	//
+	// Note that the testing harness executes the subtest
+	// _after_ the current top-level test. Hence, the subtest
+	// cannot use any ZFS state of the top-level test.
+	QueueSubtest func(id string, stf func(*Context))
+}
+
+var FailNowSentinel = fmt.Errorf("platformtest: FailNow called on context")
+
+var SkipNowSentinel = fmt.Errorf("platformtest: SkipNow called on context")
+
+var _ assert.TestingT = (*Context)(nil)
+var _ require.TestingT = (*Context)(nil)
+
+func (c *Context) Logf(format string, args ...interface{}) {
+	msg := fmt.Sprintf(format, args...)
+	GetLog(c).Info(msg)
+}
+
+func (c *Context) Errorf(format string, args ...interface{}) {
+	GetLog(c).Printf(format, args...)
+	c.FailNow()
+}
+
+func (c *Context) FailNow() {
+	panic(FailNowSentinel)
+}
+
+func (c *Context) SkipNow() {
+	panic(SkipNowSentinel)
+}

internal/platformtest/platformtest_exec.go

@@ -0,0 +1,45 @@
+package platformtest
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+
+	"github.com/zrepl/zrepl/internal/util/circlog"
+)
+
+type ex struct {
+	log Logger
+}
+
+func NewEx(log Logger) Execer {
+	return &ex{log}
+}
+
+func (e *ex) RunExpectSuccessNoOutput(ctx context.Context, cmd string, args ...string) error {
+	return e.runNoOutput(true, ctx, cmd, args...)
+}
+
+func (e *ex) RunExpectFailureNoOutput(ctx context.Context, cmd string, args ...string) error {
+	return e.runNoOutput(false, ctx, cmd, args...)
+}
+
+func (e *ex) runNoOutput(expectSuccess bool, ctx context.Context, cmd string, args ...string) error {
+	log := e.log.WithField("command", fmt.Sprintf("%q %q", cmd, args))
+	log.Debug("begin executing")
+	defer log.Debug("done executing")
+	ecmd := exec.CommandContext(ctx, cmd, args...)
+	buf, _ := circlog.NewCircularLog(32 << 10)
+	ecmd.Stdout, ecmd.Stderr = buf, buf
+	err := ecmd.Run()
+	log.WithField("output", buf.String()).Debug("command output")
+	if _, ok := err.(*exec.ExitError); err != nil && !ok {
+		panic(err)
+	}
+	if expectSuccess && err != nil {
+		return fmt.Errorf("expecting no error, got error: %s\n%s", err, buf.String())
+	} else if !expectSuccess && err == nil {
+		return fmt.Errorf("expecting error, got no error")
+	}
+	return nil
+}

internal/platformtest/platformtest_logging.go

@@ -0,0 +1,14 @@
+package platformtest
+
+import (
+	"context"
+
+	"github.com/zrepl/zrepl/internal/daemon/logging"
+	"github.com/zrepl/zrepl/internal/logger"
+)
+
+type Logger = logger.Logger
+
+func GetLog(ctx context.Context) Logger {
+	return logging.GetLogger(ctx, logging.SubsysPlatformtest)
+}

internal/platformtest/platformtest_ops.go

@@ -0,0 +1,325 @@
+package platformtest
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"unicode"
+)
+
+type Execer interface {
+	RunExpectSuccessNoOutput(ctx context.Context, cmd string, args ...string) error
+	RunExpectFailureNoOutput(ctx context.Context, cmd string, args ...string) error
+}
+
+type Stmt interface {
+	Run(context context.Context, e Execer) error
+}
+
+type Op string
+
+const (
+	Comment         Op = "#"
+	AssertExists    Op = "!E"
+	AssertNotExists Op = "!N"
+	Add             Op = "+"
+	Del             Op = "-"
+	RunCmd          Op = "R"
+	DestroyRoot     Op = "DESTROYROOT"
+	CreateRoot      Op = "CREATEROOT"
+)
+
+type DestroyRootOp struct {
+	Path string
+}
+
+func (o *DestroyRootOp) Run(ctx context.Context, e Execer) error {
+	// early-exit if it doesn't exist
+	if err := e.RunExpectSuccessNoOutput(ctx, "zfs", "get", "-H", "name", o.Path); err != nil {
+		GetLog(ctx).WithField("root_ds", o.Path).Info("assume root ds doesn't exist")
+		return nil
+	}
+	return e.RunExpectSuccessNoOutput(ctx, "zfs", "destroy", "-r", o.Path)
+}
+
+type FSOp struct {
+	Op        Op
+	Path      string
+	Encrypted bool // only for Op=Add
+}
+
+func (o *FSOp) Run(ctx context.Context, e Execer) error {
+	switch o.Op {
+	case AssertExists:
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", "get", "-H", "name", o.Path)
+	case AssertNotExists:
+		return e.RunExpectFailureNoOutput(ctx, "zfs", "get", "-H", "name", o.Path)
+	case Add:
+		opts := []string{"create"}
+		if o.Encrypted {
+			const passphraseFilePath = "/tmp/zreplplatformtest.encryption.passphrase"
+			const passphrase = "foobar2342"
+			err := os.WriteFile(passphraseFilePath, []byte(passphrase), 0600)
+			if err != nil {
+				panic(err)
+			}
+			opts = append(opts,
+				"-o", "encryption=on",
+				"-o", "keylocation=file:///"+passphraseFilePath,
+				"-o", "keyformat=passphrase",
+			)
+		}
+		opts = append(opts, o.Path)
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", opts...)
+	case Del:
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", "destroy", o.Path)
+	default:
+		panic(o.Op)
+	}
+}
+
+type SnapOp struct {
+	Op   Op
+	Path string
+}
+
+func (o *SnapOp) Run(ctx context.Context, e Execer) error {
+	switch o.Op {
+	case AssertExists:
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", "get", "-H", "name", o.Path)
+	case AssertNotExists:
+		return e.RunExpectFailureNoOutput(ctx, "zfs", "get", "-H", "name", o.Path)
+	case Add:
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", "snapshot", o.Path)
+	case Del:
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", "destroy", o.Path)
+	default:
+		panic(o.Op)
+	}
+}
+
+type BookmarkOp struct {
+	Op       Op
+	Existing string
+	Bookmark string
+}
+
+func (o *BookmarkOp) Run(ctx context.Context, e Execer) error {
+	switch o.Op {
+	case Add:
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", "bookmark", o.Existing, o.Bookmark)
+	case Del:
+		if o.Existing != "" {
+			panic("existing must be empty for destroy, got " + o.Existing)
+		}
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", "destroy", o.Bookmark)
+	default:
+		panic(o.Op)
+	}
+}
+
+type RunOp struct {
+	RootDS string
+	Script string
+}
+
+func (o *RunOp) Run(ctx context.Context, e Execer) error {
+	cmd := exec.CommandContext(ctx, "/usr/bin/env", "bash", "-c", o.Script)
+	cmd.Env = os.Environ()
+	cmd.Env = append(cmd.Env, fmt.Sprintf("ROOTDS=%s", o.RootDS))
+	log := GetLog(ctx).WithField("script", o.Script)
+	log.Info("start script")
+	defer log.Info("script done")
+	output, err := cmd.CombinedOutput()
+	if _, ok := err.(*exec.ExitError); err != nil && !ok {
+		panic(err)
+	}
+	log.Printf("script output:\n%s", output)
+	return err
+}
+
+type LineError struct {
+	Line string
+	What string
+}
+
+func (e LineError) Error() string {
+	return fmt.Sprintf("%q: %s", e.Line, e.What)
+}
+
+type RunKind int
+
+const (
+	PanicErr RunKind = 1 << iota
+	RunAll
+)
+
+func Run(ctx context.Context, rk RunKind, rootds string, stmtsStr string) {
+	stmt, err := parseSequence(rootds, stmtsStr)
+	if err != nil {
+		panic(err)
+	}
+	execer := NewEx(GetLog(ctx))
+	for _, s := range stmt {
+		err := s.Run(ctx, execer)
+		if err == nil {
+			continue
+		}
+		if rk == PanicErr {
+			panic(err)
+		} else if rk == RunAll {
+			continue
+		} else {
+			panic(rk)
+		}
+	}
+}
+
+func isNoSpace(r rune) bool {
+	return !unicode.IsSpace(r)
+}
+
+func splitQuotedWords(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	begin := bytes.IndexFunc(data, isNoSpace)
+	if begin == -1 {
+		return len(data), nil, nil
+	}
+	if data[begin] == '"' {
+		end := begin + 1
+		for end < len(data) {
+			endCandidate := bytes.Index(data[end:], []byte(`"`))
+			if endCandidate == -1 {
+				return 0, nil, nil
+			}
+			end += endCandidate
+			if data[end-1] != '\\' {
+				// unescaped quote, end of this string
+				// remove backslash-escapes
+				withBackslash := data[begin+1 : end]
+				withoutBackslash := bytes.Replace(withBackslash, []byte("\\\""), []byte("\""), -1)
+				return end + 1, withoutBackslash, nil
+			} else {
+				// continue to next quote
+				end += 1
+			}
+		}
+	} else {
+		endOffset := bytes.IndexFunc(data[begin:], unicode.IsSpace)
+		var end int
+		if endOffset == -1 {
+			if !atEOF {
+				return 0, nil, nil
+			} else {
+				end = len(data)
+			}
+		} else {
+			end = begin + endOffset
+		}
+		return end, data[begin:end], nil
+	}
+	return 0, nil, fmt.Errorf("unexpected")
+}
+
+func parseSequence(rootds, stmtsStr string) (stmts []Stmt, err error) {
+	scan := bufio.NewScanner(strings.NewReader(stmtsStr))
+nextLine:
+	for scan.Scan() {
+		if len(bytes.TrimSpace(scan.Bytes())) == 0 {
+			continue
+		}
+		comps := bufio.NewScanner(bytes.NewReader(scan.Bytes()))
+		comps.Split(splitQuotedWords)
+
+		expectMoreTokens := func() error {
+			if !comps.Scan() {
+				return &LineError{scan.Text(), "unexpected EOL"}
+			}
+			return nil
+		}
+
+		// Op
+		if err := expectMoreTokens(); err != nil {
+			return nil, err
+		}
+		var op Op
+		switch comps.Text() {
+
+		case string(RunCmd):
+			script := strings.TrimPrefix(strings.TrimSpace(scan.Text()), string(RunCmd))
+			stmts = append(stmts, &RunOp{RootDS: rootds, Script: script})
+			continue nextLine
+
+		case string(DestroyRoot):
+			if comps.Scan() {
+				return nil, &LineError{scan.Text(), "unexpected tokens at EOL"}
+			}
+			stmts = append(stmts, &DestroyRootOp{rootds})
+			continue nextLine
+
+		case string(CreateRoot):
+			if comps.Scan() {
+				return nil, &LineError{scan.Text(), "unexpected tokens at EOL"}
+			}
+			stmts = append(stmts, &FSOp{Op: Add, Path: rootds})
+			continue nextLine
+
+		case string(Add):
+			op = Add
+		case string(Del):
+			op = Del
+		case string(AssertExists):
+			op = AssertExists
+		case string(AssertNotExists):
+			op = AssertNotExists
+		case string(Comment):
+			op = Comment
+			continue
+		default:
+			return nil, &LineError{scan.Text(), fmt.Sprintf("invalid op %q", comps.Text())}
+		}
+
+		// FS / SNAP / BOOKMARK
+		if err := expectMoreTokens(); err != nil {
+			return nil, err
+		}
+		if strings.ContainsAny(comps.Text(), "@") {
+			stmts = append(stmts, &SnapOp{Op: op, Path: fmt.Sprintf("%s/%s", rootds, comps.Text())})
+		} else if strings.ContainsAny(comps.Text(), "#") {
+			bookmark := fmt.Sprintf("%s/%s", rootds, comps.Text())
+			if err := expectMoreTokens(); err != nil {
+				return nil, err
+			}
+			existing := fmt.Sprintf("%s/%s", rootds, comps.Text())
+			stmts = append(stmts, &BookmarkOp{Op: op, Existing: existing, Bookmark: bookmark})
+		} else {
+			// FS
+			fs := comps.Text()
+			var encrypted bool = false
+			if op == Add {
+				if comps.Scan() {
+					t := comps.Text()
+					switch t {
+					case "encrypted":
+						encrypted = true
+					default:
+						panic(fmt.Sprintf("unexpected token %q", t))
+					}
+				}
+			}
+			stmts = append(stmts, &FSOp{
+				Op:        op,
+				Path:      fmt.Sprintf("%s/%s", rootds, fs),
+				Encrypted: encrypted,
+			})
+		}
+
+		if comps.Scan() {
+			return nil, &LineError{scan.Text(), "unexpected tokens at EOL"}
+		}
+	}
+	return stmts, nil
+}

internal/platformtest/platformtest_parser_test.go

@@ -0,0 +1,33 @@
+package platformtest
+
+import (
+	"bufio"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSplitQuotedWords(t *testing.T) {
+	s := bufio.NewScanner(strings.NewReader(`
+	foo "bar baz" blah "foo \"with single escape" "blah baz" "\"foo" "foo\""
+	`))
+	s.Split(splitQuotedWords)
+	var words []string
+	for s.Scan() {
+		words = append(words, s.Text())
+	}
+	assert.Equal(
+		t,
+		[]string{
+			"foo",
+			"bar baz",
+			"blah",
+			"foo \"with single escape",
+			"blah baz",
+			"\"foo",
+			"foo\"",
+		},
+		words)
+
+}

internal/platformtest/platformtest_zpool.go

@@ -0,0 +1,129 @@
+package platformtest
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+var ZpoolExportTimeout time.Duration = 500 * time.Millisecond
+
+type Zpool struct {
+	args ZpoolCreateArgs
+}
+
+type ZpoolCreateArgs struct {
+	PoolName   string
+	ImagePath  string
+	ImageSize  int64
+	Mountpoint string
+}
+
+func (a ZpoolCreateArgs) Validate() error {
+	if !filepath.IsAbs(a.ImagePath) {
+		return errors.Errorf("ImagePath must be absolute, got %q", a.ImagePath)
+	}
+	const minImageSize = 1024
+	if a.ImageSize < minImageSize {
+		return errors.Errorf("ImageSize must be > %v, got %v", minImageSize, a.ImageSize)
+	}
+	if a.Mountpoint == "" || a.Mountpoint[0] != '/' {
+		return errors.Errorf("Mountpoint must be an absolute path to a directory")
+	}
+	if a.PoolName == "" {
+		return errors.Errorf("PoolName must not be empty")
+	}
+	return nil
+}
+
+func CreateOrReplaceZpool(ctx context.Context, e Execer, args ZpoolCreateArgs) (*Zpool, error) {
+	if err := args.Validate(); err != nil {
+		return nil, errors.Wrap(err, "zpool create args validation error")
+	}
+
+	// export pool if it already exists (idempotence)
+	if _, err := zfs.ZFSGetRawAnySource(ctx, args.PoolName, []string{"name"}); err != nil {
+		if _, ok := err.(*zfs.DatasetDoesNotExist); ok {
+			// we'll create it shortly
+		} else {
+			return nil, errors.Wrapf(err, "cannot determine whether test pool %q exists", args.PoolName)
+		}
+	} else {
+		// exists, export it, OpenFile will destroy it
+		if err := e.RunExpectSuccessNoOutput(ctx, "zpool", "export", args.PoolName); err != nil {
+			return nil, errors.Wrapf(err, "cannot destroy test pool %q", args.PoolName)
+		}
+	}
+
+	// clear the mountpoint dir
+	if err := os.RemoveAll(args.Mountpoint); err != nil {
+		return nil, errors.Wrapf(err, "remove mountpoint dir %q", args.Mountpoint)
+	}
+	if err := os.Mkdir(args.Mountpoint, 0700); err != nil {
+		return nil, errors.Wrapf(err, "create mountpoint dir %q", args.Mountpoint)
+	}
+
+	// idempotently (re)create the pool image
+	image, err := os.OpenFile(args.ImagePath, os.O_CREATE|os.O_RDWR, 0600)
+	if err != nil {
+		return nil, errors.Wrap(err, "create image file")
+	}
+	defer image.Close()
+	if err := image.Truncate(args.ImageSize); err != nil {
+		return nil, errors.Wrap(err, "create image: truncate")
+	}
+	image.Close()
+
+	// create the pool
+	err = e.RunExpectSuccessNoOutput(ctx, "zpool", "create", "-f",
+		"-O", fmt.Sprintf("mountpoint=%s", args.Mountpoint),
+		args.PoolName, args.ImagePath,
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "zpool create")
+	}
+
+	return &Zpool{args}, nil
+}
+
+func (p *Zpool) Name() string { return p.args.PoolName }
+
+func (p *Zpool) Destroy(ctx context.Context, e Execer) error {
+
+	exportDeadline := time.Now().Add(ZpoolExportTimeout)
+
+	for {
+		if time.Now().After(exportDeadline) {
+			return errors.Errorf("could not zpool export (got 'pool is busy'): %s", p.args.PoolName)
+		}
+		err := e.RunExpectSuccessNoOutput(ctx, "zpool", "export", p.args.PoolName)
+		if err == nil {
+			break
+		}
+		if strings.Contains(err.Error(), "pool is busy") {
+			runtime.Gosched()
+			continue
+		}
+		if err != nil {
+			return errors.Wrapf(err, "export pool %q", p.args.PoolName)
+		}
+	}
+
+	if err := os.Remove(p.args.ImagePath); err != nil {
+		return errors.Wrapf(err, "remove pool image")
+	}
+
+	if err := os.RemoveAll(p.args.Mountpoint); err != nil {
+		return errors.Wrapf(err, "remove mountpoint dir %q", p.args.Mountpoint)
+	}
+
+	return nil
+}

internal/platformtest/tests/batchDestroy.go

@@ -0,0 +1,54 @@
+package tests
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/zrepl/zrepl/internal/platformtest"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+func BatchDestroy(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@1"
+		+  "foo bar@2"
+		+  "foo bar@3"
+		R  zfs hold zrepl_platformtest "${ROOTDS}/foo bar@2"
+	`)
+
+	reqs := []*zfs.DestroySnapOp{
+		&zfs.DestroySnapOp{
+			ErrOut:     new(error),
+			Filesystem: fmt.Sprintf("%s/foo bar", ctx.RootDataset),
+			Name:       "3",
+		},
+		&zfs.DestroySnapOp{
+			ErrOut:     new(error),
+			Filesystem: fmt.Sprintf("%s/foo bar", ctx.RootDataset),
+			Name:       "2",
+		},
+	}
+	zfs.ZFSDestroyFilesystemVersions(ctx, reqs)
+	if *reqs[0].ErrOut != nil {
+		panic("expecting no error")
+	}
+	err := (*reqs[1].ErrOut).Error()
+	if !strings.Contains(err, fmt.Sprintf("%s/foo bar@2", ctx.RootDataset)) {
+		panic(fmt.Sprintf("expecting error about being unable to destroy @2: %T\n%s", err, err))
+	}
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+	!N "foo bar@3"
+	!E "foo bar@1"
+	!E "foo bar@2"
+	R zfs release zrepl_platformtest "${ROOTDS}/foo bar@2"
+	-  "foo bar@2"
+	-  "foo bar@1"
+	-  "foo bar"
+	`)
+
+}

internal/platformtest/tests/gen/gen.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"bytes"
+	"go/ast"
+	"go/format"
+	"go/parser"
+	"go/token"
+	"os"
+	"sort"
+	"strings"
+	"text/template"
+
+	"golang.org/x/tools/go/packages"
+)
+
+func check(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+type platformtestFuncDeclFinder struct {
+	testFuncs []*ast.FuncDecl
+}
+
+func isPlatformtestFunc(n *ast.FuncDecl) bool {
+	if !n.Name.IsExported() {
+		return false
+	}
+	if n.Recv != nil {
+		return false
+	}
+	if n.Type.Results.NumFields() != 0 {
+		return false
+	}
+	if n.Type.Params.NumFields() != 1 {
+		return false
+	}
+	se, ok := n.Type.Params.List[0].Type.(*ast.StarExpr)
+	if !ok {
+		return false
+	}
+	sel, ok := se.X.(*ast.SelectorExpr)
+	if !ok {
+		return false
+	}
+	x, ok := sel.X.(*ast.Ident)
+	if !ok {
+		return false
+	}
+	if x.Name != "platformtest" || sel.Sel.Name != "Context" {
+		return false
+	}
+	return true
+}
+
+func (e *platformtestFuncDeclFinder) Visit(n2 ast.Node) ast.Visitor {
+	switch n := n2.(type) {
+	case *ast.File:
+		return e
+	case *ast.FuncDecl:
+		if isPlatformtestFunc(n) {
+			e.testFuncs = append(e.testFuncs, n)
+		}
+		return nil
+	default:
+		return nil
+	}
+}
+
+func main() {
+
+	// TODO safeguards that prevent us from deleting non-generated generated_cases.go
+	os.Remove("generated_cases.go")
+	// (no error handling to easily cover the case where the file doesn't exist)
+
+	pkgs, err := packages.Load(
+		&packages.Config{
+			Mode:  packages.NeedFiles,
+			Tests: false,
+		},
+		os.Args[1],
+	)
+	check(err)
+
+	if len(pkgs) != 1 {
+		panic(pkgs)
+	}
+
+	p := pkgs[0]
+
+	var tests []*ast.FuncDecl
+
+	for _, f := range p.GoFiles {
+		s := token.NewFileSet()
+		a, err := parser.ParseFile(s, f, nil, parser.AllErrors)
+		check(err)
+		finder := &platformtestFuncDeclFinder{}
+		ast.Walk(finder, a)
+		tests = append(tests, finder.testFuncs...)
+	}
+
+	sort.Slice(tests, func(i, j int) bool {
+		return strings.Compare(tests[i].Name.Name, tests[j].Name.Name) < 0
+	})
+
+	{
+		casesTemplate := `
+// Code generated by zrepl tooling; DO NOT EDIT.
+
+package tests
+
+var Cases = []Case {
+{{- range . -}}
+	{{ .Name }},
+{{ end -}}
+}
+
+	`
+		t, err := template.New("CaseFunc").Parse(casesTemplate)
+		check(err)
+
+		var buf bytes.Buffer
+		err = t.Execute(&buf, tests)
+		check(err)
+
+		formatted, err := format.Source(buf.Bytes())
+		check(err)
+
+		err = os.WriteFile("generated_cases.go", formatted, 0664)
+		check(err)
+
+	}
+
+}

internal/platformtest/tests/generated_cases.go

@@ -0,0 +1,52 @@
+// Code generated by zrepl tooling; DO NOT EDIT.
+
+package tests
+
+var Cases = []Case{BatchDestroy,
+	CreateReplicationCursor,
+	GetNonexistent,
+	HoldsWork,
+	IdempotentBookmark,
+	IdempotentDestroy,
+	IdempotentHold,
+	ListFilesystemVersionsFilesystemNotExist,
+	ListFilesystemVersionsTypeFilteringAndPrefix,
+	ListFilesystemVersionsUserrefs,
+	ListFilesystemVersionsZeroExistIsNotAnError,
+	ListFilesystemsNoFilter,
+	ReceiveForceIntoEncryptedErr,
+	ReceiveForceRollbackWorksUnencrypted,
+	ReplicationFailingInitialParentProhibitsChildReplication,
+	ReplicationIncrementalCleansUpStaleAbstractionsWithCacheOnSecondReplication,
+	ReplicationIncrementalCleansUpStaleAbstractionsWithoutCacheOnSecondReplication,
+	ReplicationIncrementalDestroysStepHoldsIffIncrementalStepHoldsAreDisabledButStepHoldsExist,
+	ReplicationIncrementalHandlesFromVersionEqTentativeCursorCorrectly,
+	ReplicationIncrementalIsPossibleIfCommonSnapshotIsDestroyed,
+	ReplicationInitialAll,
+	ReplicationInitialFail,
+	ReplicationInitialMostRecent,
+	ReplicationIsResumableFullSend__both_GuaranteeResumability,
+	ReplicationIsResumableFullSend__initial_GuaranteeIncrementalReplication_incremental_GuaranteeIncrementalReplication,
+	ReplicationIsResumableFullSend__initial_GuaranteeResumability_incremental_GuaranteeIncrementalReplication,
+	ReplicationOfPlaceholderFilesystemsInChainedReplicationScenario,
+	ReplicationPlaceholderEncryption__EncryptOnReceiverUseCase__WorksIfConfiguredWithInherit,
+	ReplicationPlaceholderEncryption__UnspecifiedIsOkForClientIdentityPlaceholder,
+	ReplicationPlaceholderEncryption__UnspecifiedLeadsToFailureAtRuntimeWhenCreatingPlaceholders,
+	ReplicationPropertyReplicationWorks,
+	ReplicationReceiverErrorWhileStillSending,
+	ReplicationStepCompletedLostBehavior__GuaranteeIncrementalReplication,
+	ReplicationStepCompletedLostBehavior__GuaranteeResumability,
+	ResumableRecvAndTokenHandling,
+	ResumeTokenParsing,
+	SendArgsValidationEE_EncryptionAndRaw,
+	SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden__EncryptionSupported_false,
+	SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden__EncryptionSupported_true,
+	SendArgsValidationResumeTokenDifferentFilesystemForbidden,
+	SendArgsValidationResumeTokenEncryptionMismatchForbidden,
+	SendStreamCloseAfterBlockedOnPipeWrite,
+	SendStreamCloseAfterEOFRead,
+	SendStreamMultipleCloseAfterEOF,
+	SendStreamMultipleCloseBeforeEOF,
+	SendStreamNonEOFReadErrorHandling,
+	UndestroyableSnapshotParsing,
+}

internal/platformtest/tests/getNonexistent.go

@@ -0,0 +1,64 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/zrepl/zrepl/internal/platformtest"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+func GetNonexistent(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@1"
+	`)
+
+	// test raw
+	_, err := zfs.ZFSGetRawAnySource(ctx, fmt.Sprintf("%s/foo bar", ctx.RootDataset), []string{"name"})
+	if err != nil {
+		panic(err)
+	}
+
+	// test nonexistent filesystem
+	nonexistent := fmt.Sprintf("%s/nonexistent filesystem", ctx.RootDataset)
+	props, err := zfs.ZFSGetRawAnySource(ctx, nonexistent, []string{"name"})
+	if err == nil {
+		panic(props)
+	}
+	dsne, ok := err.(*zfs.DatasetDoesNotExist)
+	if !ok {
+		panic(err)
+	} else if dsne.Path != nonexistent {
+		panic(err)
+	}
+
+	// test nonexistent snapshot
+	nonexistent = fmt.Sprintf("%s/foo bar@non existent", ctx.RootDataset)
+	props, err = zfs.ZFSGetRawAnySource(ctx, nonexistent, []string{"name"})
+	if err == nil {
+		panic(props)
+	}
+	dsne, ok = err.(*zfs.DatasetDoesNotExist)
+	if !ok {
+		panic(err)
+	} else if dsne.Path != nonexistent {
+		panic(err)
+	}
+
+	// test nonexistent bookmark
+	nonexistent = fmt.Sprintf("%s/foo bar#non existent", ctx.RootDataset)
+	props, err = zfs.ZFSGetRawAnySource(ctx, nonexistent, []string{"name"})
+	if err == nil {
+		panic(props)
+	}
+	dsne, ok = err.(*zfs.DatasetDoesNotExist)
+	if !ok {
+		panic(err)
+	} else if dsne.Path != nonexistent {
+		panic(err)
+	}
+
+}

internal/platformtest/tests/helpers.go

@@ -0,0 +1,182 @@
+package tests
+
+import (
+	"io"
+	"math/rand"
+	"os"
+	"path"
+	"sort"
+	"strings"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/daemon/filters"
+	"github.com/zrepl/zrepl/internal/platformtest"
+	"github.com/zrepl/zrepl/internal/util/limitio"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+func sendArgVersion(ctx *platformtest.Context, fs, relName string) zfs.ZFSSendArgVersion {
+	guid, err := zfs.ZFSGetGUID(ctx, fs, relName)
+	if err != nil {
+		panic(err)
+	}
+	return zfs.ZFSSendArgVersion{
+		RelName: relName,
+		GUID:    guid,
+	}
+}
+
+func fsversion(ctx *platformtest.Context, fs, relname string) zfs.FilesystemVersion {
+	v, err := zfs.ZFSGetFilesystemVersion(ctx, fs+relname)
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
+func mustDatasetPath(fs string) *zfs.DatasetPath {
+	p, err := zfs.NewDatasetPath(fs)
+	if err != nil {
+		panic(err)
+	}
+	return p
+}
+
+func mustSnapshot(ctx *platformtest.Context, snap string) {
+	if err := zfs.EntityNamecheck(snap, zfs.EntityTypeSnapshot); err != nil {
+		panic(err)
+	}
+	comps := strings.Split(snap, "@")
+	if len(comps) != 2 {
+		panic(comps)
+	}
+	err := zfs.ZFSSnapshot(ctx, mustDatasetPath(comps[0]), comps[1], false)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func mustGetFilesystemVersion(ctx *platformtest.Context, snapOrBookmark string) zfs.FilesystemVersion {
+	v, err := zfs.ZFSGetFilesystemVersion(ctx, snapOrBookmark)
+	check(err)
+	return v
+}
+
+func check(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+var dummyDataRand = rand.New(rand.NewSource(99))
+
+func writeDummyData(path string, numBytes int64) {
+	r := io.LimitReader(dummyDataRand, numBytes)
+	d, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
+	check(err)
+	defer d.Close()
+	_, err = io.Copy(d, r)
+	check(err)
+}
+
+type dummySnapshotSituation struct {
+	sendFS       string
+	dummyDataLen int64
+	snapA        *zfs.ZFSSendArgVersion
+	snapB        *zfs.ZFSSendArgVersion
+}
+
+type resumeSituation struct {
+	sendArgs         zfs.ZFSSendArgsUnvalidated
+	recvOpts         zfs.RecvOptions
+	sendErr, recvErr error
+	recvErrDecoded   *zfs.RecvFailedWithResumeTokenErr
+}
+
+func makeDummyDataSnapshots(ctx *platformtest.Context, sendFS string) (situation dummySnapshotSituation) {
+
+	situation.sendFS = sendFS
+	sendFSMount, err := zfs.ZFSGetMountpoint(ctx, sendFS)
+	require.NoError(ctx, err)
+	require.True(ctx, sendFSMount.Mounted)
+
+	const dummyLen = int64(10 * (1 << 20))
+	situation.dummyDataLen = dummyLen
+
+	writeDummyData(path.Join(sendFSMount.Mountpoint, "dummy_data"), dummyLen)
+	mustSnapshot(ctx, sendFS+"@a snapshot")
+	snapA := sendArgVersion(ctx, sendFS, "@a snapshot")
+	situation.snapA = &snapA
+
+	writeDummyData(path.Join(sendFSMount.Mountpoint, "dummy_data"), dummyLen)
+	mustSnapshot(ctx, sendFS+"@b snapshot")
+	snapB := sendArgVersion(ctx, sendFS, "@b snapshot")
+	situation.snapB = &snapB
+
+	return situation
+}
+
+func makeResumeSituation(ctx *platformtest.Context, src dummySnapshotSituation, recvFS string, sendArgs zfs.ZFSSendArgsUnvalidated, recvOptions zfs.RecvOptions) *resumeSituation {
+
+	situation := &resumeSituation{}
+
+	situation.sendArgs = sendArgs
+	situation.recvOpts = recvOptions
+	require.True(ctx, recvOptions.SavePartialRecvState, "this method would be pointless otherwise")
+	require.Equal(ctx, sendArgs.FS, src.sendFS)
+	sendArgsValidated, err := sendArgs.Validate(ctx)
+	situation.sendErr = err
+	if err != nil {
+		return situation
+	}
+
+	copier, err := zfs.ZFSSend(ctx, sendArgsValidated)
+	situation.sendErr = err
+	if err != nil {
+		return situation
+	}
+
+	limitedCopier := limitio.ReadCloser(copier, src.dummyDataLen/2)
+	defer limitedCopier.Close()
+
+	require.NotNil(ctx, sendArgs.To)
+	err = zfs.ZFSRecv(ctx, recvFS, sendArgs.To, limitedCopier, recvOptions)
+	situation.recvErr = err
+	ctx.Logf("zfs recv exit with %T %s", err, err)
+
+	require.NotNil(ctx, err)
+	resumeErr, ok := err.(*zfs.RecvFailedWithResumeTokenErr)
+	require.True(ctx, ok)
+	situation.recvErrDecoded = resumeErr
+
+	return situation
+}
+
+func versionRelnamesSorted(versions []zfs.FilesystemVersion) []string {
+	var vstrs []string
+	for _, v := range versions {
+		vstrs = append(vstrs, v.RelName())
+	}
+	sort.Strings(vstrs)
+	return vstrs
+}
+
+func datasetToStringSortedTrimPrefix(prefix *zfs.DatasetPath, paths []*zfs.DatasetPath) []string {
+	var pstrs []string
+	for _, p := range paths {
+		trimmed := p.Copy()
+		trimmed.TrimPrefix(prefix)
+		if trimmed.Length() == 0 {
+			continue
+		}
+		pstrs = append(pstrs, trimmed.ToString())
+	}
+	sort.Strings(pstrs)
+	return pstrs
+}
+
+func mustAddToSFilter(ctx *platformtest.Context, f *filters.DatasetMapFilter, fs string) {
+	err := f.Add(fs, "ok")
+	require.NoError(ctx, err)
+}

internal/platformtest/tests/holds.go

@@ -0,0 +1,33 @@
+package tests
+
+import (
+	"path"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/platformtest"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+func HoldsWork(ctx *platformtest.Context) {
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@snap name"
+	`)
+
+	fs := path.Join(ctx.RootDataset, "foo bar")
+
+	err := zfs.ZFSHold(ctx, fs, fsversion(ctx, fs, "@snap name"), "tag 1")
+	require.NoError(ctx, err)
+
+	err = zfs.ZFSHold(ctx, fs, fsversion(ctx, fs, "@snap name"), "tag 2")
+	require.NoError(ctx, err)
+
+	holds, err := zfs.ZFSHolds(ctx, fs, "snap name")
+	require.NoError(ctx, err)
+	require.Len(ctx, holds, 2)
+	require.Contains(ctx, holds, "tag 1")
+	require.Contains(ctx, holds, "tag 2")
+}

internal/platformtest/tests/idempotentBookmark.go

@@ -0,0 +1,62 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/zrepl/zrepl/internal/platformtest"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+func IdempotentBookmark(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@a snap"
+		+  "foo bar@another snap"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", ctx.RootDataset)
+
+	asnap := fsversion(ctx, fs, "@a snap")
+	anotherSnap := fsversion(ctx, fs, "@another snap")
+
+	aBookmark, err := zfs.ZFSBookmark(ctx, fs, asnap, "a bookmark")
+	if err != nil {
+		panic(err)
+	}
+
+	// do it again, should be idempotent
+	aBookmarkIdemp, err := zfs.ZFSBookmark(ctx, fs, asnap, "a bookmark")
+	if err != nil {
+		panic(err)
+	}
+	assert.Equal(ctx, aBookmark, aBookmarkIdemp)
+
+	// should fail for another snapshot
+	_, err = zfs.ZFSBookmark(ctx, fs, anotherSnap, "a bookmark")
+	if err == nil {
+		panic(err)
+	}
+	if _, ok := err.(*zfs.BookmarkExists); !ok {
+		panic(fmt.Sprintf("has type %T", err))
+	}
+
+	// destroy the snapshot
+	if err := zfs.ZFSDestroy(ctx, fmt.Sprintf("%s@a snap", fs)); err != nil {
+		panic(err)
+	}
+
+	// do it again, should fail with special error type
+	_, err = zfs.ZFSBookmark(ctx, fs, asnap, "a bookmark")
+	if err == nil {
+		panic(err)
+	}
+	if _, ok := err.(*zfs.DatasetDoesNotExist); !ok {
+		panic(fmt.Sprintf("has type %T", err))
+	}
+
+}

internal/platformtest/tests/idempotentDestroy.go

@@ -0,0 +1,75 @@
+package tests
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/zrepl/zrepl/internal/platformtest"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+func IdempotentDestroy(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@a snap"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", ctx.RootDataset)
+	asnap := fsversion(ctx, fs, "@a snap")
+	_, err := zfs.ZFSBookmark(ctx, fs, asnap, "a bookmark")
+	if err != nil {
+		panic(err)
+	}
+
+	type testCase struct {
+		description, path string
+	}
+
+	cases := []testCase{
+		{"snapshot", fmt.Sprintf("%s@a snap", fs)},
+		{"bookmark", fmt.Sprintf("%s#a bookmark", fs)},
+		{"filesystem", fs},
+	}
+
+	for i := range cases {
+		func() {
+			c := cases[i]
+
+			log.Printf("SUBBEGIN testing idempotent destroy %q for path %q", c.description, c.path)
+
+			log.Println("destroy existing")
+			err = zfs.ZFSDestroy(ctx, c.path)
+			if err != nil {
+				panic(err)
+			}
+			log.Println("destroy again, non-idempotently, must error")
+			err = zfs.ZFSDestroy(ctx, c.path)
+			if _, ok := err.(*zfs.DatasetDoesNotExist); !ok {
+				panic(fmt.Sprintf("%T: %s", err, err))
+			}
+			log.Println("destroy again, idempotently, must not error")
+			err = zfs.ZFSDestroyIdempotent(ctx, c.path)
+			if err != nil {
+				panic(err)
+			}
+
+			log.Println("SUBEND")
+
+		}()
+	}
+
+	// also test idempotent destroy for cases where the parent dataset does not exist
+	err = zfs.ZFSDestroyIdempotent(ctx, fmt.Sprintf("%s/not foo bar@nonexistent snapshot", ctx.RootDataset))
+	if err != nil {
+		panic(err)
+	}
+
+	err = zfs.ZFSDestroyIdempotent(ctx, fmt.Sprintf("%s/not foo bar#nonexistent bookmark", ctx.RootDataset))
+	if err != nil {
+		panic(err)
+	}
+
+}

internal/platformtest/tests/idempotentHold.go

@@ -0,0 +1,37 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/zrepl/zrepl/internal/platformtest"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+func IdempotentHold(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@1"
+	`)
+	defer platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		R zfs release zrepl_platformtest "${ROOTDS}/foo bar@1"
+		-  "foo bar@1"
+		-  "foo bar"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", ctx.RootDataset)
+	v1 := fsversion(ctx, fs, "@1")
+
+	tag := "zrepl_platformtest"
+	err := zfs.ZFSHold(ctx, fs, v1, tag)
+	if err != nil {
+		panic(err)
+	}
+
+	err = zfs.ZFSHold(ctx, fs, v1, tag)
+	if err != nil {
+		panic(err)
+	}
+}

internal/platformtest/tests/listFilesystemVersions.go

@@ -0,0 +1,176 @@
+package tests
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/platformtest"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+func ListFilesystemVersionsTypeFilteringAndPrefix(t *platformtest.Context) {
+	platformtest.Run(t, platformtest.PanicErr, t.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@foo 1"
+		+  "foo bar#foo 1" "foo bar@foo 1"
+		+  "foo bar#bookfoo 1" "foo bar@foo 1"
+		+  "foo bar@foo 2"
+		+  "foo bar#foo 2" "foo bar@foo 2"
+		+  "foo bar#bookfoo 2" "foo bar@foo 2"
+		+  "foo bar@blup 1"
+		+  "foo bar#blup 1" "foo bar@blup 1"
+		+  "foo bar@ foo with leading whitespace"
+
+		# repeat the whole thing for a child dataset to make sure we disable recursion
+
+		+  "foo bar/child dataset"
+		+  "foo bar/child dataset@foo 1"
+		+  "foo bar/child dataset#foo 1" "foo bar/child dataset@foo 1"
+		+  "foo bar/child dataset#bookfoo 1" "foo bar/child dataset@foo 1"
+		+  "foo bar/child dataset@foo 2"
+		+  "foo bar/child dataset#foo 2" "foo bar/child dataset@foo 2"
+		+  "foo bar/child dataset#bookfoo 2" "foo bar/child dataset@foo 2"
+		+  "foo bar/child dataset@blup 1"
+		+  "foo bar/child dataset#blup 1" "foo bar/child dataset@blup 1"
+		+  "foo bar/child dataset@ foo with leading whitespace"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", t.RootDataset)
+
+	// no options := all types
+	vs, err := zfs.ZFSListFilesystemVersions(t, mustDatasetPath(fs), zfs.ListFilesystemVersionsOptions{})
+	require.NoError(t, err)
+	require.Equal(t, []string{
+		"#blup 1", "#bookfoo 1", "#bookfoo 2", "#foo 1", "#foo 2",
+		"@ foo with leading whitespace", "@blup 1", "@foo 1", "@foo 2",
+	}, versionRelnamesSorted(vs))
+
+	// just snapshots
+	vs, err = zfs.ZFSListFilesystemVersions(t, mustDatasetPath(fs), zfs.ListFilesystemVersionsOptions{
+		Types: zfs.Snapshots,
+	})
+	require.NoError(t, err)
+	require.Equal(t, []string{"@ foo with leading whitespace", "@blup 1", "@foo 1", "@foo 2"}, versionRelnamesSorted(vs))
+
+	// just bookmarks
+	vs, err = zfs.ZFSListFilesystemVersions(t, mustDatasetPath(fs), zfs.ListFilesystemVersionsOptions{
+		Types: zfs.Bookmarks,
+	})
+	require.NoError(t, err)
+	require.Equal(t, []string{"#blup 1", "#bookfoo 1", "#bookfoo 2", "#foo 1", "#foo 2"}, versionRelnamesSorted(vs))
+
+	// just with prefix foo
+	vs, err = zfs.ZFSListFilesystemVersions(t, mustDatasetPath(fs), zfs.ListFilesystemVersionsOptions{
+		ShortnamePrefix: "foo",
+	})
+	require.NoError(t, err)
+	require.Equal(t, []string{"#foo 1", "#foo 2", "@foo 1", "@foo 2"}, versionRelnamesSorted(vs))
+
+}
+
+func ListFilesystemVersionsZeroExistIsNotAnError(t *platformtest.Context) {
+	platformtest.Run(t, platformtest.PanicErr, t.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	+ "foo bar"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", t.RootDataset)
+
+	vs, err := zfs.ZFSListFilesystemVersions(t, mustDatasetPath(fs), zfs.ListFilesystemVersionsOptions{})
+	require.Empty(t, vs)
+	require.NoError(t, err)
+}
+
+func ListFilesystemVersionsFilesystemNotExist(t *platformtest.Context) {
+	platformtest.Run(t, platformtest.PanicErr, t.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	`)
+
+	nonexistentFS := fmt.Sprintf("%s/not existent", t.RootDataset)
+
+	vs, err := zfs.ZFSListFilesystemVersions(t, mustDatasetPath(nonexistentFS), zfs.ListFilesystemVersionsOptions{})
+	require.Empty(t, vs)
+	require.Error(t, err)
+	t.Logf("err = %T\n%s", err, err)
+	dsne, ok := err.(*zfs.DatasetDoesNotExist)
+	require.True(t, ok)
+	require.Equal(t, nonexistentFS, dsne.Path)
+}
+
+func ListFilesystemVersionsUserrefs(t *platformtest.Context) {
+	platformtest.Run(t, platformtest.PanicErr, t.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	+ "foo bar"
+	+ "foo bar@snap 1"
+	+ "foo bar#snap 1" "foo bar@snap 1"
+	+ "foo bar@snap 2"
+	+ "foo bar#snap 2" "foo bar@snap 2"
+	R zfs hold zrepl_platformtest "${ROOTDS}/foo bar@snap 2"
+	+ "foo bar@snap 3"
+	+ "foo bar#snap 3" "foo bar@snap 3"
+	R zfs hold zrepl_platformtest "${ROOTDS}/foo bar@snap 3"
+	R zfs hold zrepl_platformtest_second_hold "${ROOTDS}/foo bar@snap 3"
+	+ "foo bar@snap 4"
+	+ "foo bar#snap 4" "foo bar@snap 4"
+
+
+	+ "foo bar/child datset"
+	+ "foo bar/child datset@snap 1"
+	+ "foo bar/child datset#snap 1" "foo bar/child datset@snap 1"
+	+ "foo bar/child datset@snap 2"
+	+ "foo bar/child datset#snap 2" "foo bar/child datset@snap 2"
+	R zfs hold zrepl_platformtest "${ROOTDS}/foo bar/child datset@snap 2"
+	+ "foo bar/child datset@snap 3"
+	+ "foo bar/child datset#snap 3" "foo bar/child datset@snap 3"
+	R zfs hold zrepl_platformtest "${ROOTDS}/foo bar/child datset@snap 3"
+	R zfs hold zrepl_platformtest_second_hold "${ROOTDS}/foo bar/child datset@snap 3"
+	+ "foo bar/child datset@snap 4"
+	+ "foo bar/child datset#snap 4" "foo bar/child datset@snap 4"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", t.RootDataset)
+
+	vs, err := zfs.ZFSListFilesystemVersions(t, mustDatasetPath(fs), zfs.ListFilesystemVersionsOptions{})
+	require.NoError(t, err)
+
+	type expectation struct {
+		relName  string
+		userrefs zfs.OptionUint64
+	}
+
+	expect := []expectation{
+		{"#snap 1", zfs.OptionUint64{Valid: false}},
+		{"#snap 2", zfs.OptionUint64{Valid: false}},
+		{"#snap 3", zfs.OptionUint64{Valid: false}},
+		{"#snap 4", zfs.OptionUint64{Valid: false}},
+		{"@snap 1", zfs.OptionUint64{Value: 0, Valid: true}},
+		{"@snap 2", zfs.OptionUint64{Value: 1, Valid: true}},
+		{"@snap 3", zfs.OptionUint64{Value: 2, Valid: true}},
+		{"@snap 4", zfs.OptionUint64{Value: 0, Valid: true}},
+	}
+
+	sort.Slice(vs, func(i, j int) bool {
+		return strings.Compare(vs[i].RelName(), vs[j].RelName()) < 0
+	})
+
+	var expectRelNames []string
+	for _, e := range expect {
+		expectRelNames = append(expectRelNames, e.relName)
+	}
+
+	require.Equal(t, expectRelNames, versionRelnamesSorted(vs))
+
+	for i, e := range expect {
+		require.Equal(t, e.relName, vs[i].RelName())
+		require.Equal(t, e.userrefs, vs[i].UserRefs)
+	}
+
+}

internal/platformtest/tests/listFilesystems.go

@@ -0,0 +1,34 @@
+package tests
+
+import (
+	"strings"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/platformtest"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+func ListFilesystemsNoFilter(t *platformtest.Context) {
+	platformtest.Run(t, platformtest.PanicErr, t.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		R  zfs create -V 10M "${ROOTDS}/bar baz"
+		+  "foo bar"
+		+  "foo bar/bar blup"
+		+  "foo bar/blah"
+		R  zfs create -V 10M "${ROOTDS}/foo bar/blah/a volume"
+	`)
+
+	fss, err := zfs.ZFSListMapping(t, zfs.NoFilter())
+	require.NoError(t, err)
+	var onlyTestPool []*zfs.DatasetPath
+	for _, fs := range fss {
+		if strings.HasPrefix(fs.ToString(), t.RootDataset) {
+			onlyTestPool = append(onlyTestPool, fs)
+		}
+	}
+	onlyTestPoolStr := datasetToStringSortedTrimPrefix(mustDatasetPath(t.RootDataset), onlyTestPool)
+	require.Equal(t, []string{"bar baz", "foo bar", "foo bar/bar blup", "foo bar/blah", "foo bar/blah/a volume"}, onlyTestPoolStr)
+
+}

internal/platformtest/tests/recvForceIntoEncryptedErr.go

@@ -0,0 +1,58 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/platformtest"
+	"github.com/zrepl/zrepl/internal/util/nodefault"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+func ReceiveForceIntoEncryptedErr(ctx *platformtest.Context) {
+
+	supported, err := zfs.EncryptionCLISupported(ctx)
+	require.NoError(ctx, err, "encryption feature test failed")
+	if !supported {
+		ctx.SkipNow()
+		return
+	}
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar" encrypted
+		+  "sender" encrypted
+		+  "sender@1"
+	`)
+
+	rfs := fmt.Sprintf("%s/foo bar", ctx.RootDataset)
+	sfs := fmt.Sprintf("%s/sender", ctx.RootDataset)
+	sfsSnap1 := sendArgVersion(ctx, sfs, "@1")
+
+	sendArgs, err := zfs.ZFSSendArgsUnvalidated{
+		FS:   sfs,
+		From: nil,
+		To:   &sfsSnap1,
+		ZFSSendFlags: zfs.ZFSSendFlags{
+			Encrypted:   &nodefault.Bool{B: false},
+			ResumeToken: "",
+		},
+	}.Validate(ctx)
+	require.NoError(ctx, err)
+
+	sendStream, err := zfs.ZFSSend(ctx, sendArgs)
+	require.NoError(ctx, err)
+	defer sendStream.Close()
+
+	recvOpts := zfs.RecvOptions{
+		RollbackAndForceRecv: true,
+		SavePartialRecvState: false,
+	}
+	err = zfs.ZFSRecv(ctx, rfs, &zfs.ZFSSendArgVersion{RelName: "@1", GUID: sfsSnap1.GUID}, sendStream, recvOpts)
+	require.Error(ctx, err)
+	re, ok := err.(*zfs.RecvDestroyOrOverwriteEncryptedErr)
+	require.True(ctx, ok)
+	require.Contains(ctx, re.Error(), "zfs receive -F cannot be used to destroy an encrypted filesystem or overwrite an unencrypted one with an encrypted on")
+}

internal/platformtest/tests/recvRollback.go

@@ -0,0 +1,59 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/platformtest"
+	"github.com/zrepl/zrepl/internal/util/nodefault"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+func ReceiveForceRollbackWorksUnencrypted(ctx *platformtest.Context) {
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@a snap"
+		+  "foo bar@another snap"
+		+  "foo bar@snap3"
+		+  "sender"
+		+  "sender@1"
+	`)
+
+	rfs := fmt.Sprintf("%s/foo bar", ctx.RootDataset)
+	sfs := fmt.Sprintf("%s/sender", ctx.RootDataset)
+	sfsSnap1 := sendArgVersion(ctx, sfs, "@1")
+
+	sendArgs, err := zfs.ZFSSendArgsUnvalidated{
+		FS:   sfs,
+		From: nil,
+		To:   &sfsSnap1,
+		ZFSSendFlags: zfs.ZFSSendFlags{
+			Encrypted:   &nodefault.Bool{B: false},
+			ResumeToken: "",
+		},
+	}.Validate(ctx)
+	require.NoError(ctx, err)
+
+	sendStream, err := zfs.ZFSSend(ctx, sendArgs)
+	require.NoError(ctx, err)
+	defer sendStream.Close()
+
+	recvOpts := zfs.RecvOptions{
+		RollbackAndForceRecv: true,
+		SavePartialRecvState: false,
+	}
+	err = zfs.ZFSRecv(ctx, rfs, &zfs.ZFSSendArgVersion{RelName: "@1", GUID: sfsSnap1.GUID}, sendStream, recvOpts)
+	require.NoError(ctx, err)
+
+	// assert exists on receiver
+	rfsSnap1 := fsversion(ctx, rfs, "@1")
+	// assert it's the only one (rollback and force-recv should be blowing away the other filesystems)
+	rfsVersions, err := zfs.ZFSListFilesystemVersions(ctx, mustDatasetPath(rfs), zfs.ListFilesystemVersionsOptions{})
+	require.NoError(ctx, err)
+	assert.Len(ctx, rfsVersions, 1)
+	assert.Equal(ctx, rfsVersions[0], rfsSnap1)
+}

internal/platformtest/tests/replicationCursor.go

@@ -0,0 +1,80 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/endpoint"
+	"github.com/zrepl/zrepl/internal/platformtest"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+func CreateReplicationCursor(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@1 with space"
+		R  zfs bookmark "${ROOTDS}/foo bar@1 with space" "${ROOTDS}/foo bar#1 with space"
+		+  "foo bar@2 with space"
+		R  zfs bookmark "${ROOTDS}/foo bar@2 with space" "${ROOTDS}/foo bar#2 with space"
+		+  "foo bar@3 with space"
+		R  zfs bookmark "${ROOTDS}/foo bar@3 with space" "${ROOTDS}/foo bar#3 with space"
+		-  "foo bar@3 with space"
+	`)
+
+	jobid := endpoint.MustMakeJobID("zreplplatformtest")
+
+	ds, err := zfs.NewDatasetPath(ctx.RootDataset + "/foo bar")
+	if err != nil {
+		panic(err)
+	}
+
+	fs := ds.ToString()
+
+	checkCreateCursor := func(createErr error, c endpoint.Abstraction, references zfs.FilesystemVersion) {
+		assert.NoError(ctx, createErr)
+		expectName, err := endpoint.ReplicationCursorBookmarkName(fs, references.Guid, jobid)
+		assert.NoError(ctx, err)
+		require.Equal(ctx, expectName, c.GetFilesystemVersion().Name)
+	}
+
+	snap := fsversion(ctx, fs, "@1 with space")
+	book := fsversion(ctx, fs, "#1 with space")
+	book3 := fsversion(ctx, fs, "#3 with space")
+
+	// create first cursor
+	cursorOfSnap, err := endpoint.CreateReplicationCursor(ctx, fs, snap, jobid)
+	checkCreateCursor(err, cursorOfSnap, snap)
+	// check CreateReplicationCursor is idempotent (for snapshot target)
+	cursorOfSnapIdemp, err := endpoint.CreateReplicationCursor(ctx, fs, snap, jobid)
+	checkCreateCursor(err, cursorOfSnap, snap)
+	// check CreateReplicationCursor is idempotent (for bookmark target of snapshot)
+	cursorOfBook, err := endpoint.CreateReplicationCursor(ctx, fs, book, jobid)
+	checkCreateCursor(err, cursorOfBook, snap)
+	// ... for target = non-cursor bookmark
+	_, err = endpoint.CreateReplicationCursor(ctx, fs, book3, jobid)
+	assert.Equal(ctx, zfs.ErrBookmarkCloningNotSupported, err)
+	// ... for target = replication cursor bookmark to be created
+	cursorOfCursor, err := endpoint.CreateReplicationCursor(ctx, fs, cursorOfSnapIdemp.GetFilesystemVersion(), jobid)
+	checkCreateCursor(err, cursorOfCursor, cursorOfSnap.GetFilesystemVersion())
+
+	snapProps, err := zfs.ZFSGetFilesystemVersion(ctx, snap.FullPath(fs))
+	if err != nil {
+		panic(err)
+	}
+
+	bm, err := endpoint.GetMostRecentReplicationCursorOfJob(ctx, fs, jobid)
+	if err != nil {
+		panic(err)
+	}
+	if bm.CreateTXG != snapProps.CreateTXG {
+		panic(fmt.Sprintf("createtxgs do not match: %v != %v", bm.CreateTXG, snapProps.CreateTXG))
+	}
+	if bm.Guid != snapProps.Guid {
+		panic(fmt.Sprintf("guids do not match: %v != %v", bm.Guid, snapProps.Guid))
+	}
+
+}

internal/platformtest/tests/resumableRecvAndTokenHandling.go

@@ -0,0 +1,67 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/platformtest"
+	"github.com/zrepl/zrepl/internal/util/nodefault"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+func ResumableRecvAndTokenHandling(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	+	"send er"
+	`)
+
+	sendFS := fmt.Sprintf("%s/send er", ctx.RootDataset)
+	recvFS := fmt.Sprintf("%s/recv er", ctx.RootDataset)
+
+	supported, err := zfs.ResumeRecvSupported(ctx, mustDatasetPath(sendFS))
+	check(err)
+
+	src := makeDummyDataSnapshots(ctx, sendFS)
+
+	s := makeResumeSituation(ctx, src, recvFS, zfs.ZFSSendArgsUnvalidated{
+		FS: sendFS,
+		To: src.snapA,
+		ZFSSendFlags: zfs.ZFSSendFlags{
+			Encrypted:   &nodefault.Bool{B: false},
+			ResumeToken: "",
+		},
+	}, zfs.RecvOptions{
+		RollbackAndForceRecv: false, // doesnt' exist yet
+		SavePartialRecvState: true,
+	})
+
+	if !supported {
+		_, ok := s.recvErr.(*zfs.ErrRecvResumeNotSupported)
+		require.True(ctx, ok)
+
+		// we know that support on sendFS implies support on recvFS
+		// => assert that if we don't support resumed recv, the method returns ""
+		tok, err := zfs.ZFSGetReceiveResumeTokenOrEmptyStringIfNotSupported(ctx, mustDatasetPath(recvFS))
+		check(err)
+		require.Equal(ctx, "", tok)
+
+		return // nothing more to test for recv that doesn't support -s
+	}
+
+	getTokenRaw, err := zfs.ZFSGetReceiveResumeTokenOrEmptyStringIfNotSupported(ctx, mustDatasetPath(recvFS))
+	check(err)
+
+	require.NotEmpty(ctx, getTokenRaw)
+	decodedToken, err := zfs.ParseResumeToken(ctx, getTokenRaw)
+	check(err)
+
+	require.True(ctx, decodedToken.HasToGUID)
+	require.Equal(ctx, s.sendArgs.To.GUID, decodedToken.ToGUID)
+
+	recvErr := s.recvErr.(*zfs.RecvFailedWithResumeTokenErr)
+	require.Equal(ctx, recvErr.ResumeTokenRaw, getTokenRaw)
+	require.Equal(ctx, recvErr.ResumeTokenParsed, decodedToken)
+}

internal/platformtest/tests/resumeTokenParsing.go

@@ -0,0 +1,105 @@
+package tests
+
+import (
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/platformtest"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+type resumeTokenTest struct {
+	Msg         string
+	Token       string
+	ExpectToken *zfs.ResumeToken
+	ExpectError error
+}
+
+func (rtt *resumeTokenTest) Test(t *platformtest.Context) {
+
+	resumeSendSupported, err := zfs.ResumeSendSupported(t)
+	if err != nil {
+		t.Errorf("cannot determine whether resume supported: %T %s", err, err)
+		t.FailNow()
+		return
+	}
+
+	res, err := zfs.ParseResumeToken(t, rtt.Token)
+
+	// if decoding is not supported, don't bother with the expectations
+	if !resumeSendSupported {
+		require.Error(t, err)
+		require.Equal(t, zfs.ResumeTokenDecodingNotSupported, err)
+		return
+	}
+
+	if rtt.ExpectError != nil {
+		require.EqualValues(t, rtt.ExpectError, err)
+		return
+	}
+	if rtt.ExpectToken != nil {
+		require.Nil(t, err)
+		require.EqualValues(t, rtt.ExpectToken, res)
+		return
+	}
+}
+
+func ResumeTokenParsing(ctx *platformtest.Context) {
+
+	// cases generated using resumeTokensGenerate.bash on ZoL 0.8.1
+	cases := []resumeTokenTest{
+		{
+			Msg:   "zreplplatformtest/dst/full",
+			Token: "1-b338b54f3-c0-789c636064000310a500c4ec50360710e72765a52697303030419460caa7a515a796806474e0f26c48f2499525a9c540ba42430fabfe92fcf4d2cc140686c88a76d578ae45530c90e439c1f27989b9a90c0c5545a905390539892569f945b940234bf48b8b921d12c1660200c61a1aba",
+			ExpectToken: &zfs.ResumeToken{
+				HasToGUID: true,
+				ToGUID:    0x94a20a5f25877859,
+				ToName:    "zreplplatformtest/src@a",
+			},
+		},
+		{
+			Msg:   "zreplplatformtest/dst/full_raw",
+			Token: "1-e3f40c323-f8-789c636064000310a500c4ec50360710e72765a52697303030419460caa7a515a796806474e0f26c48f2499525a9c540da454f0fabfe92fcf4d2cc140686c88a76d578ae45530c90e439c1f27989b9a90c0c5545a905390539892569f945b940234bf48b8b921d12c1e6713320dc9f9c9f5b50945a5c9c9f0d119380ba07265f94580e936200004ff12141",
+			ExpectToken: &zfs.ResumeToken{
+				HasToGUID:     true,
+				ToGUID:        0x94a20a5f25877859,
+				ToName:        "zreplplatformtest/src@a",
+				HasCompressOK: true, CompressOK: true,
+				HasRawOk: true, RawOK: true,
+			},
+		},
+
+		{
+			Msg:   "zreplplatformtest/dst/inc",
+			Token: "1-eadabb296-e8-789c636064000310a501c49c50360710a715e5e7a69766a63040416445bb6a3cd7a2290a40363b92bafca4acd4e412060626a83a0cf9b4b4e2d412908c0e5c9e0d493ea9b224b518483ba8ea61d55f920f714515bf9b3fc3c396ef0648f29c60f9bcc4dc54a07c516a414e414e62495a7e512ed0c812fde2a2648724b09900d43e2191",
+			ExpectToken: &zfs.ResumeToken{
+				HasFromGUID: true, FromGUID: 0x94a20a5f25877859,
+				HasToGUID: true, ToGUID: 0xf784e1004f460f7a,
+				ToName: "zreplplatformtest/src@b",
+			},
+		},
+		{
+			Msg:   "zreplplatformtest/dst/inc_raw",
+			Token: "1-1164f8d409-120-789c636064000310a501c49c50360710a715e5e7a69766a63040416445bb6a3cd7a2290a40363b92bafca4acd4e412060626a83a0cf9b4b4e2d412908c0e5c9e0d493ea9b224b51848f368eb61d55f920f714515bf9b3fc3c396ef0648f29c60f9bcc4dc54a07c516a414e414e62495a7e512ed0c812fde2a2648724b079dc0c087f26e7e71614a51617e76743c424a0ee81c9172596c3a41800dd2c2818",
+			ExpectToken: &zfs.ResumeToken{
+				HasFromGUID: true, FromGUID: 0x94a20a5f25877859,
+				HasToGUID: true, ToGUID: 0xf784e1004f460f7a,
+				ToName:        "zreplplatformtest/src@b",
+				HasCompressOK: true, CompressOK: true,
+				HasRawOk: true, RawOK: true,
+			},
+		},
+
+		// manual test csaes
+		{
+			Msg:         "corrupted",
+			Token:       "1-1164f8d409-120-badf00d064000310a501c49c50360710a715e5e7a69766a63040416445bb6a3cd7a2290a40363b92bafca4acd4e412060626a83a0cf9b4b4e2d412908c0e5c9e0d493ea9b224b51848f368eb61d55f920f714515bf9b3fc3c396ef0648f29c60f9bcc4dc54a07c516a414e414e62495a7e512ed0c812fde2a2648724b079dc0c087f26e7e71614a51617e76743c424a0ee81c9172596c3a41800dd2c2818",
+			ExpectError: zfs.ResumeTokenCorruptError,
+		},
+	}
+
+	for _, test := range cases {
+		ctx.Logf("BEGIN SUBTEST: %s", test.Msg)
+		test.Test(ctx)
+		ctx.Logf("COMPLETE SUBTEST: %s", test.Msg)
+	}
+}

internal/platformtest/tests/resumeTokensGenerate.bash

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+
+POOLNAME="zreplplatformtest"
+POOLIMG="$1"
+
+if zpool status "$POOLNAME"; then
+    exit 1
+fi
+
+if [ -e "$POOLIMG" ]; then
+    exit 1
+fi
+fallocate -l 500M "$POOLIMG"
+zpool create "$POOLNAME" "$POOLIMG"
+
+SRC_DATASET="$POOLNAME/src"
+DST_ROOT="$POOLNAME/dst"
+
+keylocation="$(mktemp /tmp/ZREPL_PLATFORMTEST_GENERATE_TOKENS_KEYFILE_XXX)"
+echo "foobar123" > "$keylocation"
+zfs create -o encryption=on -o keylocation="file:///$keylocation" -o keyformat=passphrase "$SRC_DATASET"
+rm "$keylocation"
+
+SRC_MOUNT="$(zfs get -H -o value mountpoint "$SRC_DATASET")"
+test -d "$SRC_MOUNT"
+
+A="$SRC_DATASET"@a
+B="$SRC_DATASET"@b
+dd if=/dev/urandom of="$SRC_MOUNT"/dummy_data bs=1M count=5
+zfs snapshot "$A"
+dd if=/dev/urandom of="$SRC_MOUNT"/dummy_data bs=1M count=5
+zfs snapshot "$B"
+
+zfs create "$DST_ROOT"
+
+cutoff="dd bs=1024 count=3K"
+
+set +e
+zfs send    "$A"         | $cutoff | zfs recv -s "$DST_ROOT"/full
+
+zfs send "$A" | zfs recv "$DST_ROOT"/inc
+zfs send    -i "$A" "$B" | $cutoff | zfs recv -s "$DST_ROOT"/inc
+
+zfs send -w "$A"         | $cutoff | zfs recv -s "$DST_ROOT"/full_raw
+
+zfs send -w "$A" | zfs recv "$DST_ROOT"/inc_raw
+zfs send -w -i "$A" "$B" | $cutoff | zfs recv -s "$DST_ROOT"/inc_raw
+set -e
+
+TOKENS="$(zfs list -H -o name,receive_resume_token -r "$DST_ROOT")"
+
+echo "$TOKENS" | awk -e '//{ if ($2 == "-") {  } else { system("zfs send -nvt " + $2);} }'
+
+zpool destroy "$POOLNAME"
+rm "$POOLIMG"

internal/platformtest/tests/sendArgsValidation.go

@@ -0,0 +1,438 @@
+package tests
+
+import (
+	"fmt"
+	"path"
+
+	"github.com/kr/pretty"
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/endpoint"
+	"github.com/zrepl/zrepl/internal/platformtest"
+	"github.com/zrepl/zrepl/internal/replication/logic"
+	"github.com/zrepl/zrepl/internal/replication/logic/pdu"
+	"github.com/zrepl/zrepl/internal/replication/report"
+	"github.com/zrepl/zrepl/internal/util/nodefault"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+func SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden__EncryptionSupported_true(ctx *platformtest.Context) {
+	sendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden_impl(ctx, true)
+}
+
+func SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden__EncryptionSupported_false(ctx *platformtest.Context) {
+	sendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden_impl(ctx, false)
+}
+
+func sendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden_impl(ctx *platformtest.Context, testForEncryptionSupported bool) {
+
+	supported, err := zfs.EncryptionCLISupported(ctx)
+	check(err)
+	if supported != testForEncryptionSupported {
+		ctx.SkipNow()
+	}
+	noEncryptionCLISupport := !supported
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	+	"send er"
+	+   "send er@a snap"
+	`)
+
+	fs := fmt.Sprintf("%s/send er", ctx.RootDataset)
+	props := mustGetFilesystemVersion(ctx, fs+"@a snap")
+
+	sendArgs, err := zfs.ZFSSendArgsUnvalidated{
+		FS: fs,
+		To: &zfs.ZFSSendArgVersion{
+			RelName: "@a snap",
+			GUID:    props.Guid,
+		},
+		ZFSSendFlags: zfs.ZFSSendFlags{
+			Encrypted:   &nodefault.Bool{B: true},
+			ResumeToken: "",
+		},
+	}.Validate(ctx)
+
+	var stream *zfs.SendStream
+	if err == nil {
+		stream, err = zfs.ZFSSend(ctx, sendArgs) // no shadow
+		if err == nil {
+			defer stream.Close()
+		}
+		// fallthrough
+	}
+
+	if noEncryptionCLISupport {
+		require.Error(ctx, err)
+		saverr, ok := err.(*zfs.ZFSSendArgsValidationError)
+		require.True(ctx, ok, "%T", err)
+		require.Equal(ctx, zfs.ZFSSendArgsEncryptedSendRequestedButFSUnencrypted, saverr.What)
+		return
+	}
+	require.Error(ctx, err)
+	ctx.Logf("send err: %T %s", err, err)
+	validationErr, ok := err.(*zfs.ZFSSendArgsValidationError)
+	require.True(ctx, ok)
+	require.True(ctx, validationErr.What == zfs.ZFSSendArgsEncryptedSendRequestedButFSUnencrypted)
+}
+
+func SendArgsValidationResumeTokenEncryptionMismatchForbidden(ctx *platformtest.Context) {
+
+	supported, err := zfs.EncryptionCLISupported(ctx)
+	check(err)
+	if !supported {
+		ctx.SkipNow()
+	}
+	supported, err = zfs.ResumeSendSupported(ctx)
+	check(err)
+	if !supported {
+		ctx.SkipNow()
+	}
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	+	"send er" encrypted
+	`)
+
+	sendFS := fmt.Sprintf("%s/send er", ctx.RootDataset)
+	unencRecvFS := fmt.Sprintf("%s/unenc recv", ctx.RootDataset)
+	encRecvFS := fmt.Sprintf("%s/enc recv", ctx.RootDataset)
+
+	src := makeDummyDataSnapshots(ctx, sendFS)
+
+	unencS := makeResumeSituation(ctx, src, unencRecvFS, zfs.ZFSSendArgsUnvalidated{
+		FS:           sendFS,
+		To:           src.snapA,
+		ZFSSendFlags: zfs.ZFSSendFlags{Encrypted: &nodefault.Bool{B: false}}, // !
+	}, zfs.RecvOptions{
+		RollbackAndForceRecv: false,
+		SavePartialRecvState: true,
+	})
+
+	encS := makeResumeSituation(ctx, src, encRecvFS, zfs.ZFSSendArgsUnvalidated{
+		FS:           sendFS,
+		To:           src.snapA,
+		ZFSSendFlags: zfs.ZFSSendFlags{Encrypted: &nodefault.Bool{B: true}}, // !
+	}, zfs.RecvOptions{
+		RollbackAndForceRecv: false,
+		SavePartialRecvState: true,
+	})
+
+	// threat model: use of a crafted resume token that requests an unencrypted send
+	//               but send args require encrypted send
+	{
+		var maliciousSend zfs.ZFSSendArgsUnvalidated = encS.sendArgs
+		maliciousSend.ResumeToken = unencS.recvErrDecoded.ResumeTokenRaw
+
+		_, err := maliciousSend.Validate(ctx)
+		validationErr, ok := err.(*zfs.ZFSSendArgsValidationError)
+		require.True(ctx, ok)
+		require.Equal(ctx, validationErr.What, zfs.ZFSSendArgsResumeTokenMismatch)
+		ctx.Logf("%s", validationErr)
+
+		mismatchError, ok := validationErr.Msg.(*zfs.ZFSSendArgsResumeTokenMismatchError)
+		require.True(ctx, ok)
+		require.Equal(ctx, mismatchError.What, zfs.ZFSSendArgsResumeTokenMismatchEncryptionNotSet)
+	}
+
+	// threat model: use of a crafted resume token that requests an encrypted send
+	//               but send args require unencrypted send
+	{
+		var maliciousSend zfs.ZFSSendArgsUnvalidated = unencS.sendArgs
+		maliciousSend.ResumeToken = encS.recvErrDecoded.ResumeTokenRaw
+
+		_, err := maliciousSend.Validate(ctx)
+		require.Error(ctx, err)
+		ctx.Logf("send err: %T %s", err, err)
+		validationErr, ok := err.(*zfs.ZFSSendArgsValidationError)
+		require.True(ctx, ok)
+		require.Equal(ctx, validationErr.What, zfs.ZFSSendArgsResumeTokenMismatch)
+		ctx.Logf("%s", validationErr)
+
+		mismatchError, ok := validationErr.Msg.(*zfs.ZFSSendArgsResumeTokenMismatchError)
+		require.True(ctx, ok)
+		require.Equal(ctx, mismatchError.What, zfs.ZFSSendArgsResumeTokenMismatchEncryptionSet)
+	}
+
+}
+
+func SendArgsValidationResumeTokenDifferentFilesystemForbidden(ctx *platformtest.Context) {
+	supported, err := zfs.ResumeSendSupported(ctx)
+	check(err)
+	if !supported {
+		ctx.SkipNow()
+	}
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	+	"send er1"
+	+	"send er2"
+	`)
+
+	sendFS1 := fmt.Sprintf("%s/send er1", ctx.RootDataset)
+	sendFS2 := fmt.Sprintf("%s/send er2", ctx.RootDataset)
+	recvFS := fmt.Sprintf("%s/unenc recv", ctx.RootDataset)
+
+	src1 := makeDummyDataSnapshots(ctx, sendFS1)
+	src2 := makeDummyDataSnapshots(ctx, sendFS2)
+
+	rs := makeResumeSituation(ctx, src1, recvFS, zfs.ZFSSendArgsUnvalidated{
+		FS:           sendFS1,
+		To:           src1.snapA,
+		ZFSSendFlags: zfs.ZFSSendFlags{Encrypted: &nodefault.Bool{B: false}},
+	}, zfs.RecvOptions{
+		RollbackAndForceRecv: false,
+		SavePartialRecvState: true,
+	})
+
+	// threat model: forged resume token tries to steal a full send of snapA on fs2 by
+	//               presenting a resume token for full send of snapA on fs1
+	var maliciousSend zfs.ZFSSendArgsUnvalidated = zfs.ZFSSendArgsUnvalidated{
+		FS: sendFS2,
+		To: &zfs.ZFSSendArgVersion{
+			RelName: src2.snapA.RelName,
+			GUID:    src2.snapA.GUID,
+		},
+		ZFSSendFlags: zfs.ZFSSendFlags{
+			Encrypted:   &nodefault.Bool{B: false},
+			ResumeToken: rs.recvErrDecoded.ResumeTokenRaw,
+		},
+	}
+	_, err = maliciousSend.Validate(ctx)
+	require.Error(ctx, err)
+	ctx.Logf("send err: %T %s", err, err)
+	validationErr, ok := err.(*zfs.ZFSSendArgsValidationError)
+	require.True(ctx, ok)
+	require.Equal(ctx, validationErr.What, zfs.ZFSSendArgsResumeTokenMismatch)
+	ctx.Logf("%s", validationErr)
+
+	mismatchError, ok := validationErr.Msg.(*zfs.ZFSSendArgsResumeTokenMismatchError)
+	require.True(ctx, ok)
+	require.Equal(ctx, mismatchError.What, zfs.ZFSSendArgsResumeTokenMismatchFilesystem)
+}
+
+type sendArgsValidationEndToEndTestOutcome string
+
+const (
+	ValidationAccepts sendArgsValidationEndToEndTestOutcome = "accept"
+	ValidationRejects sendArgsValidationEndToEndTestOutcome = "rejects"
+)
+
+type sendArgsValidationEndToEndTest struct {
+	encryptedSenderFilesystem             bool
+	senderConfigHook                      func(config *endpoint.SenderConfig)
+	expectedOutcome                       sendArgsValidationEndToEndTestOutcome
+	outcomeRejectsInspectError            func(require.TestingT, *report.FilesystemReport, bool)
+	inspectReceiverFSAfterSuccessfulCycle func(rfs string)
+}
+
+func implSendArgsValidationEndToEndTest(ctx *platformtest.Context, setup sendArgsValidationEndToEndTest) {
+
+	senderEncrypted := ""
+	if setup.encryptedSenderFilesystem {
+		senderEncrypted = "encrypted"
+	}
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, fmt.Sprintf(`
+		CREATEROOT
+		+  "sender" %s
+		+  "receiver"
+		R  zfs create -p "${ROOTDS}/receiver/${ROOTDS}"
+	`, senderEncrypted))
+
+	sjid := endpoint.MustMakeJobID("sender-job")
+	rjid := endpoint.MustMakeJobID("receiver-job")
+
+	sfs := ctx.RootDataset + "/sender"
+	rfsRoot := ctx.RootDataset + "/receiver"
+
+	sfsmp, err := zfs.ZFSGetMountpoint(ctx, sfs)
+	require.NoError(ctx, err)
+	require.True(ctx, sfsmp.Mounted)
+
+	// Two cycles. one initial replication, one incremental replication.
+	// Within each cycle: interrupt replication at least once.
+	// This exercises both the no-resume-token-present and the resume-token-present validation code paths.
+initial_then_incremental:
+	for i := 0; i < 2; i++ {
+		writeDummyData(path.Join(sfsmp.Mountpoint, "dummy.data"), 2*(1<<20))
+		mustSnapshot(ctx, fmt.Sprintf("%s@%d", sfs, i))
+
+		rep := replicationInvocation{
+			sjid:             sjid,
+			rjid:             rjid,
+			sfs:              sfs,
+			rfsRoot:          rfsRoot,
+			senderConfigHook: setup.senderConfigHook,
+			interceptSender: func(e *endpoint.Sender) logic.Sender {
+				return &PartialSender{Sender: e, failAfterByteCount: 1 << 20}
+			},
+			guarantee:              pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeResumability),
+			skipSendArgsValidation: false,
+		}
+
+		rfs := rep.ReceiveSideFilesystem()
+
+		// PartialSender interrupts after 1MiB, and we wrote 2 MiB of data
+		// => Give it 3 attempts to replicate. after that, we should have a stable outcome
+		var lastReport *report.Report
+		lastResumeToken := ""
+	interrupt_current_step:
+		for j := 0; j < 3; j++ {
+
+			lastReport = rep.Do(ctx)
+			ctx.Logf("\nreport=%s", pretty.Sprint(lastReport))
+			require.Len(ctx, lastReport.Attempts, 1)
+			require.Len(ctx, lastReport.Attempts[0].Filesystems, 1)
+			lastReportFS := lastReport.Attempts[0].Filesystems[0]
+
+			var rfsExists bool
+			rfsResumeToken, err := zfs.ZFSGetReceiveResumeTokenOrEmptyStringIfNotSupported(ctx, mustDatasetPath(rfs))
+			if err != nil {
+				_, ok := err.(*zfs.DatasetDoesNotExist) // no shadow
+				require.True(ctx, ok, "no other errors expected")
+				rfsExists = false
+				rfsResumeToken = ""
+			} else {
+				rfsExists = true
+			}
+
+			if setup.expectedOutcome == ValidationRejects {
+
+				// When expecting rejection, it should manifest immediately, before sending anything.
+				// This is tested in the j=0 iteration (for both initial and incremental repl (i=0, i=1)).
+				// But we also want to assert correct behavior in case zrepl observes resume tokens.
+				// Specifically, cases where the send parameters encoded in the token conflict with the
+				// configured encryption policy.
+				// Hence, for scenarios that are expected to reject, after we validated that they reject
+				// for the non-resuming case (j==0), fabricate a resuming scenario by temporarily disabling
+				// send args validation. After fabricating the scenario, proceed into j==1 to exercise
+				// the resume token validation.
+				if j == 0 {
+					if i == 0 {
+						require.False(ctx, rfsExists, "the sender should not have sent anything")
+					} else {
+						// we fabricate a scenario where rfsExists below, hence can't assert non-existence anymore
+					}
+
+					ctx.Logf("skipping send args validation to test resuming case")
+
+					rep.skipSendArgsValidation = true
+					setupResumeReport := rep.Do(ctx)
+					ctx.Logf("setupResumeReport=%s", pretty.Sprint(setupResumeReport))
+					rep.skipSendArgsValidation = false
+					rt, err := zfs.ZFSGetReceiveResumeTokenOrEmptyStringIfNotSupported(ctx, mustDatasetPath(rfs))
+					require.NoError(ctx, err)
+					require.NotEmpty(ctx, rt, "we disabled send args validation, so the .Do above should have resulted in a resume token on rfs")
+					lastResumeToken = rt
+					continue interrupt_current_step // next iteration will test resume case with send args validation enabled
+				} else { // j > 0
+					require.Equal(ctx, lastResumeToken, rfsResumeToken, "we expect policy to refuse replication, no progress must happen")
+					_, err := zfs.ZFSGetFilesystemVersion(ctx, fmt.Sprintf("%s@%d", rfs, i))
+					_, ok := err.(*zfs.DatasetDoesNotExist)
+					require.True(ctx, ok, "another check that no progress is happening")
+				}
+
+				setup.outcomeRejectsInspectError(ctx, lastReportFS, j == 0)
+
+				// XXX: check rejection cases for incremental replication as well
+				break initial_then_incremental
+
+			} else {
+				require.Equal(ctx, ValidationAccepts, setup.expectedOutcome)
+				_, err := zfs.ZFSGetFilesystemVersion(ctx, fmt.Sprintf("%s@%d", rfs, i))
+				_, notExist := err.(*zfs.DatasetDoesNotExist)
+				if notExist {
+					require.NotEmpty(ctx, rfsResumeToken)
+					continue interrupt_current_step // next iteration will resume
+				} else {
+					require.NoError(ctx, err)
+					// version exists
+
+					// make sure all the filesystem versions we created so far were replicated by the replication loop
+					for j := 0; j <= i; j++ {
+						_ = fsversion(ctx, rfs, fmt.Sprintf("@%d", j))
+					}
+
+					setup.inspectReceiverFSAfterSuccessfulCycle(rfs)
+					continue initial_then_incremental
+				}
+			}
+		}
+
+	}
+
+}
+
+func SendArgsValidationEE_EncryptionAndRaw(ctx *platformtest.Context) {
+	type TC struct {
+		// create sender filesystem with encryption enabled yes/no
+		SFSEnc              bool
+		SndEnc              bool // send flag
+		SndRaw              bool // send flag
+		RFSEnc              bool
+		Outcome             sendArgsValidationEndToEndTestOutcome
+		RejectErrorNoResume string
+		RejectErrorResume   string
+	}
+	tcs := []TC{
+		// Sender FS is unencrypted
+		{SFSEnc: false, SndEnc: false, SndRaw: false, RFSEnc: false, Outcome: ValidationAccepts},
+		{SFSEnc: false, SndEnc: false, SndRaw: true, RFSEnc: false, Outcome: ValidationAccepts}, // allow unencrypted raw sends (#503)
+		{SFSEnc: false, SndEnc: true, SndRaw: false, RFSEnc: false, Outcome: ValidationRejects,
+			RejectErrorNoResume: `encrypted send mandated by policy, but filesystem .* is not encrypted`,
+			RejectErrorResume:   `encrypted send mandated by policy, but filesystem .* is not encrypted`,
+		},
+		{SFSEnc: false, SndEnc: true, SndRaw: true, RFSEnc: false, Outcome: ValidationRejects,
+			RejectErrorNoResume: `encrypted send mandated by policy, but filesystem .* is not encrypted`,
+			RejectErrorResume:   `encrypted send mandated by policy, but filesystem .* is not encrypted`,
+		},
+		// Sender FS is encrypted
+		{SFSEnc: true, SndEnc: false, SndRaw: false, RFSEnc: false, Outcome: ValidationAccepts}, // passes because keys are loaded, thus can send plain.
+		{SFSEnc: true, SndEnc: false, SndRaw: true, RFSEnc: false, Outcome: ValidationRejects,
+			RejectErrorNoResume: `policy mandates raw\+unencrypted sends, but filesystem .* is encrypted`,
+			RejectErrorResume:   `resume token has rawok=true which would result in encrypted send, but policy mandates unencrypted sends only`,
+		},
+		{SFSEnc: true, SndEnc: true, SndRaw: false, RFSEnc: true, Outcome: ValidationAccepts},
+		{SFSEnc: true, SndEnc: true, SndRaw: true, RFSEnc: true, Outcome: ValidationAccepts},
+	}
+
+	for _, tc := range tcs {
+		tc := tc // closure would copy by ref otherwise
+		ctx.QueueSubtest(fmt.Sprintf("%#v", tc), func(ctx *platformtest.Context) {
+			implSendArgsValidationEndToEndTest(ctx, sendArgsValidationEndToEndTest{
+				encryptedSenderFilesystem: tc.SFSEnc,
+				senderConfigHook: func(c *endpoint.SenderConfig) {
+					c.Encrypt = &nodefault.Bool{B: tc.SndEnc}
+					c.SendRaw = tc.SndRaw
+				},
+				expectedOutcome: tc.Outcome,
+				outcomeRejectsInspectError: func(ctx require.TestingT, fr *report.FilesystemReport, isResume bool) {
+					// this callback is only called for ValidationRejects
+
+					// validation should be failing during dry send => planning stage
+					// XXX mock out ZFS to ensure we never call a zfs send that would send data
+					// if we're expecting validation to fail
+					require.Equal(ctx, report.FilesystemPlanningErrored, fr.State)
+
+					if isResume {
+						require.NotEmpty(ctx, tc.RejectErrorResume)
+						require.Regexp(ctx, tc.RejectErrorResume, fr.PlanError)
+					} else {
+						require.NotEmpty(ctx, tc.RejectErrorNoResume)
+						require.Regexp(ctx, tc.RejectErrorNoResume, fr.PlanError)
+					}
+				},
+				inspectReceiverFSAfterSuccessfulCycle: func(rfs string) {
+					enabled, err := zfs.ZFSGetEncryptionEnabled(ctx, rfs)
+					require.NoError(ctx, err)
+					require.Equal(ctx, tc.RFSEnc, enabled, "receiver filesystem encryption settings unexpected")
+				},
+			})
+		})
+	}
+}

internal/platformtest/tests/sendStream.go

@@ -0,0 +1,185 @@
+package tests
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path"
+	"syscall"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+
+	"github.com/zrepl/zrepl/internal/platformtest"
+	"github.com/zrepl/zrepl/internal/util/nodefault"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+func sendStreamTest(ctx *platformtest.Context) *zfs.SendStream {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "sender"
+	`)
+
+	fs := fmt.Sprintf("%s/sender", ctx.RootDataset)
+
+	fsmpo, err := zfs.ZFSGetMountpoint(ctx, fs)
+	require.NoError(ctx, err)
+
+	writeDummyData(path.Join(fsmpo.Mountpoint, "dummy.data"), 1<<26)
+	mustSnapshot(ctx, fs+"@1")
+	snap := fsversion(ctx, fs, "@1")
+	snapSendArg := snap.ToSendArgVersion()
+
+	sendArgs, err := zfs.ZFSSendArgsUnvalidated{
+		FS:   fs,
+		From: nil,
+		To:   &snapSendArg,
+		ZFSSendFlags: zfs.ZFSSendFlags{
+			Encrypted: &nodefault.Bool{B: false},
+		},
+	}.Validate(ctx)
+	require.NoError(ctx, err)
+
+	sendStream, err := zfs.ZFSSend(ctx, sendArgs)
+	require.NoError(ctx, err)
+
+	return sendStream
+
+}
+
+func SendStreamCloseAfterBlockedOnPipeWrite(ctx *platformtest.Context) {
+
+	sendStream := sendStreamTest(ctx)
+
+	// let the pipe buffer fill and the zfs process block uninterruptibly
+
+	ctx.Logf("waiting for pipe write to block")
+	time.Sleep(5 * time.Second) // XXX need a platform-neutral way to detect that the pipe is full and the writer is blocked
+
+	ctx.Logf("closing send stream")
+	err := sendStream.Close() // this is what this test case is about
+	ctx.Logf("close error: %T %s", err, err)
+	require.NoError(ctx, err)
+
+	exitErrZfsError := sendStream.TestOnly_ExitErr()
+	require.Contains(ctx, exitErrZfsError.Error(), "signal")
+	require.Error(ctx, exitErrZfsError)
+	exitErr, ok := exitErrZfsError.WaitErr.(*exec.ExitError)
+	require.True(ctx, ok)
+	if exitErr.Exited() {
+		// some ZFS impls (FreeBSD 12) behaves that way
+		return
+	}
+
+	// ProcessState is only available after exit
+	// => use as proxy that the process was wait()ed upon and is gone
+	ctx.Logf("%#v", exitErr.ProcessState)
+	require.NotNil(ctx, exitErr.ProcessState)
+	// and let's verify that the process got killed, so that we know it was the call to .Close() above
+	waitStatus := exitErr.ProcessState.Sys().(syscall.WaitStatus)
+	ctx.Logf("wait status: %#v", waitStatus)
+	ctx.Logf("exit status: %v", waitStatus.ExitStatus())
+	require.True(ctx, waitStatus.Signaled())
+	switch waitStatus.Signal() {
+	case unix.SIGKILL:
+		fallthrough
+	case unix.SIGPIPE:
+		// ok
+
+	default:
+		ctx.Errorf("%T %s\n%v", waitStatus.Signal(), waitStatus.Signal(), waitStatus.Signal())
+		ctx.FailNow()
+	}
+}
+
+func SendStreamCloseAfterEOFRead(ctx *platformtest.Context) {
+
+	sendStream := sendStreamTest(ctx)
+
+	_, err := io.Copy(io.Discard, sendStream)
+	require.NoError(ctx, err)
+
+	var buf [128]byte
+	n, err := sendStream.Read(buf[:])
+	require.Zero(ctx, n)
+	require.Equal(ctx, io.EOF, err)
+
+	err = sendStream.Close()
+	require.NoError(ctx, err)
+
+	n, err = sendStream.Read(buf[:])
+	require.Zero(ctx, n)
+	require.Equal(ctx, os.ErrClosed, err, "same read error should be returned")
+}
+
+func SendStreamMultipleCloseAfterEOF(ctx *platformtest.Context) {
+
+	sendStream := sendStreamTest(ctx)
+
+	_, err := io.Copy(io.Discard, sendStream)
+	require.NoError(ctx, err)
+
+	var buf [128]byte
+	n, err := sendStream.Read(buf[:])
+	require.Zero(ctx, n)
+	require.Equal(ctx, io.EOF, err)
+
+	err = sendStream.Close()
+	require.NoError(ctx, err)
+
+	err = sendStream.Close()
+	require.Equal(ctx, os.ErrClosed, err)
+}
+
+func SendStreamMultipleCloseBeforeEOF(ctx *platformtest.Context) {
+
+	sendStream := sendStreamTest(ctx)
+
+	err := sendStream.Close()
+	require.NoError(ctx, err)
+
+	err = sendStream.Close()
+	require.Equal(ctx, os.ErrClosed, err)
+}
+
+type failingReadCloser struct {
+	err error
+}
+
+var _ io.ReadCloser = &failingReadCloser{}
+
+func (c *failingReadCloser) Read(p []byte) (int, error) { return 0, c.err }
+func (c *failingReadCloser) Close() error               { return c.err }
+
+func SendStreamNonEOFReadErrorHandling(ctx *platformtest.Context) {
+
+	sendStream := sendStreamTest(ctx)
+
+	var buf [128]byte
+	n, err := sendStream.Read(buf[:])
+	require.Equal(ctx, len(buf), n)
+	require.NoError(ctx, err)
+
+	var mockError = fmt.Errorf("taeghaefow4piesahwahjocu7ul5tiachaiLipheijae8ooZ8Pies8shohGee9feeTeirai5aiFeiyaecai4kiaLoh4azeih0tea")
+	mock := &failingReadCloser{err: mockError}
+	orig := sendStream.TestOnly_ReplaceStdoutReader(mock)
+
+	n, err = sendStream.Read(buf[:])
+	require.Equal(ctx, 0, n)
+	require.Equal(ctx, mockError, err)
+
+	if sendStream.TestOnly_ReplaceStdoutReader(orig) != mock {
+		panic("incorrect test impl")
+	}
+
+	err = sendStream.Close()
+	require.NoError(ctx, err) // if we can't kill the child then this will be a flaky test, but let's assume we can kill the child
+
+	err = sendStream.Close()
+	require.Equal(ctx, os.ErrClosed, err)
+}

internal/platformtest/tests/tests.go

@@ -0,0 +1,16 @@
+package tests
+
+import (
+	"reflect"
+	"runtime"
+
+	"github.com/zrepl/zrepl/internal/platformtest"
+)
+
+type Case func(*platformtest.Context)
+
+func (c Case) String() string {
+	return runtime.FuncForPC(reflect.ValueOf(c).Pointer()).Name()
+}
+
+//go:generate go run ./gen github.com/zrepl/zrepl/platformtest/tests

internal/platformtest/tests/undestroyableSnapshotParsing.go

@@ -0,0 +1,37 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/platformtest"
+	"github.com/zrepl/zrepl/internal/zfs"
+)
+
+func UndestroyableSnapshotParsing(t *platformtest.Context) {
+	platformtest.Run(t, platformtest.PanicErr, t.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@1 2 3"
+		+  "foo bar@4 5 6"
+		+  "foo bar@7 8 9"
+		R  zfs hold zrepl_platformtest "${ROOTDS}/foo bar@4 5 6"
+	`)
+
+	err := zfs.ZFSDestroy(t, fmt.Sprintf("%s/foo bar@1 2 3,4 5 6,7 8 9", t.RootDataset))
+	if err == nil {
+		panic("expecting destroy error due to hold")
+	}
+	if dse, ok := err.(*zfs.DestroySnapshotsError); !ok {
+		panic(fmt.Sprintf("expecting *zfs.DestroySnapshotsError, got %T\n%v\n%s", err, err, err))
+	} else {
+		if dse.Filesystem != fmt.Sprintf("%s/foo bar", t.RootDataset) {
+			panic(dse.Filesystem)
+		}
+		require.Equal(t, []string{"4 5 6"}, dse.Undestroyable)
+		require.Equal(t, []string{"dataset is busy"}, dse.Reason)
+	}
+
+}