move implementation to internal/ directory (#828)

internal/zfs/datasetpath_visitor.go

@@ -0,0 +1,132 @@
+package zfs
+
+type DatasetPathForest struct {
+	roots []*datasetPathTree
+}
+
+func NewDatasetPathForest() *DatasetPathForest {
+	return &DatasetPathForest{
+		make([]*datasetPathTree, 0),
+	}
+}
+
+func (f *DatasetPathForest) Add(p *DatasetPath) {
+	if len(p.comps) <= 0 {
+		panic("dataset path too short. must have length > 0")
+	}
+
+	// Find its root
+	var root *datasetPathTree
+	for _, r := range f.roots {
+		if r.Add(p.comps) {
+			root = r
+			break
+		}
+	}
+	if root == nil {
+		root = newDatasetPathTree(p.comps)
+		f.roots = append(f.roots, root)
+	}
+}
+
+type DatasetPathVisit struct {
+	Path *DatasetPath
+	// If true, the dataset referenced by Path was not in the list of datasets to traverse
+	FilledIn bool
+	Parent   *DatasetPathVisit
+}
+
+type DatasetPathsVisitor func(v *DatasetPathVisit) (visitChildTree bool)
+
+// Traverse a list of DatasetPaths top down, i.e. given a set of datasets with same
+// path prefix, those with shorter prefix are traversed first.
+// If there are gaps, i.e. the intermediary component a/b between a and a/b/c,
+// those gaps are still visited but the FilledIn property of the visit is set to true.
+func (f *DatasetPathForest) WalkTopDown(visitor DatasetPathsVisitor) {
+
+	for _, r := range f.roots {
+		r.WalkTopDown(&DatasetPathVisit{
+			Path:     &DatasetPath{nil},
+			FilledIn: true,
+			Parent:   nil,
+		}, visitor)
+	}
+
+}
+
+/* PRIVATE IMPLEMENTATION */
+
+type datasetPathTree struct {
+	Component string
+	FilledIn  bool
+	Children  []*datasetPathTree
+}
+
+func (t *datasetPathTree) Add(p []string) bool {
+
+	if len(p) == 0 {
+		return true
+	}
+
+	if p[0] == t.Component {
+
+		remainder := p[1:]
+		if len(remainder) == 0 {
+			t.FilledIn = false
+			return true
+		}
+
+		for _, c := range t.Children {
+			if c.Add(remainder) {
+				return true
+			}
+		}
+
+		t.Children = append(t.Children, newDatasetPathTree(remainder))
+		return true
+
+	} else {
+
+		return false
+
+	}
+
+}
+
+func (t *datasetPathTree) WalkTopDown(parent *DatasetPathVisit, visitor DatasetPathsVisitor) {
+
+	thisVisitPath := parent.Path.Copy()
+	thisVisitPath.Extend(&DatasetPath{[]string{t.Component}})
+
+	thisVisit := &DatasetPathVisit{
+		thisVisitPath,
+		t.FilledIn,
+		parent,
+	}
+	visitChildTree := visitor(thisVisit)
+
+	if visitChildTree {
+		for _, c := range t.Children {
+			c.WalkTopDown(thisVisit, visitor)
+		}
+	}
+
+}
+
+func newDatasetPathTree(initialComps []string) (t *datasetPathTree) {
+	t = &datasetPathTree{}
+	cur := t
+	for i, comp := range initialComps {
+		cur.Component = comp
+		cur.FilledIn = true
+		cur.Children = make([]*datasetPathTree, 0, 1)
+		if i == len(initialComps)-1 {
+			cur.FilledIn = false // last component is not filled in
+			break
+		}
+		child := &datasetPathTree{}
+		cur.Children = append(cur.Children, child)
+		cur = child
+	}
+	return t
+}

internal/zfs/datasetpath_visitor_test.go

@@ -0,0 +1,116 @@
+package zfs
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewDatasetPathTree(t *testing.T) {
+
+	r := newDatasetPathTree(toDatasetPath("pool1/foo/bar").comps)
+
+	assert.Equal(t, "pool1", r.Component)
+	assert.True(t, len(r.Children) == 1)
+	cr := r.Children[0]
+	assert.Equal(t, "foo", cr.Component)
+	assert.True(t, len(cr.Children) == 1)
+	ccr := cr.Children[0]
+	assert.Equal(t, "bar", ccr.Component)
+
+}
+
+type visitRecorder struct {
+	visits []DatasetPathVisit
+}
+
+func makeVisitRecorder() (v DatasetPathsVisitor, rec *visitRecorder) {
+	rec = &visitRecorder{
+		visits: make([]DatasetPathVisit, 0),
+	}
+	v = func(v *DatasetPathVisit) bool {
+		rec.visits = append(rec.visits, *v)
+		return true
+	}
+	return
+}
+
+func buildForest(paths []*DatasetPath) (f *DatasetPathForest) {
+	f = NewDatasetPathForest()
+	for _, p := range paths {
+		f.Add(p)
+	}
+	return
+}
+
+type expectedDatasetPathVisit struct {
+	expectPath   *DatasetPath
+	expectFillIn bool
+}
+
+func (e *expectedDatasetPathVisit) AssertEqual(t *testing.T, actual DatasetPathVisit) {
+	assert.Equal(t, e.expectPath, actual.Path)
+	assert.Equal(t, e.expectFillIn, actual.FilledIn)
+	if actual.Parent != nil {
+		assert.Equal(t, actual.Parent.Path.Length()+1, e.expectPath.Length())
+		assert.True(t, e.expectPath.HasPrefix(actual.Parent.Path))
+	}
+}
+
+func expectedDatasetPathVisits(t *testing.T, expected []expectedDatasetPathVisit, actual []DatasetPathVisit) {
+	assert.Equal(t, len(expected), len(actual))
+	for i := range expected {
+		expected[i].AssertEqual(t, actual[i])
+	}
+}
+
+func TestDatasetPathForestWalkTopDown(t *testing.T) {
+
+	paths := []*DatasetPath{
+		toDatasetPath("pool1"),
+		toDatasetPath("pool1/foo/bar"),
+		toDatasetPath("pool1/foo/bar/looloo"),
+		toDatasetPath("pool2/test/bar"),
+	}
+
+	v, rec := makeVisitRecorder()
+
+	buildForest(paths).WalkTopDown(v)
+
+	expectedVisits := []expectedDatasetPathVisit{
+		{toDatasetPath("pool1"), false},
+		{toDatasetPath("pool1/foo"), true},
+		{toDatasetPath("pool1/foo/bar"), false},
+		{toDatasetPath("pool1/foo/bar/looloo"), false},
+		{toDatasetPath("pool2"), true},
+		{toDatasetPath("pool2/test"), true},
+		{toDatasetPath("pool2/test/bar"), false},
+	}
+	expectedDatasetPathVisits(t, expectedVisits, rec.visits)
+}
+
+func TestDatasetPathWalkTopDownWorksUnordered(t *testing.T) {
+
+	paths := []*DatasetPath{
+		toDatasetPath("pool1"),
+		toDatasetPath("pool1/foo/bar/looloo"),
+		toDatasetPath("pool1/foo/bar"),
+		toDatasetPath("pool1/bang/baz"),
+	}
+
+	v, rec := makeVisitRecorder()
+
+	buildForest(paths).WalkTopDown(v)
+
+	expectedVisits := []expectedDatasetPathVisit{
+		{toDatasetPath("pool1"), false},
+		{toDatasetPath("pool1/foo"), true},
+		{toDatasetPath("pool1/foo/bar"), false},
+		{toDatasetPath("pool1/foo/bar/looloo"), false},
+		{toDatasetPath("pool1/bang"), true},
+		{toDatasetPath("pool1/bang/baz"), false},
+	}
+
+	expectedDatasetPathVisits(t, expectedVisits, rec.visits)
+
+}

internal/zfs/encryption.go

@@ -0,0 +1,72 @@
+package zfs
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"strings"
+	"sync"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/internal/util/envconst"
+	"github.com/zrepl/zrepl/internal/zfs/zfscmd"
+)
+
+var encryptionCLISupport struct {
+	once      sync.Once
+	supported bool
+	err       error
+}
+
+func EncryptionCLISupported(ctx context.Context) (bool, error) {
+	encryptionCLISupport.once.Do(func() {
+		// "feature discovery"
+		cmd := zfscmd.CommandContext(ctx, "zfs", "load-key")
+		output, err := cmd.CombinedOutput()
+		if ee, ok := err.(*exec.ExitError); !ok || ok && !ee.Exited() {
+			encryptionCLISupport.err = errors.Wrap(err, "native encryption cli support feature check failed")
+		}
+		def := strings.Contains(string(output), "load-key") && strings.Contains(string(output), "keylocation")
+		encryptionCLISupport.supported = envconst.Bool("ZREPL_EXPERIMENTAL_ZFS_ENCRYPTION_CLI_SUPPORTED", def)
+		debug("encryption cli feature check complete %#v", &encryptionCLISupport)
+	})
+	return encryptionCLISupport.supported, encryptionCLISupport.err
+}
+
+// returns false, nil if encryption is not supported
+func ZFSGetEncryptionEnabled(ctx context.Context, fs string) (enabled bool, err error) {
+	defer func(e *error) {
+		if *e != nil {
+			*e = fmt.Errorf("zfs get encryption enabled fs=%q: %s", fs, *e)
+		}
+	}(&err)
+	if supp, err := EncryptionCLISupported(ctx); err != nil {
+		return false, err
+	} else if !supp {
+		return false, nil
+	}
+
+	if err := validateZFSFilesystem(fs); err != nil {
+		return false, err
+	}
+
+	props, err := zfsGet(ctx, fs, []string{"encryption"}, SourceAny)
+	if err != nil {
+		return false, errors.Wrap(err, "cannot get `encryption` property")
+	}
+	val := props.Get("encryption")
+	switch val {
+	case "":
+		panic("zfs get should return a value for `encryption`")
+	case "-":
+		return false, errors.New("`encryption` property should never be \"-\"")
+	case "off":
+		return false, nil
+	default:
+		// we don't want to hardcode the cipher list, and we checked for != 'off'
+		// ==> assume any other value means encryption is enabled
+		// TODO add test to OpenZFS test suite
+		return true, nil
+	}
+}

internal/zfs/filesystemplaceholdercreateencryptionvalue_enumer.go

@@ -0,0 +1,50 @@
+// Code generated by "enumer -type=FilesystemPlaceholderCreateEncryptionValue -trimprefix=FilesystemPlaceholderCreateEncryption"; DO NOT EDIT.
+
+package zfs
+
+import (
+	"fmt"
+)
+
+const _FilesystemPlaceholderCreateEncryptionValueName = "InheritOff"
+
+var _FilesystemPlaceholderCreateEncryptionValueIndex = [...]uint8{0, 7, 10}
+
+func (i FilesystemPlaceholderCreateEncryptionValue) String() string {
+	i -= 1
+	if i < 0 || i >= FilesystemPlaceholderCreateEncryptionValue(len(_FilesystemPlaceholderCreateEncryptionValueIndex)-1) {
+		return fmt.Sprintf("FilesystemPlaceholderCreateEncryptionValue(%d)", i+1)
+	}
+	return _FilesystemPlaceholderCreateEncryptionValueName[_FilesystemPlaceholderCreateEncryptionValueIndex[i]:_FilesystemPlaceholderCreateEncryptionValueIndex[i+1]]
+}
+
+var _FilesystemPlaceholderCreateEncryptionValueValues = []FilesystemPlaceholderCreateEncryptionValue{1, 2}
+
+var _FilesystemPlaceholderCreateEncryptionValueNameToValueMap = map[string]FilesystemPlaceholderCreateEncryptionValue{
+	_FilesystemPlaceholderCreateEncryptionValueName[0:7]:  1,
+	_FilesystemPlaceholderCreateEncryptionValueName[7:10]: 2,
+}
+
+// FilesystemPlaceholderCreateEncryptionValueString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func FilesystemPlaceholderCreateEncryptionValueString(s string) (FilesystemPlaceholderCreateEncryptionValue, error) {
+	if val, ok := _FilesystemPlaceholderCreateEncryptionValueNameToValueMap[s]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to FilesystemPlaceholderCreateEncryptionValue values", s)
+}
+
+// FilesystemPlaceholderCreateEncryptionValueValues returns all values of the enum
+func FilesystemPlaceholderCreateEncryptionValueValues() []FilesystemPlaceholderCreateEncryptionValue {
+	return _FilesystemPlaceholderCreateEncryptionValueValues
+}
+
+// IsAFilesystemPlaceholderCreateEncryptionValue returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i FilesystemPlaceholderCreateEncryptionValue) IsAFilesystemPlaceholderCreateEncryptionValue() bool {
+	for _, v := range _FilesystemPlaceholderCreateEncryptionValueValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}

internal/zfs/holds.go

@@ -0,0 +1,134 @@
+package zfs
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"syscall"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/internal/util/envconst"
+	"github.com/zrepl/zrepl/internal/zfs/zfscmd"
+)
+
+// no need for feature tests, holds have been around forever
+
+func validateNotEmpty(field, s string) error {
+	if s == "" {
+		return fmt.Errorf("`%s` must not be empty", field)
+	}
+	return nil
+}
+
+// returned err != nil is guaranteed to represent invalid hold tag
+func ValidHoldTag(tag string) error {
+	maxlen := envconst.Int("ZREPL_ZFS_MAX_HOLD_TAG_LEN", 256-1) // 256 include NULL byte, from module/zfs/dsl_userhold.c
+	if len(tag) > maxlen {
+		return fmt.Errorf("hold tag %q exceeds max length of %d", tag, maxlen)
+	}
+	return nil
+}
+
+// Idemptotent: does not return an error if the tag already exists
+func ZFSHold(ctx context.Context, fs string, v FilesystemVersion, tag string) error {
+	if !v.IsSnapshot() {
+		return errors.Errorf("can only hold snapshots, got %s", v.RelName())
+	}
+
+	if err := validateNotEmpty("tag", tag); err != nil {
+		return err
+	}
+	fullPath := v.FullPath(fs)
+	output, err := zfscmd.CommandContext(ctx, "zfs", "hold", tag, fullPath).CombinedOutput()
+	if err != nil {
+		if bytes.Contains(output, []byte("tag already exists on this dataset")) {
+			goto success
+		}
+		return &ZFSError{output, errors.Wrapf(err, "cannot hold %q", fullPath)}
+	}
+success:
+	return nil
+}
+
+func ZFSHolds(ctx context.Context, fs, snap string) ([]string, error) {
+	if err := validateZFSFilesystem(fs); err != nil {
+		return nil, errors.Wrap(err, "`fs` is not a valid filesystem path")
+	}
+	if snap == "" {
+		return nil, fmt.Errorf("`snap` must not be empty")
+	}
+	dp := fmt.Sprintf("%s@%s", fs, snap)
+	output, err := zfscmd.CommandContext(ctx, "zfs", "holds", "-H", dp).CombinedOutput()
+	if err != nil {
+		return nil, &ZFSError{output, errors.Wrap(err, "zfs holds failed")}
+	}
+	scan := bufio.NewScanner(bytes.NewReader(output))
+	var tags []string
+	for scan.Scan() {
+		// NAME              TAG  TIMESTAMP
+		comps := strings.SplitN(scan.Text(), "\t", 3)
+		if len(comps) != 3 {
+			return nil, fmt.Errorf("zfs holds: unexpected output\n%s", output)
+		}
+		if comps[0] != dp {
+			return nil, fmt.Errorf("zfs holds: unexpected output: expecting %q as first component, got %q\n%s", dp, comps[0], output)
+		}
+		tags = append(tags, comps[1])
+	}
+	return tags, nil
+}
+
+// Idempotent: if the hold doesn't exist, this is not an error
+func ZFSRelease(ctx context.Context, tag string, snaps ...string) error {
+	cumLens := make([]int, len(snaps))
+	for i := 1; i < len(snaps); i++ {
+		cumLens[i] = cumLens[i-1] + len(snaps[i])
+	}
+	maxInvocationLen := 12 * os.Getpagesize()
+	var noSuchTagLines, otherLines []string
+	for i := 0; i < len(snaps); {
+		var j = i
+		for ; j < len(snaps); j++ {
+			if cumLens[j]-cumLens[i] > maxInvocationLen {
+				break
+			}
+		}
+		args := []string{"release", tag}
+		args = append(args, snaps[i:j]...)
+		output, err := zfscmd.CommandContext(ctx, "zfs", args...).CombinedOutput()
+		if pe, ok := err.(*os.PathError); err != nil && ok && pe.Err == syscall.E2BIG {
+			maxInvocationLen = maxInvocationLen / 2
+			continue
+		}
+		// further error handling part of error scraper below
+
+		maxInvocationLen = maxInvocationLen + os.Getpagesize()
+		i = j
+
+		// even if release fails for datasets where there's no hold with the tag
+		// the hold is still released on datasets which have a hold with the tag
+		// FIXME verify this in a platformtest
+		// => screen-scrape
+		scan := bufio.NewScanner(bytes.NewReader(output))
+		for scan.Scan() {
+			line := scan.Text()
+			if strings.Contains(line, "no such tag on this dataset") {
+				noSuchTagLines = append(noSuchTagLines, line)
+			} else {
+				otherLines = append(otherLines, line)
+			}
+		}
+
+	}
+	if debugEnabled {
+		debug("zfs release: no such tag lines=%v otherLines=%v", noSuchTagLines, otherLines)
+	}
+	if len(otherLines) > 0 {
+		return fmt.Errorf("unknown zfs error while releasing hold with tag %q:\n%s", tag, strings.Join(otherLines, "\n"))
+	}
+	return nil
+}

internal/zfs/mapping.go

@@ -0,0 +1,107 @@
+package zfs
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/zrepl/zrepl/internal/zfs/zfscmd"
+)
+
+type DatasetFilter interface {
+	Filter(p *DatasetPath) (pass bool, err error)
+	// The caller owns the returned set.
+	// Implementations should return a copy.
+	UserSpecifiedDatasets() UserSpecifiedDatasetsSet
+}
+
+// A set of dataset names that the user specified in the configuration file.
+type UserSpecifiedDatasetsSet map[string]bool
+
+// Returns a DatasetFilter that does not filter (passes all paths)
+func NoFilter() DatasetFilter {
+	return noFilter{}
+}
+
+type noFilter struct{}
+
+var _ DatasetFilter = noFilter{}
+
+func (noFilter) Filter(p *DatasetPath) (pass bool, err error)    { return true, nil }
+func (noFilter) UserSpecifiedDatasets() UserSpecifiedDatasetsSet { return nil }
+
+func ZFSListMapping(ctx context.Context, filter DatasetFilter) (datasets []*DatasetPath, err error) {
+	res, err := ZFSListMappingProperties(ctx, filter, nil)
+	if err != nil {
+		return nil, err
+	}
+	datasets = make([]*DatasetPath, len(res))
+	for i, r := range res {
+		datasets[i] = r.Path
+	}
+	return datasets, nil
+}
+
+type ZFSListMappingPropertiesResult struct {
+	Path *DatasetPath
+	// Guaranteed to have the same length as properties in the originating call
+	Fields []string
+}
+
+// properties must not contain 'name'
+func ZFSListMappingProperties(ctx context.Context, filter DatasetFilter, properties []string) (datasets []ZFSListMappingPropertiesResult, err error) {
+
+	if filter == nil {
+		panic("filter must not be nil")
+	}
+
+	for _, p := range properties {
+		if p == "name" {
+			panic("properties must not contain 'name'")
+		}
+	}
+	newProps := make([]string, len(properties)+1)
+	newProps[0] = "name"
+	copy(newProps[1:], properties)
+	properties = newProps
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	rchan := make(chan ZFSListResult)
+
+	go ZFSListChan(ctx, rchan, properties, nil, "-r", "-t", "filesystem,volume")
+
+	unmatchedUserSpecifiedDatasets := filter.UserSpecifiedDatasets()
+	datasets = make([]ZFSListMappingPropertiesResult, 0)
+	for r := range rchan {
+
+		if r.Err != nil {
+			err = r.Err
+			return
+		}
+
+		var path *DatasetPath
+		if path, err = NewDatasetPath(r.Fields[0]); err != nil {
+			return
+		}
+
+		delete(unmatchedUserSpecifiedDatasets, path.ToString())
+
+		pass, filterErr := filter.Filter(path)
+		if filterErr != nil {
+			return nil, fmt.Errorf("error calling filter: %s", filterErr)
+		}
+		if pass {
+			datasets = append(datasets, ZFSListMappingPropertiesResult{
+				Path:   path,
+				Fields: r.Fields[1:],
+			})
+		}
+
+	}
+
+	jobid := zfscmd.GetJobIDOrDefault(ctx, "__nojobid")
+	metric := prom.ZFSListUnmatchedUserSpecifiedDatasetCount.WithLabelValues(jobid)
+	metric.Add(float64(len(unmatchedUserSpecifiedDatasets)))
+
+	return
+}

internal/zfs/namecheck.go

@@ -0,0 +1,194 @@
+package zfs
+
+import (
+	"bytes"
+	"fmt"
+	"regexp"
+	"strings"
+	"unicode"
+)
+
+const MaxDatasetNameLen = 256 - 1
+
+type EntityType string
+
+const (
+	EntityTypeFilesystem EntityType = "filesystem"
+	EntityTypeVolume     EntityType = "volume"
+	EntityTypeSnapshot   EntityType = "snapshot"
+	EntityTypeBookmark   EntityType = "bookmark"
+)
+
+func (e EntityType) Validate() error {
+	switch e {
+	case EntityTypeFilesystem:
+		return nil
+	case EntityTypeVolume:
+		return nil
+	case EntityTypeSnapshot:
+		return nil
+	case EntityTypeBookmark:
+		return nil
+	default:
+		return fmt.Errorf("invalid entity type %q", string(e))
+	}
+}
+
+func (e EntityType) MustValidate() {
+	if err := e.Validate(); err != nil {
+		panic(err)
+	}
+}
+
+func (e EntityType) String() string {
+	e.MustValidate()
+	return string(e)
+}
+
+var componentValidChar = regexp.MustCompile(`^[0-9a-zA-Z-_\.: ]+$`)
+
+// From module/zcommon/zfs_namecheck.c
+//
+// Snapshot names must be made up of alphanumeric characters plus the following
+// characters:
+//
+//	[-_.: ]
+func ComponentNamecheck(datasetPathComponent string) error {
+	if len(datasetPathComponent) == 0 {
+		return fmt.Errorf("must not be empty")
+	}
+	if len(datasetPathComponent) > MaxDatasetNameLen {
+		return fmt.Errorf("must not be longer than %d chars", MaxDatasetNameLen)
+	}
+
+	if !(isASCII(datasetPathComponent)) {
+		return fmt.Errorf("must be ASCII")
+	}
+
+	if !componentValidChar.MatchString(datasetPathComponent) {
+		return fmt.Errorf("must only contain alphanumeric chars and any in %q", "-_.: ")
+	}
+
+	if datasetPathComponent == "." || datasetPathComponent == ".." {
+		return fmt.Errorf("must not be '%s'", datasetPathComponent)
+	}
+
+	return nil
+}
+
+type PathValidationError struct {
+	path       string
+	entityType EntityType
+	msg        string
+}
+
+func (e *PathValidationError) Path() string { return e.path }
+
+func (e *PathValidationError) Error() string {
+	return fmt.Sprintf("invalid %s %q: %s", e.entityType, e.path, e.msg)
+}
+
+// combines
+//
+//	lib/libzfs/libzfs_dataset.c: zfs_validate_name
+//	module/zcommon/zfs_namecheck.c: entity_namecheck
+//
+// The '%' character is not allowed because it's reserved for zfs-internal use
+func EntityNamecheck(path string, t EntityType) (err *PathValidationError) {
+	pve := func(msg string) *PathValidationError {
+		return &PathValidationError{path: path, entityType: t, msg: msg}
+	}
+
+	t.MustValidate()
+
+	// delimiter checks
+	if t != EntityTypeSnapshot && strings.Contains(path, "@") {
+		return pve("snapshot delimiter '@' is not expected here")
+	}
+	if t == EntityTypeSnapshot && !strings.Contains(path, "@") {
+		return pve("missing '@' delimiter in snapshot name")
+	}
+	if t != EntityTypeBookmark && strings.Contains(path, "#") {
+		return pve("bookmark delimiter '#' is not expected here")
+	}
+	if t == EntityTypeBookmark && !strings.Contains(path, "#") {
+		return pve("missing '#' delimiter in bookmark name")
+	}
+
+	// EntityTypeVolume and EntityTypeFilesystem are already covered above
+
+	if strings.Contains(path, "%") {
+		return pve("invalid character '%' in name")
+	}
+
+	// mimic module/zcommon/zfs_namecheck.c: entity_namecheck
+
+	if len(path) > MaxDatasetNameLen {
+		return pve("name too long")
+	}
+
+	if len(path) == 0 {
+		return pve("must not be empty")
+	}
+
+	if !isASCII(path) {
+		return pve("must be ASCII")
+	}
+
+	slashComps := bytes.Split([]byte(path), []byte("/"))
+	bookmarkOrSnapshotDelims := 0
+	for compI, comp := range slashComps {
+
+		snapCount := bytes.Count(comp, []byte("@"))
+		bookCount := bytes.Count(comp, []byte("#"))
+
+		if !(snapCount*bookCount == 0) {
+			panic("implementation error: delimiter checks before this loop must ensure this cannot happen")
+		}
+		bookmarkOrSnapshotDelims += snapCount + bookCount
+		if bookmarkOrSnapshotDelims > 1 {
+			return pve("multiple delimiters '@' or '#' are not allowed")
+		}
+		if bookmarkOrSnapshotDelims == 1 && compI != len(slashComps)-1 {
+			return pve("snapshot or bookmark must not contain '/'")
+		}
+
+		if bookmarkOrSnapshotDelims == 0 {
+			// hot path, all but last component
+			component := string(comp)
+			if err := ComponentNamecheck(component); err != nil {
+				return pve(fmt.Sprintf("component %q: %s", component, err.Error()))
+			}
+			continue
+		}
+
+		subComps := bytes.FieldsFunc(comp, func(r rune) bool {
+			return r == '#' || r == '@'
+		})
+		if len(subComps) > 2 {
+			panic("implementation error: delimiter checks above should ensure a single bookmark or snapshot delimiter per component")
+		}
+		if len(subComps) != 2 {
+			return pve("empty component, bookmark or snapshot name not allowed")
+		}
+
+		for _, comp := range subComps {
+			component := string(comp)
+			if err := ComponentNamecheck(component); err != nil {
+				return pve(fmt.Sprintf("component %q: %s", component, err.Error()))
+			}
+		}
+
+	}
+
+	return nil
+}
+
+func isASCII(s string) bool {
+	for i := 0; i < len(s); i++ {
+		if s[i] > unicode.MaxASCII {
+			return false
+		}
+	}
+	return true
+}

internal/zfs/namecheck_test.go

@@ -0,0 +1,68 @@
+package zfs
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestEntityNamecheck(t *testing.T) {
+
+	type testcase struct {
+		input      string
+		entityType EntityType
+		ok         bool
+	}
+
+	tcs := []testcase{
+		{"/", EntityTypeFilesystem, false},
+		{"/foo", EntityTypeFilesystem, false},
+		{"/foo@bar", EntityTypeSnapshot, false},
+		{"foo", EntityTypeBookmark, false},
+		{"foo", EntityTypeSnapshot, false},
+		{"foo@bar", EntityTypeBookmark, false},
+		{"foo#bar", EntityTypeSnapshot, false},
+		{"foo#book", EntityTypeBookmark, true},
+		{"foo#book@bar", EntityTypeBookmark, false},
+		{"foo/book@bar", EntityTypeSnapshot, true},
+		{"foo/book#bar", EntityTypeBookmark, true},
+		{"foo/for%idden", EntityTypeFilesystem, false},
+		{"foo/bår", EntityTypeFilesystem, false},
+		{"", EntityTypeFilesystem, false},
+		{"foo/bar@", EntityTypeSnapshot, false},
+		{"foo/bar#", EntityTypeBookmark, false},
+		{"foo/bar#@blah", EntityTypeBookmark, false},
+		{"foo bar/baz bar@blah foo", EntityTypeSnapshot, true},
+		{"foo bar/baz bar@#lah foo", EntityTypeSnapshot, false},
+		{"foo bar/baz bar@@lah foo", EntityTypeSnapshot, false},
+		{"foo bar/baz bar##lah foo", EntityTypeBookmark, false},
+		{"foo bar/baz@blah/foo", EntityTypeSnapshot, false},
+		{"foo bar/baz@blah/foo", EntityTypeFilesystem, false},
+		{"foo/b\tr@ba\tz", EntityTypeSnapshot, false},
+		{"foo/b\tr@baz", EntityTypeSnapshot, false},
+		{"foo/bar@ba\tz", EntityTypeSnapshot, false},
+		{"foo/./bar", EntityTypeFilesystem, false},
+		{"foo/../bar", EntityTypeFilesystem, false},
+		{"foo/bar@..", EntityTypeFilesystem, false},
+		{"foo/bar@.", EntityTypeFilesystem, false},
+		{strings.Repeat("a", MaxDatasetNameLen), EntityTypeFilesystem, true},
+		{strings.Repeat("a", MaxDatasetNameLen) + "a", EntityTypeFilesystem, false},
+		{strings.Repeat("a", MaxDatasetNameLen-2) + "/a", EntityTypeFilesystem, true},
+		{strings.Repeat("a", MaxDatasetNameLen-4) + "/a@b", EntityTypeSnapshot, true},
+		{strings.Repeat("a", MaxDatasetNameLen) + "/a@b", EntityTypeSnapshot, false},
+		// + is not allowed, and particularly relevant to test here because
+		// common timestamp formats usually use `+` as a delimiter for numeric timezone offset
+		// => cf with package `timestamp_formatting`
+		{"foo/bar@23+42", EntityTypeSnapshot, false},
+	}
+
+	for idx := range tcs {
+		t.Run(tcs[idx].input, func(t *testing.T) {
+			tc := tcs[idx]
+			err := EntityNamecheck(tc.input, tc.entityType)
+			if !((err == nil && tc.ok) || (err != nil && !tc.ok)) {
+				t.Errorf("expecting ok=%v but got err=%v", tc.ok, err)
+			}
+		})
+	}
+
+}

internal/zfs/placeholder.go

@@ -0,0 +1,197 @@
+package zfs
+
+import (
+	"context"
+	"crypto/sha512"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/internal/zfs/zfscmd"
+)
+
+const (
+	// For a placeholder filesystem to be a placeholder, the property source must be local,
+	// i.e. not inherited.
+	PlaceholderPropertyName string = "zrepl:placeholder"
+	placeholderPropertyOn   string = "on"
+	placeholderPropertyOff  string = "off"
+)
+
+// computeLegacyPlaceholderPropertyValue is a legacy-compatibility function.
+//
+// In the 0.0.x series, the value stored in the PlaceholderPropertyName user property
+// was a hash value of the dataset path.
+// A simple `on|off` value could not be used at the time because `zfs list` was used to
+// list all filesystems and their placeholder state with a single command: due to property
+// inheritance, `zfs list` would print the placeholder state for all (non-placeholder) children
+// of a dataset, so the hash value was used to distinguish whether the property was local or
+// inherited.
+//
+// One of the drawbacks of the above approach is that `zfs rename` renders a placeholder filesystem
+// a non-placeholder filesystem if any of the parent path components change.
+//
+// We `zfs get` nowadays, which returns the property source, making the hash value no longer
+// necessary. However, we want to keep legacy compatibility.
+func computeLegacyHashBasedPlaceholderPropertyValue(p *DatasetPath) string {
+	ps := []byte(p.ToString())
+	sum := sha512.Sum512_256(ps)
+	return hex.EncodeToString(sum[:])
+}
+
+// the caller asserts that placeholderPropertyValue is sourceLocal
+func isLocalPlaceholderPropertyValuePlaceholder(p *DatasetPath, placeholderPropertyValue string) (isPlaceholder bool) {
+	legacy := computeLegacyHashBasedPlaceholderPropertyValue(p)
+	switch placeholderPropertyValue {
+	case legacy:
+		return true
+	case placeholderPropertyOn:
+		return true
+	default:
+		return false
+	}
+}
+
+type FilesystemPlaceholderState struct {
+	FS                    string
+	FSExists              bool
+	IsPlaceholder         bool
+	RawLocalPropertyValue string
+}
+
+// ZFSGetFilesystemPlaceholderState is the authoritative way to determine whether a filesystem
+// is a placeholder. Note that the property source must be `local` for the returned value to be valid.
+//
+// For nonexistent FS, err == nil and state.FSExists == false
+func ZFSGetFilesystemPlaceholderState(ctx context.Context, p *DatasetPath) (state *FilesystemPlaceholderState, err error) {
+	state = &FilesystemPlaceholderState{FS: p.ToString()}
+	state.FS = p.ToString()
+	props, err := zfsGet(ctx, p.ToString(), []string{PlaceholderPropertyName}, SourceLocal)
+	var _ error = (*DatasetDoesNotExist)(nil) // weak assertion on zfsGet's interface
+	if _, ok := err.(*DatasetDoesNotExist); ok {
+		return state, nil
+	} else if err != nil {
+		return state, err
+	}
+	state.FSExists = true
+	state.RawLocalPropertyValue = props.Get(PlaceholderPropertyName)
+	state.IsPlaceholder = isLocalPlaceholderPropertyValuePlaceholder(p, state.RawLocalPropertyValue)
+	return state, nil
+}
+
+//go:generate enumer -type=FilesystemPlaceholderCreateEncryptionValue -trimprefix=FilesystemPlaceholderCreateEncryption
+type FilesystemPlaceholderCreateEncryptionValue int
+
+const (
+	FilesystemPlaceholderCreateEncryptionInherit FilesystemPlaceholderCreateEncryptionValue = 1 << iota
+	FilesystemPlaceholderCreateEncryptionOff
+)
+
+func ZFSCreatePlaceholderFilesystem(ctx context.Context, fs *DatasetPath, parent *DatasetPath, encryption FilesystemPlaceholderCreateEncryptionValue) (err error) {
+	if fs.Length() == 1 {
+		return fmt.Errorf("cannot create %q: pools cannot be created with zfs create", fs.ToString())
+	}
+
+	cmdline := []string{
+		"create",
+		"-o", fmt.Sprintf("%s=%s", PlaceholderPropertyName, placeholderPropertyOn),
+		"-o", "mountpoint=none",
+	}
+
+	if !encryption.IsAFilesystemPlaceholderCreateEncryptionValue() {
+		panic(encryption)
+	}
+	switch encryption {
+	case FilesystemPlaceholderCreateEncryptionInherit:
+		// no-op
+	case FilesystemPlaceholderCreateEncryptionOff:
+		cmdline = append(cmdline, "-o", "encryption=off")
+	default:
+		panic(encryption)
+	}
+
+	cmdline = append(cmdline, fs.ToString())
+	cmd := zfscmd.CommandContext(ctx, ZFS_BINARY, cmdline...)
+
+	stdio, err := cmd.CombinedOutput()
+	if err != nil {
+		err = &ZFSError{
+			Stderr:  stdio,
+			WaitErr: err,
+		}
+	}
+
+	return
+}
+
+func ZFSSetPlaceholder(ctx context.Context, p *DatasetPath, isPlaceholder bool) error {
+	prop := placeholderPropertyOff
+	if isPlaceholder {
+		prop = placeholderPropertyOn
+	}
+	props := map[string]string{PlaceholderPropertyName: prop}
+	return zfsSet(ctx, p.ToString(), props)
+}
+
+type MigrateHashBasedPlaceholderReport struct {
+	OriginalState     FilesystemPlaceholderState
+	NeedsModification bool
+}
+
+// fs must exist, will panic otherwise
+func ZFSMigrateHashBasedPlaceholderToCurrent(ctx context.Context, fs *DatasetPath, dryRun bool) (*MigrateHashBasedPlaceholderReport, error) {
+	st, err := ZFSGetFilesystemPlaceholderState(ctx, fs)
+	if err != nil {
+		return nil, fmt.Errorf("error getting placeholder state: %s", err)
+	}
+	if !st.FSExists {
+		panic("inconsistent placeholder state returned: fs must exist")
+	}
+
+	report := MigrateHashBasedPlaceholderReport{
+		OriginalState: *st,
+	}
+	report.NeedsModification = st.IsPlaceholder && st.RawLocalPropertyValue != placeholderPropertyOn
+
+	if dryRun || !report.NeedsModification {
+		return &report, nil
+	}
+
+	err = ZFSSetPlaceholder(ctx, fs, st.IsPlaceholder)
+	if err != nil {
+		return nil, fmt.Errorf("error re-writing placeholder property: %s", err)
+	}
+	return &report, nil
+}
+
+func ZFSListPlaceholderFilesystemsWithAdditionalProps(ctx context.Context, root string, additionalProps []string) (map[string]*ZFSProperties, error) {
+
+	props := []string{PlaceholderPropertyName}
+	if len(additionalProps) > 0 {
+		props = append(props, additionalProps...)
+	}
+
+	propsByFS, err := zfsGetRecursive(ctx, root, -1, []string{"filesystem", "volume"}, props, SourceAny)
+	if err != nil {
+		return nil, errors.Wrapf(err, "cannot get placeholder filesystems under %q", root)
+	}
+
+	filtered := make(map[string]*ZFSProperties)
+	for fs, props := range propsByFS {
+		details := props.GetDetails(PlaceholderPropertyName)
+		if details.Source != SourceLocal {
+			continue
+		}
+		fsp, err := NewDatasetPath(fs)
+		if err != nil {
+			return nil, errors.Wrapf(err, "zfs get returned invalid dataset path %q", fs)
+		}
+		if !isLocalPlaceholderPropertyValuePlaceholder(fsp, details.Value) {
+			continue
+		}
+		filtered[fs] = props
+	}
+
+	return filtered, nil
+}

internal/zfs/prometheus.go

@@ -0,0 +1,66 @@
+package zfs
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var prom struct {
+	ZFSListFilesystemVersionDuration          *prometheus.HistogramVec
+	ZFSSnapshotDuration                       *prometheus.HistogramVec
+	ZFSBookmarkDuration                       *prometheus.HistogramVec
+	ZFSDestroyDuration                        *prometheus.HistogramVec
+	ZFSListUnmatchedUserSpecifiedDatasetCount *prometheus.GaugeVec
+}
+
+func init() {
+	prom.ZFSListFilesystemVersionDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "zrepl",
+		Subsystem: "zfs",
+		Name:      "list_filesystem_versions_duration",
+		Help:      "Seconds it took for listing the versions of a given filesystem",
+	}, []string{"filesystem"})
+	prom.ZFSSnapshotDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "zrepl",
+		Subsystem: "zfs",
+		Name:      "snapshot_duration",
+		Help:      "Seconds it took to create a snapshot a given filesystem",
+	}, []string{"filesystem"})
+	prom.ZFSBookmarkDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "zrepl",
+		Subsystem: "zfs",
+		Name:      "bookmark_duration",
+		Help:      "Duration it took to bookmark a given snapshot",
+	}, []string{"filesystem"})
+	prom.ZFSDestroyDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "zrepl",
+		Subsystem: "zfs",
+		Name:      "destroy_duration",
+		Help:      "Duration it took to destroy a dataset",
+	}, []string{"dataset_type", "filesystem"})
+	prom.ZFSListUnmatchedUserSpecifiedDatasetCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "zrepl",
+		Subsystem: "zfs",
+		Name:      "list_unmatched_user_specified_dataset_count",
+		Help: "When evaluating a DatsetFilter against zfs list output, this counter " +
+			"is incremented for every DatasetFilter rule that did not match any " +
+			"filesystem name in the zfs list output. Monitor for increases to detect filesystem " +
+			"filter rules that have no effect because they don't match any local filesystem.",
+	}, []string{"jobid"})
+}
+
+func PrometheusRegister(registry prometheus.Registerer) error {
+	if err := registry.Register(prom.ZFSListFilesystemVersionDuration); err != nil {
+		return err
+	}
+	if err := registry.Register(prom.ZFSBookmarkDuration); err != nil {
+		return err
+	}
+	if err := registry.Register(prom.ZFSSnapshotDuration); err != nil {
+		return err
+	}
+	if err := registry.Register(prom.ZFSDestroyDuration); err != nil {
+		return err
+	}
+	if err := registry.Register(prom.ZFSListUnmatchedUserSpecifiedDatasetCount); err != nil {
+		return err
+	}
+	return nil
+}

internal/zfs/property/property.go

@@ -0,0 +1,33 @@
+package property
+
+import (
+	"fmt"
+	"regexp"
+)
+
+type Property string
+
+// Check property name conforms to zfsprops(8), section "User Properties"
+// Keep regex and error message in sync!
+var (
+	propertyValidNameChars    = regexp.MustCompile(`^[0-9a-zA-Z-_\.:]+$`)
+	propertyValidNameCharsErr = fmt.Errorf("property name must only contain alphanumeric chars and any in %q", "-_.:")
+)
+
+func (p Property) Validate() error {
+	const PROPERTYNAMEMAXLEN int = 256
+
+	if len(p) < 1 {
+		return fmt.Errorf("property name cannot be empty")
+	}
+	if len(p) > PROPERTYNAMEMAXLEN {
+		return fmt.Errorf("property name longer than %d characters", PROPERTYNAMEMAXLEN)
+	}
+	if p[0] == '-' {
+		return fmt.Errorf("property name cannot start with '-'")
+	}
+	if !propertyValidNameChars.MatchString(string(p)) {
+		return propertyValidNameCharsErr
+	}
+	return nil
+}

internal/zfs/propertysource_enumer.go

@@ -0,0 +1,81 @@
+// Code generated by "enumer -type=PropertySource -trimprefix=Source"; DO NOT EDIT.
+
+package zfs
+
+import (
+	"fmt"
+)
+
+const (
+	_PropertySourceName_0 = "LocalDefault"
+	_PropertySourceName_1 = "Inherited"
+	_PropertySourceName_2 = "None"
+	_PropertySourceName_3 = "Temporary"
+	_PropertySourceName_4 = "Received"
+	_PropertySourceName_5 = "Any"
+)
+
+var (
+	_PropertySourceIndex_0 = [...]uint8{0, 5, 12}
+	_PropertySourceIndex_1 = [...]uint8{0, 9}
+	_PropertySourceIndex_2 = [...]uint8{0, 4}
+	_PropertySourceIndex_3 = [...]uint8{0, 9}
+	_PropertySourceIndex_4 = [...]uint8{0, 8}
+	_PropertySourceIndex_5 = [...]uint8{0, 3}
+)
+
+func (i PropertySource) String() string {
+	switch {
+	case 1 <= i && i <= 2:
+		i -= 1
+		return _PropertySourceName_0[_PropertySourceIndex_0[i]:_PropertySourceIndex_0[i+1]]
+	case i == 4:
+		return _PropertySourceName_1
+	case i == 8:
+		return _PropertySourceName_2
+	case i == 16:
+		return _PropertySourceName_3
+	case i == 32:
+		return _PropertySourceName_4
+	case i == 4294967295:
+		return _PropertySourceName_5
+	default:
+		return fmt.Sprintf("PropertySource(%d)", i)
+	}
+}
+
+var _PropertySourceValues = []PropertySource{1, 2, 4, 8, 16, 32, 4294967295}
+
+var _PropertySourceNameToValueMap = map[string]PropertySource{
+	_PropertySourceName_0[0:5]:  1,
+	_PropertySourceName_0[5:12]: 2,
+	_PropertySourceName_1[0:9]:  4,
+	_PropertySourceName_2[0:4]:  8,
+	_PropertySourceName_3[0:9]:  16,
+	_PropertySourceName_4[0:8]:  32,
+	_PropertySourceName_5[0:3]:  4294967295,
+}
+
+// PropertySourceString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func PropertySourceString(s string) (PropertySource, error) {
+	if val, ok := _PropertySourceNameToValueMap[s]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to PropertySource values", s)
+}
+
+// PropertySourceValues returns all values of the enum
+func PropertySourceValues() []PropertySource {
+	return _PropertySourceValues
+}
+
+// IsAPropertySource returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i PropertySource) IsAPropertySource() bool {
+	for _, v := range _PropertySourceValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}

internal/zfs/resume_token.go

@@ -0,0 +1,306 @@
+package zfs
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/internal/util/envconst"
+	"github.com/zrepl/zrepl/internal/zfs/zfscmd"
+)
+
+// NOTE: Update ZFSSendARgs.Validate when changing fields (potentially SECURITY SENSITIVE)
+type ResumeToken struct {
+	HasFromGUID, HasToGUID        bool
+	FromGUID, ToGUID              uint64
+	ToName                        string
+	HasCompressOK, CompressOK     bool
+	HasRawOk, RawOK               bool
+	HasLargeBlockOK, LargeBlockOK bool
+	HasEmbedOk, EmbedOK           bool
+	HasSavedOk, SavedOk           bool
+}
+
+var resumeTokenNVListRE = regexp.MustCompile(`\t(\S+) = (.*)`)
+
+var resumeTokenContentsRE = regexp.MustCompile(`resume token contents:\nnvlist version: 0`)
+var resumeTokenIsCorruptRE = regexp.MustCompile(`resume token is corrupt`)
+
+var ResumeTokenCorruptError = errors.New("resume token is corrupt")
+var ResumeTokenDecodingNotSupported = errors.New("zfs binary does not allow decoding resume token or zrepl cannot scrape zfs output")
+var ResumeTokenParsingError = errors.New("zrepl cannot parse resume token values")
+
+var resumeSendSupportedCheck struct {
+	once      sync.Once
+	supported bool
+	err       error
+}
+
+func ResumeSendSupported(ctx context.Context) (bool, error) {
+	resumeSendSupportedCheck.once.Do(func() {
+		// "feature discovery"
+		cmd := zfscmd.CommandContext(ctx, "zfs", "send")
+		output, err := cmd.CombinedOutput()
+		if ee, ok := err.(*exec.ExitError); !ok || ok && !ee.Exited() {
+			resumeSendSupportedCheck.err = errors.Wrap(err, "resumable send cli support feature check failed")
+		}
+		def := strings.Contains(string(output), "receive_resume_token")
+		resumeSendSupportedCheck.supported = envconst.Bool("ZREPL_EXPERIMENTAL_ZFS_SEND_RESUME_SUPPORTED", def)
+		debug("resume send feature check complete %#v", &resumeSendSupportedCheck)
+	})
+	return resumeSendSupportedCheck.supported, resumeSendSupportedCheck.err
+}
+
+var resumeRecvPoolSupportRecheckTimeout = envconst.Duration("ZREPL_ZFS_RESUME_RECV_POOL_SUPPORT_RECHECK_TIMEOUT", 30*time.Second)
+
+type resumeRecvPoolSupportedResult struct {
+	lastCheck time.Time
+	supported bool
+	err       error
+}
+
+var resumeRecvSupportedCheck struct {
+	mtx         sync.RWMutex
+	flagSupport struct {
+		checked   bool
+		supported bool
+		err       error
+	}
+	poolSupported map[string]resumeRecvPoolSupportedResult
+}
+
+// fs == nil only checks for CLI support
+func ResumeRecvSupported(ctx context.Context, fs *DatasetPath) (bool, error) {
+	sup := &resumeRecvSupportedCheck
+	sup.mtx.RLock()
+	defer sup.mtx.RUnlock()
+	upgradeWhile := func(cb func()) {
+		sup.mtx.RUnlock()
+		defer sup.mtx.RLock()
+		sup.mtx.Lock()
+		defer sup.mtx.Unlock()
+		cb()
+	}
+
+	if !sup.flagSupport.checked {
+		output, err := zfscmd.CommandContext(ctx, ZFS_BINARY, "receive").CombinedOutput()
+		upgradeWhile(func() {
+			sup.flagSupport.checked = true
+			if ee, ok := err.(*exec.ExitError); err != nil && (!ok || ok && !ee.Exited()) {
+				sup.flagSupport.err = err
+			} else {
+				sup.flagSupport.supported = strings.Contains(string(output), "-A <filesystem|volume>")
+			}
+			debug("resume recv cli flag feature check result: %#v", sup.flagSupport)
+		})
+		// fallthrough
+	}
+
+	if sup.flagSupport.err != nil {
+		return false, errors.Wrap(sup.flagSupport.err, "zfs recv feature check for resumable send & recv failed")
+	} else if !sup.flagSupport.supported || fs == nil {
+		return sup.flagSupport.supported, nil
+	}
+
+	// Flag is supported and pool-support is request
+	// Now check for pool support
+
+	pool, err := fs.Pool()
+	if err != nil {
+		return false, errors.Wrap(err, "resume recv check requires pool of dataset")
+	}
+
+	if sup.poolSupported == nil {
+		upgradeWhile(func() {
+			sup.poolSupported = make(map[string]resumeRecvPoolSupportedResult)
+		})
+	}
+
+	var poolSup resumeRecvPoolSupportedResult
+	var ok bool
+	if poolSup, ok = sup.poolSupported[pool]; !ok || // shadow
+		(!poolSup.supported && time.Since(poolSup.lastCheck) > resumeRecvPoolSupportRecheckTimeout) {
+
+		output, err := zfscmd.CommandContext(ctx, "zpool", "get", "-H", "-p", "-o", "value", "feature@extensible_dataset", pool).CombinedOutput()
+		if err != nil {
+			debug("resume recv pool support check result: %#v", sup.flagSupport)
+			poolSup.supported = false
+			poolSup.err = err
+		} else {
+			poolSup.err = nil
+			o := strings.TrimSpace(string(output))
+			poolSup.supported = o == "active" || o == "enabled"
+		}
+		poolSup.lastCheck = time.Now()
+
+		// we take the lock late, so two updaters might check simultaneously, but that shouldn't hurt
+		upgradeWhile(func() {
+			sup.poolSupported[pool] = poolSup
+		})
+		// fallthrough
+	}
+
+	if poolSup.err != nil {
+		return false, errors.Wrapf(poolSup.err, "pool %q check for feature@extensible_dataset feature failed", pool)
+	}
+
+	return poolSup.supported, nil
+}
+
+// Abuse 'zfs send' to decode the resume token
+//
+// FIXME: implement nvlist unpacking in Go and read through libzfs_sendrecv.c
+func ParseResumeToken(ctx context.Context, token string) (*ResumeToken, error) {
+
+	if supported, err := ResumeSendSupported(ctx); err != nil {
+		return nil, err
+	} else if !supported {
+		return nil, ResumeTokenDecodingNotSupported
+	}
+
+	// Example resume tokens:
+	//
+	// From a non-incremental send
+	// 1-bf31b879a-b8-789c636064000310a500c4ec50360710e72765a5269740f80cd8e4d3d28a534b18e00024cf86249f5459925acc802a8facbf243fbd3433858161f5ddb9ab1ae7c7466a20c97382e5f312735319180af2f3730cf58166953824c2cc0200cde81651
+
+	// From an incremental send
+	// 1-c49b979a2-e0-789c636064000310a501c49c50360710a715e5e7a69766a63040c1eabb735735ce8f8d5400b2d991d4e52765a5269740f82080219f96569c5ac2000720793624f9a4ca92d46206547964fd25f91057f09e37babb88c9bf5503499e132c9f97989bcac050909f9f63a80f34abc421096616007c881d4c
+
+	// Resulting output of zfs send -nvt <token>
+	//
+	//resume token contents:
+	//nvlist version: 0
+	//	fromguid = 0x595d9f81aa9dddab
+	//	object = 0x1
+	//	offset = 0x0
+	//	bytes = 0x0
+	//	toguid = 0x854f02a2dd32cf0d
+	//	toname = pool1/test@b
+	//cannot resume send: 'pool1/test@b' used in the initial send no longer exists
+
+	cmd := zfscmd.CommandContext(ctx, ZFS_BINARY, "send", "-nvt", string(token))
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			if !exitErr.Exited() {
+				return nil, err
+			}
+			// we abuse zfs send for decoding, the exit error may be due to
+			// a) the token being from a third machine
+			// b) it no longer exists on the machine where
+		} else {
+			return nil, err
+		}
+	}
+
+	if !resumeTokenContentsRE.Match(output) {
+		if resumeTokenIsCorruptRE.Match(output) {
+			return nil, ResumeTokenCorruptError
+		}
+		return nil, ResumeTokenDecodingNotSupported
+	}
+
+	matches := resumeTokenNVListRE.FindAllStringSubmatch(string(output), -1)
+	if matches == nil {
+		return nil, ResumeTokenDecodingNotSupported
+	}
+
+	rt := &ResumeToken{}
+
+	for _, m := range matches {
+		attr, val := m[1], m[2]
+		switch attr {
+		case "fromguid":
+			rt.FromGUID, err = strconv.ParseUint(val, 0, 64)
+			if err != nil {
+				return nil, ResumeTokenParsingError
+			}
+			rt.HasFromGUID = true
+		case "toguid":
+			rt.ToGUID, err = strconv.ParseUint(val, 0, 64)
+			if err != nil {
+				return nil, ResumeTokenParsingError
+			}
+			rt.HasToGUID = true
+		case "toname":
+			rt.ToName = val
+		case "rawok":
+			rt.HasRawOk = true
+			rt.RawOK, err = strconv.ParseBool(val)
+			if err != nil {
+				return nil, ResumeTokenParsingError
+			}
+		case "compressok":
+			rt.HasCompressOK = true
+			rt.CompressOK, err = strconv.ParseBool(val)
+			if err != nil {
+				return nil, ResumeTokenParsingError
+			}
+		case "embedok":
+			rt.HasEmbedOk = true
+			rt.EmbedOK, err = strconv.ParseBool(val)
+			if err != nil {
+				return nil, ResumeTokenParsingError
+			}
+		case "largeblockok":
+			rt.HasLargeBlockOK = true
+			rt.LargeBlockOK, err = strconv.ParseBool(val)
+			if err != nil {
+				return nil, ResumeTokenParsingError
+			}
+		case "savedok":
+			rt.HasSavedOk = true
+			rt.SavedOk, err = strconv.ParseBool(val)
+			if err != nil {
+				return nil, ResumeTokenParsingError
+			}
+		}
+	}
+
+	if !rt.HasToGUID {
+		return nil, ResumeTokenDecodingNotSupported
+	}
+
+	return rt, nil
+
+}
+
+// if string is empty and err == nil, the feature is not supported
+func ZFSGetReceiveResumeTokenOrEmptyStringIfNotSupported(ctx context.Context, fs *DatasetPath) (string, error) {
+	if supported, err := ResumeRecvSupported(ctx, fs); err != nil {
+		return "", errors.Wrap(err, "cannot determine zfs recv resume support")
+	} else if !supported {
+		return "", nil
+	}
+	const prop_receive_resume_token = "receive_resume_token"
+	props, err := ZFSGet(ctx, fs, []string{prop_receive_resume_token})
+	if err != nil {
+		return "", err
+	}
+	res := props.Get(prop_receive_resume_token)
+	debug("%q receive_resume_token=%q", fs.ToString(), res)
+	if res == "-" {
+		return "", nil
+	} else {
+		return res, nil
+	}
+}
+
+func (t *ResumeToken) ToNameSplit() (fs *DatasetPath, snapName string, err error) {
+	comps := strings.SplitN(t.ToName, "@", 2)
+	if len(comps) != 2 {
+		return nil, "", fmt.Errorf("resume token field `toname` does not contain @: %q", t.ToName)
+	}
+	dp, err := NewDatasetPath(comps[0])
+	if err != nil {
+		return nil, "", errors.Wrap(err, "resume token field `toname` dataset path invalid")
+	}
+	return dp, comps[1], nil
+}

internal/zfs/test_helpers/exec_mapping_good.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+
+
+while read line # we know this is not always correct...
+do
+    if [[ "$line" =~ nomap* ]]; then
+        echo "NOMAP"
+        continue
+    fi
+
+    echo "didmap/${line}"
+
+    if [ "$1" == "stop" ]; then
+        break
+    fi
+
+done

internal/zfs/test_helpers/zfs_failer.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "error: this is a mock" 1>&2
+exit 1

internal/zfs/versions.go

@@ -0,0 +1,281 @@
+package zfs
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type VersionType string
+
+const (
+	Bookmark VersionType = "bookmark"
+	Snapshot VersionType = "snapshot"
+)
+
+type VersionTypeSet map[VersionType]bool
+
+var (
+	AllVersionTypes = VersionTypeSet{
+		Bookmark: true,
+		Snapshot: true,
+	}
+	Bookmarks = VersionTypeSet{
+		Bookmark: true,
+	}
+	Snapshots = VersionTypeSet{
+		Snapshot: true,
+	}
+)
+
+func (s VersionTypeSet) zfsListTFlagRepr() string {
+	var types []string
+	for t := range s {
+		types = append(types, t.String())
+	}
+	sort.StringSlice(types).Sort()
+	return strings.Join(types, ",")
+}
+func (s VersionTypeSet) String() string { return s.zfsListTFlagRepr() }
+
+func (t VersionType) DelimiterChar() string {
+	switch t {
+	case Bookmark:
+		return "#"
+	case Snapshot:
+		return "@"
+	default:
+		panic(fmt.Sprintf("unexpected VersionType %#v", t))
+	}
+}
+
+func (t VersionType) String() string {
+	return string(t)
+}
+
+func DecomposeVersionString(v string) (fs string, versionType VersionType, name string, err error) {
+	if len(v) < 3 {
+		err = fmt.Errorf("snapshot or bookmark name implausibly short: %s", v)
+		return
+	}
+
+	snapSplit := strings.SplitN(v, "@", 2)
+	bookmarkSplit := strings.SplitN(v, "#", 2)
+	if len(snapSplit)*len(bookmarkSplit) != 2 {
+		err = fmt.Errorf("dataset cannot be snapshot and bookmark at the same time: %s", v)
+		return
+	}
+
+	if len(snapSplit) == 2 {
+		return snapSplit[0], Snapshot, snapSplit[1], nil
+	} else {
+		return bookmarkSplit[0], Bookmark, bookmarkSplit[1], nil
+	}
+}
+
+// The data in a FilesystemVersion is guaranteed to stem from a ZFS CLI invocation.
+type FilesystemVersion struct {
+	Type VersionType
+
+	// Display name. Should not be used for identification, only for user output
+	Name string
+
+	// GUID as exported by ZFS. Uniquely identifies a snapshot across pools
+	Guid uint64
+
+	// The TXG in which the snapshot was created. For bookmarks,
+	// this is the GUID of the snapshot it was initially tied to.
+	CreateTXG uint64
+
+	// The time the dataset was created
+	Creation time.Time
+
+	// userrefs field (snapshots only)
+	UserRefs OptionUint64
+}
+
+type OptionUint64 struct {
+	Value uint64
+	Valid bool
+}
+
+func (v FilesystemVersion) GetCreateTXG() uint64 { return v.CreateTXG }
+func (v FilesystemVersion) GetGUID() uint64      { return v.Guid }
+func (v FilesystemVersion) GetGuid() uint64      { return v.Guid }
+func (v FilesystemVersion) GetName() string      { return v.Name }
+func (v FilesystemVersion) IsSnapshot() bool     { return v.Type == Snapshot }
+func (v FilesystemVersion) IsBookmark() bool     { return v.Type == Bookmark }
+func (v FilesystemVersion) RelName() string {
+	return fmt.Sprintf("%s%s", v.Type.DelimiterChar(), v.Name)
+}
+func (v FilesystemVersion) String() string { return v.RelName() }
+
+// Only takes into account those attributes of FilesystemVersion that
+// are immutable over time in ZFS.
+func FilesystemVersionEqualIdentity(a, b FilesystemVersion) bool {
+	// .Name is mutable
+	return a.Guid == b.Guid && a.CreateTXG == b.CreateTXG && a.Creation == b.Creation
+}
+
+func (v FilesystemVersion) ToAbsPath(p *DatasetPath) string {
+	var b bytes.Buffer
+	b.WriteString(p.ToString())
+	b.WriteString(v.Type.DelimiterChar())
+	b.WriteString(v.Name)
+	return b.String()
+}
+
+func (v FilesystemVersion) FullPath(fs string) string {
+	return fmt.Sprintf("%s%s", fs, v.RelName())
+}
+
+func (v FilesystemVersion) ToSendArgVersion() ZFSSendArgVersion {
+	return ZFSSendArgVersion{
+		RelName: v.RelName(),
+		GUID:    v.Guid,
+	}
+}
+
+type ParseFilesystemVersionArgs struct {
+	fullname                            string
+	guid, createtxg, creation, userrefs string
+}
+
+func ParseFilesystemVersion(args ParseFilesystemVersionArgs) (v FilesystemVersion, err error) {
+	_, v.Type, v.Name, err = DecomposeVersionString(args.fullname)
+	if err != nil {
+		return v, err
+	}
+
+	if v.Guid, err = strconv.ParseUint(args.guid, 10, 64); err != nil {
+		err = errors.Wrapf(err, "cannot parse GUID %q", args.guid)
+		return v, err
+	}
+
+	if v.CreateTXG, err = strconv.ParseUint(args.createtxg, 10, 64); err != nil {
+		err = errors.Wrapf(err, "cannot parse CreateTXG %q", args.createtxg)
+		return v, err
+	}
+
+	creationUnix, err := strconv.ParseInt(args.creation, 10, 64)
+	if err != nil {
+		err = errors.Wrapf(err, "cannot parse creation date %q", args.creation)
+		return v, err
+	} else {
+		v.Creation = time.Unix(creationUnix, 0)
+	}
+
+	switch v.Type {
+	case Bookmark:
+		if args.userrefs != "-" {
+			return v, errors.Errorf("expecting %q for bookmark property userrefs, got %q", "-", args.userrefs)
+		}
+		v.UserRefs = OptionUint64{Valid: false}
+	case Snapshot:
+		if v.UserRefs.Value, err = strconv.ParseUint(args.userrefs, 10, 64); err != nil {
+			err = errors.Wrapf(err, "cannot parse userrefs %q", args.userrefs)
+			return v, err
+		}
+		v.UserRefs.Valid = true
+	default:
+		panic(v.Type)
+	}
+
+	return v, nil
+}
+
+type ListFilesystemVersionsOptions struct {
+	// the prefix of the version name, without the delimiter char
+	// empty means any prefix matches
+	ShortnamePrefix string
+
+	// which types should be returned
+	// nil or len(0) means any prefix matches
+	Types VersionTypeSet
+}
+
+func (o *ListFilesystemVersionsOptions) typesFlagArgs() string {
+	if len(o.Types) == 0 {
+		return AllVersionTypes.zfsListTFlagRepr()
+	} else {
+		return o.Types.zfsListTFlagRepr()
+	}
+}
+
+func (o *ListFilesystemVersionsOptions) matches(v FilesystemVersion) bool {
+	return (len(o.Types) == 0 || o.Types[v.Type]) && strings.HasPrefix(v.Name, o.ShortnamePrefix)
+}
+
+// returned versions are sorted by createtxg FIXME drop sort by createtxg requirement
+func ZFSListFilesystemVersions(ctx context.Context, fs *DatasetPath, options ListFilesystemVersionsOptions) (res []FilesystemVersion, err error) {
+	listResults := make(chan ZFSListResult)
+
+	promTimer := prometheus.NewTimer(prom.ZFSListFilesystemVersionDuration.WithLabelValues(fs.ToString()))
+	defer promTimer.ObserveDuration()
+
+	// Note: we don't create a separate trace.Task here because our loop that consumes
+	// the goroutine's output doesn't use ctx.
+	ctx, cancel := context.WithCancel(ctx)
+	// make sure the goroutine doesn't outlive this function call
+	var wg sync.WaitGroup
+	wg.Add(1)
+	defer wg.Wait()
+	defer cancel() // on exit, cancel list process before waiting for it
+	go func() {
+		defer wg.Done()
+		ZFSListChan(ctx, listResults,
+			[]string{"name", "guid", "createtxg", "creation", "userrefs"},
+			fs,
+			"-r", "-d", "1",
+			"-t", options.typesFlagArgs(),
+			"-s", "createtxg", fs.ToString())
+	}()
+
+	res = make([]FilesystemVersion, 0)
+	for listResult := range listResults {
+		if listResult.Err != nil {
+			return nil, listResult.Err
+		}
+
+		line := listResult.Fields
+		args := ParseFilesystemVersionArgs{
+			fullname:  line[0],
+			guid:      line[1],
+			createtxg: line[2],
+			creation:  line[3],
+			userrefs:  line[4],
+		}
+		v, err := ParseFilesystemVersion(args)
+		if err != nil {
+			return nil, err
+		}
+
+		if options.matches(v) {
+			res = append(res, v)
+		}
+
+	}
+	return
+}
+
+func ZFSGetFilesystemVersion(ctx context.Context, ds string) (v FilesystemVersion, _ error) {
+	props, err := zfsGet(ctx, ds, []string{"createtxg", "guid", "creation", "userrefs"}, SourceAny)
+	if err != nil {
+		return v, err
+	}
+	return ParseFilesystemVersion(ParseFilesystemVersionArgs{
+		fullname:  ds,
+		createtxg: props.Get("createtxg"),
+		guid:      props.Get("guid"),
+		creation:  props.Get("creation"),
+		userrefs:  props.Get("userrefs"),
+	})
+}

internal/zfs/versions_destroy.go

@@ -0,0 +1,240 @@
+package zfs
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"sort"
+	"strings"
+	"sync"
+	"syscall"
+
+	"github.com/zrepl/zrepl/internal/util/envconst"
+	"github.com/zrepl/zrepl/internal/zfs/zfscmd"
+)
+
+func ZFSDestroyFilesystemVersion(ctx context.Context, filesystem *DatasetPath, version *FilesystemVersion) (err error) {
+
+	datasetPath := version.ToAbsPath(filesystem)
+
+	// Sanity check...
+	if !strings.ContainsAny(datasetPath, "@#") {
+		return fmt.Errorf("sanity check failed: no @ or # character found in %q", datasetPath)
+	}
+
+	return ZFSDestroy(ctx, datasetPath)
+}
+
+var destroyerSingleton = destroyerImpl{}
+
+type DestroySnapOp struct {
+	Filesystem string
+	Name       string
+	ErrOut     *error
+}
+
+func (o *DestroySnapOp) String() string {
+	return fmt.Sprintf("destroy operation %s@%s", o.Filesystem, o.Name)
+}
+
+func ZFSDestroyFilesystemVersions(ctx context.Context, reqs []*DestroySnapOp) {
+	doDestroy(ctx, reqs, destroyerSingleton)
+}
+
+func setDestroySnapOpErr(b []*DestroySnapOp, err error) {
+	for _, r := range b {
+		*r.ErrOut = err
+	}
+}
+
+type destroyer interface {
+	Destroy(ctx context.Context, args []string) error
+	DestroySnapshotsCommaSyntaxSupported(context.Context) (bool, error)
+}
+
+func doDestroy(ctx context.Context, reqs []*DestroySnapOp, e destroyer) {
+
+	var validated []*DestroySnapOp
+	for _, req := range reqs {
+		// Filesystem and Snapshot should not be empty
+		// ZFS will generally fail because those are invalid destroy arguments,
+		// but we'd rather apply defensive programming here (doing destroy after all)
+		if req.Filesystem == "" {
+			*req.ErrOut = fmt.Errorf("Filesystem must not be an empty string")
+		} else if req.Name == "" {
+			*req.ErrOut = fmt.Errorf("Name must not be an empty string")
+		} else {
+			validated = append(validated, req)
+		}
+	}
+	reqs = validated
+
+	commaSupported, err := e.DestroySnapshotsCommaSyntaxSupported(ctx)
+	if err != nil {
+		debug("destroy: comma syntax support detection failed: %s", err)
+		setDestroySnapOpErr(reqs, err)
+		return
+	}
+
+	if !commaSupported {
+		doDestroySeq(ctx, reqs, e)
+	} else {
+		doDestroyBatched(ctx, reqs, e)
+	}
+}
+
+func doDestroySeq(ctx context.Context, reqs []*DestroySnapOp, e destroyer) {
+	for _, r := range reqs {
+		*r.ErrOut = e.Destroy(ctx, []string{fmt.Sprintf("%s@%s", r.Filesystem, r.Name)})
+	}
+}
+
+func doDestroyBatched(ctx context.Context, reqs []*DestroySnapOp, d destroyer) {
+	perFS := buildBatches(reqs)
+	for _, fsbatch := range perFS {
+		doDestroyBatchedRec(ctx, fsbatch, d)
+	}
+}
+
+func buildBatches(reqs []*DestroySnapOp) [][]*DestroySnapOp {
+	if len(reqs) == 0 {
+		return nil
+	}
+	sorted := make([]*DestroySnapOp, len(reqs))
+	copy(sorted, reqs)
+	sort.SliceStable(sorted, func(i, j int) bool {
+		// by filesystem, then snap name
+		fscmp := strings.Compare(sorted[i].Filesystem, sorted[j].Filesystem)
+		if fscmp != 0 {
+			return fscmp == -1
+		}
+		return strings.Compare(sorted[i].Name, sorted[j].Name) == -1
+	})
+
+	// group by fs
+	var perFS [][]*DestroySnapOp
+	consumed := 0
+	maxBatchSize := envconst.Int("ZREPL_DESTROY_MAX_BATCH_SIZE", 0)
+	for consumed < len(sorted) {
+		batchConsumedUntil := consumed
+		for ; batchConsumedUntil < len(sorted) && (maxBatchSize < 1 || batchConsumedUntil-consumed < maxBatchSize) && sorted[batchConsumedUntil].Filesystem == sorted[consumed].Filesystem; batchConsumedUntil++ {
+		}
+		perFS = append(perFS, sorted[consumed:batchConsumedUntil])
+		consumed = batchConsumedUntil
+	}
+	return perFS
+}
+
+// batch must be on same Filesystem, panics otherwise
+func tryBatch(ctx context.Context, batch []*DestroySnapOp, d destroyer) error {
+	if len(batch) == 0 {
+		return nil
+	}
+
+	batchFS := batch[0].Filesystem
+	batchNames := make([]string, len(batch))
+	for i := range batchNames {
+		batchNames[i] = batch[i].Name
+		if batchFS != batch[i].Filesystem {
+			panic("inconsistent batch")
+		}
+	}
+	batchArg := fmt.Sprintf("%s@%s", batchFS, strings.Join(batchNames, ","))
+	return d.Destroy(ctx, []string{batchArg})
+}
+
+// fsbatch must be on same filesystem
+func doDestroyBatchedRec(ctx context.Context, fsbatch []*DestroySnapOp, d destroyer) {
+	if len(fsbatch) <= 1 {
+		doDestroySeq(ctx, fsbatch, d)
+		return
+	}
+
+	err := tryBatch(ctx, fsbatch, d)
+	if err == nil {
+		setDestroySnapOpErr(fsbatch, nil)
+		return
+	}
+
+	if pe, ok := err.(*os.PathError); ok && pe.Err == syscall.E2BIG {
+		// see TestExcessiveArgumentsResultInE2BIG
+		// try halving batch size, assuming snapshots names are roughly the same length
+		debug("batch destroy: E2BIG encountered: %s", err)
+		doDestroyBatchedRec(ctx, fsbatch[0:len(fsbatch)/2], d)
+		doDestroyBatchedRec(ctx, fsbatch[len(fsbatch)/2:], d)
+		return
+	}
+
+	singleRun := fsbatch // the destroys that will be tried sequentially after "smart" error handling below
+
+	if err, ok := err.(*DestroySnapshotsError); ok {
+		// eliminate undestroyable datasets from batch and try it once again
+		strippedBatch, remaining := make([]*DestroySnapOp, 0, len(fsbatch)), make([]*DestroySnapOp, 0, len(fsbatch))
+
+		for _, b := range fsbatch {
+			isUndestroyable := false
+			for _, undestroyable := range err.Undestroyable {
+				if undestroyable == b.Name {
+					isUndestroyable = true
+					break
+				}
+			}
+			if isUndestroyable {
+				remaining = append(remaining, b)
+			} else {
+				strippedBatch = append(strippedBatch, b)
+			}
+		}
+
+		err := tryBatch(ctx, strippedBatch, d)
+		if err != nil {
+			// run entire batch sequentially if the stripped one fails
+			// (it shouldn't because we stripped erroneous datasets)
+			singleRun = fsbatch // shadow
+		} else {
+			setDestroySnapOpErr(strippedBatch, nil) // these ones worked
+			singleRun = remaining                   // shadow
+		}
+		// fallthrough
+	}
+
+	doDestroySeq(ctx, singleRun, d)
+
+}
+
+type destroyerImpl struct{}
+
+func (d destroyerImpl) Destroy(ctx context.Context, args []string) error {
+	if len(args) != 1 {
+		// we have no use case for this at the moment, so let's crash (safer than destroying something unexpectedly)
+		panic(fmt.Sprintf("unexpected number of arguments: %v", args))
+	}
+	// we know that we are only using this for snapshots, so also sanity check for an @ in args[0]
+	if !strings.ContainsAny(args[0], "@") {
+		panic(fmt.Sprintf("sanity check: expecting '@' in call to Destroy, got %q", args[0]))
+	}
+	return ZFSDestroy(ctx, args[0])
+}
+
+var batchDestroyFeatureCheck struct {
+	once   sync.Once
+	enable bool
+	err    error
+}
+
+func (d destroyerImpl) DestroySnapshotsCommaSyntaxSupported(ctx context.Context) (bool, error) {
+	batchDestroyFeatureCheck.once.Do(func() {
+		// "feature discovery"
+		cmd := zfscmd.CommandContext(ctx, ZFS_BINARY, "destroy")
+		output, err := cmd.CombinedOutput()
+		if _, ok := err.(*exec.ExitError); !ok {
+			debug("destroy feature check failed: %T %s", err, err)
+			batchDestroyFeatureCheck.err = err
+		}
+		def := strings.Contains(string(output), "<filesystem|volume>@<snap>[%<snap>][,...]")
+		batchDestroyFeatureCheck.enable = envconst.Bool("ZREPL_EXPERIMENTAL_ZFS_COMMA_SYNTAX_SUPPORTED", def)
+		debug("destroy feature check complete %#v", &batchDestroyFeatureCheck)
+	})
+	return batchDestroyFeatureCheck.enable, batchDestroyFeatureCheck.err
+}

internal/zfs/versions_destroy_test.go

@@ -0,0 +1,296 @@
+package zfs
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+	"syscall"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/zrepl/zrepl/internal/util/chainlock"
+)
+
+type mockBatchDestroy struct {
+	mtx              chainlock.L
+	calls            []string
+	commaUnsupported bool
+	undestroyable    *regexp.Regexp
+	randomerror      string
+	e2biglen         int
+}
+
+func (m *mockBatchDestroy) DestroySnapshotsCommaSyntaxSupported(_ context.Context) (bool, error) {
+	return !m.commaUnsupported, nil
+}
+
+func (m *mockBatchDestroy) Destroy(ctx context.Context, args []string) error {
+	defer m.mtx.Lock().Unlock()
+	if len(args) != 1 {
+		panic("unexpected use of Destroy")
+	}
+	a := args[0]
+	if m.e2biglen > 0 && len(a) > m.e2biglen {
+		return &os.PathError{Err: syscall.E2BIG} // TestExcessiveArgumentsResultInE2BIG checks that this errors is produced
+	}
+	m.calls = append(m.calls, a)
+
+	var snapnames []string
+	if m.commaUnsupported {
+		snapnames = append(snapnames, a)
+		if strings.Contains(a, ",") {
+			return fmt.Errorf("unsupported syntax mock error")
+		}
+	} else {
+		snapnames = append(snapnames, strings.Split(a, ",")...)
+	}
+	fs, vt, firstsnapname, err := DecomposeVersionString(snapnames[0])
+	if err != nil {
+		panic(err)
+	}
+	if vt != Snapshot {
+		panic(vt)
+	}
+	snapnames[0] = firstsnapname
+
+	var undestroyable []string
+	if m.undestroyable != nil {
+		for _, snap := range snapnames {
+			if m.undestroyable.MatchString(snap) {
+				undestroyable = append(undestroyable, snap)
+			}
+		}
+	}
+	if len(undestroyable) > 0 {
+		return &DestroySnapshotsError{
+			Filesystem:    fs,
+			Undestroyable: undestroyable,
+			Reason:        []string{"undestroyable mock with regexp " + m.undestroyable.String()},
+		}
+	}
+	if m.randomerror != "" && strings.Contains(a, m.randomerror) {
+		return fmt.Errorf("randomerror")
+	}
+	return nil
+}
+
+func TestBatchDestroySnaps(t *testing.T) {
+
+	errs := make([]error, 10)
+	nilErrs := func() {
+		for i := range errs {
+			errs[i] = nil
+		}
+	}
+	opsTemplate := []*DestroySnapOp{
+		&DestroySnapOp{"zroot/z", "foo", &errs[0]},
+		&DestroySnapOp{"zroot/a", "foo", &errs[1]},
+		&DestroySnapOp{"zroot/a", "bar", &errs[2]},
+		&DestroySnapOp{"zroot/b", "bar", &errs[3]},
+		&DestroySnapOp{"zroot/b", "zab", &errs[4]},
+		&DestroySnapOp{"zroot/b", "undestroyable", &errs[5]},
+		&DestroySnapOp{"zroot/c", "baz", &errs[6]},
+		&DestroySnapOp{"zroot/c", "randomerror", &errs[7]},
+		&DestroySnapOp{"zroot/c", "bar", &errs[8]},
+		&DestroySnapOp{"zroot/d", "blup", &errs[9]},
+	}
+
+	t.Run("single_undestroyable_dataset", func(t *testing.T) {
+		nilErrs()
+		mock := &mockBatchDestroy{
+			commaUnsupported: false,
+			undestroyable:    regexp.MustCompile(`undestroyable`),
+			randomerror:      "randomerror",
+		}
+
+		doDestroy(context.TODO(), opsTemplate, mock)
+
+		assert.NoError(t, errs[0])
+		assert.NoError(t, errs[1])
+		assert.NoError(t, errs[2])
+		assert.NoError(t, errs[3])
+		assert.NoError(t, errs[4])
+		assert.Error(t, errs[5], "undestroyable")
+		assert.NoError(t, errs[6])
+		assert.Error(t, errs[7], "randomerror")
+		assert.NoError(t, errs[8])
+		assert.NoError(t, errs[9])
+
+		defer mock.mtx.Lock().Unlock()
+		assert.Equal(
+			t,
+			[]string{
+				"zroot/a@bar,foo", // reordered snaps in lexicographical order
+				"zroot/b@bar,undestroyable,zab",
+				"zroot/b@bar,zab", // eliminate undestroyables, try others again
+				"zroot/b@undestroyable",
+				"zroot/c@bar,baz,randomerror",
+				"zroot/c@bar", // fallback to single-snapshot on non DestroyError
+				"zroot/c@baz",
+				"zroot/c@randomerror",
+				"zroot/d@blup",
+				"zroot/z@foo", // ordered at last position
+			},
+			mock.calls,
+		)
+	})
+
+	t.Run("all_undestroyable", func(t *testing.T) {
+
+		opsTemplate := []*DestroySnapOp{
+			&DestroySnapOp{"zroot/a", "foo", new(error)},
+			&DestroySnapOp{"zroot/a", "bar", new(error)},
+		}
+
+		mock := &mockBatchDestroy{
+			commaUnsupported: false,
+			undestroyable:    regexp.MustCompile(`.*`),
+		}
+
+		doDestroy(context.TODO(), opsTemplate, mock)
+
+		defer mock.mtx.Lock().Unlock()
+		assert.Equal(
+			t,
+			[]string{
+				"zroot/a@bar,foo", // reordered snaps in lexicographical order
+				"zroot/a@bar",
+				"zroot/a@foo",
+			},
+			mock.calls,
+		)
+	})
+
+	t.Run("comma_syntax_unsupported", func(t *testing.T) {
+		nilErrs()
+		mock := &mockBatchDestroy{
+			commaUnsupported: true,
+			undestroyable:    regexp.MustCompile(`undestroyable`),
+			randomerror:      "randomerror",
+		}
+
+		doDestroy(context.TODO(), opsTemplate, mock)
+
+		assert.NoError(t, errs[0])
+		assert.NoError(t, errs[1])
+		assert.NoError(t, errs[2])
+		assert.NoError(t, errs[3])
+		assert.NoError(t, errs[4])
+		assert.Error(t, errs[5], "undestroyable")
+		assert.NoError(t, errs[6])
+		assert.Error(t, errs[7], "randomerror")
+		assert.NoError(t, errs[8])
+		assert.NoError(t, errs[9])
+
+		defer mock.mtx.Lock().Unlock()
+		assert.Equal(
+			t,
+			[]string{
+				// expect exactly argument order
+				"zroot/z@foo",
+				"zroot/a@foo",
+				"zroot/a@bar",
+				"zroot/b@bar",
+				"zroot/b@zab",
+				"zroot/b@undestroyable",
+				"zroot/c@baz",
+				"zroot/c@randomerror",
+				"zroot/c@bar",
+				"zroot/d@blup",
+			},
+			mock.calls,
+		)
+	})
+
+	t.Run("empty_ops", func(t *testing.T) {
+		mock := &mockBatchDestroy{}
+		doDestroy(context.TODO(), nil, mock)
+		defer mock.mtx.Lock().Unlock()
+		assert.Empty(t, mock.calls)
+	})
+
+	t.Run("ops_without_snapnames", func(t *testing.T) {
+		mock := &mockBatchDestroy{}
+		var err error
+		ops := []*DestroySnapOp{&DestroySnapOp{"somefs", "", &err}}
+		doDestroy(context.TODO(), ops, mock)
+		assert.Error(t, err)
+		defer mock.mtx.Lock().Unlock()
+		assert.Empty(t, mock.calls)
+	})
+
+	t.Run("ops_without_fsnames", func(t *testing.T) {
+		mock := &mockBatchDestroy{}
+		var err error
+		ops := []*DestroySnapOp{&DestroySnapOp{"", "fsname", &err}}
+		doDestroy(context.TODO(), ops, mock)
+		assert.Error(t, err)
+		defer mock.mtx.Lock().Unlock()
+		assert.Empty(t, mock.calls)
+	})
+
+	t.Run("splits_up_batches_at_e2big", func(t *testing.T) {
+		mock := &mockBatchDestroy{
+			e2biglen: 10,
+		}
+
+		var dummy error
+		reqs := []*DestroySnapOp{
+			// should fit (1111@a,b,c)
+			&DestroySnapOp{"1111", "a", &dummy},
+			&DestroySnapOp{"1111", "b", &dummy},
+			&DestroySnapOp{"1111", "c", &dummy},
+
+			// should split
+			&DestroySnapOp{"2222", "01", &dummy},
+			&DestroySnapOp{"2222", "02", &dummy},
+			&DestroySnapOp{"2222", "03", &dummy},
+			&DestroySnapOp{"2222", "04", &dummy},
+			&DestroySnapOp{"2222", "05", &dummy},
+			&DestroySnapOp{"2222", "06", &dummy},
+			&DestroySnapOp{"2222", "07", &dummy},
+			&DestroySnapOp{"2222", "08", &dummy},
+			&DestroySnapOp{"2222", "09", &dummy},
+			&DestroySnapOp{"2222", "10", &dummy},
+		}
+
+		doDestroy(context.TODO(), reqs, mock)
+
+		defer mock.mtx.Lock().Unlock()
+		assert.Equal(
+			t,
+			[]string{
+				"1111@a,b,c",
+				"2222@01,02",
+				"2222@03",
+				"2222@04,05",
+				"2222@06,07",
+				"2222@08",
+				"2222@09,10",
+			},
+			mock.calls,
+		)
+
+	})
+
+}
+
+func TestExcessiveArgumentsResultInE2BIG(t *testing.T) {
+	// FIXME dynamic value
+	const maxArgumentLength = 1 << 20 // higher than any OS we know, should always fail
+	t.Logf("maxArgumentLength=%v", maxArgumentLength)
+	maxArg := bytes.Repeat([]byte("a"), maxArgumentLength)
+	cmd := exec.Command("/bin/sh", "-c", "echo -n $1; echo -n $2", "cmdname", string(maxArg), string(maxArg))
+	output, err := cmd.CombinedOutput()
+	if pe, ok := err.(*os.PathError); ok && pe.Err == syscall.E2BIG {
+		t.Logf("ok, system returns E2BIG")
+	} else {
+		t.Errorf("system did not return E2BIG, but err=%T: %v ", err, err)
+		t.Logf("output:\n%s", output)
+	}
+}

internal/zfs/zfs_debug.go

@@ -0,0 +1,21 @@
+package zfs
+
+import (
+	"fmt"
+	"os"
+)
+
+var debugEnabled bool = false
+
+func init() {
+	if os.Getenv("ZREPL_ZFS_DEBUG") != "" {
+		debugEnabled = true
+	}
+}
+
+//nolint:deadcode,unused
+func debug(format string, args ...interface{}) {
+	if debugEnabled {
+		fmt.Fprintf(os.Stderr, "zfs: %s\n", fmt.Sprintf(format, args...))
+	}
+}

internal/zfs/zfs_pipe.go

@@ -0,0 +1,23 @@
+//go:build !linux
+// +build !linux
+
+package zfs
+
+import (
+	"os"
+	"sync"
+)
+
+func getPipeCapacityHint(envvar string) int {
+	return 0 // not supported
+}
+
+var zfsPipeCapacityNotSupported sync.Once
+
+func trySetPipeCapacity(p *os.File, capacity int) {
+	if debugEnabled && capacity != 0 {
+		zfsPipeCapacityNotSupported.Do(func() {
+			debug("trySetPipeCapacity error: OS does not support setting pipe capacity")
+		})
+	}
+}

internal/zfs/zfs_pipe_linux.go

@@ -0,0 +1,42 @@
+package zfs
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/zrepl/zrepl/internal/util/envconst"
+)
+
+func getPipeCapacityHint(envvar string) int {
+	var capacity int64 = 1 << 25
+
+	// Work around a race condition in Linux >= 5.8 related to pipe resizing.
+	// https://github.com/zrepl/zrepl/issues/424#issuecomment-800370928
+	// https://bugzilla.kernel.org/show_bug.cgi?id=212295
+	if _, err := os.Stat("/proc/sys/fs/pipe-max-size"); err == nil {
+		if dat, err := os.ReadFile("/proc/sys/fs/pipe-max-size"); err == nil {
+			if capacity, err = strconv.ParseInt(strings.TrimSpace(string(dat)), 10, 64); err != nil {
+				capacity = 1 << 25
+			}
+		}
+	}
+
+	return int(envconst.Int64(envvar, capacity))
+}
+
+func trySetPipeCapacity(p *os.File, capacity int) {
+	res, err := unix.FcntlInt(p.Fd(), unix.F_SETPIPE_SZ, capacity)
+	if err != nil {
+		err = fmt.Errorf("cannot set pipe capacity to %v", capacity)
+	} else if res == -1 {
+		err = errors.New("cannot set pipe capacity: fcntl returned -1")
+	}
+	if debugEnabled && err != nil {
+		debug("trySetPipeCapacity error: %s\n", err)
+	}
+}

internal/zfs/zfs_test.go

@@ -0,0 +1,452 @@
+package zfs
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/zrepl/zrepl/internal/util/nodefault"
+	zfsprop "github.com/zrepl/zrepl/internal/zfs/property"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// FIXME make this a platformtest
+func TestZFSListHandlesProducesZFSErrorOnNonZeroExit(t *testing.T) {
+	t.SkipNow() // FIXME ZFS_BINARY does not work if tests run in parallel
+
+	var err error
+
+	ctx := context.Background()
+
+	ZFS_BINARY = "./test_helpers/zfs_failer.sh"
+
+	_, err = ZFSList(ctx, []string{"fictionalprop"}, "nonexistent/dataset")
+
+	assert.Error(t, err)
+	zfsError, ok := err.(*ZFSError)
+	assert.True(t, ok)
+	assert.Equal(t, "error: this is a mock\n", string(zfsError.Stderr))
+}
+
+func TestDatasetPathTrimNPrefixComps(t *testing.T) {
+	p, err := NewDatasetPath("foo/bar/a/b")
+	assert.Nil(t, err)
+	p.TrimNPrefixComps(2)
+	assert.True(t, p.Equal(toDatasetPath("a/b")))
+	p.TrimNPrefixComps((2))
+	assert.True(t, p.Empty())
+	p.TrimNPrefixComps((1))
+	assert.True(t, p.Empty(), "empty trimming shouldn't do harm")
+}
+
+func TestZFSPropertySource(t *testing.T) {
+
+	tcs := []struct {
+		in  PropertySource
+		exp []string
+	}{
+		{
+			in: SourceAny,
+			// although empty prefix matches any source
+			exp: []string{"local", "default", "inherited", "-", "temporary", "received", ""},
+		},
+		{
+			in:  SourceTemporary,
+			exp: []string{"temporary"},
+		},
+		{
+			in:  SourceLocal | SourceInherited,
+			exp: []string{"local", "inherited"},
+		},
+	}
+
+	toSet := func(in []string) map[string]struct{} {
+		m := make(map[string]struct{}, len(in))
+		for _, s := range in {
+			m[s] = struct{}{}
+		}
+		return m
+	}
+
+	for _, tc := range tcs {
+
+		t.Logf("TEST CASE %v", tc)
+
+		// give the parsing code some coverage
+		for _, e := range tc.exp {
+			if e == "" {
+				continue // "" is the prefix that matches SourceAny
+			}
+			s, err := parsePropertySource(e)
+			assert.NoError(t, err)
+			t.Logf("s: %x %s", s, s)
+			t.Logf("in: %x %s", tc.in, tc.in)
+			assert.True(t, s&tc.in != 0)
+		}
+
+		// prefix matching
+		res := tc.in.zfsGetSourceFieldPrefixes()
+		resSet := toSet(res)
+		expSet := toSet(tc.exp)
+		assert.Equal(t, expSet, resSet)
+	}
+
+}
+
+func TestDrySendRegexesHaveSameCaptureGroupCount(t *testing.T) {
+	assert.Equal(t, sendDryRunInfoLineRegexFull.NumSubexp(), sendDryRunInfoLineRegexIncremental.NumSubexp())
+}
+
+func TestDrySendInfo(t *testing.T) {
+
+	// # full send
+	// $ zfs send -Pnv -t 1-9baebea70-b8-789c636064000310a500c4ec50360710e72765a52697303030419460caa7a515a79680647ce0f26c48f2499525a9c5405ac3c90fabfe92fcf4d2cc140686b30972c7850efd0cd24092e704cbe725e6a632305415e5e797e803cd2ad14f743084b805001b201795
+	fullSend := `
+resume token contents:
+nvlist version: 0
+	object = 0x2
+	offset = 0x4c0000
+	bytes = 0x4e4228
+	toguid = 0x52f9c212c71e60cd
+	toname = zroot/test/a@1
+full	zroot/test/a@1	5389768
+`
+
+	// # incremental send with token
+	// $ zfs send -nvP -t 1-ef01e717e-e0-789c636064000310a501c49c50360710a715e5e7a69766a63040c1d904b9e342877e062900d9ec48eaf293b252934b181898a0ea30e4d3d28a534b40323e70793624f9a4ca92d46220fdc1ce0fabfe927c882bc46c8a0a9f71ad3baf8124cf0996cf4bcc4d6560a82acacf2fd1079a55a29fe86004710b00d8ae1f93
+	incSend := `
+resume token contents:
+nvlist version: 0
+	fromguid = 0x52f9c212c71e60cd
+	object = 0x2
+	offset = 0x4c0000
+	bytes = 0x4e3ef0
+	toguid = 0xcfae0ae671723c16
+	toname = zroot/test/a@2
+incremental	zroot/test/a@1	zroot/test/a@2	5383936
+`
+
+	// # incremental send with token + bookmark
+	// $ zfs send -nvP -t 1-ef01e717e-e0-789c636064000310a501c49c50360710a715e5e7a69766a63040c1d904b9e342877e062900d9ec48eaf293b252934b181898a0ea30e4d3d28a534b40323e70793624f9a4ca92d46220fdc1ce0fabfe927c882bc46c8a0a9f71ad3baf8124cf0996cf4bcc4d6560a82acacf2fd1079a55a29fe86004710b00d8ae1f93
+	incSendBookmark := `
+resume token contents:
+nvlist version: 0
+	fromguid = 0x52f9c212c71e60cd
+	object = 0x2
+	offset = 0x4c0000
+	bytes = 0x4e3ef0
+	toguid = 0xcfae0ae671723c16
+	toname = zroot/test/a@2
+incremental	zroot/test/a#1	zroot/test/a@2	5383312
+
+`
+
+	// incremental send without token
+	// $ sudo zfs send -nvP -i @1 zroot/test/a@2
+	incNoToken := `
+incremental	1	zroot/test/a@2	10511856
+size	10511856
+`
+	// full send without token
+	// $ sudo zfs send -nvP  zroot/test/a@3
+	fullNoToken := `
+full	zroot/test/a@3	10518512
+size	10518512
+`
+
+	// zero-length incremental send on ZoL 0.7.12
+	// (it omits the size field as well as the size line if size is 0)
+	// see https://github.com/zrepl/zrepl/issues/289
+	// fixed in https://github.com/openzfs/zfs/commit/835db58592d7d947e5818eb7281882e2a46073e0#diff-66bd524398bcd2ac70d90925ab6d8073L1245
+	incZeroSized_0_7_12 := `
+incremental	p1 with/ spaces d1@1 with space	p1 with/ spaces d1@2 with space
+`
+
+	fullZeroSized_0_7_12 := `
+full	p1 with/ spaces d1@2 with space
+`
+
+	fullWithSpaces := "\nfull\tpool1/otherjob/ds with spaces@blaffoo\t12912\nsize\t12912\n"
+	fullWithSpacesInIntermediateComponent := "\nfull\tpool1/otherjob/another ds with spaces/childfs@blaffoo\t12912\nsize\t12912\n"
+	incrementalWithSpaces := "\nincremental\tblaffoo\tpool1/otherjob/another ds with spaces@blaffoo2\t624\nsize\t624\n"
+	incrementalWithSpacesInIntermediateComponent := "\nincremental\tblaffoo\tpool1/otherjob/another ds with spaces/childfs@blaffoo2\t624\nsize\t624\n"
+
+	type tc struct {
+		name   string
+		in     string
+		exp    *DrySendInfo
+		expErr bool
+	}
+
+	tcs := []tc{
+		{
+			name: "fullSend", in: fullSend,
+			exp: &DrySendInfo{
+				Type:         DrySendTypeFull,
+				Filesystem:   "zroot/test/a",
+				From:         "",
+				To:           "zroot/test/a@1",
+				SizeEstimate: 5389768,
+			},
+		},
+		{
+			name: "incSend", in: incSend,
+			exp: &DrySendInfo{
+				Type:         DrySendTypeIncremental,
+				Filesystem:   "zroot/test/a",
+				From:         "zroot/test/a@1",
+				To:           "zroot/test/a@2",
+				SizeEstimate: 5383936,
+			},
+		},
+		{
+			name: "incSendBookmark", in: incSendBookmark,
+			exp: &DrySendInfo{
+				Type:         DrySendTypeIncremental,
+				Filesystem:   "zroot/test/a",
+				From:         "zroot/test/a#1",
+				To:           "zroot/test/a@2",
+				SizeEstimate: 5383312,
+			},
+		},
+		{
+			name: "incNoToken", in: incNoToken,
+			exp: &DrySendInfo{
+				Type:       DrySendTypeIncremental,
+				Filesystem: "zroot/test/a",
+				// as can be seen in the string incNoToken,
+				// we cannot infer whether the incremental source is a snapshot or bookmark
+				From:         "1", // yes, this is actually correct on ZoL 0.7.11
+				To:           "zroot/test/a@2",
+				SizeEstimate: 10511856,
+			},
+		},
+		{
+			name: "fullNoToken", in: fullNoToken,
+			exp: &DrySendInfo{
+				Type:         DrySendTypeFull,
+				Filesystem:   "zroot/test/a",
+				From:         "",
+				To:           "zroot/test/a@3",
+				SizeEstimate: 10518512,
+			},
+		},
+		{
+			name: "fullWithSpaces", in: fullWithSpaces,
+			exp: &DrySendInfo{
+				Type:         DrySendTypeFull,
+				Filesystem:   "pool1/otherjob/ds with spaces",
+				From:         "",
+				To:           "pool1/otherjob/ds with spaces@blaffoo",
+				SizeEstimate: 12912,
+			},
+		},
+		{
+			name: "fullWithSpacesInIntermediateComponent", in: fullWithSpacesInIntermediateComponent,
+			exp: &DrySendInfo{
+				Type:         DrySendTypeFull,
+				Filesystem:   "pool1/otherjob/another ds with spaces/childfs",
+				From:         "",
+				To:           "pool1/otherjob/another ds with spaces/childfs@blaffoo",
+				SizeEstimate: 12912,
+			},
+		},
+		{
+			name: "incrementalWithSpaces", in: incrementalWithSpaces,
+			exp: &DrySendInfo{
+				Type:         DrySendTypeIncremental,
+				Filesystem:   "pool1/otherjob/another ds with spaces",
+				From:         "blaffoo",
+				To:           "pool1/otherjob/another ds with spaces@blaffoo2",
+				SizeEstimate: 624,
+			},
+		},
+		{
+			name: "incrementalWithSpacesInIntermediateComponent", in: incrementalWithSpacesInIntermediateComponent,
+			exp: &DrySendInfo{
+				Type:         DrySendTypeIncremental,
+				Filesystem:   "pool1/otherjob/another ds with spaces/childfs",
+				From:         "blaffoo",
+				To:           "pool1/otherjob/another ds with spaces/childfs@blaffoo2",
+				SizeEstimate: 624,
+			},
+		},
+		{
+			name: "incrementalZeroSizedOpenZFS_pre0.7.12", in: incZeroSized_0_7_12,
+			exp: &DrySendInfo{
+				Type:         DrySendTypeIncremental,
+				Filesystem:   "p1 with/ spaces d1",
+				From:         "p1 with/ spaces d1@1 with space",
+				To:           "p1 with/ spaces d1@2 with space",
+				SizeEstimate: 0,
+			},
+		},
+		{
+			name: "fullZeroSizedOpenZFS_pre0.7.12", in: fullZeroSized_0_7_12,
+			exp: &DrySendInfo{
+				Type:         DrySendTypeFull,
+				Filesystem:   "p1 with/ spaces d1",
+				To:           "p1 with/ spaces d1@2 with space",
+				SizeEstimate: 0,
+			},
+		},
+	}
+
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+
+			in := tc.in[1:] // strip first newline
+			var si DrySendInfo
+			err := si.unmarshalZFSOutput([]byte(in))
+			t.Logf("%#v", &si)
+			t.Logf("err=%T %s", err, err)
+
+			if tc.expErr {
+				assert.Error(t, err)
+			}
+			if tc.exp != nil {
+				assert.Equal(t, tc.exp, &si)
+			}
+
+		})
+	}
+}
+
+func TestTryRecvDestroyOrOverwriteEncryptedErr(t *testing.T) {
+	msg := "cannot receive new filesystem stream: zfs receive -F cannot be used to destroy an encrypted filesystem or overwrite an unencrypted one with an encrypted one\n"
+	assert.GreaterOrEqual(t, RecvStderrBufSiz, len(msg))
+
+	err := tryRecvDestroyOrOverwriteEncryptedErr([]byte(msg))
+	require.NotNil(t, err)
+	assert.EqualError(t, err, strings.TrimSpace(msg))
+}
+
+func TestZFSSendArgsBuildSendFlags(t *testing.T) {
+
+	type args = ZFSSendFlags
+	type SendTest struct {
+		conf         args
+		exactMatch   bool
+		flagsInclude []string
+		flagsExclude []string
+	}
+
+	withEncrypted := func(justFlags ZFSSendFlags) ZFSSendFlags {
+		if justFlags.Encrypted == nil {
+			justFlags.Encrypted = &nodefault.Bool{B: false}
+		}
+		return justFlags
+	}
+
+	var sendTests = map[string]SendTest{
+		"Empty Args": {
+			conf:         withEncrypted(args{}),
+			flagsInclude: []string{},
+			flagsExclude: []string{"-w", "-p", "-b"},
+		},
+		"Raw": {
+			conf:         withEncrypted(args{Raw: true}),
+			flagsInclude: []string{"-w"},
+			flagsExclude: []string{},
+		},
+		"Encrypted": {
+			conf:         withEncrypted(args{Encrypted: &nodefault.Bool{B: true}}),
+			flagsInclude: []string{"-w"},
+			flagsExclude: []string{},
+		},
+		"Unencrypted_And_Raw": {
+			conf:         withEncrypted(args{Encrypted: &nodefault.Bool{B: false}, Raw: true}),
+			flagsInclude: []string{"-w"},
+			flagsExclude: []string{},
+		},
+		"Encrypted_And_Raw": {
+			conf:         withEncrypted(args{Encrypted: &nodefault.Bool{B: true}, Raw: true}),
+			flagsInclude: []string{"-w"},
+			flagsExclude: []string{},
+		},
+		"Send properties": {
+			conf:         withEncrypted(args{Properties: true}),
+			flagsInclude: []string{"-p"},
+			flagsExclude: []string{},
+		},
+		"Send backup properties": {
+			conf:         withEncrypted(args{BackupProperties: true}),
+			flagsInclude: []string{"-b"},
+			flagsExclude: []string{},
+		},
+		"Send -b and -p": {
+			conf:         withEncrypted(args{Properties: true, BackupProperties: true}),
+			flagsInclude: []string{"-p", "-b"},
+			flagsExclude: []string{},
+		},
+		"Send resume state": {
+			conf:         withEncrypted(args{Saved: true}),
+			flagsInclude: []string{"-S"},
+		},
+		"Resume token wins if not empty": {
+			conf:         withEncrypted(args{ResumeToken: "$theresumetoken$", Compressed: true}),
+			flagsInclude: []string{"-t", "$theresumetoken$"},
+			exactMatch:   true,
+		},
+	}
+
+	for testName, test := range sendTests {
+		t.Run(testName, func(t *testing.T) {
+			flags := test.conf.buildSendFlagsUnchecked()
+			assert.GreaterOrEqual(t, len(flags), len(test.flagsInclude))
+			assert.Subset(t, flags, test.flagsInclude)
+			if test.exactMatch {
+				assert.Equal(t, flags, test.flagsInclude)
+			}
+			for flag := range flags {
+				assert.NotContains(t, test.flagsExclude, flag)
+			}
+		})
+	}
+}
+
+func TestZFSCommonRecvArgsBuild(t *testing.T) {
+	type RecvTest struct {
+		conf         RecvOptions
+		flagsInclude []string
+		flagsExclude []string
+	}
+	var recvTests = map[string]RecvTest{
+		"Empty Args": {
+			conf:         RecvOptions{},
+			flagsInclude: []string{},
+			flagsExclude: []string{"-x", "-o", "-F", "-s"},
+		},
+		"ForceRollback": {
+			conf:         RecvOptions{RollbackAndForceRecv: true},
+			flagsInclude: []string{"-F"},
+			flagsExclude: []string{"-x", "-o", "-s"},
+		},
+		"PartialSend": {
+			conf:         RecvOptions{SavePartialRecvState: true},
+			flagsInclude: []string{"-s"},
+			flagsExclude: []string{"-x", "-o", "-F"},
+		},
+		"Override properties": {
+			conf:         RecvOptions{OverrideProperties: map[zfsprop.Property]string{zfsprop.Property("abc"): "123"}},
+			flagsInclude: []string{"-o", "abc=123"},
+			flagsExclude: []string{"-x", "-F", "-s"},
+		},
+		"Exclude/inherit properties": {
+			conf:         RecvOptions{InheritProperties: []zfsprop.Property{"abc", "123"}},
+			flagsInclude: []string{"-x", "abc", "123"}, flagsExclude: []string{"-o", "-F", "-s"},
+		},
+	}
+
+	for testName, test := range recvTests {
+		t.Run(testName, func(t *testing.T) {
+			flags := test.conf.buildRecvFlags()
+			assert.Subset(t, flags, test.flagsInclude)
+			for flag := range flags {
+				assert.NotContains(t, test.flagsExclude, flag)
+			}
+		})
+	}
+}

internal/zfs/zfscmd/zfscmd-logging-scraper/README.md

@@ -0,0 +1,11 @@
+The tool in this package (`go run . -h`) scrapes log lines produces by the `github.com/zrepl/zrepl/zfs/zfscmd` package
+into a stream of JSON objects.
+
+The `analysis.ipynb` then runs some basic analysis on the collected log output.
+
+## Deps for the `scrape_graylog_csv.bash` script
+
+```
+pip install --upgrade git+https://github.com/lk-jeffpeck/csvfilter.git@ec433f14330fbbf5d41f56febfeedac22868a949
+```
+

internal/zfs/zfscmd/zfscmd-logging-scraper/analysis.ipynb

@@ -0,0 +1,260 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import seaborn as sns\n",
+    "import matplotlib.pyplot as plt\n",
+    "import re\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ./parsed.json is the stdout of the scraper tool in this directory\n",
+    "df = pd.read_json(\"./parsed.json\", lines=True)\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def parse_ds(entity):\n",
+    "    m = re.search(r\"(?P<dataset>[^@#]*)([@#].+)?\", entity)\n",
+    "    return m.group(\"dataset\")\n",
+    "    \n",
+    "def parse_cmd(row):\n",
+    "    cmd  = row.Cmd\n",
+    "    binary, verb, *tail = re.split(r\"\\s+\", cmd) # NOTE whitespace in dataset names => don't use comp\n",
+    "    \n",
+    "    dataset = None\n",
+    "    if binary == \"zfs\":\n",
+    "        if verb == \"send\":      \n",
+    "            if len(tail) == 0:\n",
+    "                verb = \"send-feature-test\"\n",
+    "            else:\n",
+    "                dataset = parse_ds(tail[-1])\n",
+    "                if \"-n\" in tail:\n",
+    "                    verb = \"send-dry\"\n",
+    "        elif verb == \"recv\" or verb == \"receive\":\n",
+    "            verb = \"receive\"\n",
+    "            if len(tail) > 0:\n",
+    "                dataset = parse_ds(tail[-1])\n",
+    "            else:\n",
+    "                verb = \"receive-CLI-test\"\n",
+    "        elif verb == \"get\":\n",
+    "            dataset = parse_ds(tail[-1])\n",
+    "        elif verb == \"list\":\n",
+    "            if \"-r\" in tail and \"-d\" in tail and \"1\" in tail:\n",
+    "                dataset = parse_ds(tail[-1])\n",
+    "                verb = \"list-single-dataset\"\n",
+    "            else:\n",
+    "                dataset = \"!ALL_POOLS!\"\n",
+    "                verb = \"list-all-filesystems\"\n",
+    "        elif verb == \"bookmark\":\n",
+    "            dataset = parse_ds(tail[-2])\n",
+    "        elif verb == \"hold\":\n",
+    "            dataset = parse_ds(tail[-1])\n",
+    "        elif verb == \"snapshot\":\n",
+    "            dataset = parse_ds(tail[-1])\n",
+    "        elif verb == \"release\":\n",
+    "            dss = tail[-1].split(\",\")\n",
+    "            if len(dss) > 1:\n",
+    "                raise Exception(\"cannot handle batch-release\")\n",
+    "            dataset = parse_ds(dss[0])\n",
+    "        elif verb == \"holds\" and \"-H\" in tail:\n",
+    "            dss = tail[-1].split(\",\")\n",
+    "            if len(dss) > 1:\n",
+    "                raise Exception(\"cannot handle batch-holds\")\n",
+    "            dataset = parse_ds(dss[0])\n",
+    "        elif verb == \"destroy\":\n",
+    "            dss = tail[-1].split(\",\")\n",
+    "            if len(dss) > 1:\n",
+    "                raise Exception(\"cannot handle batch-holds\")\n",
+    "            dataset = parse_ds(dss[0])\n",
+    "    \n",
+    "    return {'action':binary + \"-\" + verb, 'dataset': dataset }\n",
+    "    \n",
+    "    \n",
+    "res = df.apply(parse_cmd, axis='columns', result_type='expand')\n",
+    "res = pd.concat([df, res], axis='columns')\n",
+    "for cat in [\"action\", \"dataset\"]:\n",
+    "    res[cat] = res[cat].astype('category')\n",
+    "\n",
+    "res[\"LogTimeUnix\"] = pd.to_datetime(res.LogTime)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res[\"OtherTime\"] = res.TotalTime - res.Usertime - res.Systime\n",
+    "x = res.melt(id_vars=[\"action\", \"dataset\"], value_vars=[\"TotalTime\", \"OtherTime\", \"Usertime\", \"Systime\"])\n",
+    "x"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(\"commands with NaN values\")\n",
+    "set(x[x.isna().any(axis=1)].action.values)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# (~x.action.astype('str').isin([\"zfs-send\", \"zfs-recv\"]))\n",
+    "totaltimes = x[(x.variable == \"TotalTime\")].groupby([\"action\", \"dataset\"]).sum().reset_index()\n",
+    "display(totaltimes)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "totaltimes_by_action = totaltimes.groupby(\"action\").sum().sort_values(by=\"value\")\n",
+    "totaltimes_by_action.plot.barh()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "totaltimes.groupby(\"dataset\").sum().plot.barh(fontsize=5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "most_expensive_action = totaltimes_by_action.idxmax().value\n",
+    "display(most_expensive_action)\n",
+    "most_expensive_action_by_dataset = totaltimes[totaltimes.action == most_expensive_action].groupby(\"dataset\").sum().sort_values(by=\"value\")\n",
+    "most_expensive_action_by_dataset.plot.barh(rot=50, fontsize=5, figsize=(10, 20))\n",
+    "plt.savefig('most-expensive-command.pdf')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# %matplotlib notebook \n",
+    "\n",
+    "# res.index = res.LogTimeUnix\n",
+    "\n",
+    "# resampled = res.pivot(columns='action', values='TotalTime').resample(\"1s\").sum()\n",
+    "# resampled.cumsum().plot()\n",
+    "# res[\"BeginTime\"] = res.LogTimeUnix.dt.total_seconds()\n",
+    "# holds = res[res.action == \"zfs-holds\"]\n",
+    "# sns.stripplot(x=\"LogTimeUnix\", y=\"action\", data=res)\n",
+    "# res[\"LogTimeUnix\"].resample(\"20min\").sum()\n",
+    "# res[res.action == \"zfs-holds\"].plot.scatter(x=\"LogTimeUnix\", y=\"TotalTime\")\n",
+    "\n",
+    "#res[res.action == \"zfs-holds\"].pivot(columns='action', values=['TotalTime', 'Systime', \"Usertime\"]).resample(\"1s\").sum().cumsum().plot()\n",
+    "pivoted = res.reset_index(drop=True).pivot_table(values=['TotalTime', 'Systime', \"Usertime\"], index=\"LogTimeUnix\", columns=\"action\")\n",
+    "pivoted"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pivoted.cumsum()[[(\"TotalTime\", \"zfs-holds\"),(\"Systime\", \"zfs-holds\"),(\"Usertime\", \"zfs-holds\")]].plot()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pivoted = res.reset_index(drop=True).pivot_table(values=['TotalTime'], index=\"LogTimeUnix\", columns=\"action\")\n",
+    "cum_invocation_counts_per_action = pivoted.isna().astype(int).cumsum()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cum_invocation_counts_per_action"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# zfs-get as reference value\n",
+    "cum_invocation_counts_per_action[[(\"TotalTime\",\"zfs-holds\"),(\"TotalTime\",\"zfs-get\")]].plot()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

internal/zfs/zfscmd/zfscmd-logging-scraper/scrape_graylog_csv.bash

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# This script converts output that was produced by zrepl and captured by Graylog
+# back to something that the scraper in this package's main can understand
+# Intended for human syslog
+# logging:
+# - type: syslog
+#   level: debug
+#   format: human
+
+
+csvfilter --skip 1 -f 0,2 -q '"' --out-quotechar=' ' /dev/stdin | sed -E 's/^\s*([^,]*), /\1 [LEVEL]/' | \
+    go run . -v \
+    --dateRE '^([^\[]+) (\[.*)' \
+    --dateFormat '2006-01-02T15:04:05.999999999Z'
+

internal/zfs/zfscmd/zfscmd-logging-scraper/zfscmd_logging_scraper.go

@@ -0,0 +1,129 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/go-logfmt/logfmt"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/internal/daemon/logging"
+)
+
+type RuntimeLine struct {
+	LogTime                      time.Time
+	Cmd                          string
+	TotalTime, Usertime, Systime time.Duration
+	Error                        string
+}
+
+var humanFormatterLineRE = regexp.MustCompile(`^(\[[^\]]+\]){2}\[zfs.cmd\]\[[^\]]+\]:\s+command\s+exited\s+(with|without)\s+error\s+(.+)`)
+
+func parseSecs(s string) (time.Duration, error) {
+	d, err := time.ParseDuration(s + "s")
+	if err != nil {
+		return 0, errors.Wrapf(err, "parse duration %q", s)
+	}
+	return d, nil
+}
+
+func parseHumanFormatterNodate(line string) (l RuntimeLine, err error) {
+	m := humanFormatterLineRE.FindStringSubmatch(line)
+	if m == nil {
+		return l, errors.New("human formatter regex does not match")
+	}
+
+	d := logfmt.NewDecoder(strings.NewReader(m[3]))
+	for d.ScanRecord() {
+		for d.ScanKeyval() {
+			k := string(d.Key())
+			v := string(d.Value())
+			switch k {
+			case "cmd":
+				l.Cmd = v
+			case "total_time_s":
+				l.TotalTime, err = parseSecs(v)
+			case "usertime_s":
+				l.Usertime, err = parseSecs(v)
+			case "systemtime_s":
+				l.Systime, err = parseSecs(v)
+			case "err":
+				l.Error = v
+			case "invocation":
+				continue // pass
+			default:
+				return l, errors.Errorf("unknown key %q", k)
+			}
+			if err != nil {
+				return l, err
+			}
+		}
+	}
+	if d.Err() != nil {
+		return l, errors.Wrap(d.Err(), "decode key value pairs")
+	}
+	return l, nil
+}
+
+func parseLogLine(line string) (l RuntimeLine, err error) {
+	m := dateRegex.FindStringSubmatch(line)
+	if len(m) != 3 {
+		return l, errors.Errorf("invalid date regex match %v", m)
+	}
+	dateTrimmed := strings.TrimSpace(m[1])
+	date, err := time.Parse(dateFormat, dateTrimmed)
+	if err != nil {
+		panic(fmt.Sprintf("cannot parse date %q: %s", dateTrimmed, err))
+	}
+	logLine := m[2]
+
+	l, err = parseHumanFormatterNodate(strings.TrimSpace(logLine))
+	l.LogTime = date
+	return l, err
+}
+
+var verbose bool
+var dateRegexArg string
+var dateRegex = regexp.MustCompile(`^([^\[]+)(.*)`)
+var dateFormat = logging.HumanFormatterDateFormat
+
+func main() {
+
+	pflag.StringVarP(&dateRegexArg, "dateRE", "d", "", "date regex")
+	pflag.StringVar(&dateFormat, "dateFormat", logging.HumanFormatterDateFormat, "go date format")
+	pflag.BoolVarP(&verbose, "verbose", "v", false, "verbose")
+	pflag.Parse()
+
+	if dateRegexArg != "" {
+		dateRegex = regexp.MustCompile(dateRegexArg)
+	}
+
+	input := bufio.NewScanner(os.Stdin)
+	input.Split(bufio.ScanLines)
+
+	enc := json.NewEncoder(os.Stdout)
+	for input.Scan() {
+
+		l, err := parseLogLine(input.Text())
+		if err != nil && verbose {
+			fmt.Fprintf(os.Stderr, "ignoring line after error %v\n", err)
+			fmt.Fprintf(os.Stderr, "offending line was: %s\n", input.Text())
+		}
+		if err == nil {
+			if err := enc.Encode(l); err != nil {
+				panic(err)
+			}
+		}
+	}
+
+	if input.Err() != nil {
+		panic(input.Err())
+	}
+
+}

internal/zfs/zfscmd/zfscmd-logging-scraper/zfscmd_logging_scrapter_test.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/daemon/logging"
+)
+
+func TestParseHumanFormatter(t *testing.T) {
+
+	type testCase struct {
+		Name      string
+		Input     string
+		Expect    *RuntimeLine
+		ExpectErr string
+	}
+
+	secs := func(s string) time.Duration {
+		d, err := parseSecs(s)
+		require.NoError(t, err)
+		return d
+	}
+
+	logTime, err := time.Parse(logging.HumanFormatterDateFormat, "2020-04-04T00:00:05+02:00")
+	require.NoError(t, err)
+
+	tcs := []testCase{
+		{
+			Name:  "human-formatter-noerror",
+			Input: `2020-04-04T00:00:05+02:00 [DEBG][jobname][zfs.cmd][task$stack$span.stack]: command exited without error usertime_s="0.008445" cmd="zfs list -H -p -o name -r -t filesystem,volume" systemtime_s="0.033783" invocation="84" total_time_s="0.037828619"`,
+			Expect: &RuntimeLine{
+				Cmd:       "zfs list -H -p -o name -r -t filesystem,volume",
+				TotalTime: secs("0.037828619"),
+				Usertime:  secs("0.008445"),
+				Systime:   secs("0.033783"),
+				Error:     "",
+				LogTime:   logTime,
+			},
+		},
+		{
+			Name:  "human-formatter-witherror",
+			Input: `2020-04-04T00:00:05+02:00 [DEBG][jobname][zfs.cmd][task$stack$span.stack]: command exited with error usertime_s="0.008445" cmd="zfs list -H -p -o name -r -t filesystem,volume" systemtime_s="0.033783" invocation="84" total_time_s="0.037828619" err="some error"`,
+			Expect: &RuntimeLine{
+				Cmd:       "zfs list -H -p -o name -r -t filesystem,volume",
+				TotalTime: secs("0.037828619"),
+				Usertime:  secs("0.008445"),
+				Systime:   secs("0.033783"),
+				Error:     "some error",
+				LogTime:   logTime,
+			},
+		},
+		{
+			Name:  "from graylog",
+			Input: `2020-04-04T00:00:05+02:00 [DEBG][csnas][zfs.cmd][task$stack$span.stack]:  command  exited  without  error  usertime_s="0"  cmd="zfs  send  -i  zroot/ezjail/synapse-12@zrepl_20200329_095518_000  zroot/ezjail/synapse-12@zrepl_20200329_102454_000"  total_time_s="0.101598591"  invocation="85"  systemtime_s="0.041581"`,
+			Expect: &RuntimeLine{
+				Cmd:       "zfs  send  -i  zroot/ezjail/synapse-12@zrepl_20200329_095518_000  zroot/ezjail/synapse-12@zrepl_20200329_102454_000",
+				TotalTime: secs("0.101598591"),
+				Systime:   secs("0.041581"),
+				Usertime:  secs("0"),
+				Error:     "",
+				LogTime:   logTime,
+			},
+		},
+	}
+
+	for _, c := range tcs {
+		t.Run(c.Name, func(t *testing.T) {
+			l, err := parseLogLine(c.Input)
+			t.Logf("l=%v", l)
+			t.Logf("err=%T %v", err, err)
+			if (c.Expect != nil && c.ExpectErr != "") || (c.Expect == nil && c.ExpectErr == "") {
+				t.Fatal("bad test case", c)
+			}
+			if c.Expect != nil {
+				require.Equal(t, *c.Expect, l)
+			}
+			if c.ExpectErr != "" {
+				require.EqualError(t, err, c.ExpectErr)
+			}
+		})
+	}
+
+}

internal/zfs/zfscmd/zfscmd.go

@@ -0,0 +1,215 @@
+// Package zfscmd provides a wrapper around packate os/exec.
+// Functionality provided by the wrapper:
+// - logging start and end of command execution
+// - status report of active commands
+// - prometheus metrics of runtimes
+package zfscmd
+
+import (
+	"context"
+	"io"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/zrepl/zrepl/internal/daemon/logging/trace"
+	"github.com/zrepl/zrepl/internal/util/circlog"
+)
+
+type Cmd struct {
+	cmd                                      *exec.Cmd
+	ctx                                      context.Context
+	mtx                                      sync.RWMutex
+	startedAt, waitStartedAt, waitReturnedAt time.Time
+	waitReturnEndSpanCb                      trace.DoneFunc
+}
+
+func CommandContext(ctx context.Context, name string, arg ...string) *Cmd {
+	cmd := exec.CommandContext(ctx, name, arg...)
+	return &Cmd{cmd: cmd, ctx: ctx}
+}
+
+// err.(*exec.ExitError).Stderr will NOT be set
+func (c *Cmd) CombinedOutput() (o []byte, err error) {
+	c.startPre(false)
+	c.startPost(nil)
+	c.waitPre()
+	o, err = c.cmd.CombinedOutput()
+	c.waitPost(err)
+	return
+}
+
+// err.(*exec.ExitError).Stderr will be set
+func (c *Cmd) Output() (o []byte, err error) {
+	c.startPre(false)
+	c.startPost(nil)
+	c.waitPre()
+	o, err = c.cmd.Output()
+	c.waitPost(err)
+	return
+}
+
+// Careful: err.(*exec.ExitError).Stderr will not be set, even if you don't open an StderrPipe
+func (c *Cmd) StdoutPipeWithErrorBuf() (p io.ReadCloser, errBuf *circlog.CircularLog, err error) {
+	p, err = c.cmd.StdoutPipe()
+	errBuf = circlog.MustNewCircularLog(1 << 15)
+	c.cmd.Stderr = errBuf
+	return p, errBuf, err
+}
+
+type Stdio struct {
+	Stdin  io.ReadCloser
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+func (c *Cmd) SetStdio(stdio Stdio) {
+	c.cmd.Stdin = stdio.Stdin
+	c.cmd.Stderr = stdio.Stderr
+	c.cmd.Stdout = stdio.Stdout
+}
+
+func (c *Cmd) String() string {
+	return strings.Join(c.cmd.Args, " ") // includes argv[0] if initialized with CommandContext, that's the only way we o it
+}
+
+func (c *Cmd) log() Logger {
+	return getLogger(c.ctx).WithField("cmd", c.String())
+}
+
+// Start the command.
+//
+// This creates a new trace.WithTask as a child task of the ctx passed to CommandContext.
+// If the process is successfully started (err == nil), it is the CALLER'S RESPONSIBILITY to ensure that
+// the spawned process does not outlive the ctx's trace.Task.
+//
+// If this method returns an error, the Cmd instance is invalid. Start must not be called repeatedly.
+func (c *Cmd) Start() (err error) {
+	c.startPre(true)
+	err = c.cmd.Start()
+	c.startPost(err)
+	return err
+}
+
+// Get the underlying os.Process.
+//
+// Only call this method after a successful call to .Start().
+func (c *Cmd) Process() *os.Process {
+	if c.startedAt.IsZero() {
+		panic("calling Process() only allowed after successful call to Start()")
+	}
+	return c.cmd.Process
+}
+
+// Blocking wait for the process to exit.
+// May be called concurrently and repeatly (exec.Cmd.Wait() semantics apply).
+//
+// Only call this method after a successful call to .Start().
+func (c *Cmd) Wait() (err error) {
+	c.waitPre()
+	err = c.cmd.Wait()
+	c.waitPost(err)
+	return err
+}
+
+func (c *Cmd) startPre(newTask bool) {
+	if newTask {
+		// avoid explosion of tasks with name c.String()
+		c.ctx, c.waitReturnEndSpanCb = trace.WithTaskAndSpan(c.ctx, "zfscmd", c.String())
+	} else {
+		c.ctx, c.waitReturnEndSpanCb = trace.WithSpan(c.ctx, c.String())
+	}
+	startPreLogging(c, time.Now())
+}
+
+func (c *Cmd) startPost(err error) {
+
+	now := time.Now()
+	c.startedAt = now
+
+	startPostReport(c, err, now)
+	startPostLogging(c, err, now)
+
+	if err != nil {
+		c.waitReturnEndSpanCb()
+	}
+}
+
+func (c *Cmd) waitPre() {
+	now := time.Now()
+
+	// ignore duplicate waits
+	c.mtx.Lock()
+	// ignore duplicate waits
+	if !c.waitStartedAt.IsZero() {
+		c.mtx.Unlock()
+		return
+	}
+	c.waitStartedAt = now
+	c.mtx.Unlock()
+
+	waitPreLogging(c, now)
+}
+
+type usage struct {
+	total_secs, system_secs, user_secs float64
+}
+
+func (c *Cmd) waitPost(err error) {
+	now := time.Now()
+
+	c.mtx.Lock()
+	// ignore duplicate waits
+	if !c.waitReturnedAt.IsZero() {
+		c.mtx.Unlock()
+		return
+	}
+	c.waitReturnedAt = now
+	c.mtx.Unlock()
+
+	// build usage
+	var u usage
+	{
+		var s *os.ProcessState
+		if err == nil {
+			s = c.cmd.ProcessState
+		} else if ee, ok := err.(*exec.ExitError); ok {
+			s = ee.ProcessState
+		}
+
+		if s == nil {
+			u = usage{
+				total_secs:  c.Runtime().Seconds(),
+				system_secs: -1,
+				user_secs:   -1,
+			}
+		} else {
+			u = usage{
+				total_secs:  c.Runtime().Seconds(),
+				system_secs: s.SystemTime().Seconds(),
+				user_secs:   s.UserTime().Seconds(),
+			}
+		}
+	}
+
+	waitPostReport(c, u, now)
+	waitPostLogging(c, u, err, now)
+	waitPostPrometheus(c, u, err, now)
+
+	// must be last because c.ctx might be used by other waitPost calls
+	c.waitReturnEndSpanCb()
+}
+
+// returns 0 if the command did not yet finish
+func (c *Cmd) Runtime() time.Duration {
+	if c.waitReturnedAt.IsZero() {
+		return 0
+	}
+	return c.waitReturnedAt.Sub(c.startedAt)
+}
+
+func (c *Cmd) TestOnly_ExecCmd() *exec.Cmd {
+	return c.cmd
+}

internal/zfs/zfscmd/zfscmd_context.go

@@ -0,0 +1,36 @@
+package zfscmd
+
+import (
+	"context"
+
+	"github.com/zrepl/zrepl/internal/daemon/logging"
+	"github.com/zrepl/zrepl/internal/logger"
+)
+
+type contextKey int
+
+const (
+	contextKeyJobID contextKey = 1 + iota
+)
+
+type Logger = logger.Logger
+
+func WithJobID(ctx context.Context, jobID string) context.Context {
+	return context.WithValue(ctx, contextKeyJobID, jobID)
+}
+
+func GetJobIDOrDefault(ctx context.Context, def string) string {
+	return getJobIDOrDefault(ctx, def)
+}
+
+func getJobIDOrDefault(ctx context.Context, def string) string {
+	ret, ok := ctx.Value(contextKeyJobID).(string)
+	if !ok {
+		return def
+	}
+	return ret
+}
+
+func getLogger(ctx context.Context) Logger {
+	return logging.GetLogger(ctx, logging.SubsysZFSCmd)
+}

internal/zfs/zfscmd/zfscmd_logging.go

@@ -0,0 +1,42 @@
+package zfscmd
+
+import (
+	"time"
+)
+
+// Implementation Note:
+//
+// Pre-events logged with debug
+// Post-event without error logged with info
+// Post-events with error _also_ logged with info
+// (Not all errors we observe at this layer) are actual errors in higher-level layers)
+
+func startPreLogging(c *Cmd, now time.Time) {
+	c.log().Debug("starting command")
+}
+
+func startPostLogging(c *Cmd, err error, now time.Time) {
+	if err == nil {
+		c.log().Info("started command")
+	} else {
+		c.log().WithError(err).Error("cannot start command")
+	}
+}
+
+func waitPreLogging(c *Cmd, now time.Time) {
+	c.log().Debug("start waiting")
+}
+
+func waitPostLogging(c *Cmd, u usage, err error, now time.Time) {
+
+	log := c.log().
+		WithField("total_time_s", u.total_secs).
+		WithField("systemtime_s", u.system_secs).
+		WithField("usertime_s", u.user_secs)
+
+	if err == nil {
+		log.Info("command exited without error")
+	} else {
+		log.WithError(err).Info("command exited with error")
+	}
+}

internal/zfs/zfscmd/zfscmd_platform_test.bash

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "to stderr" 1>&2
+echo "to stdout"
+
+exit "$1"

internal/zfs/zfscmd/zfscmd_platform_test.go

@@ -0,0 +1,125 @@
+package zfscmd
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"io"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/internal/util/circlog"
+)
+
+const testBin = "./zfscmd_platform_test.bash"
+
+func TestCmdStderrBehaviorOutput(t *testing.T) {
+
+	stdout, err := exec.Command(testBin, "0").Output()
+	require.NoError(t, err)
+	require.Equal(t, []byte("to stdout\n"), stdout)
+
+	stdout, err = exec.Command(testBin, "1").Output()
+	assert.Equal(t, []byte("to stdout\n"), stdout)
+	require.Error(t, err)
+	ee, ok := err.(*exec.ExitError)
+	require.True(t, ok)
+	require.Equal(t, ee.Stderr, []byte("to stderr\n"))
+}
+
+func TestCmdStderrBehaviorCombinedOutput(t *testing.T) {
+
+	stdio, err := exec.Command(testBin, "0").CombinedOutput()
+	require.NoError(t, err)
+	require.Equal(t, "to stderr\nto stdout\n", string(stdio))
+
+	stdio, err = exec.Command(testBin, "1").CombinedOutput()
+	require.Equal(t, "to stderr\nto stdout\n", string(stdio))
+	require.Error(t, err)
+	ee, ok := err.(*exec.ExitError)
+	require.True(t, ok)
+	require.Empty(t, ee.Stderr) // !!!! maybe not what one would expect
+}
+
+func TestCmdStderrBehaviorStdoutPipe(t *testing.T) {
+	cmd := exec.Command(testBin, "1")
+	stdoutPipe, err := cmd.StdoutPipe()
+	require.NoError(t, err)
+	err = cmd.Start()
+	require.NoError(t, err)
+	defer cmd.Wait()
+	var stdout bytes.Buffer
+	_, err = io.Copy(&stdout, stdoutPipe)
+	require.NoError(t, err)
+	require.Equal(t, "to stdout\n", stdout.String())
+
+	err = cmd.Wait()
+	require.Error(t, err)
+	ee, ok := err.(*exec.ExitError)
+	require.True(t, ok)
+	require.Empty(t, ee.Stderr) // !!!!! probably not what one would expect if we only redirect stdout
+}
+
+func TestCmdProcessState(t *testing.T) {
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	cmd := exec.CommandContext(ctx, "bash", "-c", "echo running; sleep 3600")
+	stdout, err := cmd.StdoutPipe()
+	require.NoError(t, err)
+	err = cmd.Start()
+	require.NoError(t, err)
+
+	r := bufio.NewReader(stdout)
+	line, err := r.ReadString('\n')
+	require.NoError(t, err)
+	require.Equal(t, "running\n", line)
+
+	// we know it's running and sleeping
+	cancel()
+	err = cmd.Wait()
+	t.Logf("wait err %T\n%s", err, err)
+	require.Error(t, err)
+	ee, ok := err.(*exec.ExitError)
+	require.True(t, ok)
+	require.NotNil(t, ee.ProcessState)
+	require.Contains(t, ee.Error(), "killed")
+}
+
+func TestSigpipe(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	cmd := CommandContext(ctx, "bash", "-c", "sleep 5; echo invalid input; exit 23")
+	r, w, err := os.Pipe()
+	require.NoError(t, err)
+	output := circlog.MustNewCircularLog(1 << 20)
+	cmd.SetStdio(Stdio{
+		Stdin:  r,
+		Stdout: output,
+		Stderr: output,
+	})
+	err = cmd.Start()
+	require.NoError(t, err)
+	err = r.Close()
+	require.NoError(t, err)
+
+	// the script doesn't read stdin, but this input is almost certainly smaller than the pipe buffer
+	const LargerThanPipeBuffer = 1 << 21
+	_, err = io.Copy(w, bytes.NewBuffer(bytes.Repeat([]byte("i"), LargerThanPipeBuffer)))
+	// => io.Copy is going to block because the pipe buffer is full and the
+	//    script is not reading from it
+	// => the script is going to exit after 5s
+	// => we should expect a broken pipe error from the copier's perspective
+	t.Logf("copy err = %T: %s", err, err)
+	require.NotNil(t, err)
+	require.True(t, strings.Contains(err.Error(), "broken pipe"))
+
+	err = cmd.Wait()
+	require.EqualError(t, err, "exit status 23")
+
+}

internal/zfs/zfscmd/zfscmd_prometheus.go

@@ -0,0 +1,73 @@
+package zfscmd
+
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var metrics struct {
+	totaltime  *prometheus.HistogramVec
+	systemtime *prometheus.HistogramVec
+	usertime   *prometheus.HistogramVec
+}
+
+var timeLabels = []string{"jobid", "zfsbinary", "zfsverb"}
+var timeBuckets = []float64{0.01, 0.1, 0.2, 0.5, 0.75, 1, 2, 5, 10, 60}
+
+func init() {
+	metrics.totaltime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "zrepl",
+		Subsystem: "zfscmd",
+		Name:      "runtime",
+		Help:      "number of seconds that the command took from start until wait returned",
+		Buckets:   timeBuckets,
+	}, timeLabels)
+	metrics.systemtime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "zrepl",
+		Subsystem: "zfscmd",
+		Name:      "systemtime",
+		Help:      "https://golang.org/pkg/os/#ProcessState.SystemTime",
+		Buckets:   timeBuckets,
+	}, timeLabels)
+	metrics.usertime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "zrepl",
+		Subsystem: "zfscmd",
+		Name:      "usertime",
+		Help:      "https://golang.org/pkg/os/#ProcessState.UserTime",
+		Buckets:   timeBuckets,
+	}, timeLabels)
+
+}
+
+func RegisterMetrics(r prometheus.Registerer) {
+	r.MustRegister(metrics.totaltime)
+	r.MustRegister(metrics.systemtime)
+	r.MustRegister(metrics.usertime)
+}
+
+func waitPostPrometheus(c *Cmd, u usage, err error, now time.Time) {
+
+	if len(c.cmd.Args) < 2 {
+		getLogger(c.ctx).WithField("args", c.cmd.Args).
+			Warn("prometheus: cannot turn zfs command into metric")
+		return
+	}
+
+	// Note: do not start parsing other aspects
+	// of the ZFS command line. This is not the suitable layer
+	// for such a task.
+
+	jobid := getJobIDOrDefault(c.ctx, "_nojobid")
+
+	labelValues := []string{jobid, c.cmd.Args[0], c.cmd.Args[1]}
+
+	metrics.totaltime.
+		WithLabelValues(labelValues...).
+		Observe(u.total_secs)
+	metrics.systemtime.WithLabelValues(labelValues...).
+		Observe(u.system_secs)
+	metrics.usertime.WithLabelValues(labelValues...).
+		Observe(u.user_secs)
+
+}

internal/zfs/zfscmd/zfscmd_report.go

@@ -0,0 +1,68 @@
+package zfscmd
+
+import (
+	"fmt"
+	"sync"
+	"time"
+)
+
+type Report struct {
+	Active []ActiveCommand
+}
+
+type ActiveCommand struct {
+	Path      string
+	Args      []string
+	StartedAt time.Time
+}
+
+func GetReport() *Report {
+	active.mtx.RLock()
+	defer active.mtx.RUnlock()
+	var activeCommands []ActiveCommand
+	for c := range active.cmds {
+		c.mtx.RLock()
+		activeCommands = append(activeCommands, ActiveCommand{
+			Path:      c.cmd.Path,
+			Args:      c.cmd.Args,
+			StartedAt: c.startedAt,
+		})
+		c.mtx.RUnlock()
+	}
+	return &Report{
+		Active: activeCommands,
+	}
+}
+
+var active struct {
+	mtx  sync.RWMutex
+	cmds map[*Cmd]bool
+}
+
+func init() {
+	active.cmds = make(map[*Cmd]bool)
+}
+
+func startPostReport(c *Cmd, err error, now time.Time) {
+	if err != nil {
+		return
+	}
+
+	active.mtx.Lock()
+	prev := active.cmds[c]
+	if prev {
+		panic("impl error: duplicate active command")
+	}
+	active.cmds[c] = true
+	active.mtx.Unlock()
+}
+
+func waitPostReport(c *Cmd, _ usage, now time.Time) {
+	active.mtx.Lock()
+	defer active.mtx.Unlock()
+	prev := active.cmds[c]
+	if !prev {
+		panic(fmt.Sprintf("impl error: onWaitDone must only be called on an active command: %s", c))
+	}
+	delete(active.cmds, c)
+}