WIP

.circleci/config.yml

@@ -150,7 +150,7 @@ parameters:
 
   release_docker_baseimage_tag:
     type: string
-    default: "1.17"
+    default: "1.16"
 
 workflows:
   version: 2
@@ -160,15 +160,15 @@ workflows:
     jobs:
       - quickcheck-docs
       - quickcheck-go: &quickcheck-go-smoketest
-          name: quickcheck-go-amd64-linux-1.17
-          goversion: &latest-go-release "1.17"
+          name: quickcheck-go-amd64-linux-1.16
+          goversion: &latest-go-release "1.16"
           goos: linux
           goarch: amd64
       - test-go-on-latest-go-release:
           goversion: *latest-go-release
       - quickcheck-go:
           requires:
-            - quickcheck-go-amd64-linux-1.17 #quickcheck-go-smoketest.name
+            - quickcheck-go-amd64-linux-1.16 #quickcheck-go-smoketest.name
           matrix: &quickcheck-go-matrix
             alias: quickcheck-go-matrix
             parameters:
@@ -231,6 +231,8 @@ jobs:
       - install-docdep
       - run: make docs
 
+      - store_artifacts:
+          path: artifacts
       - download-and-install-minio-client
       - upload-minio:
           src: artifacts
@@ -356,7 +358,7 @@ jobs:
           subcommand: docdep
       - run:
           command: |
-            git config --global user.email "zreplbot@cschwarz.com"
+            git config --global user.email "me@cschwarz.com"
             git config --global user.name "zrepl-github-io-ci"
 
       # https://circleci.com/docs/2.0/add-ssh-key/#adding-multiple-keys-with-blank-hostnames

Makefile

@@ -28,7 +28,7 @@ GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
 GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
 GOLANGCI_LINT := golangci-lint
 GOCOVMERGE := gocovmerge
-RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.17
+RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.16
 RELEASE_DOCKER_BASEIMAGE ?= golang:$(RELEASE_DOCKER_BASEIMAGE_TAG)
 
 ifneq ($(GOARM),)

build/tools.go

@@ -1,4 +1,3 @@
-//go:build tools
 // +build tools
 
 package main

client/reset.go

@@ -0,0 +1,88 @@
+package client
+
+import (
+	"context"
+	"strconv"
+	"time"
+
+	"github.com/kr/pretty"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+	"github.com/zrepl/zrepl/daemon/job"
+)
+
+var resetCmdArgs struct {
+	verbose  bool
+	interval time.Duration
+	token    string
+}
+
+var ResetCmd = &cli.Subcommand{
+	Use:   "reset [-t TOKEN | JOB INVOCATION [replication|snapshotting|prune_sender|prune_receiver]]",
+	Short: "",
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		return runResetCmd(subcommand.Config(), args)
+	},
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.BoolVarP(&resetCmdArgs.verbose, "verbose", "v", false, "verbose output")
+		f.DurationVarP(&resetCmdArgs.interval, "poll-interval", "i", 100*time.Millisecond, "poll interval")
+		f.StringVarP(&resetCmdArgs.token, "token", "t", "", "token produced by 'signal' subcommand")
+	},
+}
+
+func runResetCmd(config *config.Config, args []string) error {
+
+	httpc, err := controlHttpClient(config.Global.Control.SockPath)
+	if err != nil {
+		return err
+	}
+
+	var req daemon.ControlJobEndpointResetActiveRequest
+	if resetCmdArgs.token != "" {
+		var token TriggerToken
+		err := token.Decode(resetCmdArgs.token)
+		if err != nil {
+			return errors.Wrap(err, "cannot decode token")
+		}
+		req = token.ToReset()
+	} else {
+		jobName := args[0]
+
+		invocationId, err := strconv.ParseUint(args[1], 10, 64)
+		if err != nil {
+			return errors.Wrap(err, "parse invocation id")
+		}
+
+		// what := args[2]
+
+		// updated by subsequent requests
+		req = daemon.ControlJobEndpointResetActiveRequest{
+			Job: jobName,
+			ActiveSideResetRequest: job.ActiveSideResetRequest{
+				InvocationId: invocationId,
+			},
+		}
+	}
+
+	var res job.ActiveSideResetResponse
+	if resetCmdArgs.verbose {
+		pretty.Println("making request", req)
+	}
+	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointResetActive,
+		req,
+		&res,
+	)
+	if err != nil {
+		return err
+	}
+
+	if resetCmdArgs.verbose {
+		pretty.Println("got response", res)
+	}
+
+	return nil
+}

client/signal.go

@@ -1,42 +0,0 @@
-package client
-
-import (
-	"context"
-
-	"github.com/pkg/errors"
-
-	"github.com/zrepl/zrepl/cli"
-	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon"
-)
-
-var SignalCmd = &cli.Subcommand{
-	Use:   "signal [wakeup|reset] JOB",
-	Short: "wake up a job from wait state or abort its current invocation",
-	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
-		return runSignalCmd(subcommand.Config(), args)
-	},
-}
-
-func runSignalCmd(config *config.Config, args []string) error {
-	if len(args) != 2 {
-		return errors.Errorf("Expected 2 arguments: [wakeup|reset] JOB")
-	}
-
-	httpc, err := controlHttpClient(config.Global.Control.SockPath)
-	if err != nil {
-		return err
-	}
-
-	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointSignal,
-		struct {
-			Name string
-			Op   string
-		}{
-			Name: args[1],
-			Op:   args[0],
-		},
-		struct{}{},
-	)
-	return err
-}

client/status/client/client.go

@@ -11,6 +11,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/zrepl/zrepl/daemon"
+	"github.com/zrepl/zrepl/daemon/job"
 )
 
 type Client struct {
@@ -42,21 +43,27 @@ func (c *Client) StatusRaw() ([]byte, error) {
 	return r, nil
 }
 
-func (c *Client) signal(job, sig string) error {
-	return jsonRequestResponse(c.h, daemon.ControlJobEndpointSignal,
+func (c *Client) signal(jobName, sig string) error {
+	return jsonRequestResponse(c.h, daemon.ControlJobEndpointTriggerActive,
 		struct {
-			Name string
-			Op   string
+			Job string
+			job.ActiveSideTriggerRequest
 		}{
-			Name: job,
-			Op:   sig,
+			Job: jobName,
+			ActiveSideTriggerRequest: job.ActiveSideTriggerRequest{
+				What: sig,
+			},
 		},
 		struct{}{},
 	)
 }
 
-func (c *Client) SignalWakeup(job string) error {
-	return c.signal(job, "wakeup")
+func (c *Client) SignalReplication(job string) error {
+	return c.signal(job, "replication")
+}
+
+func (c *Client) SignalSnapshot(job string) error {
+	return c.signal(job, "snapshot")
 }
 
 func (c *Client) SignalReset(job string) error {

client/status/status.go

@@ -19,7 +19,8 @@ import (
 type Client interface {
 	Status() (daemon.Status, error)
 	StatusRaw() ([]byte, error)
-	SignalWakeup(job string) error
+	SignalReplication(job string) error
+	SignalSnapshot(job string) error
 	SignalReset(job string) error
 }
 
@@ -70,16 +71,12 @@ func runStatusV2Command(ctx context.Context, config *config.Config, args []strin
 
 	mode := statusv2Flags.Mode.Value().(statusv2Mode)
 
-	if !isatty.IsTerminal(os.Stdout.Fd()) && mode != StatusV2ModeDump && mode != StatusV2ModeRaw {
-		dumpmode, err := statusv2Flags.Mode.InputForChoice(StatusV2ModeDump)
+	if !isatty.IsTerminal(os.Stdout.Fd()) && mode != StatusV2ModeDump {
+		usemode, err := statusv2Flags.Mode.InputForChoice(StatusV2ModeDump)
 		if err != nil {
 			panic(err)
 		}
-		rawmode, err := statusv2Flags.Mode.InputForChoice(StatusV2ModeRaw)
-		if err != nil {
-			panic(err)
-		}
-		return errors.Errorf("error: stdout is not a tty, please use --mode %s or --mode %s", dumpmode, rawmode)
+		return errors.Errorf("error: stdout is not a tty, please use --mode %s", usemode)
 	}
 
 	switch mode {

client/status/status_interactive.go

@@ -281,8 +281,8 @@ func interactive(c Client, flag statusFlags) error {
 			if !ok {
 				return nil
 			}
-			signals := []string{"wakeup", "reset"}
-			clientFuncs := []func(job string) error{c.SignalWakeup, c.SignalReset}
+			signals := []string{"replication", "snapshot", "reset"}
+			clientFuncs := []func(job string) error{c.SignalReplication, c.SignalSnapshot, c.SignalReset}
 			sigMod := tview.NewModal()
 			sigMod.SetBackgroundColor(tcell.ColorDefault)
 			sigMod.SetBorder(true)

client/status/viewmodel/bytecountbinary.go

@@ -2,22 +2,14 @@ package viewmodel
 
 import (
 	"fmt"
-	"math"
 )
 
-func ByteCountBinaryUint(b uint64) string {
-	if b > math.MaxInt64 {
-		panic(b)
-	}
-	return ByteCountBinary(int64(b))
-}
-
 func ByteCountBinary(b int64) string {
 	const unit = 1024
 	if b < unit {
 		return fmt.Sprintf("%d B", b)
 	}
-	div, exp := unit, 0
+	div, exp := int64(unit), 0
 	for n := b / unit; n >= unit; n /= unit {
 		div *= unit
 		exp++

client/status/viewmodel/bytesprogresshistory.go

@@ -4,16 +4,17 @@ import "time"
 
 type byteProgressMeasurement struct {
 	time time.Time
-	val  uint64
+	val  int64
 }
 
 type bytesProgressHistory struct {
 	last        *byteProgressMeasurement // pointer as poor man's optional
 	changeCount int
 	lastChange  time.Time
+	bpsAvg      float64
 }
 
-func (p *bytesProgressHistory) Update(currentVal uint64) (bytesPerSecondAvg int64, changeCount int) {
+func (p *bytesProgressHistory) Update(currentVal int64) (bytesPerSecondAvg int64, changeCount int) {
 
 	if p.last == nil {
 		p.last = &byteProgressMeasurement{
@@ -33,17 +34,15 @@ func (p *bytesProgressHistory) Update(currentVal uint64) (bytesPerSecondAvg int6
 		return 0, 0
 	}
 
-	var deltaV int64
-	if currentVal >= p.last.val {
-		deltaV = int64(currentVal - p.last.val)
-	} else {
-		deltaV = -int64(p.last.val - currentVal)
-	}
+	deltaV := currentVal - p.last.val
 	deltaT := time.Since(p.last.time)
 	rate := float64(deltaV) / deltaT.Seconds()
 
+	factor := 0.3
+	p.bpsAvg = (1-factor)*p.bpsAvg + factor*rate
+
 	p.last.time = time.Now()
 	p.last.val = currentVal
 
-	return int64(rate), p.changeCount
+	return int64(p.bpsAvg), p.changeCount
 }

client/status/viewmodel/render.go

@@ -230,7 +230,7 @@ func printFilesystemStatus(t *stringbuilder.B, rep *report.FilesystemReport, act
 	status := fmt.Sprintf("%s (step %d/%d, %s/%s)%s",
 		strings.ToUpper(string(rep.State)),
 		rep.CurrentStep, len(rep.Steps),
-		ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected),
+		ByteCountBinary(replicated), ByteCountBinary(expected),
 		sizeEstimationImpreciseNotice,
 	)
 
@@ -356,17 +356,9 @@ func renderReplicationReport(t *stringbuilder.B, rep *report.Report, history *by
 		// Progress: [---------------]
 		expected, replicated, containsInvalidSizeEstimates := latest.BytesSum()
 		rate, changeCount := history.Update(replicated)
-		eta := time.Duration(0)
-		if rate > 0 {
-			eta = time.Duration((float64(expected)-float64(replicated))/float64(rate)) * time.Second
-		}
-
 		t.Write("Progress: ")
 		t.DrawBar(50, replicated, expected, changeCount)
-		t.Write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected), ByteCountBinary(rate)))
-		if eta != 0 {
-			t.Write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
-		}
+		t.Write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinary(replicated), ByteCountBinary(expected), ByteCountBinary(rate)))
 		t.Newline()
 		if containsInvalidSizeEstimates {
 			t.Write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
@@ -391,30 +383,6 @@ func renderReplicationReport(t *stringbuilder.B, rep *report.Report, history *by
 	}
 }
 
-func humanizeDuration(duration time.Duration) string {
-	days := int64(duration.Hours() / 24)
-	hours := int64(math.Mod(duration.Hours(), 24))
-	minutes := int64(math.Mod(duration.Minutes(), 60))
-	seconds := int64(math.Mod(duration.Seconds(), 60))
-
-	var parts []string
-
-	force := false
-	chunks := []int64{days, hours, minutes, seconds}
-	for i, chunk := range chunks {
-		if force || chunk > 0 {
-			padding := 0
-			if force {
-				padding = 2
-			}
-			parts = append(parts, fmt.Sprintf("%*d%c", padding, chunk, "dhms"[i]))
-			force = true
-		}
-	}
-
-	return strings.Join(parts, " ")
-}
-
 func renderPrunerReport(t *stringbuilder.B, r *pruner.Report, fsfilter FilterFunc) {
 	if r == nil {
 		t.Printf("...\n")

client/status/viewmodel/stringbuilder/stringbuilder.go

@@ -99,11 +99,11 @@ func RightPad(str string, length int, pad string) string {
 }
 
 // changeCount = 0 indicates stall / no progress
-func (w *B) DrawBar(length int, bytes, totalBytes uint64, changeCount int) {
+func (w *B) DrawBar(length int, bytes, totalBytes int64, changeCount int) {
 	const arrowPositions = `>\|/`
 	var completedLength int
 	if totalBytes > 0 {
-		completedLength = int(uint64(length) * bytes / totalBytes)
+		completedLength = int(int64(length) * bytes / totalBytes)
 		if completedLength > length {
 			completedLength = length
 		}

client/trigger.go

@@ -0,0 +1,94 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+	"github.com/zrepl/zrepl/daemon/job"
+)
+
+var TriggerCmd = &cli.Subcommand{
+	Use:   "trigger JOB [replication|snapshot]",
+	Short: "",
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		return runTriggerCmd(subcommand.Config(), args)
+	},
+}
+
+type TriggerToken struct {
+	// TODO version, daemon invocation id, etc.
+	Job          string
+	InvocationId uint64
+}
+
+func (t TriggerToken) Encode() string {
+	j, err := json.Marshal(t)
+	if err != nil {
+		panic(err)
+	}
+	return string(j)
+}
+
+func (t *TriggerToken) Decode(s string) error {
+	return json.Unmarshal([]byte(s), t)
+}
+
+func (t TriggerToken) ToReset() daemon.ControlJobEndpointResetActiveRequest {
+	return daemon.ControlJobEndpointResetActiveRequest{
+		Job: t.Job,
+		ActiveSideResetRequest: job.ActiveSideResetRequest{
+			InvocationId: t.InvocationId,
+		},
+	}
+}
+
+func (t TriggerToken) ToWait() daemon.ControlJobEndpointWaitActiveRequest {
+	return daemon.ControlJobEndpointWaitActiveRequest{
+		Job: t.Job,
+		ActiveSidePollRequest: job.ActiveSidePollRequest{
+			InvocationId: t.InvocationId,
+		},
+	}
+}
+
+func runTriggerCmd(config *config.Config, args []string) error {
+	if len(args) != 2 {
+		return errors.Errorf("Expected 2 arguments: [replication|reset|snapshot] JOB")
+	}
+
+	httpc, err := controlHttpClient(config.Global.Control.SockPath)
+	if err != nil {
+		return err
+	}
+
+	jobName := args[0]
+	what := args[1]
+
+	var res job.ActiveSideSignalResponse
+	err = jsonRequestResponse(httpc, daemon.ControlJobEndpointTriggerActive,
+		struct {
+			Job string
+			job.ActiveSideTriggerRequest
+		}{
+			Job: jobName,
+			ActiveSideTriggerRequest: job.ActiveSideTriggerRequest{
+				What: what,
+			},
+		},
+		&res,
+	)
+
+	token := TriggerToken{
+		Job:          jobName,
+		InvocationId: res.InvocationId,
+	}
+
+	fmt.Println(token.Encode())
+	return err
+}

client/wait.go

@@ -0,0 +1,110 @@
+package client
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/kr/pretty"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/config"
+	"github.com/zrepl/zrepl/daemon"
+	"github.com/zrepl/zrepl/daemon/job"
+)
+
+var waitCmdArgs struct {
+	verbose  bool
+	interval time.Duration
+	token    string
+}
+
+var WaitCmd = &cli.Subcommand{
+	Use:   "wait [-t TOKEN | JOB INVOCATION [replication|snapshotting|prune_sender|prune_receiver]]",
+	Short: "",
+	Run: func(ctx context.Context, subcommand *cli.Subcommand, args []string) error {
+		return runWaitCmd(subcommand.Config(), args)
+	},
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.BoolVarP(&waitCmdArgs.verbose, "verbose", "v", false, "verbose output")
+		f.DurationVarP(&waitCmdArgs.interval, "poll-interval", "i", 100*time.Millisecond, "poll interval")
+		f.StringVarP(&waitCmdArgs.token, "token", "t", "", "token produced by 'signal' subcommand")
+	},
+}
+
+func runWaitCmd(config *config.Config, args []string) error {
+
+	httpc, err := controlHttpClient(config.Global.Control.SockPath)
+	if err != nil {
+		return err
+	}
+
+	var req daemon.ControlJobEndpointWaitActiveRequest
+	if waitCmdArgs.token != "" {
+		var token TriggerToken
+		err := token.Decode(resetCmdArgs.token)
+		if err != nil {
+			return errors.Wrap(err, "cannot decode token")
+		}
+		req = token.ToWait()
+	} else {
+
+		jobName := args[0]
+
+		invocationId, err := strconv.ParseUint(args[1], 10, 64)
+		if err != nil {
+			return errors.Wrap(err, "parse invocation id")
+		}
+
+		// updated by subsequent requests
+		req = daemon.ControlJobEndpointWaitActiveRequest{
+			Job: jobName,
+			ActiveSidePollRequest: job.ActiveSidePollRequest{
+				InvocationId: invocationId,
+			},
+		}
+	}
+
+	doneErr := fmt.Errorf("done")
+
+	pollOnce := func() error {
+		var res job.ActiveSidePollResponse
+		if waitCmdArgs.verbose {
+			pretty.Println("making poll request", req)
+		}
+		err = jsonRequestResponse(httpc, daemon.ControlJobEndpointPollActive,
+			req,
+			&res,
+		)
+		if err != nil {
+			return err
+		}
+
+		if waitCmdArgs.verbose {
+			pretty.Println("got poll response", res)
+		}
+
+		if res.Done {
+			return doneErr
+		}
+
+		req.InvocationId = res.InvocationId
+
+		return nil
+	}
+
+	t := time.NewTicker(waitCmdArgs.interval)
+	for range t.C {
+		err := pollOnce()
+		if err == doneErr {
+			return nil
+		} else if err != nil {
+			return err
+		}
+	}
+
+	return err
+}

config/config.go

@@ -13,7 +13,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/zrepl/yaml-config"
 
-	"github.com/zrepl/zrepl/util/datasizeunit"
 	zfsprop "github.com/zrepl/zrepl/zfs/property"
 )
 
@@ -88,8 +87,6 @@ type SendOptions struct {
 	Compressed       bool `yaml:"compressed,optional,default=false"`
 	EmbeddedData     bool `yaml:"embbeded_data,optional,default=false"`
 	Saved            bool `yaml:"saved,optional,default=false"`
-
-	BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
 }
 
 type RecvOptions struct {
@@ -99,15 +96,6 @@ type RecvOptions struct {
 	// Reencrypt bool `yaml:"reencrypt"`
 
 	Properties *PropertyRecvOptions `yaml:"properties,fromdefaults"`
-
-	BandwidthLimit *BandwidthLimit `yaml:"bandwidth_limit,optional,fromdefaults"`
-}
-
-var _ yaml.Unmarshaler = &datasizeunit.Bits{}
-
-type BandwidthLimit struct {
-	Max            datasizeunit.Bits `yaml:"max,default=-1 B"`
-	BucketCapacity datasizeunit.Bits `yaml:"bucket_capacity,default=128 KiB"`
 }
 
 type Replication struct {
@@ -125,6 +113,10 @@ type ReplicationOptionsConcurrency struct {
 	SizeEstimates int `yaml:"size_estimates,optional,default=4"`
 }
 
+func (l *RecvOptions) SetDefault() {
+	*l = RecvOptions{Properties: &PropertyRecvOptions{}}
+}
+
 type PropertyRecvOptions struct {
 	Inherit  []zfsprop.Property          `yaml:"inherit,optional"`
 	Override map[zfsprop.Property]string `yaml:"override,optional"`

config/config_global_test.go

@@ -70,9 +70,9 @@ func TestPrometheusMonitoring(t *testing.T) {
 global:
   monitoring:
     - type: prometheus
-      listen: ':9811'
+      listen: ':9091'
 `)
-	assert.Equal(t, ":9811", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
+	assert.Equal(t, ":9091", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen)
 }
 
 func TestSyslogLoggingOutletFacility(t *testing.T) {

config/samples/bandwidth_limit.yml

@@ -1,41 +0,0 @@
-
-jobs:
-  - type: sink
-    name: "limited_sink"
-    root_fs: "fs0"
-    recv:
-      bandwidth_limit:
-        max: 12345 B
-    serve:
-      type: local
-      listener_name: localsink
-
-  - type: push
-    name: "limited_push"
-    connect:
-      type: local
-      listener_name: localsink
-      client_identity: local_backup
-    filesystems: {
-      "root<": true,
-    }
-    send:
-      bandwidth_limit:
-        max: 54321 B
-        bucket_capacity: 1024 B
-    snapshotting:
-      type: manual
-    pruning:
-      keep_sender:
-        - type: last_n
-          count: 1
-      keep_receiver:
-        - type: last_n
-          count: 1
-
-  - type: sink
-    name: "nolimit_sink"
-    root_fs: "fs1"
-    serve:
-      type: local
-      listener_name: localsink

config/samples/quickstart_backup_to_external_disk.yml

@@ -5,7 +5,7 @@
 # quick start section which inlines this example.
 #
 # CUSTOMIZATIONS YOU WILL LIKELY WANT TO APPLY:
-# - adjust the name of the production pool `system` in the `filesystems` filter of jobs `snapjob` and `push_to_drive`
+# - adjust the name of the production pool `system` in the `filesystems` filter of jobs `snapjob` and `push_to_derive`
 # - adjust the name of the backup pool `backuppool` in the `backuppool_sink` job
 # - adjust the occurences of `myhostname` to the name of the system you are backing up (cannot be easily changed once you start replicating)
 # - make sure the `zrepl_` prefix is not being used by any other zfs tools you might have installed (it likely isn't)
@@ -40,7 +40,7 @@ jobs:
 
 # This job pushes to the local sink defined in job `backuppool_sink`.
 # We trigger replication manually from the command line / udev rules using
-#  `zrepl signal wakeup push_to_drive`
+#  `zrepl signal replication push_to_drive`
 - type: push
   name: push_to_drive
   connect:
@@ -85,4 +85,4 @@ jobs:
   root_fs: "backuppool/zrepl/sink"
   serve:
     type: local
-    listener_name: backuppool_sink
+    listener_name: backuppool_sink

daemon/control.go

@@ -73,12 +73,29 @@ func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) {
 }
 
 const (
-	ControlJobEndpointPProf   string = "/debug/pprof"
-	ControlJobEndpointVersion string = "/version"
-	ControlJobEndpointStatus  string = "/status"
-	ControlJobEndpointSignal  string = "/signal"
+	ControlJobEndpointPProf         string = "/debug/pprof"
+	ControlJobEndpointVersion       string = "/version"
+	ControlJobEndpointStatus        string = "/status"
+	ControlJobEndpointTriggerActive string = "/signal/active"
+	ControlJobEndpointPollActive    string = "/poll/active"
+	ControlJobEndpointResetActive   string = "/reset/active"
 )
 
+type ControlJobEndpointTriggerActiveRequest struct {
+	Job string
+	job.ActiveSideTriggerRequest
+}
+
+type ControlJobEndpointResetActiveRequest struct {
+	Job string
+	job.ActiveSideResetRequest
+}
+
+type ControlJobEndpointWaitActiveRequest struct {
+	Job string
+	job.ActiveSidePollRequest
+}
+
 func (j *controlJob) Run(ctx context.Context) {
 
 	log := job.GetLogger(ctx)
@@ -130,29 +147,106 @@ func (j *controlJob) Run(ctx context.Context) {
 			return s, nil
 		}})
 
-	mux.Handle(ControlJobEndpointSignal,
-		requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) {
+	mux.Handle(ControlJobEndpointPollActive, requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (v interface{}, err error) {
+		var req ControlJobEndpointWaitActiveRequest
+		if decoder(&req) != nil {
+			return nil, errors.Errorf("decode failed")
+		}
+
+		j.jobs.m.RLock()
+
+		jo, ok := j.jobs.jobs[req.Job]
+		if !ok {
+			j.jobs.m.RUnlock()
+			return struct{}{}, fmt.Errorf("unknown job name %q", req.Job)
+		}
+
+		ajo, ok := jo.(*job.ActiveSide)
+		if !ok {
+			v, err = struct{}{}, fmt.Errorf("job %q is not an active side (it's a %T)", jo.Name(), jo)
+			j.jobs.m.RUnlock()
+			return v, err
+		}
+
+		res, err := ajo.Poll(req.ActiveSidePollRequest)
+
+		j.jobs.m.RUnlock()
+
+		return res, err
+	}}})
+
+	mux.Handle(ControlJobEndpointTriggerActive,
+		requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (v interface{}, err error) {
 			type reqT struct {
-				Name string
-				Op   string
+				Job string
+				job.ActiveSideTriggerRequest
 			}
 			var req reqT
 			if decoder(&req) != nil {
 				return nil, errors.Errorf("decode failed")
 			}
 
-			var err error
-			switch req.Op {
-			case "wakeup":
-				err = j.jobs.wakeup(req.Name)
-			case "reset":
-				err = j.jobs.reset(req.Name)
-			default:
-				err = fmt.Errorf("operation %q is invalid", req.Op)
+			// FIXME dedup the following code with ControlJobEndpointPollActive
+
+			j.jobs.m.RLock()
+
+			jo, ok := j.jobs.jobs[req.Job]
+			if !ok {
+				j.jobs.m.RUnlock()
+				return struct{}{}, fmt.Errorf("unknown job name %q", req.Job)
 			}
 
-			return struct{}{}, err
+			ajo, ok := jo.(*job.ActiveSide)
+			if !ok {
+				v, err = struct{}{}, fmt.Errorf("job %q is not an active side (it's a %T)", jo.Name(), jo)
+				j.jobs.m.RUnlock()
+				return v, err
+			}
+
+			res, err := ajo.Trigger(req.ActiveSideTriggerRequest)
+
+			j.jobs.m.RUnlock()
+
+			return res, err
+
 		}}})
+
+	mux.Handle(ControlJobEndpointResetActive,
+		requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (v interface{}, err error) {
+			type reqT struct {
+				Job string
+				job.ActiveSideResetRequest
+			}
+			var req reqT
+			if decoder(&req) != nil {
+				return nil, errors.Errorf("decode failed")
+			}
+
+			// FIXME dedup the following code with ControlJobEndpointPollActive
+
+			j.jobs.m.RLock()
+
+			jo, ok := j.jobs.jobs[req.Job]
+			if !ok {
+				j.jobs.m.RUnlock()
+				return struct{}{}, fmt.Errorf("unknown job name %q", req.Job)
+			}
+
+			ajo, ok := jo.(*job.ActiveSide)
+			if !ok {
+				v, err = struct{}{}, fmt.Errorf("job %q is not an active side (it's a %T)", jo.Name(), jo)
+				j.jobs.m.RUnlock()
+				return v, err
+			}
+
+			res, err := ajo.Reset(req.ActiveSideResetRequest)
+
+			j.jobs.m.RUnlock()
+
+			return res, err
+
+		}}})
+
 	server := http.Server{
 		Handler: mux,
 		// control socket is local, 1s timeout should be more than sufficient, even on a loaded system

daemon/daemon.go

@@ -20,8 +20,6 @@ import (
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/job"
-	"github.com/zrepl/zrepl/daemon/job/reset"
-	"github.com/zrepl/zrepl/daemon/job/wakeup"
 	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/version"
@@ -131,17 +129,13 @@ type jobs struct {
 	wg sync.WaitGroup
 
 	// m protects all fields below it
-	m       sync.RWMutex
-	wakeups map[string]wakeup.Func // by Job.Name
-	resets  map[string]reset.Func  // by Job.Name
-	jobs    map[string]job.Job
+	m    sync.RWMutex
+	jobs map[string]job.Job
 }
 
 func newJobs() *jobs {
 	return &jobs{
-		wakeups: make(map[string]wakeup.Func),
-		resets:  make(map[string]reset.Func),
-		jobs:    make(map[string]job.Job),
+		jobs: make(map[string]job.Job),
 	}
 }
 
@@ -190,28 +184,6 @@ func (s *jobs) status() map[string]*job.Status {
 	return ret
 }
 
-func (s *jobs) wakeup(job string) error {
-	s.m.RLock()
-	defer s.m.RUnlock()
-
-	wu, ok := s.wakeups[job]
-	if !ok {
-		return errors.Errorf("Job %s does not exist", job)
-	}
-	return wu()
-}
-
-func (s *jobs) reset(job string) error {
-	s.m.RLock()
-	defer s.m.RUnlock()
-
-	wu, ok := s.resets[job]
-	if !ok {
-		return errors.Errorf("Job %s does not exist", job)
-	}
-	return wu()
-}
-
 const (
 	jobNamePrometheus = "_prometheus"
 	jobNameControl    = "_control"
@@ -242,10 +214,6 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
 
 	s.jobs[jobName] = j
 	ctx = zfscmd.WithJobID(ctx, j.Name())
-	ctx, wakeup := wakeup.Context(ctx)
-	ctx, resetFunc := reset.Context(ctx)
-	s.wakeups[jobName] = wakeup
-	s.resets[jobName] = resetFunc
 
 	s.wg.Add(1)
 	go func() {

daemon/job/active.go

@@ -8,14 +8,11 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/common/log"
 
 	"github.com/zrepl/zrepl/daemon/logging/trace"
 	"github.com/zrepl/zrepl/util/envconst"
 
 	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/daemon/job/reset"
-	"github.com/zrepl/zrepl/daemon/job/wakeup"
 	"github.com/zrepl/zrepl/daemon/pruner"
 	"github.com/zrepl/zrepl/daemon/snapper"
 	"github.com/zrepl/zrepl/endpoint"
@@ -43,8 +40,12 @@ type ActiveSide struct {
 	promBytesReplicated   *prometheus.CounterVec   // labels: filesystem
 	promReplicationErrors prometheus.Gauge
 
-	tasksMtx sync.Mutex
-	tasks    activeSideTasks
+	tasksMtx           sync.Mutex
+	tasks              activeSideTasks
+	nextInvocationId   uint64
+	activeInvocationId uint64 // 0 <=> inactive
+	trigger            chan struct{}
+	reset              chan uint64
 }
 
 //go:generate enumer -type=ActiveSideState
@@ -57,18 +58,17 @@ const (
 	ActiveSideDone // also errors
 )
 
-type activeSideTasks struct {
+type activeSideReplicationAndTriggerRemotePruneSequence struct {
 	state ActiveSideState
 
 	// valid for state ActiveSideReplicating, ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
 	replicationReport driver.ReportFunc
 	replicationCancel context.CancelFunc
+	replicationDone   *report.Report
 
 	// valid for state ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
-	prunerSender, prunerReceiver *pruner.Pruner
-
-	// valid for state ActiveSidePruneReceiver, ActiveSideDone
-	prunerSenderCancel, prunerReceiverCancel context.CancelFunc
+	pruneRemote *pruner.Pruner
+	pruneRemoteCancel context.CancelFunc
 }
 
 func (a *ActiveSide) updateTasks(u func(*activeSideTasks)) activeSideTasks {
@@ -89,7 +89,7 @@ type activeMode interface {
 	SenderReceiver() (logic.Sender, logic.Receiver)
 	Type() Type
 	PlannerPolicy() logic.PlannerPolicy
-	RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{})
+	RunPeriodic(ctx context.Context, wakePeriodic <-chan struct{}, replicationCommon chan<- struct{})
 	SnapperReport() *snapper.Report
 	ResetConnectBackoff()
 }
@@ -131,8 +131,8 @@ func (m *modePush) Type() Type { return TypePush }
 
 func (m *modePush) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
 
-func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
-	m.snapper.Run(ctx, wakeUpCommon)
+func (m *modePush) RunPeriodic(ctx context.Context, wakePeriodic <-chan struct{}, replicationCommon chan<- struct{}) {
+	m.snapper.Run(ctx, replicationCommon)
 }
 
 func (m *modePush) SnapperReport() *snapper.Report {
@@ -214,10 +214,10 @@ func (*modePull) Type() Type { return TypePull }
 
 func (m *modePull) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
 
-func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
+func (m *modePull) RunPeriodic(ctx context.Context, wakePeriodic <-chan struct{}, replicationCommon chan<- struct{}) {
 	if m.interval.Manual {
 		GetLogger(ctx).Info("manual pull configured, periodic pull disabled")
-		// "waiting for wakeups" is printed in common ActiveSide.do
+		// "waiting for wakeup replications" is printed in common ActiveSide.do
 		return
 	}
 	t := time.NewTicker(m.interval.Interval)
@@ -226,12 +226,12 @@ func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}
 		select {
 		case <-t.C:
 			select {
-			case wakeUpCommon <- struct{}{}:
+			case replicationCommon <- struct{}{}:
 			default:
 				GetLogger(ctx).
 					WithField("pull_interval", m.interval).
 					Warn("pull job took longer than pull interval")
-				wakeUpCommon <- struct{}{} // block anyways, to queue up the wakeup
+				replicationCommon <- struct{}{} // block anyways, to queue up the wakeup replication
 			}
 		case <-ctx.Done():
 			return
@@ -425,50 +425,205 @@ func (j *ActiveSide) Run(ctx context.Context) {
 
 	defer log.Info("job exiting")
 
+	type Activity interface {
+		Trigger() (interface{}, error)
+	}
+
+	var periodicActivity Activity
+	var replicationActivity Activity
+	
 	periodicDone := make(chan struct{})
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 	periodicCtx, endTask := trace.WithTask(ctx, "periodic")
 	defer endTask()
-	go j.mode.RunPeriodic(periodicCtx, periodicDone)
 
-	invocationCount := 0
-outer:
+	wakePeriodic := make(chan struct{})
+	go j.mode.RunPeriodic(periodicCtx, wakePeriodic, periodicDone)
+
+	j.trigger = make(chan struct{})
+	j.reset = make(chan uint64)
+	j.nextInvocationId = 1
+
+	type WaitTriggerResult interface {
+		
+	}
+	var t interface{
+		WaitForTrigger(context.Context) (context.Context, <-chan WaitTriggerResult)
+	}
+
 	for {
-		log.Info("wait for wakeups")
-		select {
-		case <-ctx.Done():
-			log.WithError(ctx.Err()).Info("context")
-			break outer
+		log.Info("wait for triggers")
 
-		case <-wakeup.Wait(ctx):
-			j.mode.ResetConnectBackoff()
-		case <-periodicDone:
+			// j.tasksMtx.Lock()
+		// j.activeInvocationId = j.nextInvocationId
+		// j.nextInvocationId++
+		// thisInvocation := j.activeInvocationId // stack-local, for use in reset-handler goroutine below
+		// j.tasksMtx.Unlock()
+		
+		// // setup the goroutine that waits for task resets
+		// // Task resets are converted into cancellations of the invocation context.
+		
+		invocationCtx, cancelInvocation := context.WithCancel(invocationCtx)
+		// waitForResetCtx, stopWaitForReset := context.WithCancel(ctx)
+		// var wg sync.WaitGroup
+		// wg.Add(1)
+		// go func() {
+		// 	defer wg.Done()
+		// 	select {
+		// 	case <-waitForResetCtx.Done():
+		// 		return
+		// 	case reqResetInvocation := <-j.reset:
+		// 		l := log.WithField("requested_invocation_id", reqResetInvocation).
+		// 			WithField("this_invocation_id", thisInvocation)
+		// 		if reqResetInvocation == thisInvocation {
+		// 			l.Info("reset received, cancelling current invocation")
+		// 			cancelInvocation()
+		// 		} else {
+		// 			l.Debug("received reset for invocation id that is not us, discarding request")
+		// 		}
+		// 	}
+		// }()
+
+		// j.tasksMtx.Lock()
+		// j.activeInvocationId = 0
+		// j.tasksMtx.Unlock()
+
+		invocationCtx, err := t.WaitForTrigger(ctx)
+		if err != nil {
+			log.WithError(ctx.Err()).Info("error waiting for trigger")
+			break
 		}
-		invocationCount++
-		invocationCtx, endSpan := trace.WithSpan(ctx, fmt.Sprintf("invocation-%d", invocationCount))
+
+		j.mode.ResetConnectBackoff()
+
+		// setup the invocation context
+		invocationCtx, endSpan := trace.WithSpan(invocationCtx, fmt.Sprintf("invocation-%d", j.nextInvocationId))
+		
+
 		j.do(invocationCtx)
+		stopWaitForReset()
+		wg.Wait()
+
+	
+
 		endSpan()
 	}
 }
 
+type ActiveSidePollRequest struct {
+	InvocationId uint64
+}
+
+type ActiveSidePollResponse struct {
+	Done         bool
+	InvocationId uint64
+}
+
+func (j *ActiveSide) Poll(req ActiveSidePollRequest) (*ActiveSidePollResponse, error) {
+	j.tasksMtx.Lock()
+	defer j.tasksMtx.Unlock()
+
+	waitForId := req.InvocationId
+	if req.InvocationId == 0 {
+		// handle the case where the client doesn't know what the current invocation id is
+		if j.activeInvocationId != 0 {
+			waitForId = j.activeInvocationId
+		} else {
+			waitForId = j.nextInvocationId
+		}
+	}
+
+	var done bool
+	if j.activeInvocationId == 0 {
+		done = waitForId < j.nextInvocationId
+	} else {
+		done = waitForId < j.activeInvocationId
+	}
+	res := &ActiveSidePollResponse{Done: done, InvocationId: waitForId}
+	return res, nil
+}
+
+type ActiveSideResetRequest struct {
+	InvocationId uint64
+}
+
+type ActiveSideResetResponse struct {
+	InvocationId uint64
+}
+
+func (j *ActiveSide) Reset(req ActiveSideResetRequest) (*ActiveSideResetResponse, error) {
+	j.tasksMtx.Lock()
+	defer j.tasksMtx.Unlock()
+
+	resetId := req.InvocationId
+	if req.InvocationId == 0 {
+		// handle the case where the client doesn't know what the current invocation id is
+		resetId = j.activeInvocationId
+	}
+
+	if resetId == 0 {
+		return nil, fmt.Errorf("no active invocation")
+	}
+
+	if resetId != j.activeInvocationId {
+		return nil, fmt.Errorf("active invocation (%d) is not the invocation requested for reset (%d); (active invocation '0' indicates no active invocation)", j.activeInvocationId, resetId)
+	}
+
+	// non-blocking send (.Run() must not hold mutex while waiting for resets)
+	select {
+	case j.reset <- resetId:
+	default:
+	}
+
+	return &ActiveSideResetResponse{InvocationId: resetId}, nil
+}
+
+type ActiveSideTriggerRequest struct {
+}
+
+type ActiveSideSignalResponse struct {
+	InvocationId uint64
+	
+}
+
+func (j *ActiveSide) Trigger(req ActiveSideTriggerRequest) (*ActiveSideSignalResponse, error) {
+	// switch req.What {
+	// case "replication":
+	// 	invocationId, err = j.jobs.doreplication(req.Name)
+	// case "reset":
+	// 	err = j.jobs.reset(req.Name)
+	// case "snapshot":
+	// 	err = j.jobs.dosnapshot(req.Name)
+	// default:
+	// 	err = fmt.Errorf("operation %q is invalid", req.Op)
+	// }
+
+	j.tasksMtx.Lock()
+	var invocationId uint64
+	if j.activeInvocationId != 0 {
+		invocationId = j.activeInvocationId
+	} else {
+		invocationId = j.nextInvocationId
+	}
+	// non-blocking send (.Run() must not hold mutex while waiting for signals)
+	select {
+	case j.trigger <- struct{}{}:
+	default:
+	}
+	j.tasksMtx.Unlock()
+	return &ActiveSideSignalResponse{InvocationId: invocationId}, nil
+}
+
+type ReplicationPlusRemotePruneSequence struct {
+
+}
+
 func (j *ActiveSide) do(ctx context.Context) {
 
 	j.mode.ConnectEndpoints(ctx, j.connecter)
 	defer j.mode.DisconnectEndpoints()
 
-	// allow cancellation of an invocation (this function)
-	ctx, cancelThisRun := context.WithCancel(ctx)
-	defer cancelThisRun()
-	go func() {
-		select {
-		case <-reset.Wait(ctx):
-			log.Info("reset received, cancelling current invocation")
-			cancelThisRun()
-		case <-ctx.Done():
-		}
-	}()
-
 	sender, receiver := j.mode.SenderReceiver()
 
 	{
@@ -492,10 +647,11 @@ func (j *ActiveSide) do(ctx context.Context) {
 		GetLogger(ctx).Info("start replication")
 		repWait(true) // wait blocking
 		repCancel()   // always cancel to free up context resources
-
 		replicationReport := j.tasks.replicationReport()
 		j.promReplicationErrors.Set(float64(replicationReport.GetFailedFilesystemsCountInLatestAttempt()))
-
+		j.updateTasks(func(tasks *activeSideTasks) {
+			tasks.replicationDone = replicationReport
+		})
 		endSpan()
 	}
 

daemon/job/build_jobs.go

@@ -8,7 +8,6 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/zrepl/zrepl/config"
-	"github.com/zrepl/zrepl/util/bandwidthlimit"
 )
 
 func JobsFromConfig(c *config.Config) ([]Job, error) {
@@ -108,13 +107,3 @@ func validateReceivingSidesDoNotOverlap(receivingRootFSs []string) error {
 	}
 	return nil
 }
-
-func buildBandwidthLimitConfig(in *config.BandwidthLimit) (c bandwidthlimit.Config, _ error) {
-	if in.Max.ToBytes() > 0 && int64(in.Max.ToBytes()) == 0 {
-		return c, fmt.Errorf("bandwidth limit `max` is too small, must at least specify one byte")
-	}
-	return bandwidthlimit.Config{
-		Max:            int64(in.Max.ToBytes()),
-		BucketCapacity: int64(in.BucketCapacity.ToBytes()),
-	}, nil
-}

daemon/job/build_jobs_sendrecvoptions.go

@@ -22,12 +22,7 @@ func buildSenderConfig(in SendingJobConfig, jobID endpoint.JobID) (*endpoint.Sen
 		return nil, errors.Wrap(err, "cannot build filesystem filter")
 	}
 	sendOpts := in.GetSendOptions()
-	bwlim, err := buildBandwidthLimitConfig(sendOpts.BandwidthLimit)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot build bandwith limit config")
-	}
-
-	sc := &endpoint.SenderConfig{
+	return &endpoint.SenderConfig{
 		FSF:   fsf,
 		JobID: jobID,
 
@@ -39,15 +34,7 @@ func buildSenderConfig(in SendingJobConfig, jobID endpoint.JobID) (*endpoint.Sen
 		SendCompressed:       sendOpts.Compressed,
 		SendEmbeddedData:     sendOpts.EmbeddedData,
 		SendSaved:            sendOpts.Saved,
-
-		BandwidthLimit: bwlim,
-	}
-
-	if err := sc.Validate(); err != nil {
-		return nil, errors.Wrap(err, "cannot build sender config")
-	}
-
-	return sc, nil
+	}, nil
 }
 
 type ReceivingJobConfig interface {
@@ -66,12 +53,6 @@ func buildReceiverConfig(in ReceivingJobConfig, jobID endpoint.JobID) (rc endpoi
 	}
 
 	recvOpts := in.GetRecvOptions()
-
-	bwlim, err := buildBandwidthLimitConfig(recvOpts.BandwidthLimit)
-	if err != nil {
-		return rc, errors.Wrap(err, "cannot build bandwith limit config")
-	}
-
 	rc = endpoint.ReceiverConfig{
 		JobID:                      jobID,
 		RootWithoutClientComponent: rootFs,
@@ -79,8 +60,6 @@ func buildReceiverConfig(in ReceivingJobConfig, jobID endpoint.JobID) (rc endpoi
 
 		InheritProperties:  recvOpts.Properties.Inherit,
 		OverrideProperties: recvOpts.Properties.Override,
-
-		BandwidthLimit: bwlim,
 	}
 	if err := rc.Validate(); err != nil {
 		return rc, errors.Wrap(err, "cannot build receiver config")

daemon/job/build_jobs_test.go

@@ -119,14 +119,6 @@ func TestSampleConfigsAreBuiltWithoutErrors(t *testing.T) {
 		t.Errorf("glob failed: %+v", err)
 	}
 
-	type additionalCheck struct {
-		state int
-		test  func(t *testing.T, jobs []Job)
-	}
-	additionalChecks := map[string]*additionalCheck{
-		"bandwidth_limit.yml": {test: testSampleConfig_BandwidthLimit},
-	}
-
 	for _, p := range paths {
 
 		if path.Ext(p) != ".yml" {
@@ -134,20 +126,10 @@ func TestSampleConfigsAreBuiltWithoutErrors(t *testing.T) {
 			continue
 		}
 
-		filename := path.Base(p)
-		t.Logf("checking for presence additonal checks for file %q", filename)
-		additionalCheck := additionalChecks[filename]
-		if additionalCheck == nil {
-			t.Logf("no additional checks")
-		} else {
-			t.Logf("additional check present")
-			additionalCheck.state = 1
-		}
-
 		t.Run(p, func(t *testing.T) {
 			c, err := config.ParseConfig(p)
 			if err != nil {
-				t.Fatalf("error parsing %s:\n%+v", p, err)
+				t.Errorf("error parsing %s:\n%+v", p, err)
 			}
 
 			t.Logf("file: %s", p)
@@ -156,57 +138,11 @@ func TestSampleConfigsAreBuiltWithoutErrors(t *testing.T) {
 			tls.FakeCertificateLoading(t)
 			jobs, err := JobsFromConfig(c)
 			t.Logf("jobs: %#v", jobs)
-			require.NoError(t, err)
-
-			if additionalCheck != nil {
-				additionalCheck.test(t, jobs)
-				additionalCheck.state = 2
-			}
+			assert.NoError(t, err)
 		})
 
 	}
 
-	for basename, c := range additionalChecks {
-		if c.state == 0 {
-			panic("univisited additional check " + basename)
-		}
-	}
-
-}
-
-func testSampleConfig_BandwidthLimit(t *testing.T, jobs []Job) {
-	require.Len(t, jobs, 3)
-
-	{
-		limitedSink, ok := jobs[0].(*PassiveSide)
-		require.True(t, ok, "%T", jobs[0])
-		limitedSinkMode, ok := limitedSink.mode.(*modeSink)
-		require.True(t, ok, "%T", limitedSink)
-
-		assert.Equal(t, int64(12345), limitedSinkMode.receiverConfig.BandwidthLimit.Max)
-		assert.Equal(t, int64(1<<17), limitedSinkMode.receiverConfig.BandwidthLimit.BucketCapacity)
-	}
-
-	{
-		limitedPush, ok := jobs[1].(*ActiveSide)
-		require.True(t, ok, "%T", jobs[1])
-		limitedPushMode, ok := limitedPush.mode.(*modePush)
-		require.True(t, ok, "%T", limitedPush)
-
-		assert.Equal(t, int64(54321), limitedPushMode.senderConfig.BandwidthLimit.Max)
-		assert.Equal(t, int64(1024), limitedPushMode.senderConfig.BandwidthLimit.BucketCapacity)
-	}
-
-	{
-		unlimitedSink, ok := jobs[2].(*PassiveSide)
-		require.True(t, ok, "%T", jobs[2])
-		unlimitedSinkMode, ok := unlimitedSink.mode.(*modeSink)
-		require.True(t, ok, "%T", unlimitedSink)
-
-		max := unlimitedSinkMode.receiverConfig.BandwidthLimit.Max
-		assert.Less(t, max, int64(0), max, "unlimited mode <=> negative value for .Max, see bandwidthlimit.Config")
-	}
-
 }
 
 func TestReplicationOptions(t *testing.T) {

daemon/job/job.go

@@ -19,6 +19,7 @@ func GetLogger(ctx context.Context) Logger {
 	return logging.GetLogger(ctx, logging.SubsysJob)
 }
 
+
 type Job interface {
 	Name() string
 	Run(ctx context.Context)

daemon/job/reset/reset.go

@@ -1,35 +0,0 @@
-package reset
-
-import (
-	"context"
-	"errors"
-)
-
-type contextKey int
-
-const contextKeyReset contextKey = iota
-
-func Wait(ctx context.Context) <-chan struct{} {
-	wc, ok := ctx.Value(contextKeyReset).(chan struct{})
-	if !ok {
-		wc = make(chan struct{})
-	}
-	return wc
-}
-
-type Func func() error
-
-var AlreadyReset = errors.New("already reset")
-
-func Context(ctx context.Context) (context.Context, Func) {
-	wc := make(chan struct{})
-	wuf := func() error {
-		select {
-		case wc <- struct{}{}:
-			return nil
-		default:
-			return AlreadyReset
-		}
-	}
-	return context.WithValue(ctx, contextKeyReset, wc), wuf
-}

daemon/job/snapjob.go

@@ -10,12 +10,10 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/zrepl/zrepl/daemon/logging/trace"
-	"github.com/zrepl/zrepl/util/bandwidthlimit"
 	"github.com/zrepl/zrepl/util/nodefault"
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/filters"
-	"github.com/zrepl/zrepl/daemon/job/wakeup"
 	"github.com/zrepl/zrepl/daemon/pruner"
 	"github.com/zrepl/zrepl/daemon/snapper"
 	"github.com/zrepl/zrepl/endpoint"
@@ -113,13 +111,13 @@ func (j *SnapJob) Run(ctx context.Context) {
 	invocationCount := 0
 outer:
 	for {
-		log.Info("wait for wakeups")
+		log.Info("wait for replications")
 		select {
 		case <-ctx.Done():
 			log.WithError(ctx.Err()).Info("context")
 			break outer
 
-		case <-wakeup.Wait(ctx):
+		// case <-doreplication.Wait(ctx):
 		case <-periodicDone:
 		}
 		invocationCount++
@@ -147,7 +145,7 @@ type alwaysUpToDateReplicationCursorHistory struct {
 	target pruner.Target
 }
 
-var _ pruner.Sender = (*alwaysUpToDateReplicationCursorHistory)(nil)
+var _ pruner.History = (*alwaysUpToDateReplicationCursorHistory)(nil)
 
 func (h alwaysUpToDateReplicationCursorHistory) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
 	fsvReq := &pdu.ListFilesystemVersionsReq{
@@ -180,11 +178,8 @@ func (j *SnapJob) doPrune(ctx context.Context) {
 	sender := endpoint.NewSender(endpoint.SenderConfig{
 		JobID: j.name,
 		FSF:   j.fsfilter,
-		// FIXME the following config fields are irrelevant for SnapJob
-		// because the endpoint is only used as pruner.Target.
-		// However, the implementation requires them to be set.
-		Encrypt:        &nodefault.Bool{B: true},
-		BandwidthLimit: bandwidthlimit.NoLimitConfig(),
+		// FIXME encryption setting is irrelevant for SnapJob because the endpoint is only used as pruner.Target
+		Encrypt: &nodefault.Bool{B: true},
 	})
 	j.prunerMtx.Lock()
 	j.pruner = j.prunerFactory.BuildLocalPruner(ctx, sender, alwaysUpToDateReplicationCursorHistory{sender})

daemon/job/trigger/trigger.go

@@ -0,0 +1,184 @@
+//
+//
+// Alternative Design (in "RustGo")
+//
+//	enum InternalMsg {
+//	    Trigger((), chan (TriggerResponse, error)),
+//	    Poll(PollRequest, chan PollResponse),
+//	    Reset(ResetRequest, chan (ResetResponse, error)),
+//	}
+//
+//	enum State {
+//	    Running{
+//	        invocationId: u32,
+//	        cancelCurrentInvocation: context.CancelFunc
+//	    }
+//	    Waiting{
+//	        nextInvocationId: u32,
+//	    }
+//	}
+//
+//	for msg := <- t.internalMsgs {
+//	    match (msg, state) {
+//	        ...
+//	    }
+//	}
+package trigger
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"sync"
+)
+
+type T struct {
+	mtx sync.Mutex
+	cv  sync.Cond
+
+	nextInvocationId        uint64
+	activeInvocationId      uint64 // 0 <=> inactive
+	triggerPending          bool
+	contextDone             bool
+	reset                   chan uint64
+	stopWaitForReset        chan struct{}
+	cancelCurrentInvocation context.CancelFunc
+}
+
+func New() *T {
+	t := &T{
+		activeInvocationId: math.MaxUint64,
+		nextInvocationId:   1,
+	}
+	t.cv.L = &t.mtx
+	return t
+}
+
+func (t *T) WaitForTrigger(ctx context.Context) (rctx context.Context, err error) {
+	t.mtx.Lock()
+	defer t.mtx.Unlock()
+
+	if t.activeInvocationId == 0 {
+		return nil, fmt.Errorf("must be running when calling this function")
+	}
+	t.activeInvocationId = 0
+	t.cancelCurrentInvocation = nil
+
+	if t.contextDone == true {
+		panic("implementation error: this variable is only true while in WaitForTrigger, and that's a mutually exclusive function")
+	}
+	stopWaitingForDone := make(chan struct{})
+	go func() {
+		select {
+		case <-stopWaitingForDone:
+		case <-ctx.Done():
+			t.mtx.Lock()
+			t.contextDone = true
+			t.cv.Broadcast()
+			t.mtx.Unlock()
+		}
+	}()
+
+	defer func() {
+		t.triggerPending = false
+		t.contextDone = false
+	}()
+	for !t.triggerPending && !t.contextDone {
+		t.cv.Wait()
+	}
+	close(stopWaitingForDone)
+	if t.contextDone {
+		if ctx.Err() == nil {
+			panic("implementation error: contextDone <=> ctx.Err() != nil")
+		}
+		return nil, ctx.Err()
+	}
+
+	t.activeInvocationId = t.nextInvocationId
+	t.nextInvocationId++
+	rctx, t.cancelCurrentInvocation = context.WithCancel(ctx)
+
+	return rctx, nil
+}
+
+type TriggerResponse struct {
+	InvocationId uint64
+}
+
+func (t *T) Trigger() (TriggerResponse, error) {
+	t.mtx.Lock()
+	defer t.mtx.Unlock()
+	var invocationId uint64
+	if t.activeInvocationId != 0 {
+		invocationId = t.activeInvocationId
+	} else {
+		invocationId = t.nextInvocationId
+	}
+	// non-blocking send (.Run() must not hold mutex while waiting for signals)
+	t.triggerPending = true
+	t.cv.Broadcast()
+	return TriggerResponse{InvocationId: invocationId}, nil
+}
+
+type PollRequest struct {
+	InvocationId uint64
+}
+
+type PollResponse struct {
+	Done         bool
+	InvocationId uint64
+}
+
+func (t *T) Poll(req PollRequest) (res PollResponse) {
+	t.mtx.Lock()
+	defer t.mtx.Unlock()
+
+	waitForId := req.InvocationId
+	if req.InvocationId == 0 {
+		// handle the case where the client doesn't know what the current invocation id is
+		if t.activeInvocationId != 0 {
+			waitForId = t.activeInvocationId
+		} else {
+			waitForId = t.nextInvocationId
+		}
+	}
+
+	var done bool
+	if t.activeInvocationId == 0 {
+		done = waitForId < t.nextInvocationId
+	} else {
+		done = waitForId < t.activeInvocationId
+	}
+	return PollResponse{Done: done, InvocationId: waitForId}
+}
+
+type ResetRequest struct {
+	InvocationId uint64
+}
+
+type ResetResponse struct {
+	InvocationId uint64
+}
+
+func (t *T) Reset(req ResetRequest) (*ResetResponse, error) {
+	t.mtx.Lock()
+	defer t.mtx.Unlock()
+
+	resetId := req.InvocationId
+	if req.InvocationId == 0 {
+		// handle the case where the client doesn't know what the current invocation id is
+		resetId = t.activeInvocationId
+	}
+
+	if resetId == 0 {
+		return nil, fmt.Errorf("no active invocation")
+	}
+
+	if resetId != t.activeInvocationId {
+		return nil, fmt.Errorf("active invocation (%d) is not the invocation requested for reset (%d); (active invocation '0' indicates no active invocation)", t.activeInvocationId, resetId)
+	}
+
+	t.cancelCurrentInvocation()
+
+	return &ResetResponse{InvocationId: resetId}, nil
+}

daemon/job/trigger/trigger_test.go

@@ -0,0 +1,205 @@
+package trigger_test
+
+import (
+	"context"
+	"net/http"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/zrepl/zrepl/daemon/job/trigger"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+	"github.com/zrepl/zrepl/replication"
+	"github.com/zrepl/zrepl/replication/driver"
+	"github.com/zrepl/zrepl/replication/logic"
+)
+
+func TestBasics(t *testing.T) {
+
+	var wg sync.WaitGroup
+	defer wg.Wait()
+
+	tr := trigger.New()
+
+	triggered := make(chan int)
+	waitForTriggerError := make(chan error)
+	waitForResetCallToBeMadeByMainGoroutine := make(chan struct{})
+	postResetAssertionsDone := make(chan struct{})
+
+	taskCtx := context.Background()
+	taskCtx, cancelTaskCtx := context.WithCancel(taskCtx)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		taskCtx := context.WithValue(taskCtx, "mykey", "myvalue")
+
+		triggers := 0
+
+	outer:
+		for {
+			invocationCtx, err := tr.WaitForTrigger(taskCtx)
+			if err != nil {
+				waitForTriggerError <- err
+				return
+			}
+			require.Equal(t, invocationCtx.Value("mykey"), "myvalue")
+
+			triggers++
+			triggered <- triggers
+
+			switch triggers {
+			case 1:
+				continue outer
+			case 2:
+				<-waitForResetCallToBeMadeByMainGoroutine
+				require.Equal(t, context.Canceled, invocationCtx.Err(), "Reset() cancels invocation context")
+				require.Nil(t, taskCtx.Err(), "Reset() does not cancel task context")
+				close(postResetAssertionsDone)
+			}
+
+		}
+
+	}()
+
+	t.Logf("trigger 1")
+	_, err := tr.Trigger()
+	require.NoError(t, err)
+	v := <-triggered
+	require.Equal(t, 1, v)
+
+	t.Logf("trigger 2")
+	triggerResponse, err := tr.Trigger()
+	require.NoError(t, err)
+	v = <-triggered
+	require.Equal(t, 2, v)
+
+	t.Logf("reset")
+	resetResponse, err := tr.Reset(trigger.ResetRequest{InvocationId: triggerResponse.InvocationId})
+	require.NoError(t, err)
+	t.Logf("reset response: %#v", resetResponse)
+	close(waitForResetCallToBeMadeByMainGoroutine)
+	<-postResetAssertionsDone
+
+	t.Logf("cancel the context")
+	cancelTaskCtx()
+	wfte := <-waitForTriggerError
+	require.Equal(t, taskCtx.Err(), wfte)
+
+}
+
+type PushJob struct {
+	snap *Snapshotter
+	repl *ReplicationAndTriggerRemotePruningSequence
+}
+
+func (j *PushJob) Handle(w http.ResponseWriter, r *http.Request) {
+	panic("unimplemented")
+}
+
+func (j *PushJob) HandleTrigger(w http.ResponseWriter, r *http.Request) {
+	panic("unimplemented")
+}
+
+func (j *PushJob) Run(ctx context.Context) {
+
+}
+
+type SnapshotSequence struct {
+
+}
+
+type ReplicationAndTriggerRemotePruningSequence struct {
+
+}
+
+func (s ReplicationAndTriggerRemotePruningSequence) Run(ctx context.Context) {
+
+	j.mode.ConnectEndpoints(ctx, j.connecter)
+	defer j.mode.DisconnectEndpoints()
+
+	sender, receiver := j.mode.SenderReceiver()
+
+	{
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+		ctx, endSpan := trace.WithSpan(ctx, "replication")
+		ctx, repCancel := context.WithCancel(ctx)
+		var repWait driver.WaitFunc
+		j.updateTasks(func(tasks *activeSideTasks) {
+			// reset it
+			*tasks = activeSideTasks{}
+			tasks.replicationCancel = func() { repCancel(); endSpan() }
+			tasks.replicationReport, repWait = replication.Do(
+				ctx, j.replicationDriverConfig, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver, j.mode.PlannerPolicy()),
+			)
+			tasks.state = ActiveSideReplicating
+		})
+		GetLogger(ctx).Info("start replication")
+		repWait(true) // wait blocking
+		repCancel()   // always cancel to free up context resources
+		replicationReport := j.tasks.replicationReport()
+		j.promReplicationErrors.Set(float64(replicationReport.GetFailedFilesystemsCountInLatestAttempt()))
+		j.updateTasks(func(tasks *activeSideTasks) {
+			tasks.replicationDone = replicationReport
+		})
+		endSpan()
+	}
+
+	{
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+		ctx, endSpan := trace.WithSpan(ctx, "prune_sender")
+		ctx, senderCancel := context.WithCancel(ctx)
+		tasks := j.updateTasks(func(tasks *activeSideTasks) {
+			tasks.prunerSender = j.prunerFactory.BuildSenderPruner(ctx, sender, sender)
+			tasks.prunerSenderCancel = func() { senderCancel(); endSpan() }
+			tasks.state = ActiveSidePruneSender
+		})
+		GetLogger(ctx).Info("start pruning sender")
+		tasks.prunerSender.Prune()
+		GetLogger(ctx).Info("finished pruning sender")
+		senderCancel()
+		endSpan()
+	}
+	{
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+		ctx, endSpan := trace.WithSpan(ctx, "prune_recever")
+		ctx, receiverCancel := context.WithCancel(ctx)
+		tasks := j.updateTasks(func(tasks *activeSideTasks) {
+			tasks.prunerReceiver = j.prunerFactory.BuildReceiverPruner(ctx, receiver, sender)
+			tasks.prunerReceiverCancel = func() { receiverCancel(); endSpan() }
+			tasks.state = ActiveSidePruneReceiver
+		})
+		GetLogger(ctx).Info("start pruning receiver")
+		tasks.prunerReceiver.Prune()
+		GetLogger(ctx).Info("finished pruning receiver")
+		receiverCancel()
+		endSpan()
+	}
+
+	j.updateTasks(func(tasks *activeSideTasks) {
+		tasks.state = ActiveSideDone
+	})
+
+}
+
+
+func TestUseCase(t *testing.T) {
+
+	var as ActiveSide
+
+	as.
+	
+}

daemon/job/wakeup/wakeup.go

@@ -1,35 +0,0 @@
-package wakeup
-
-import (
-	"context"
-	"errors"
-)
-
-type contextKey int
-
-const contextKeyWakeup contextKey = iota
-
-func Wait(ctx context.Context) <-chan struct{} {
-	wc, ok := ctx.Value(contextKeyWakeup).(chan struct{})
-	if !ok {
-		wc = make(chan struct{})
-	}
-	return wc
-}
-
-type Func func() error
-
-var AlreadyWokenUp = errors.New("already woken up")
-
-func Context(ctx context.Context) (context.Context, Func) {
-	wc := make(chan struct{})
-	wuf := func() error {
-		select {
-		case wc <- struct{}{}:
-			return nil
-		default:
-			return AlreadyWokenUp
-		}
-	}
-	return context.WithValue(ctx, contextKeyWakeup, wc), wuf
-}

daemon/pruner/pruner.go

@@ -19,20 +19,12 @@ import (
 	"github.com/zrepl/zrepl/util/envconst"
 )
 
-// The sender in the replication setup.
-// The pruner uses the Sender to determine which of the Target's filesystems need to be pruned.
-// Also, it asks the Sender about the replication cursor of each filesystem
-// to enable the 'not_replicated' pruning rule.
-//
 // Try to keep it compatible with github.com/zrepl/zrepl/endpoint.Endpoint
-type Sender interface {
+type History interface {
 	ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error)
 	ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
 }
 
-// The pruning target, i.e., on which snapshots are destroyed.
-// This can be a replication sender or receiver.
-//
 // Try to keep it compatible with github.com/zrepl/zrepl/endpoint.Endpoint
 type Target interface {
 	ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
@@ -56,7 +48,7 @@ func GetLogger(ctx context.Context) Logger {
 type args struct {
 	ctx                            context.Context
 	target                         Target
-	sender                         Sender
+	receiver                       History
 	rules                          []pruning.KeepRule
 	retryWait                      time.Duration
 	considerSnapAtCursorReplicated bool
@@ -140,12 +132,12 @@ func NewPrunerFactory(in config.PruningSenderReceiver, promPruneSecs *prometheus
 	return f, nil
 }
 
-func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, sender Sender) *Pruner {
+func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, receiver History) *Pruner {
 	p := &Pruner{
 		args: args{
 			context.WithValue(ctx, contextKeyPruneSide, "sender"),
 			target,
-			sender,
+			receiver,
 			f.senderRules,
 			f.retryWait,
 			f.considerSnapAtCursorReplicated,
@@ -156,12 +148,12 @@ func (f *PrunerFactory) BuildSenderPruner(ctx context.Context, target Target, se
 	return p
 }
 
-func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target, sender Sender) *Pruner {
+func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target, receiver History) *Pruner {
 	p := &Pruner{
 		args: args{
 			context.WithValue(ctx, contextKeyPruneSide, "receiver"),
 			target,
-			sender,
+			receiver,
 			f.receiverRules,
 			f.retryWait,
 			false, // senseless here anyways
@@ -172,12 +164,12 @@ func (f *PrunerFactory) BuildReceiverPruner(ctx context.Context, target Target,
 	return p
 }
 
-func (f *LocalPrunerFactory) BuildLocalPruner(ctx context.Context, target Target, history Sender) *Pruner {
+func (f *LocalPrunerFactory) BuildLocalPruner(ctx context.Context, target Target, receiver History) *Pruner {
 	p := &Pruner{
 		args: args{
 			context.WithValue(ctx, contextKeyPruneSide, "local"),
 			target,
-			history,
+			receiver,
 			f.keepRules,
 			f.retryWait,
 			false, // considerSnapAtCursorReplicated is not relevant for local pruning
@@ -351,9 +343,9 @@ func (s snapshot) Date() time.Time { return s.date }
 
 func doOneAttempt(a *args, u updater) {
 
-	ctx, target, sender := a.ctx, a.target, a.sender
+	ctx, target, receiver := a.ctx, a.target, a.receiver
 
-	sfssres, err := sender.ListFilesystems(ctx, &pdu.ListFilesystemReq{})
+	sfssres, err := receiver.ListFilesystems(ctx, &pdu.ListFilesystemReq{})
 	if err != nil {
 		u(func(p *Pruner) {
 			p.state = PlanErr
@@ -418,7 +410,7 @@ tfss_loop:
 		rcReq := &pdu.ReplicationCursorReq{
 			Filesystem: tfs.Path,
 		}
-		rc, err := sender.ReplicationCursor(ctx, rcReq)
+		rc, err := receiver.ReplicationCursor(ctx, rcReq)
 		if err != nil {
 			pfsPlanErrAndLog(err, "cannot get replication cursor bookmark")
 			continue tfss_loop
@@ -464,7 +456,7 @@ tfss_loop:
 			})
 		}
 		if preCursor {
-			pfsPlanErrAndLog(fmt.Errorf("prune target has no snapshot that corresponds to sender replication cursor bookmark"), "")
+			pfsPlanErrAndLog(fmt.Errorf("replication cursor not found in prune target filesystem versions"), "")
 			continue tfss_loop
 		}
 

daemon/snapper/snapper.go

@@ -210,6 +210,10 @@ func syncUp(a args, u updater) state {
 		return u(func(s *Snapper) {
 			s.state = Planning
 		}).sf()
+	// case <-dosnapshot.Wait(a.ctx):
+	// 	return u(func(s *Snapper) {
+	// 		s.state = Planning
+	// 	}).sf()
 	case <-a.ctx.Done():
 		return onMainCtxDone(a.ctx, u)
 	}
@@ -378,6 +382,10 @@ func wait(a args, u updater) state {
 		return u(func(snapper *Snapper) {
 			snapper.state = Planning
 		}).sf()
+	// case <-dosnapshot.Wait(a.ctx):
+	// 	return u(func(snapper *Snapper) {
+	// 		snapper.state = Planning
+	// 	}).sf()
 	case <-a.ctx.Done():
 		return onMainCtxDone(a.ctx, u)
 	}

daemon/snapper/snapper_all.go

@@ -18,9 +18,9 @@ type PeriodicOrManual struct {
 	s *Snapper
 }
 
-func (s *PeriodicOrManual) Run(ctx context.Context, wakeUpCommon chan<- struct{}) {
+func (s *PeriodicOrManual) Run(ctx context.Context, replicationCommon chan<- struct{}) {
 	if s.s != nil {
-		s.s.Run(ctx, wakeUpCommon)
+		s.s.Run(ctx, replicationCommon)
 	}
 }
 

docs/changelog.rst

@@ -32,6 +32,7 @@ We use the following annotations for classifying changes:
 0.4.0
 -----
 
+* |break| Change syntax to trigger a job replication, rename ``zrepl signal wakeup JOB`` to ``zrepl signal replication JOB``
 * |feature| support setting zfs send / recv flags in the config (send: ``-wLcepbS`` , recv: ``-ox`` ).
   Config docs :ref:`here <job-send-options>` and :ref:`here <job-recv-options>` .
 * |feature| parallel replication is now configurable (disabled by default, :ref:`config docs here <replication-option-concurrency>` ).

docs/configuration/jobs.rst

@@ -78,7 +78,7 @@ Job Type ``pull``
         ``$root_fs/$source_path``
     * - ``interval``
       - | Interval at which to pull from the source job (e.g. ``10m``).
-        | ``manual`` disables periodic pulling, replication then only happens on :ref:`wakeup <cli-signal-wakeup>`.
+        | ``manual`` disables periodic pulling, replication then only happens on :ref:`replication <cli-signal-replication>`.
     * - ``pruning``
       - |pruning-spec|
 

docs/configuration/monitoring.rst

@@ -13,7 +13,7 @@ Prometheus & Grafana
 --------------------
 
 zrepl can expose `Prometheus metrics <https://prometheus.io/docs/instrumenting/exposition_formats/>`_ via HTTP.
-The ``listen`` attribute is a `net.Listen <https://golang.org/pkg/net/#Listen>`_  string for tcp, e.g. ``:9811`` or ``127.0.0.1:9811`` (port 9811 was reserved to zrepl `on the official list <https://github.com/prometheus/prometheus/wiki/Default-port-allocations/_compare/43e495dd251ee328ac0d08b58084665b5c0f7a7e...459195059b55b414193ebeb80c5ba463d2606951>`_).
+The ``listen`` attribute is a `net.Listen <https://golang.org/pkg/net/#Listen>`_  string for tcp, e.g. ``:9091`` or ``127.0.0.1:9091``.
 The ``listen_freebind`` attribute is :ref:`explained here <listen-freebind-explanation>`.
 The Prometheus monitoring job appears in the ``zrepl control`` job list and may be specified **at most once**.
 
@@ -30,7 +30,7 @@ The dashboard also contains some advice on which metrics are important to monito
     global:
       monitoring:
         - type: prometheus
-          listen: ':9811'
+          listen: ':9091'
           listen_freebind: true # optional, default false
 
 

docs/configuration/prune.rst

@@ -67,9 +67,8 @@ Policy ``not_replicated``
      ...
 
 ``not_replicated`` keeps all snapshots that have not been replicated to the receiving side.
-It only makes sense to specify this rule for the ``keep_sender``.
-The reason is that, by definition, all snapshots on the receiver have already been replicated to there from the sender.
-To determine whether a sender-side snapshot has already been replicated, zrepl uses the :ref:`replication cursor bookmark <replication-cursor-and-last-received-hold>` which corresponds to the most recent successfully replicated snapshot.
+It only makes sense to specify this rule on a sender (source or push job).
+The state required to evaluate this rule is stored in the :ref:`replication cursor bookmark <replication-cursor-and-last-received-hold>` on the sending side.
 
 .. _prune-keep-retention-grid:
 
@@ -125,51 +124,43 @@ The syntax to describe the bucket list is as follows:
 
 ::
 
-   Assume the following grid spec:
+          This grid spec produces the following list of adjacent buckets. For the sake of simplicity,
+          we subject all snapshots to the grid pruning policy by settings `regex: .*`.
 
-      grid: 1x1h(keep=all) | 2x2h | 1x3h
+          `
+           grid: 1x1h(keep=all) | 2x2h | 1x3h
+           regex: .*
+          `
 
-   This grid spec produces the following constellation of buckets:
-
-   0h        1h        2h        3h        4h        5h        6h        7h        8h        9h
-   |         |         |         |         |         |         |         |         |         |
-   |-Bucket1-|-----Bucket2-------|------Bucket3------|-----------Bucket4-----------|
-   | keep=all|      keep=1       |       keep=1      |            keep=1           |
+          0h        1h        2h        3h        4h        5h        6h        7h        8h        9h
+          |         |         |         |         |         |         |         |         |         |
+          |-Bucket1-|-----Bucket 2------|------Bucket 3-----|-----------Bucket 4----------|
+          | keep=all|      keep=1       |       keep=1      |            keep=1           |
 
 
 
-   Now assume that we have a set of snapshots @a, @b, ..., @D.
-   Snapshot @a is the most recent snapshot.
-   Snapshot @D is the oldest snapshot, it is almost 9 hours older than snapshot @a.
-   We place the snapshots on the same timeline as the buckets:
+          Let us consider the following set of snapshots @a-zA-C:
 
 
-   0h        1h        2h        3h        4h        5h        6h        7h        8h        9h
-   |         |         |         |         |         |         |         |         |         |
-   |-Bucket1-|-----Bucket2-------|------Bucket3------|-----------Bucket4-----------|
-   | keep=all|      keep=1       |       keep=1      |            keep=1           |
-   |         |                   |                   |                             |
-   |  a  b  c| d  e  f  g  h  i  j  k  l  m  n  o  p |q  r  s  t  u  v  w  x  y  z |A  B  C  D
+          |  a  b  c  d  e  f  g  h  i  j  k  l  m  n  o  p  q  r  s  t  u  v  w  x  y  z  A  B  C  D        |
 
-   The result is the following mapping of snapshots to buckets:
+          The `grid` algorithm maps them to their respective buckets:
 
-   Bucket1:   a, b, c
-   Bucket2:   d,e,f,g,h,i,j
-   Bucket3:   k,l,m,n,o,p
-   Bucket4:   q,r,s,t,u,v,w,x,y,z
-   No bucket: A,B,C,D
+          Bucket 1: a, b, c
+          Bucket 2: d,e,f,g,h,i,j
+          Bucket 3: k,l,m,n,o,p
+          Bucket 4: q,r, q,r,s,t,u,v,w,x,y,z
+          None:     A,B,C,D
 
-   For each bucket, we now prune snapshots until it only contains `keep` snapshots.
-   Newer snapshots are destroyed first.
-   Snapshots that do not fall into a bucket are always destroyed.
+          It then applies the per-bucket pruning logic described above which resulting in the
+          following list of remaining snapshots.
 
-   Result after pruning:
+          |  a  b  c                    j                 p                             z                    |
+
+          Note that it only makes sense to grow (not shorten) the interval duration for buckets
+          further in the past since each bucket acts like a low-pass filter for incoming snapshots
+          and adding a less-low-pass-filter after a low-pass one has no effect.
 
-   0h        1h        2h        3h        4h        5h        6h        7h        8h        9h
-   |         |         |         |         |         |         |         |         |         |
-   |-Bucket1-|-----Bucket2-------|------Bucket3------|-----------Bucket4-----------|
-   |         |                   |                   |                             |
-   |  a  b  c|                   j                 p |                           z  |
 
 .. _prune-keep-last-n:
 

docs/configuration/sendrecvoptions.rst

@@ -36,9 +36,6 @@ See the `upstream man page <https://openzfs.github.io/openzfs-docs/man/8/zfs-sen
     * - ``encrypted``
       -
       - Specific to zrepl, :ref:`see below <job-send-options-encrypted>`.
-    * - ``bandwidth_limit``
-      -
-      - Specific to zrepl, :ref:`see below <job-send-recv-options-bandwidth-limit>`.
     * - ``raw``
       - ``-w``
       - Use ``encrypted`` to only allow encrypted sends.
@@ -141,7 +138,6 @@ Recv Options
          override: {
            "org.openzfs.systemd:ignore": "on"
          }
-       bandwidth_limit: ... # see below
      ...
 
 .. _job-recv-options--inherit-and-override:
@@ -216,25 +212,3 @@ and property replication is enabled, the receiver must :ref:`inherit the followi
 * ``keylocation``
 * ``keyformat``
 * ``encryption``
-
-Common Options
-~~~~~~~~~~~~~~
-
-.. _job-send-recv-options-bandwidth-limit:
-
-Bandwidth Limit (send & recv)
------------------------------
-
-::
-
-   bandwidth_limit:
-     max: 23.5 MiB # -1 is the default and disabled rate limiting
-     bucket_capacity: # token bucket capacity in bytes; defaults to 128KiB
-
-Both ``send`` and ``recv`` can be limited to a maximum bandwidth through ``bandwidth_limit``.
-For most users, it should be sufficient to just set ``bandwidth_limit.max``.
-The ``bandwidth_limit.bucket_capacity`` refers to the `token bucket size <https://github.com/juju/ratelimit>`_.
-
-The bandwidth limit only applies to the payload data, i.e., the ZFS send stream.
-It does not account for transport protocol overheads.
-The scope is the job level, i.e., all :ref:`concurrent <replication-option-concurrency>` sends or incoming receives of a job share the bandwidth limit.

docs/configuration/snapshotting.rst

@@ -15,7 +15,7 @@ A filesystem that does not have snapshots by the snapshotter has lower priority
 
 For ``push`` jobs, replication is automatically triggered after all filesystems have been snapshotted.
 
-Note that the ``zrepl signal wakeup JOB`` subcommand does not trigger snapshotting.
+Note that the ``zrepl signal replication JOB`` subcommand does not trigger snapshotting.
 
 
 ::
@@ -38,7 +38,7 @@ There is also a ``manual`` snapshotting type, which covers the following use cas
 * Existing infrastructure for automatic snapshots: you only want to use this zrepl job for replication.
 * Handling snapshotting through a separate ``snap`` job.
 
-Note that you will have to trigger replication manually using the ``zrepl signal wakeup JOB`` subcommand in that case.
+Note that you will have to trigger replication manually using the ``zrepl signal replication JOB`` subcommand in that case.
 
 ::
 

docs/installation/apt-repos.rst

@@ -13,13 +13,13 @@ The following snippet configure the repository for your Debian or Ubuntu release
 
 ::
 
-    sudo apt update && sudo apt install curl gnupg lsb-release; \
+    apt update && apt install curl gnupg lsb-release; \
     ARCH="$(dpkg --print-architecture)"; \
     CODENAME="$(lsb_release -i -s | tr '[:upper:]' '[:lower:]') $(lsb_release -c -s | tr '[:upper:]' '[:lower:]')"; \
     echo "Using Distro and Codename: $CODENAME"; \
-    (curl https://zrepl.cschwarz.com/apt/apt-key.asc | sudo apt-key add -) && \
-    (echo "deb [arch=$ARCH] https://zrepl.cschwarz.com/apt/$CODENAME main" | sudo tee /etc/apt/sources.list.d/zrepl.list) && \
-    sudo apt update
+    (curl https://zrepl.cschwarz.com/apt/apt-key.asc | apt-key add -) && \
+    (echo "deb [arch=$ARCH] https://zrepl.cschwarz.com/apt/$CODENAME main" > /etc/apt/sources.list.d/zrepl.list) && \
+    apt update
 
 
 .. NOTE::

docs/publish.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
-set -euo pipefail
+set -eo pipefail
+
 
 NON_INTERACTIVE=false
 DO_CLONE=false
@@ -12,7 +13,7 @@ while getopts "ca" arg; do
             DO_CLONE=true
             ;;
         *)
-            echo "invalid option '-$arg'"
+            echo invalid option
             exit 1
             ;;
     esac
@@ -26,6 +27,11 @@ checkout_repo_msg() {
     echo "clone ${GHPAGESREPO} to ${PUBLICDIR}:"
 }
 
+exit_msg() {
+    echo "error, exiting..."
+}
+trap exit_msg EXIT
+
 if ! type sphinx-versioning >/dev/null; then
     echo "install sphinx-versioning and come back"
     exit 1
@@ -65,9 +71,6 @@ git reset --hard origin/master
 
 echo "cleaning GitHub pages repo"
 git rm -rf .
-cat > .gitignore <<EOF
-**/.doctrees
-EOF
 
 popd
 
@@ -93,13 +96,7 @@ COMMIT_MSG="sphinx-versioning render from publish.sh - $(date -u) - ${CURRENT_CO
 pushd "$PUBLICDIR"
 
 echo "adding and commiting all changes in GitHub pages repo"
-git add .gitignore
 git add -A
-if [ "$(git status --porcelain)" != "" ]; then
-    git commit -m "$COMMIT_MSG"
-else
-    echo "nothing to commit"
-fi
-echo "pushing to GitHub pages repo"
+git commit -m "$COMMIT_MSG"
 git push origin master
 

docs/quickstart.rst

@@ -55,8 +55,8 @@ Watch it Work
 =============
 
 Run ``zrepl status`` on the active side of the replication setup to monitor snaphotting, replication and pruning activity.
-To re-trigger replication (snapshots are separate!), use ``zrepl signal wakeup JOBNAME``.
-(refer to the example use case document if you are uncertain which job you want to wake up).
+To re-trigger replication (snapshots are separate!), use ``zrepl signal replication JOBNAME``.
+(refer to the example use case document if you are uncertain which job you want to start replication).
 
 You can also use basic UNIX tools to inspect see what's going on.
 If you like tmux, here is a handy script that works on FreeBSD: ::

docs/supporters.rst

@@ -32,8 +32,6 @@ We would like to thank the following people and organizations for supporting zre
 
    <div class="fa fa-code" style="width: 1em;"></div>
 
-* |supporter-gold| Prominic.NET, Inc.
-* |supporter-std| Torsten Blum
 * |supporter-gold| Cyberiada GmbH
 * |supporter-std| `Gordon Schulz <https://github.com/azmodude>`_
 * |supporter-std| `@jwittlincohen <https://github.com/jwittlincohen>`_

docs/usage.rst

@@ -13,7 +13,7 @@ CLI Overview
     The zrepl binary is self-documenting:
     run ``zrepl help`` for an overview of the available subcommands or ``zrepl SUBCOMMAND --help`` for information on available flags, etc.
 
-.. _cli-signal-wakeup:
+.. _cli-signal-replication:
 
 .. list-table::
     :widths: 30 70
@@ -29,7 +29,7 @@ CLI Overview
       - show job activity, or with ``--raw`` for JSON output
     * - ``zrepl stdinserver``
       - see :ref:`transport-ssh+stdinserver`
-    * - ``zrepl signal wakeup JOB``
+    * - ``zrepl signal replication JOB``
       - manually trigger replication + pruning of JOB
     * - ``zrepl signal reset JOB``
       - manually abort current replication + pruning of JOB

endpoint/endpoint.go

@@ -14,7 +14,6 @@ import (
 	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/replication/logic/pdu"
-	"github.com/zrepl/zrepl/util/bandwidthlimit"
 	"github.com/zrepl/zrepl/util/chainedio"
 	"github.com/zrepl/zrepl/util/chainlock"
 	"github.com/zrepl/zrepl/util/envconst"
@@ -35,8 +34,6 @@ type SenderConfig struct {
 	SendCompressed       bool
 	SendEmbeddedData     bool
 	SendSaved            bool
-
-	BandwidthLimit bandwidthlimit.Config
 }
 
 func (c *SenderConfig) Validate() error {
@@ -47,9 +44,6 @@ func (c *SenderConfig) Validate() error {
 	if _, err := StepHoldTag(c.JobID); err != nil {
 		return fmt.Errorf("JobID cannot be used for hold tag: %s", err)
 	}
-	if err := bandwidthlimit.ValidateConfig(c.BandwidthLimit); err != nil {
-		return errors.Wrap(err, "`BandwidthLimit` field invalid")
-	}
 	return nil
 }
 
@@ -60,21 +54,16 @@ type Sender struct {
 	FSFilter zfs.DatasetFilter
 	jobId    JobID
 	config   SenderConfig
-	bwLimit  bandwidthlimit.Wrapper
 }
 
 func NewSender(conf SenderConfig) *Sender {
 	if err := conf.Validate(); err != nil {
 		panic("invalid config" + err.Error())
 	}
-
-	ratelimiter := bandwidthlimit.WrapperFromConfig(conf.BandwidthLimit)
-
 	return &Sender{
 		FSFilter: conf.FSF,
 		jobId:    conf.JobID,
 		config:   conf,
-		bwLimit:  ratelimiter,
 	}
 }
 
@@ -159,11 +148,12 @@ func sendArgsFromPDUAndValidateExistsAndGetVersion(ctx context.Context, fs strin
 	return version, nil
 }
 
-func (s *Sender) sendMakeArgs(ctx context.Context, r *pdu.SendReq) (sendArgs zfs.ZFSSendArgsValidated, _ error) {
+func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) {
+	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
 
 	_, err := s.filterCheckFS(r.Filesystem)
 	if err != nil {
-		return sendArgs, err
+		return nil, nil, err
 	}
 	switch r.Encrypted {
 	case pdu.Tri_DontCare:
@@ -171,16 +161,16 @@ func (s *Sender) sendMakeArgs(ctx context.Context, r *pdu.SendReq) (sendArgs zfs
 		// ok, fallthrough outer
 	case pdu.Tri_False:
 		if s.config.Encrypt.B {
-			return sendArgs, errors.New("only encrypted sends allowed (send -w + encryption!= off), but unencrypted send requested")
+			return nil, nil, errors.New("only encrypted sends allowed (send -w + encryption!= off), but unencrypted send requested")
 		}
 		// fallthrough outer
 	case pdu.Tri_True:
 		if !s.config.Encrypt.B {
-			return sendArgs, errors.New("only unencrypted sends allowed, but encrypted send requested")
+			return nil, nil, errors.New("only unencrypted sends allowed, but encrypted send requested")
 		}
 		// fallthrough outer
 	default:
-		return sendArgs, fmt.Errorf("unknown pdu.Tri variant %q", r.Encrypted)
+		return nil, nil, fmt.Errorf("unknown pdu.Tri variant %q", r.Encrypted)
 	}
 
 	sendArgsUnvalidated := zfs.ZFSSendArgsUnvalidated{
@@ -200,19 +190,30 @@ func (s *Sender) sendMakeArgs(ctx context.Context, r *pdu.SendReq) (sendArgs zfs
 		},
 	}
 
-	sendArgs, err = sendArgsUnvalidated.Validate(ctx)
+	sendArgs, err := sendArgsUnvalidated.Validate(ctx)
 	if err != nil {
-		return sendArgs, errors.Wrap(err, "validate send arguments")
+		return nil, nil, errors.Wrap(err, "validate send arguments")
 	}
-	return sendArgs, nil
-}
 
-func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error) {
-	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
-
-	sendArgs, err := s.sendMakeArgs(ctx, r)
+	si, err := zfs.ZFSSendDry(ctx, sendArgs)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errors.Wrap(err, "zfs send dry failed")
+	}
+
+	// From now on, assume that sendArgs has been validated by ZFSSendDry
+	// (because validation involves shelling out, it's actually a little expensive)
+
+	var expSize int64 = 0      // protocol says 0 means no estimate
+	if si.SizeEstimate != -1 { // but si returns -1 for no size estimate
+		expSize = si.SizeEstimate
+	}
+	res := &pdu.SendRes{
+		ExpectedSize:    expSize,
+		UsedResumeToken: r.ResumeToken != "",
+	}
+
+	if r.DryRun {
+		return res, nil, nil
 	}
 
 	// create holds or bookmarks of `From` and `To` to guarantee one of the following:
@@ -300,47 +301,15 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.Rea
 		abstractionsCacheSingleton.TryBatchDestroy(ctx, s.jobId, sendArgs.FS, destroyTypes, keep, check)
 	}()
 
-	var sendStream io.ReadCloser
-	sendStream, err = zfs.ZFSSend(ctx, sendArgs)
+	sendStream, err := zfs.ZFSSend(ctx, sendArgs)
 	if err != nil {
 		// it's ok to not destroy the abstractions we just created here, a new send attempt will take care of it
 		return nil, nil, errors.Wrap(err, "zfs send failed")
 	}
 
-	// apply rate limit
-	sendStream = s.bwLimit.WrapReadCloser(sendStream)
-
-	res := &pdu.SendRes{
-		ExpectedSize:    0,
-		UsedResumeToken: r.ResumeToken != "",
-	}
-
 	return res, sendStream, nil
 }
 
-func (s *Sender) SendDry(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, error) {
-	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
-
-	sendArgs, err := s.sendMakeArgs(ctx, r)
-	if err != nil {
-		return nil, err
-	}
-
-	si, err := zfs.ZFSSendDry(ctx, sendArgs)
-	if err != nil {
-		return nil, errors.Wrap(err, "zfs send dry failed")
-	}
-
-	// From now on, assume that sendArgs has been validated by ZFSSendDry
-	// (because validation involves shelling out, it's actually a little expensive)
-
-	res := &pdu.SendRes{
-		ExpectedSize:    si.SizeEstimate,
-		UsedResumeToken: r.ResumeToken != "",
-	}
-	return res, nil
-}
-
 func (p *Sender) SendCompleted(ctx context.Context, r *pdu.SendCompletedReq) (*pdu.SendCompletedRes, error) {
 	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
 
@@ -470,8 +439,6 @@ type ReceiverConfig struct {
 
 	InheritProperties  []zfsprop.Property
 	OverrideProperties map[zfsprop.Property]string
-
-	BandwidthLimit bandwidthlimit.Config
 }
 
 func (c *ReceiverConfig) copyIn() {
@@ -508,11 +475,6 @@ func (c *ReceiverConfig) Validate() error {
 	if c.RootWithoutClientComponent.Length() <= 0 {
 		return errors.New("RootWithoutClientComponent must not be an empty dataset path")
 	}
-
-	if err := bandwidthlimit.ValidateConfig(c.BandwidthLimit); err != nil {
-		return errors.Wrap(err, "`BandwidthLimit` field invalid")
-	}
-
 	return nil
 }
 
@@ -522,8 +484,6 @@ type Receiver struct {
 
 	conf ReceiverConfig // validated
 
-	bwLimit bandwidthlimit.Wrapper
-
 	recvParentCreationMtx *chainlock.L
 }
 
@@ -535,7 +495,6 @@ func NewReceiver(config ReceiverConfig) *Receiver {
 	return &Receiver{
 		conf:                  config,
 		recvParentCreationMtx: chainlock.New(),
-		bwLimit:               bandwidthlimit.WrapperFromConfig(config.BandwidthLimit),
 	}
 }
 
@@ -709,11 +668,6 @@ func (s *Receiver) Send(ctx context.Context, req *pdu.SendReq) (*pdu.SendRes, io
 	return nil, nil, fmt.Errorf("receiver does not implement Send()")
 }
 
-func (s *Receiver) SendDry(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, error) {
-	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
-	return nil, fmt.Errorf("receiver does not implement SendDry()")
-}
-
 func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive io.ReadCloser) (*pdu.ReceiveRes, error) {
 	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
 
@@ -833,9 +787,6 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive io.
 		return nil, errors.Wrap(err, "cannot determine whether we can use resumable send & recv")
 	}
 
-	// apply rate limit
-	receive = s.bwLimit.WrapReadCloser(receive)
-
 	var peek bytes.Buffer
 	var MaxPeek = envconst.Int64("ZREPL_ENDPOINT_RECV_PEEK_SIZE", 1<<20)
 	log.WithField("max_peek_bytes", MaxPeek).Info("peeking incoming stream")

go.mod

@@ -8,13 +8,13 @@ require (
 	github.com/gdamore/tcell/v2 v2.2.0
 	github.com/gitchander/permutation v0.0.0-20181107151852-9e56b92e9909
 	github.com/go-logfmt/logfmt v0.4.0
+	github.com/go-playground/universal-translator v0.17.0 // indirect
 	github.com/go-playground/validator v9.31.0+incompatible
 	github.com/go-playground/validator/v10 v10.4.1
 	github.com/go-sql-driver/mysql v1.4.1-0.20190907122137-b2c03bcae3d4
 	github.com/golang/protobuf v1.4.3
 	github.com/google/uuid v1.1.2
 	github.com/jinzhu/copier v0.0.0-20170922082739-db4671f3a9b8
-	github.com/juju/ratelimit v1.0.1
 	github.com/kisielk/gotool v1.0.0 // indirect
 	github.com/kr/pretty v0.1.0
 	github.com/leodido/go-urn v1.2.1 // indirect
@@ -43,6 +43,7 @@ require (
 	golang.org/x/net v0.0.0-20210119194325-5f4716e94777
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
+	golang.org/x/text v0.3.5 // indirect
 	golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135
 	gonum.org/v1/gonum v0.7.0 // indirect
 	google.golang.org/genproto v0.0.0-20210122163508-8081c04a3579 // indirect

go.sum

@@ -164,8 +164,6 @@ github.com/jinzhu/copier v0.0.0-20170922082739-db4671f3a9b8 h1:+dKzeuiDYbD/Cfi/s
 github.com/jinzhu/copier v0.0.0-20170922082739-db4671f3a9b8/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY=
-github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=

main.go

@@ -11,7 +11,9 @@ import (
 func init() {
 	cli.AddSubcommand(daemon.DaemonCmd)
 	cli.AddSubcommand(status.Subcommand)
-	cli.AddSubcommand(client.SignalCmd)
+	cli.AddSubcommand(client.TriggerCmd)
+	cli.AddSubcommand(client.ResetCmd)
+	cli.AddSubcommand(client.WaitCmd)
 	cli.AddSubcommand(client.StdinserverCmd)
 	cli.AddSubcommand(client.ConfigcheckCmd)
 	cli.AddSubcommand(client.VersionCmd)

packaging/deb/Dockerfile

@@ -1,13 +1,9 @@
 FROM debian:latest
 
-# binutils are for cross-compilation to work in bullseye
 RUN apt-get update && apt-get install -y \
 	build-essential \
     devscripts \
-    dh-exec \
-	binutils-aarch64-linux-gnu \
-	binutils-arm-linux-gnueabihf  \
-	binutils-i686-linux-gnu
+    dh-exec
 
 RUN mkdir -p /build/src && chmod -R 0777 /build
 

platformtest/tests/generated_cases.go

@@ -34,10 +34,5 @@ var Cases = []Case{BatchDestroy,
 	SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden__EncryptionSupported_true,
 	SendArgsValidationResumeTokenDifferentFilesystemForbidden,
 	SendArgsValidationResumeTokenEncryptionMismatchForbidden,
-	SendStreamCloseAfterBlockedOnPipeWrite,
-	SendStreamCloseAfterEOFRead,
-	SendStreamMultipleCloseAfterEOF,
-	SendStreamMultipleCloseBeforeEOF,
-	SendStreamNonEOFReadErrorHandling,
 	UndestroyableSnapshotParsing,
 }

platformtest/tests/recvRollback.go

@@ -40,7 +40,6 @@ func ReceiveForceRollbackWorksUnencrypted(ctx *platformtest.Context) {
 
 	sendStream, err := zfs.ZFSSend(ctx, sendArgs)
 	require.NoError(ctx, err)
-	defer sendStream.Close()
 
 	recvOpts := zfs.RecvOptions{
 		RollbackAndForceRecv: true,

platformtest/tests/replication.go

@@ -7,7 +7,6 @@ import (
 	"os"
 	"path"
 	"sort"
-	"strings"
 	"time"
 
 	"github.com/kr/pretty"
@@ -21,7 +20,6 @@ import (
 	"github.com/zrepl/zrepl/replication/logic"
 	"github.com/zrepl/zrepl/replication/logic/pdu"
 	"github.com/zrepl/zrepl/replication/report"
-	"github.com/zrepl/zrepl/util/bandwidthlimit"
 	"github.com/zrepl/zrepl/util/limitio"
 	"github.com/zrepl/zrepl/util/nodefault"
 	"github.com/zrepl/zrepl/zfs"
@@ -65,10 +63,9 @@ func (i replicationInvocation) Do(ctx *platformtest.Context) *report.Report {
 	}
 
 	senderConfig := endpoint.SenderConfig{
-		FSF:            i.sfilter.AsFilter(),
-		Encrypt:        &nodefault.Bool{B: false},
-		JobID:          i.sjid,
-		BandwidthLimit: bandwidthlimit.NoLimitConfig(),
+		FSF:     i.sfilter.AsFilter(),
+		Encrypt: &nodefault.Bool{B: false},
+		JobID:   i.sjid,
 	}
 	if i.senderConfigHook != nil {
 		i.senderConfigHook(&senderConfig)
@@ -78,7 +75,6 @@ func (i replicationInvocation) Do(ctx *platformtest.Context) *report.Report {
 		JobID:                      i.rjid,
 		AppendClientIdentity:       false,
 		RootWithoutClientComponent: mustDatasetPath(i.rfsRoot),
-		BandwidthLimit:             bandwidthlimit.NoLimitConfig(),
 	}
 	if i.receiverConfigHook != nil {
 		i.receiverConfigHook(&receiverConfig)
@@ -94,7 +90,6 @@ func (i replicationInvocation) Do(ctx *platformtest.Context) *report.Report {
 		ReplicationConfig: &pdu.ReplicationConfig{
 			Protection: i.guarantee,
 		},
-		SizeEstimationConcurrency: 1,
 	}
 
 	report, wait := replication.Do(
@@ -809,22 +804,13 @@ type NeverEndingSender struct {
 	*endpoint.Sender
 }
 
-func (s *NeverEndingSender) SendDry(ctx context.Context, req *pdu.SendReq) (r *pdu.SendRes, err error) {
-	r, _, err = s.sendImpl(ctx, req, true)
-	return r, err
-}
-
 func (s *NeverEndingSender) Send(ctx context.Context, req *pdu.SendReq) (r *pdu.SendRes, stream io.ReadCloser, _ error) {
-	return s.sendImpl(ctx, req, false)
-}
-
-func (s *NeverEndingSender) sendImpl(ctx context.Context, req *pdu.SendReq, dry bool) (r *pdu.SendRes, stream io.ReadCloser, _ error) {
 	stream = nil
 	r = &pdu.SendRes{
 		UsedResumeToken: false,
 		ExpectedSize:    1 << 30,
 	}
-	if dry {
+	if req.DryRun {
 		return r, stream, nil
 	}
 	dz, err := os.Open("/dev/zero")
@@ -1068,9 +1054,6 @@ func ReplicationPropertyReplicationWorks(ctx *platformtest.Context) {
 		rep := fsByName[fs]
 		require.Len(ctx, rep.Steps, 1)
 		require.Nil(ctx, rep.PlanError)
-		if rep.StepError != nil && strings.Contains(rep.StepError.Error(), "invalid option 'x'") {
-			ctx.SkipNow() // XXX feature detection
-		}
 		require.Nil(ctx, rep.StepError)
 		require.Len(ctx, rep.Steps, 1)
 		require.Equal(ctx, 1, rep.CurrentStep)

platformtest/tests/sendStream.go

@@ -1,186 +0,0 @@
-package tests
-
-import (
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"path"
-	"syscall"
-	"time"
-
-	"github.com/stretchr/testify/require"
-	"golang.org/x/sys/unix"
-
-	"github.com/zrepl/zrepl/platformtest"
-	"github.com/zrepl/zrepl/util/nodefault"
-	"github.com/zrepl/zrepl/zfs"
-)
-
-func sendStreamTest(ctx *platformtest.Context) *zfs.SendStream {
-
-	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
-		DESTROYROOT
-		CREATEROOT
-		+  "sender"
-	`)
-
-	fs := fmt.Sprintf("%s/sender", ctx.RootDataset)
-
-	fsmpo, err := zfs.ZFSGetMountpoint(ctx, fs)
-	require.NoError(ctx, err)
-
-	writeDummyData(path.Join(fsmpo.Mountpoint, "dummy.data"), 1<<26)
-	mustSnapshot(ctx, fs+"@1")
-	snap := fsversion(ctx, fs, "@1")
-	snapSendArg := snap.ToSendArgVersion()
-
-	sendArgs, err := zfs.ZFSSendArgsUnvalidated{
-		FS:   fs,
-		From: nil,
-		To:   &snapSendArg,
-		ZFSSendFlags: zfs.ZFSSendFlags{
-			Encrypted: &nodefault.Bool{B: false},
-		},
-	}.Validate(ctx)
-	require.NoError(ctx, err)
-
-	sendStream, err := zfs.ZFSSend(ctx, sendArgs)
-	require.NoError(ctx, err)
-
-	return sendStream
-
-}
-
-func SendStreamCloseAfterBlockedOnPipeWrite(ctx *platformtest.Context) {
-
-	sendStream := sendStreamTest(ctx)
-
-	// let the pipe buffer fill and the zfs process block uninterruptibly
-
-	ctx.Logf("waiting for pipe write to block")
-	time.Sleep(5 * time.Second) // XXX need a platform-neutral way to detect that the pipe is full and the writer is blocked
-
-	ctx.Logf("closing send stream")
-	err := sendStream.Close() // this is what this test case is about
-	ctx.Logf("close error: %T %s", err, err)
-	require.NoError(ctx, err)
-
-	exitErrZfsError := sendStream.TestOnly_ExitErr()
-	require.Contains(ctx, exitErrZfsError.Error(), "signal")
-	require.Error(ctx, exitErrZfsError)
-	exitErr, ok := exitErrZfsError.WaitErr.(*exec.ExitError)
-	require.True(ctx, ok)
-	if exitErr.Exited() {
-		// some ZFS impls (FreeBSD 12) behaves that way
-		return
-	}
-
-	// ProcessState is only available after exit
-	// => use as proxy that the process was wait()ed upon and is gone
-	ctx.Logf("%#v", exitErr.ProcessState)
-	require.NotNil(ctx, exitErr.ProcessState)
-	// and let's verify that the process got killed, so that we know it was the call to .Close() above
-	waitStatus := exitErr.ProcessState.Sys().(syscall.WaitStatus)
-	ctx.Logf("wait status: %#v", waitStatus)
-	ctx.Logf("exit status: %v", waitStatus.ExitStatus())
-	require.True(ctx, waitStatus.Signaled())
-	switch waitStatus.Signal() {
-	case unix.SIGKILL:
-		fallthrough
-	case unix.SIGPIPE:
-		// ok
-
-	default:
-		ctx.Errorf("%T %s\n%v", waitStatus.Signal(), waitStatus.Signal(), waitStatus.Signal())
-		ctx.FailNow()
-	}
-}
-
-func SendStreamCloseAfterEOFRead(ctx *platformtest.Context) {
-
-	sendStream := sendStreamTest(ctx)
-
-	_, err := io.Copy(ioutil.Discard, sendStream)
-	require.NoError(ctx, err)
-
-	var buf [128]byte
-	n, err := sendStream.Read(buf[:])
-	require.Zero(ctx, n)
-	require.Equal(ctx, io.EOF, err)
-
-	err = sendStream.Close()
-	require.NoError(ctx, err)
-
-	n, err = sendStream.Read(buf[:])
-	require.Zero(ctx, n)
-	require.Equal(ctx, os.ErrClosed, err, "same read error should be returned")
-}
-
-func SendStreamMultipleCloseAfterEOF(ctx *platformtest.Context) {
-
-	sendStream := sendStreamTest(ctx)
-
-	_, err := io.Copy(ioutil.Discard, sendStream)
-	require.NoError(ctx, err)
-
-	var buf [128]byte
-	n, err := sendStream.Read(buf[:])
-	require.Zero(ctx, n)
-	require.Equal(ctx, io.EOF, err)
-
-	err = sendStream.Close()
-	require.NoError(ctx, err)
-
-	err = sendStream.Close()
-	require.Equal(ctx, os.ErrClosed, err)
-}
-
-func SendStreamMultipleCloseBeforeEOF(ctx *platformtest.Context) {
-
-	sendStream := sendStreamTest(ctx)
-
-	err := sendStream.Close()
-	require.NoError(ctx, err)
-
-	err = sendStream.Close()
-	require.Equal(ctx, os.ErrClosed, err)
-}
-
-type failingReadCloser struct {
-	err error
-}
-
-var _ io.ReadCloser = &failingReadCloser{}
-
-func (c *failingReadCloser) Read(p []byte) (int, error) { return 0, c.err }
-func (c *failingReadCloser) Close() error               { return c.err }
-
-func SendStreamNonEOFReadErrorHandling(ctx *platformtest.Context) {
-
-	sendStream := sendStreamTest(ctx)
-
-	var buf [128]byte
-	n, err := sendStream.Read(buf[:])
-	require.Equal(ctx, len(buf), n)
-	require.NoError(ctx, err)
-
-	var mockError = fmt.Errorf("taeghaefow4piesahwahjocu7ul5tiachaiLipheijae8ooZ8Pies8shohGee9feeTeirai5aiFeiyaecai4kiaLoh4azeih0tea")
-	mock := &failingReadCloser{err: mockError}
-	orig := sendStream.TestOnly_ReplaceStdoutReader(mock)
-
-	n, err = sendStream.Read(buf[:])
-	require.Equal(ctx, 0, n)
-	require.Equal(ctx, mockError, err)
-
-	if sendStream.TestOnly_ReplaceStdoutReader(orig) != mock {
-		panic("incorrect test impl")
-	}
-
-	err = sendStream.Close()
-	require.NoError(ctx, err) // if we can't kill the child then this will be a flaky test, but let's assume we can kill the child
-
-	err = sendStream.Close()
-	require.Equal(ctx, os.ErrClosed, err)
-}

pruning/keep_last_n.go

@@ -57,11 +57,6 @@ func (k KeepLastN) KeepRule(snaps []Snapshot) (destroyList []Snapshot) {
 		// then lexicographically descending (e.g. b, a)
 		return strings.Compare(matching[i].Name(), matching[j].Name()) == 1
 	})
-
-	n := k.n
-	if n > len(matching) {
-		n = len(matching)
-	}
-	destroyList = append(destroyList, matching[n:]...)
+	destroyList = append(destroyList, matching[k.n:]...)
 	return destroyList
 }

pruning/keep_last_n_test.go

@@ -83,19 +83,6 @@ func TestKeepLastN(t *testing.T) {
 				"b1": true,
 			},
 		},
-		"keep_more_than_matching": {
-			inputs: []Snapshot{
-				stubSnap{"a1", false, o(10)},
-				stubSnap{"b1", false, o(11)},
-				stubSnap{"a2", false, o(12)},
-			},
-			rules: []KeepRule{
-				MustKeepLastN(3, "a"),
-			},
-			expDestroy: map[string]bool{
-				"b1": true,
-			},
-		},
 	}
 
 	testTable(tcs, t)

replication/logic/pdu/pdu.pb.go

@@ -518,7 +518,8 @@ type SendReq struct {
 	// encoded in the ResumeToken. Otherwise, the Sender MUST return an error.
 	ResumeToken       string             `protobuf:"bytes,4,opt,name=ResumeToken,proto3" json:"ResumeToken,omitempty"`
 	Encrypted         Tri                `protobuf:"varint,5,opt,name=Encrypted,proto3,enum=Tri" json:"Encrypted,omitempty"`
-	ReplicationConfig *ReplicationConfig `protobuf:"bytes,6,opt,name=ReplicationConfig,proto3" json:"ReplicationConfig,omitempty"`
+	DryRun            bool               `protobuf:"varint,6,opt,name=DryRun,proto3" json:"DryRun,omitempty"`
+	ReplicationConfig *ReplicationConfig `protobuf:"bytes,7,opt,name=ReplicationConfig,proto3" json:"ReplicationConfig,omitempty"`
 }
 
 func (x *SendReq) Reset() {
@@ -588,6 +589,13 @@ func (x *SendReq) GetEncrypted() Tri {
 	return Tri_DontCare
 }
 
+func (x *SendReq) GetDryRun() bool {
+	if x != nil {
+		return x.DryRun
+	}
+	return false
+}
+
 func (x *SendReq) GetReplicationConfig() *ReplicationConfig {
 	if x != nil {
 		return x.ReplicationConfig
@@ -758,11 +766,12 @@ type SendRes struct {
 	unknownFields protoimpl.UnknownFields
 
 	// Whether the resume token provided in the request has been used or not.
-	// If the SendReq.ResumeToken == "", this field MUST be false.
-	UsedResumeToken bool `protobuf:"varint,1,opt,name=UsedResumeToken,proto3" json:"UsedResumeToken,omitempty"`
+	// If the SendReq.ResumeToken == "", this field has no meaning.
+	UsedResumeToken bool `protobuf:"varint,2,opt,name=UsedResumeToken,proto3" json:"UsedResumeToken,omitempty"`
 	// Expected stream size determined by dry run, not exact.
 	// 0 indicates that for the given SendReq, no size estimate could be made.
-	ExpectedSize uint64 `protobuf:"varint,2,opt,name=ExpectedSize,proto3" json:"ExpectedSize,omitempty"`
+	ExpectedSize int64       `protobuf:"varint,3,opt,name=ExpectedSize,proto3" json:"ExpectedSize,omitempty"`
+	Properties   []*Property `protobuf:"bytes,4,rep,name=Properties,proto3" json:"Properties,omitempty"`
 }
 
 func (x *SendRes) Reset() {
@@ -804,13 +813,20 @@ func (x *SendRes) GetUsedResumeToken() bool {
 	return false
 }
 
-func (x *SendRes) GetExpectedSize() uint64 {
+func (x *SendRes) GetExpectedSize() int64 {
 	if x != nil {
 		return x.ExpectedSize
 	}
 	return 0
 }
 
+func (x *SendRes) GetProperties() []*Property {
+	if x != nil {
+		return x.Properties
+	}
+	return nil
+}
+
 type SendCompletedReq struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -1427,7 +1443,7 @@ var file_pdu_proto_rawDesc = []byte{
 	0x72, 0x65, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x29, 0x0a, 0x0b, 0x56, 0x65, 0x72, 0x73, 0x69,
 	0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68,
 	0x6f, 0x74, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x6f, 0x6f, 0x6b, 0x6d, 0x61, 0x72, 0x6b,
-	0x10, 0x01, 0x22, 0xfd, 0x01, 0x0a, 0x07, 0x53, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x71, 0x12, 0x1e,
+	0x10, 0x01, 0x22, 0x95, 0x02, 0x0a, 0x07, 0x53, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x71, 0x12, 0x1e,
 	0x0a, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01,
 	0x28, 0x09, 0x52, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x12, 0x26,
 	0x0a, 0x04, 0x46, 0x72, 0x6f, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x46,
@@ -1439,119 +1455,122 @@ var file_pdu_proto_rawDesc = []byte{
 	0x0b, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x22, 0x0a, 0x09,
 	0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32,
 	0x04, 0x2e, 0x54, 0x72, 0x69, 0x52, 0x09, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64,
-	0x12, 0x40, 0x0a, 0x11, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43,
-	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x52, 0x65,
-	0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52,
+	0x12, 0x16, 0x0a, 0x06, 0x44, 0x72, 0x79, 0x52, 0x75, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08,
+	0x52, 0x06, 0x44, 0x72, 0x79, 0x52, 0x75, 0x6e, 0x12, 0x40, 0x0a, 0x11, 0x52, 0x65, 0x70, 0x6c,
+	0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x07, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f,
+	0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x11, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x51, 0x0a, 0x11, 0x52, 0x65,
+	0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12,
+	0x3c, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f,
+	0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50, 0x72, 0x6f, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f,
+	0x6e, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x8f, 0x01,
+	0x0a, 0x1b, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e,
+	0x66, 0x69, 0x67, 0x50, 0x72, 0x6f, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x33, 0x0a,
+	0x07, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19,
+	0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x47, 0x75, 0x61, 0x72,
+	0x61, 0x6e, 0x74, 0x65, 0x65, 0x4b, 0x69, 0x6e, 0x64, 0x52, 0x07, 0x49, 0x6e, 0x69, 0x74, 0x69,
+	0x61, 0x6c, 0x12, 0x3b, 0x0a, 0x0b, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61,
+	0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63,
+	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x47, 0x75, 0x61, 0x72, 0x61, 0x6e, 0x74, 0x65, 0x65, 0x4b, 0x69,
+	0x6e, 0x64, 0x52, 0x0b, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x22,
+	0x34, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x4e,
+	0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12,
+	0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05,
+	0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x82, 0x01, 0x0a, 0x07, 0x53, 0x65, 0x6e, 0x64, 0x52, 0x65,
+	0x73, 0x12, 0x28, 0x0a, 0x0f, 0x55, 0x73, 0x65, 0x64, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x54,
+	0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x55, 0x73, 0x65, 0x64,
+	0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x45,
+	0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28,
+	0x03, 0x52, 0x0c, 0x45, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x12,
+	0x29, 0x0a, 0x0a, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20,
+	0x03, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x52, 0x0a,
+	0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x22, 0x3e, 0x0a, 0x10, 0x53, 0x65,
+	0x6e, 0x64, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x12, 0x2a,
+	0x0a, 0x0b, 0x4f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x71, 0x52, 0x0b, 0x4f,
+	0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x22, 0x12, 0x0a, 0x10, 0x53, 0x65,
+	0x6e, 0x64, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x22, 0xbe,
+	0x01, 0x0a, 0x0a, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x52, 0x65, 0x71, 0x12, 0x1e, 0x0a,
+	0x0a, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x12, 0x22, 0x0a,
+	0x02, 0x54, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x46, 0x69, 0x6c, 0x65,
+	0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x02, 0x54,
+	0x6f, 0x12, 0x2a, 0x0a, 0x10, 0x43, 0x6c, 0x65, 0x61, 0x72, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65,
+	0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x43, 0x6c, 0x65,
+	0x61, 0x72, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x40, 0x0a,
 	0x11, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66,
-	0x69, 0x67, 0x22, 0x51, 0x0a, 0x11, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f,
-	0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3c, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x74, 0x65,
-	0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x52, 0x65,
-	0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50,
-	0x72, 0x6f, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x74, 0x65,
-	0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x8f, 0x01, 0x0a, 0x1b, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63,
-	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50, 0x72, 0x6f, 0x74, 0x65,
-	0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x33, 0x0a, 0x07, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61,
-	0x74, 0x69, 0x6f, 0x6e, 0x47, 0x75, 0x61, 0x72, 0x61, 0x6e, 0x74, 0x65, 0x65, 0x4b, 0x69, 0x6e,
-	0x64, 0x52, 0x07, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x12, 0x3b, 0x0a, 0x0b, 0x49, 0x6e,
-	0x63, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32,
-	0x19, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x47, 0x75, 0x61,
-	0x72, 0x61, 0x6e, 0x74, 0x65, 0x65, 0x4b, 0x69, 0x6e, 0x64, 0x52, 0x0b, 0x49, 0x6e, 0x63, 0x72,
-	0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x22, 0x34, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x70, 0x65,
-	0x72, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x57, 0x0a,
-	0x07, 0x53, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x73, 0x12, 0x28, 0x0a, 0x0f, 0x55, 0x73, 0x65, 0x64,
-	0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28,
-	0x08, 0x52, 0x0f, 0x55, 0x73, 0x65, 0x64, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x54, 0x6f, 0x6b,
-	0x65, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x45, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x53, 0x69,
-	0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x45, 0x78, 0x70, 0x65, 0x63, 0x74,
-	0x65, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x3e, 0x0a, 0x10, 0x53, 0x65, 0x6e, 0x64, 0x43, 0x6f,
-	0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x12, 0x2a, 0x0a, 0x0b, 0x4f, 0x72,
-	0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
-	0x08, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x71, 0x52, 0x0b, 0x4f, 0x72, 0x69, 0x67, 0x69,
-	0x6e, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x22, 0x12, 0x0a, 0x10, 0x53, 0x65, 0x6e, 0x64, 0x43, 0x6f,
-	0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x22, 0xbe, 0x01, 0x0a, 0x0a, 0x52,
-	0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x52, 0x65, 0x71, 0x12, 0x1e, 0x0a, 0x0a, 0x46, 0x69, 0x6c,
-	0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x46,
-	0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x12, 0x22, 0x0a, 0x02, 0x54, 0x6f, 0x18,
-	0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74,
-	0x65, 0x6d, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x02, 0x54, 0x6f, 0x12, 0x2a, 0x0a,
-	0x10, 0x43, 0x6c, 0x65, 0x61, 0x72, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x54, 0x6f, 0x6b, 0x65,
-	0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x43, 0x6c, 0x65, 0x61, 0x72, 0x52, 0x65,
-	0x73, 0x75, 0x6d, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x40, 0x0a, 0x11, 0x52, 0x65, 0x70,
-	0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x04,
-	0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69,
-	0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x11, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63,
-	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x0c, 0x0a, 0x0a, 0x52,
-	0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x52, 0x65, 0x73, 0x22, 0x67, 0x0a, 0x13, 0x44, 0x65, 0x73,
-	0x74, 0x72, 0x6f, 0x79, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x52, 0x65, 0x71,
+	0x69, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69,
+	0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x11, 0x52, 0x65,
+	0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22,
+	0x0c, 0x0a, 0x0a, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x52, 0x65, 0x73, 0x22, 0x67, 0x0a,
+	0x13, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74,
+	0x73, 0x52, 0x65, 0x71, 0x12, 0x1e, 0x0a, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74,
+	0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79,
+	0x73, 0x74, 0x65, 0x6d, 0x12, 0x30, 0x0a, 0x09, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74,
+	0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79,
+	0x73, 0x74, 0x65, 0x6d, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x53, 0x6e, 0x61,
+	0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x22, 0x5a, 0x0a, 0x12, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f,
+	0x79, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x73, 0x12, 0x2e, 0x0a, 0x08,
+	0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12,
+	0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x56, 0x65, 0x72, 0x73, 0x69,
+	0x6f, 0x6e, 0x52, 0x08, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x14, 0x0a, 0x05,
+	0x45, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x45, 0x72, 0x72,
+	0x6f, 0x72, 0x22, 0x44, 0x0a, 0x13, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x53, 0x6e, 0x61,
+	0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x52, 0x65, 0x73, 0x12, 0x2d, 0x0a, 0x07, 0x52, 0x65, 0x73,
+	0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x44, 0x65, 0x73,
+	0x74, 0x72, 0x6f, 0x79, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x73, 0x52,
+	0x07, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x22, 0x36, 0x0a, 0x14, 0x52, 0x65, 0x70, 0x6c,
+	0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x52, 0x65, 0x71,
 	0x12, 0x1e, 0x0a, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18, 0x01,
 	0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d,
-	0x12, 0x30, 0x0a, 0x09, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x18, 0x02, 0x20,
-	0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d,
-	0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f,
-	0x74, 0x73, 0x22, 0x5a, 0x0a, 0x12, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x53, 0x6e, 0x61,
-	0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x73, 0x12, 0x2e, 0x0a, 0x08, 0x53, 0x6e, 0x61, 0x70,
-	0x73, 0x68, 0x6f, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x46, 0x69, 0x6c,
-	0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x08,
-	0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x45, 0x72, 0x72, 0x6f,
-	0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x44,
-	0x0a, 0x13, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f,
-	0x74, 0x73, 0x52, 0x65, 0x73, 0x12, 0x2d, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73,
-	0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79,
-	0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x73, 0x52, 0x07, 0x52, 0x65, 0x73,
-	0x75, 0x6c, 0x74, 0x73, 0x22, 0x36, 0x0a, 0x14, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74,
-	0x69, 0x6f, 0x6e, 0x43, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x12, 0x1e, 0x0a, 0x0a,
-	0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x22, 0x54, 0x0a, 0x14,
-	0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x75, 0x72, 0x73, 0x6f,
-	0x72, 0x52, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x04, 0x47, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01,
-	0x28, 0x04, 0x48, 0x00, 0x52, 0x04, 0x47, 0x75, 0x69, 0x64, 0x12, 0x1c, 0x0a, 0x08, 0x4e, 0x6f,
-	0x74, 0x65, 0x78, 0x69, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x08,
-	0x4e, 0x6f, 0x74, 0x65, 0x78, 0x69, 0x73, 0x74, 0x42, 0x08, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75,
-	0x6c, 0x74, 0x22, 0x23, 0x0a, 0x07, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x12, 0x18, 0x0a,
-	0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07,
-	0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x1d, 0x0a, 0x07, 0x50, 0x69, 0x6e, 0x67, 0x52,
-	0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x45, 0x63, 0x68, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x52, 0x04, 0x45, 0x63, 0x68, 0x6f, 0x2a, 0x28, 0x0a, 0x03, 0x54, 0x72, 0x69, 0x12, 0x0c, 0x0a,
-	0x08, 0x44, 0x6f, 0x6e, 0x74, 0x43, 0x61, 0x72, 0x65, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x46,
-	0x61, 0x6c, 0x73, 0x65, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x54, 0x72, 0x75, 0x65, 0x10, 0x02,
-	0x2a, 0x86, 0x01, 0x0a, 0x18, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e,
-	0x47, 0x75, 0x61, 0x72, 0x61, 0x6e, 0x74, 0x65, 0x65, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x14, 0x0a,
-	0x10, 0x47, 0x75, 0x61, 0x72, 0x61, 0x6e, 0x74, 0x65, 0x65, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69,
-	0x64, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x47, 0x75, 0x61, 0x72, 0x61, 0x6e, 0x74, 0x65, 0x65,
-	0x52, 0x65, 0x73, 0x75, 0x6d, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x10, 0x01, 0x12, 0x23,
-	0x0a, 0x1f, 0x47, 0x75, 0x61, 0x72, 0x61, 0x6e, 0x74, 0x65, 0x65, 0x49, 0x6e, 0x63, 0x72, 0x65,
-	0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f,
-	0x6e, 0x10, 0x02, 0x12, 0x14, 0x0a, 0x10, 0x47, 0x75, 0x61, 0x72, 0x61, 0x6e, 0x74, 0x65, 0x65,
-	0x4e, 0x6f, 0x74, 0x68, 0x69, 0x6e, 0x67, 0x10, 0x03, 0x32, 0x8f, 0x03, 0x0a, 0x0b, 0x52, 0x65,
-	0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1a, 0x0a, 0x04, 0x50, 0x69, 0x6e,
-	0x67, 0x12, 0x08, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x1a, 0x08, 0x2e, 0x50, 0x69,
-	0x6e, 0x67, 0x52, 0x65, 0x73, 0x12, 0x39, 0x0a, 0x0f, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x69, 0x6c,
-	0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x73, 0x12, 0x12, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46,
-	0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x52, 0x65, 0x71, 0x1a, 0x12, 0x2e, 0x4c,
-	0x69, 0x73, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x52, 0x65, 0x73,
-	0x12, 0x50, 0x0a, 0x16, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74,
-	0x65, 0x6d, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1a, 0x2e, 0x4c, 0x69, 0x73,
+	0x22, 0x54, 0x0a, 0x14, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43,
+	0x75, 0x72, 0x73, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x04, 0x47, 0x75, 0x69, 0x64,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x48, 0x00, 0x52, 0x04, 0x47, 0x75, 0x69, 0x64, 0x12, 0x1c,
+	0x0a, 0x08, 0x4e, 0x6f, 0x74, 0x65, 0x78, 0x69, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08,
+	0x48, 0x00, 0x52, 0x08, 0x4e, 0x6f, 0x74, 0x65, 0x78, 0x69, 0x73, 0x74, 0x42, 0x08, 0x0a, 0x06,
+	0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, 0x23, 0x0a, 0x07, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65,
+	0x71, 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x1d, 0x0a, 0x07, 0x50,
+	0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x45, 0x63, 0x68, 0x6f, 0x18, 0x01,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x45, 0x63, 0x68, 0x6f, 0x2a, 0x28, 0x0a, 0x03, 0x54, 0x72,
+	0x69, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x6f, 0x6e, 0x74, 0x43, 0x61, 0x72, 0x65, 0x10, 0x00, 0x12,
+	0x09, 0x0a, 0x05, 0x46, 0x61, 0x6c, 0x73, 0x65, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x54, 0x72,
+	0x75, 0x65, 0x10, 0x02, 0x2a, 0x86, 0x01, 0x0a, 0x18, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x47, 0x75, 0x61, 0x72, 0x61, 0x6e, 0x74, 0x65, 0x65, 0x4b, 0x69, 0x6e,
+	0x64, 0x12, 0x14, 0x0a, 0x10, 0x47, 0x75, 0x61, 0x72, 0x61, 0x6e, 0x74, 0x65, 0x65, 0x49, 0x6e,
+	0x76, 0x61, 0x6c, 0x69, 0x64, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x47, 0x75, 0x61, 0x72, 0x61,
+	0x6e, 0x74, 0x65, 0x65, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79,
+	0x10, 0x01, 0x12, 0x23, 0x0a, 0x1f, 0x47, 0x75, 0x61, 0x72, 0x61, 0x6e, 0x74, 0x65, 0x65, 0x49,
+	0x6e, 0x63, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x6c, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63,
+	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x10, 0x02, 0x12, 0x14, 0x0a, 0x10, 0x47, 0x75, 0x61, 0x72, 0x61,
+	0x6e, 0x74, 0x65, 0x65, 0x4e, 0x6f, 0x74, 0x68, 0x69, 0x6e, 0x67, 0x10, 0x03, 0x32, 0xf0, 0x02,
+	0x0a, 0x0b, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1a, 0x0a,
+	0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, 0x08, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x1a,
+	0x08, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x12, 0x39, 0x0a, 0x0f, 0x4c, 0x69, 0x73,
+	0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x73, 0x12, 0x12, 0x2e, 0x4c,
+	0x69, 0x73, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x52, 0x65, 0x71,
+	0x1a, 0x12, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65,
+	0x6d, 0x52, 0x65, 0x73, 0x12, 0x50, 0x0a, 0x16, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x69, 0x6c, 0x65,
+	0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1a,
+	0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x56,
+	0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x1a, 0x1a, 0x2e, 0x4c, 0x69, 0x73,
 	0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x56, 0x65, 0x72, 0x73, 0x69,
-	0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x1a, 0x1a, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x69, 0x6c,
-	0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x52,
-	0x65, 0x73, 0x12, 0x3e, 0x0a, 0x10, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x53, 0x6e, 0x61,
-	0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x12, 0x14, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79,
-	0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x52, 0x65, 0x71, 0x1a, 0x14, 0x2e, 0x44,
-	0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x52,
-	0x65, 0x73, 0x12, 0x41, 0x0a, 0x11, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f,
-	0x6e, 0x43, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x12, 0x15, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63,
-	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x15,
-	0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x75, 0x72, 0x73,
-	0x6f, 0x72, 0x52, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x07, 0x53, 0x65, 0x6e, 0x64, 0x44, 0x72, 0x79,
-	0x12, 0x08, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x71, 0x1a, 0x08, 0x2e, 0x53, 0x65, 0x6e,
-	0x64, 0x52, 0x65, 0x73, 0x12, 0x35, 0x0a, 0x0d, 0x53, 0x65, 0x6e, 0x64, 0x43, 0x6f, 0x6d, 0x70,
-	0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x11, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x43, 0x6f, 0x6d, 0x70,
-	0x6c, 0x65, 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x1a, 0x11, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x43,
-	0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x42, 0x07, 0x5a, 0x05, 0x2e,
-	0x3b, 0x70, 0x64, 0x75, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x12, 0x3e, 0x0a, 0x10, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f,
+	0x79, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x12, 0x14, 0x2e, 0x44, 0x65, 0x73,
+	0x74, 0x72, 0x6f, 0x79, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x52, 0x65, 0x71,
+	0x1a, 0x14, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68,
+	0x6f, 0x74, 0x73, 0x52, 0x65, 0x73, 0x12, 0x41, 0x0a, 0x11, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63,
+	0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x12, 0x15, 0x2e, 0x52, 0x65,
+	0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x52,
+	0x65, 0x71, 0x1a, 0x15, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e,
+	0x43, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x12, 0x35, 0x0a, 0x0d, 0x53, 0x65, 0x6e,
+	0x64, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x11, 0x2e, 0x53, 0x65, 0x6e,
+	0x64, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x1a, 0x11, 0x2e,
+	0x53, 0x65, 0x6e, 0x64, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73,
+	0x42, 0x07, 0x5a, 0x05, 0x2e, 0x3b, 0x70, 0x64, 0x75, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x33,
 }
 
 var (
@@ -1606,31 +1625,30 @@ var file_pdu_proto_depIdxs = []int32{
 	11, // 7: ReplicationConfig.protection:type_name -> ReplicationConfigProtection
 	1,  // 8: ReplicationConfigProtection.Initial:type_name -> ReplicationGuaranteeKind
 	1,  // 9: ReplicationConfigProtection.Incremental:type_name -> ReplicationGuaranteeKind
-	9,  // 10: SendCompletedReq.OriginalReq:type_name -> SendReq
-	8,  // 11: ReceiveReq.To:type_name -> FilesystemVersion
-	10, // 12: ReceiveReq.ReplicationConfig:type_name -> ReplicationConfig
-	8,  // 13: DestroySnapshotsReq.Snapshots:type_name -> FilesystemVersion
-	8,  // 14: DestroySnapshotRes.Snapshot:type_name -> FilesystemVersion
-	19, // 15: DestroySnapshotsRes.Results:type_name -> DestroySnapshotRes
-	23, // 16: Replication.Ping:input_type -> PingReq
-	3,  // 17: Replication.ListFilesystems:input_type -> ListFilesystemReq
-	6,  // 18: Replication.ListFilesystemVersions:input_type -> ListFilesystemVersionsReq
-	18, // 19: Replication.DestroySnapshots:input_type -> DestroySnapshotsReq
-	21, // 20: Replication.ReplicationCursor:input_type -> ReplicationCursorReq
-	9,  // 21: Replication.SendDry:input_type -> SendReq
+	12, // 10: SendRes.Properties:type_name -> Property
+	9,  // 11: SendCompletedReq.OriginalReq:type_name -> SendReq
+	8,  // 12: ReceiveReq.To:type_name -> FilesystemVersion
+	10, // 13: ReceiveReq.ReplicationConfig:type_name -> ReplicationConfig
+	8,  // 14: DestroySnapshotsReq.Snapshots:type_name -> FilesystemVersion
+	8,  // 15: DestroySnapshotRes.Snapshot:type_name -> FilesystemVersion
+	19, // 16: DestroySnapshotsRes.Results:type_name -> DestroySnapshotRes
+	23, // 17: Replication.Ping:input_type -> PingReq
+	3,  // 18: Replication.ListFilesystems:input_type -> ListFilesystemReq
+	6,  // 19: Replication.ListFilesystemVersions:input_type -> ListFilesystemVersionsReq
+	18, // 20: Replication.DestroySnapshots:input_type -> DestroySnapshotsReq
+	21, // 21: Replication.ReplicationCursor:input_type -> ReplicationCursorReq
 	14, // 22: Replication.SendCompleted:input_type -> SendCompletedReq
 	24, // 23: Replication.Ping:output_type -> PingRes
 	4,  // 24: Replication.ListFilesystems:output_type -> ListFilesystemRes
 	7,  // 25: Replication.ListFilesystemVersions:output_type -> ListFilesystemVersionsRes
 	20, // 26: Replication.DestroySnapshots:output_type -> DestroySnapshotsRes
 	22, // 27: Replication.ReplicationCursor:output_type -> ReplicationCursorRes
-	13, // 28: Replication.SendDry:output_type -> SendRes
-	15, // 29: Replication.SendCompleted:output_type -> SendCompletedRes
-	23, // [23:30] is the sub-list for method output_type
-	16, // [16:23] is the sub-list for method input_type
-	16, // [16:16] is the sub-list for extension type_name
-	16, // [16:16] is the sub-list for extension extendee
-	0,  // [0:16] is the sub-list for field type_name
+	15, // 28: Replication.SendCompleted:output_type -> SendCompletedRes
+	23, // [23:29] is the sub-list for method output_type
+	17, // [17:23] is the sub-list for method input_type
+	17, // [17:17] is the sub-list for extension type_name
+	17, // [17:17] is the sub-list for extension extendee
+	0,  // [0:17] is the sub-list for field type_name
 }
 
 func init() { file_pdu_proto_init() }

replication/logic/pdu/pdu.proto

@@ -8,7 +8,6 @@ service Replication {
       returns (ListFilesystemVersionsRes);
   rpc DestroySnapshots(DestroySnapshotsReq) returns (DestroySnapshotsRes);
   rpc ReplicationCursor(ReplicationCursorReq) returns (ReplicationCursorRes);
-  rpc SendDry(SendReq) returns (SendRes);
   rpc SendCompleted(SendCompletedReq) returns (SendCompletedRes);
   // for Send and Recv, see package rpc
 }
@@ -61,7 +60,9 @@ message SendReq {
   string ResumeToken = 4;
   Tri Encrypted = 5;
 
-  ReplicationConfig ReplicationConfig = 6;
+  bool DryRun = 6;
+
+  ReplicationConfig ReplicationConfig = 7;
 }
 
 message ReplicationConfig {
@@ -88,12 +89,14 @@ message Property {
 
 message SendRes {
   // Whether the resume token provided in the request has been used or not.
-  // If the SendReq.ResumeToken == "", this field MUST be false.
-  bool UsedResumeToken = 1;
+  // If the SendReq.ResumeToken == "", this field has no meaning.
+  bool UsedResumeToken = 2;
 
   // Expected stream size determined by dry run, not exact.
   // 0 indicates that for the given SendReq, no size estimate could be made.
-  uint64 ExpectedSize = 2;
+  int64 ExpectedSize = 3;
+
+  repeated Property Properties = 4;
 }
 
 message SendCompletedReq {

replication/logic/pdu/pdu_grpc.pb.go

@@ -23,7 +23,6 @@ type ReplicationClient interface {
 	ListFilesystemVersions(ctx context.Context, in *ListFilesystemVersionsReq, opts ...grpc.CallOption) (*ListFilesystemVersionsRes, error)
 	DestroySnapshots(ctx context.Context, in *DestroySnapshotsReq, opts ...grpc.CallOption) (*DestroySnapshotsRes, error)
 	ReplicationCursor(ctx context.Context, in *ReplicationCursorReq, opts ...grpc.CallOption) (*ReplicationCursorRes, error)
-	SendDry(ctx context.Context, in *SendReq, opts ...grpc.CallOption) (*SendRes, error)
 	SendCompleted(ctx context.Context, in *SendCompletedReq, opts ...grpc.CallOption) (*SendCompletedRes, error)
 }
 
@@ -80,15 +79,6 @@ func (c *replicationClient) ReplicationCursor(ctx context.Context, in *Replicati
 	return out, nil
 }
 
-func (c *replicationClient) SendDry(ctx context.Context, in *SendReq, opts ...grpc.CallOption) (*SendRes, error) {
-	out := new(SendRes)
-	err := c.cc.Invoke(ctx, "/Replication/SendDry", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
 func (c *replicationClient) SendCompleted(ctx context.Context, in *SendCompletedReq, opts ...grpc.CallOption) (*SendCompletedRes, error) {
 	out := new(SendCompletedRes)
 	err := c.cc.Invoke(ctx, "/Replication/SendCompleted", in, out, opts...)
@@ -107,7 +97,6 @@ type ReplicationServer interface {
 	ListFilesystemVersions(context.Context, *ListFilesystemVersionsReq) (*ListFilesystemVersionsRes, error)
 	DestroySnapshots(context.Context, *DestroySnapshotsReq) (*DestroySnapshotsRes, error)
 	ReplicationCursor(context.Context, *ReplicationCursorReq) (*ReplicationCursorRes, error)
-	SendDry(context.Context, *SendReq) (*SendRes, error)
 	SendCompleted(context.Context, *SendCompletedReq) (*SendCompletedRes, error)
 	mustEmbedUnimplementedReplicationServer()
 }
@@ -131,9 +120,6 @@ func (UnimplementedReplicationServer) DestroySnapshots(context.Context, *Destroy
 func (UnimplementedReplicationServer) ReplicationCursor(context.Context, *ReplicationCursorReq) (*ReplicationCursorRes, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method ReplicationCursor not implemented")
 }
-func (UnimplementedReplicationServer) SendDry(context.Context, *SendReq) (*SendRes, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method SendDry not implemented")
-}
 func (UnimplementedReplicationServer) SendCompleted(context.Context, *SendCompletedReq) (*SendCompletedRes, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method SendCompleted not implemented")
 }
@@ -240,24 +226,6 @@ func _Replication_ReplicationCursor_Handler(srv interface{}, ctx context.Context
 	return interceptor(ctx, in, info, handler)
 }
 
-func _Replication_SendDry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(SendReq)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ReplicationServer).SendDry(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/Replication/SendDry",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ReplicationServer).SendDry(ctx, req.(*SendReq))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
 func _Replication_SendCompleted_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(SendCompletedReq)
 	if err := dec(in); err != nil {
@@ -303,10 +271,6 @@ var Replication_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "ReplicationCursor",
 			Handler:    _Replication_ReplicationCursor_Handler,
 		},
-		{
-			MethodName: "SendDry",
-			Handler:    _Replication_SendDry_Handler,
-		},
 		{
 			MethodName: "SendCompleted",
 			Handler:    _Replication_SendCompleted_Handler,

replication/logic/replication_logic.go

@@ -41,7 +41,6 @@ type Sender interface {
 	// any next call to the parent github.com/zrepl/zrepl/replication.Endpoint.
 	// If the send request is for dry run the io.ReadCloser will be nil
 	Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error)
-	SendDry(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, error)
 	SendCompleted(ctx context.Context, r *pdu.SendCompletedReq) (*pdu.SendCompletedRes, error)
 	ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error)
 }
@@ -159,7 +158,7 @@ type Step struct {
 	encrypt     tri
 	resumeToken string // empty means no resume token shall be used
 
-	expectedSize uint64 // 0 means no size estimate present / possible
+	expectedSize int64 // 0 means no size estimate present / possible
 
 	// byteCounter is nil initially, and set later in Step.doReplication
 	// => concurrent read of that pointer from Step.ReportInfo must be protected
@@ -190,7 +189,7 @@ func (s *Step) Step(ctx context.Context) error {
 func (s *Step) ReportInfo() *report.StepInfo {
 
 	// get current byteCounter value
-	var byteCounter uint64
+	var byteCounter int64
 	s.byteCounterMtx.Lock()
 	if s.byteCounter != nil {
 		byteCounter = s.byteCounter.Count()
@@ -547,10 +546,10 @@ func (s *Step) updateSizeEstimate(ctx context.Context) error {
 
 	log := getLogger(ctx)
 
-	sr := s.buildSendRequest()
+	sr := s.buildSendRequest(true)
 
 	log.Debug("initiate dry run send request")
-	sres, err := s.sender.SendDry(ctx, sr)
+	sres, _, err := s.sender.Send(ctx, sr)
 	if err != nil {
 		log.WithError(err).Error("dry run send request failed")
 		return err
@@ -564,7 +563,7 @@ func (s *Step) updateSizeEstimate(ctx context.Context) error {
 	return nil
 }
 
-func (s *Step) buildSendRequest() (sr *pdu.SendReq) {
+func (s *Step) buildSendRequest(dryRun bool) (sr *pdu.SendReq) {
 	fs := s.parent.Path
 	sr = &pdu.SendReq{
 		Filesystem:        fs,
@@ -572,6 +571,7 @@ func (s *Step) buildSendRequest() (sr *pdu.SendReq) {
 		To:                s.to,
 		Encrypted:         s.encrypt.ToPDU(),
 		ResumeToken:       s.resumeToken,
+		DryRun:            dryRun,
 		ReplicationConfig: s.parent.policy.ReplicationConfig,
 	}
 	return sr
@@ -582,7 +582,7 @@ func (s *Step) doReplication(ctx context.Context) error {
 	fs := s.parent.Path
 
 	log := getLogger(ctx).WithField("filesystem", fs)
-	sr := s.buildSendRequest()
+	sr := s.buildSendRequest(false)
 
 	log.Debug("initiate send request")
 	sres, stream, err := s.sender.Send(ctx, sr)

replication/report/replication_report.go

@@ -97,11 +97,11 @@ type StepInfo struct {
 	From, To        string
 	Resumed         bool
 	Encrypted       EncryptedEnum
-	BytesExpected   uint64
-	BytesReplicated uint64
+	BytesExpected   int64
+	BytesReplicated int64
 }
 
-func (a *AttemptReport) BytesSum() (expected, replicated uint64, containsInvalidSizeEstimates bool) {
+func (a *AttemptReport) BytesSum() (expected, replicated int64, containsInvalidSizeEstimates bool) {
 	for _, fs := range a.Filesystems {
 		e, r, fsContainsInvalidEstimate := fs.BytesSum()
 		containsInvalidSizeEstimates = containsInvalidSizeEstimates || fsContainsInvalidEstimate
@@ -111,7 +111,7 @@ func (a *AttemptReport) BytesSum() (expected, replicated uint64, containsInvalid
 	return expected, replicated, containsInvalidSizeEstimates
 }
 
-func (f *FilesystemReport) BytesSum() (expected, replicated uint64, containsInvalidSizeEstimates bool) {
+func (f *FilesystemReport) BytesSum() (expected, replicated int64, containsInvalidSizeEstimates bool) {
 	for _, step := range f.Steps {
 		expected += step.Info.BytesExpected
 		replicated += step.Info.BytesReplicated

rpc/dataconn/dataconn_client.go

@@ -114,6 +114,12 @@ func (c *Client) ReqSend(ctx context.Context, req *pdu.SendReq) (*pdu.SendRes, i
 	if err != nil {
 		return nil, nil, err
 	}
+	putWireOnReturn := true
+	defer func() {
+		if putWireOnReturn {
+			c.putWire(conn)
+		}
+	}()
 
 	if err := c.send(ctx, conn, EndpointSend, req, nil); err != nil {
 		return nil, nil, err
@@ -125,10 +131,14 @@ func (c *Client) ReqSend(ctx context.Context, req *pdu.SendReq) (*pdu.SendRes, i
 	}
 
 	var stream io.ReadCloser
-	stream, err = conn.ReadStream(ZFSStream, true) // no shadow
-	if err != nil {
-		return nil, nil, err
+	if !req.DryRun {
+		putWireOnReturn = false
+		stream, err = conn.ReadStream(ZFSStream, true) // no shadow
+		if err != nil {
+			return nil, nil, err
+		}
 	}
+
 	return &res, stream, nil
 }
 

rpc/dataconn/dataconn_server.go

@@ -172,15 +172,6 @@ func (s *Server) serveConnRequest(ctx context.Context, endpoint string, c *strea
 			return
 		}
 		res, sendStream, handlerErr = s.h.Send(ctx, &req) // SHADOWING
-		// ensure that we always close the sendStream
-		if sendStream != nil {
-			defer func() {
-				err := sendStream.Close()
-				if err != nil {
-					s.log.WithError(err).Error("cannot close send stream")
-				}
-			}()
-		}
 	case EndpointRecv:
 		var req pdu.ReceiveReq
 		if err := proto.Unmarshal(reqStructured, &req); err != nil {
@@ -248,9 +239,12 @@ func (s *Server) serveConnRequest(ctx context.Context, endpoint string, c *strea
 
 	if sendStream != nil {
 		err := c.SendStream(ctx, sendStream, ZFSStream)
+		closeErr := sendStream.Close()
+		if closeErr != nil {
+			s.log.WithError(err).Error("cannot close send stream")
+		}
 		if err != nil {
 			s.log.WithError(err).Error("cannot write send stream")
 		}
-		// sendStream.Close() done via defer above
 	}
 }

rpc/dataconn/timeoutconn/timeoutconn_readv_unsupported.go

@@ -1,4 +1,3 @@
-//go:build illumos || solaris
 // +build illumos solaris
 
 package timeoutconn

rpc/dataconn/timeoutconn/timoutconn_readv.go

@@ -1,5 +1,5 @@
-//go:build !illumos && !solaris
-// +build !illumos,!solaris
+// +build !illumos
+// +build !solaris
 
 package timeoutconn
 

rpc/rpc_client.go

@@ -108,13 +108,6 @@ func (c *Client) Receive(ctx context.Context, req *pdu.ReceiveReq, stream io.Rea
 	return c.dataClient.ReqRecv(ctx, req, stream)
 }
 
-func (c *Client) SendDry(ctx context.Context, in *pdu.SendReq) (*pdu.SendRes, error) {
-	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.SendDry")
-	defer endSpan()
-
-	return c.controlClient.SendDry(ctx, in)
-}
-
 func (c *Client) ListFilesystems(ctx context.Context, in *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error) {
 	ctx, endSpan := trace.WithSpan(ctx, "rpc.client.ListFilesystems")
 	defer endSpan()

rpc/versionhandshake/versionhandshake.go

@@ -152,7 +152,7 @@ func (m *HandshakeMessage) DecodeReader(r io.Reader, maxLen int) error {
 
 func DoHandshakeCurrentVersion(conn net.Conn, deadline time.Time) *HandshakeError {
 	// current protocol version is hardcoded here
-	return DoHandshakeVersion(conn, deadline, 6)
+	return DoHandshakeVersion(conn, deadline, 5)
 }
 
 const HandshakeMessageMaxLen = 16 * 4096

transport/tcp/serve_tcp_ipmap_test.go

@@ -91,12 +91,15 @@ func TestIPMap(t *testing.T) {
 				"fde4:8dba:82e1::/64": "sub64-*",
 			},
 			expect: map[string]testCaseExpect{
-				"10.1.2.3":      {expectNoMapping: true},
-				"192.168.23.1":  {expectIdent: "db-192.168.23.1"},
-				"192.168.42.1":  {expectIdent: "web-192.168.42.1"},
-				"192.168.23.23": {expectIdent: "db-twentythree"},
-				"10.1.4.5":      {expectIdent: "my-10.1.4.5-server"},
+				"10.1.2.3":        {expectNoMapping: true},
+				"192.168.23.1":    {expectIdent: "db-192.168.23.1"},
+				"192.168.23.23":   {expectIdent: "db-twentythree"},
+				"192.168.023.001": {expectIdent: "db-192.168.23.1"},
+				"10.1.4.5":        {expectIdent: "my-10.1.4.5-server"},
 
+				// normalization
+				"192.168.42.1":    {expectIdent: "web-192.168.42.1"},
+				"192.168.042.001": {expectIdent: "web-192.168.42.1"},
 				// v6 matching
 				"fe80::23:42%eth1": {expectIdent: "san-fe80::23:42-eth1"},
 				"fe80::23:42%eth2": {expectNoMapping: true},
@@ -176,8 +179,7 @@ func TestIPMap(t *testing.T) {
 			for input, expect := range c.expect {
 				// reuse newIPMapEntry to parse test case input
 				// "test" is not used during testing but must not be empty.
-				ipMapEntry, err := newIPMapEntry(input, "test")
-				require.NoError(t, err)
+				ipMapEntry, _ := newIPMapEntry(input, "test")
 				ones, bits := ipMapEntry.subnet.Mask.Size()
 				require.Equal(t, bits, net.IPv6len*8, "and we know ipMapEntry always expands its IPs to 16bytes")
 				require.Equal(t, ones, net.IPv6len*8, "test case addresses must be fully specified")

util/bandwidthlimit/bandwidthlimit.go

@@ -1,79 +0,0 @@
-package bandwidthlimit
-
-import (
-	"errors"
-	"io"
-
-	"github.com/juju/ratelimit"
-)
-
-type Wrapper interface {
-	WrapReadCloser(io.ReadCloser) io.ReadCloser
-}
-
-type Config struct {
-	// Units in this struct are in _bytes_.
-
-	Max            int64 // < 0 means no limit, BucketCapacity is irrelevant then
-	BucketCapacity int64
-}
-
-func NoLimitConfig() Config {
-	return Config{
-		Max:            -1,
-		BucketCapacity: -1,
-	}
-}
-
-func ValidateConfig(conf Config) error {
-	if conf.BucketCapacity == 0 {
-		return errors.New("BucketCapacity must not be zero")
-	}
-	return nil
-}
-
-func WrapperFromConfig(conf Config) Wrapper {
-	if err := ValidateConfig(conf); err != nil {
-		panic(err)
-	}
-
-	if conf.Max < 0 {
-		return noLimit{}
-	}
-
-	return &withLimit{
-		bucket: ratelimit.NewBucketWithRate(float64(conf.Max), conf.BucketCapacity),
-	}
-}
-
-type noLimit struct{}
-
-func (_ noLimit) WrapReadCloser(rc io.ReadCloser) io.ReadCloser { return rc }
-
-type withLimit struct {
-	bucket *ratelimit.Bucket
-}
-
-func (l *withLimit) WrapReadCloser(rc io.ReadCloser) io.ReadCloser {
-	return WrapReadCloser(rc, l.bucket)
-}
-
-type withLimitReadCloser struct {
-	orig    io.Closer
-	limited io.Reader
-}
-
-func (r *withLimitReadCloser) Read(buf []byte) (int, error) {
-	return r.limited.Read(buf)
-}
-
-func (r *withLimitReadCloser) Close() error {
-	return r.orig.Close()
-}
-
-func WrapReadCloser(rc io.ReadCloser, bucket *ratelimit.Bucket) io.ReadCloser {
-	return &withLimitReadCloser{
-		limited: ratelimit.Reader(rc, bucket),
-		orig:    rc,
-	}
-}

util/bandwidthlimit/bandwidthlimit_test.go

@@ -1,19 +0,0 @@
-package bandwidthlimit
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestNoLimitConfig(t *testing.T) {
-
-	conf := NoLimitConfig()
-
-	err := ValidateConfig(conf)
-	require.NoError(t, err)
-
-	require.NotPanics(t, func() {
-		_ = WrapperFromConfig(conf)
-	})
-}

util/bytecounter/bytecounter_readcloser.go

@@ -9,7 +9,7 @@ import (
 // its interface and counting the bytes written to during copying.
 type ReadCloser interface {
 	io.ReadCloser
-	Count() uint64
+	Count() int64
 }
 
 // NewReadCloser wraps rc.
@@ -19,11 +19,11 @@ func NewReadCloser(rc io.ReadCloser) ReadCloser {
 
 type readCloser struct {
 	rc    io.ReadCloser
-	count uint64
+	count int64
 }
 
-func (r *readCloser) Count() uint64 {
-	return atomic.LoadUint64(&r.count)
+func (r *readCloser) Count() int64 {
+	return atomic.LoadInt64(&r.count)
 }
 
 var _ io.ReadCloser = &readCloser{}
@@ -34,9 +34,6 @@ func (r *readCloser) Close() error {
 
 func (r *readCloser) Read(p []byte) (int, error) {
 	n, err := r.rc.Read(p)
-	if n < 0 {
-		panic("expecting n >= 0")
-	}
-	atomic.AddUint64(&r.count, uint64(n))
+	atomic.AddInt64(&r.count, int64(n))
 	return n, err
 }

util/datasizeunit/datasizeunit.go

@@ -1,86 +0,0 @@
-package datasizeunit
-
-import (
-	"errors"
-	"fmt"
-	"math"
-	"regexp"
-	"strconv"
-	"strings"
-)
-
-type Bits struct {
-	bits float64
-}
-
-func (b Bits) ToBits() float64    { return b.bits }
-func (b Bits) ToBytes() float64   { return b.bits / 8 }
-func FromBytesInt64(i int64) Bits { return Bits{float64(i) * 8} }
-
-var datarateRegex = regexp.MustCompile(`^([-0-9\.]*)\s*(bit|(|K|Ki|M|Mi|G|Gi|T|Ti)([bB]))$`)
-
-func (r *Bits) UnmarshalYAML(u func(interface{}, bool) error) (_ error) {
-
-	var s string
-	err := u(&s, false)
-	if err != nil {
-		return err
-	}
-
-	genericErr := func(err error) error {
-		var buf strings.Builder
-		fmt.Fprintf(&buf, "cannot parse %q using regex %s", s, datarateRegex)
-		if err != nil {
-			fmt.Fprintf(&buf, ": %s", err)
-		}
-		return errors.New(buf.String())
-	}
-
-	match := datarateRegex.FindStringSubmatch(s)
-	if match == nil {
-		return genericErr(nil)
-	}
-
-	bps, err := strconv.ParseFloat(match[1], 64)
-	if err != nil {
-		return genericErr(err)
-	}
-
-	if match[2] == "bit" {
-		if math.Round(bps) != bps {
-			return genericErr(fmt.Errorf("unit bit must be an integer value"))
-		}
-		r.bits = bps
-		return nil
-	}
-
-	factorMap := map[string]uint64{
-		"": 1,
-
-		"K": 1e3,
-		"M": 1e6,
-		"G": 1e9,
-		"T": 1e12,
-
-		"Ki": 1 << 10,
-		"Mi": 1 << 20,
-		"Gi": 1 << 30,
-		"Ti": 1 << 40,
-	}
-	factor, ok := factorMap[match[3]]
-	if !ok {
-		panic(match)
-	}
-
-	baseUnitFactorMap := map[string]uint64{
-		"b": 1,
-		"B": 8,
-	}
-	baseUnitFactor, ok := baseUnitFactorMap[match[4]]
-	if !ok {
-		panic(match)
-	}
-
-	r.bits = bps * float64(factor) * float64(baseUnitFactor)
-	return nil
-}

util/datasizeunit/datasizeunit_test.go

@@ -1,57 +0,0 @@
-package datasizeunit
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"github.com/zrepl/yaml-config"
-)
-
-func TestBits(t *testing.T) {
-
-	tcs := []struct {
-		input      string
-		expectRate float64
-		expectErr  string
-	}{
-		{`23 bit`, 23, ""}, // bit special case works
-		{`23bit`, 23, ""},  // also without space
-
-		{`10MiB`, 10 * (1 << 20) * 8, ""},  // integer unit without space
-		{`10 MiB`, 8 * 10 * (1 << 20), ""}, // integer unit with space
-
-		{`10.5 Kib`, 10.5 * (1 << 10), ""}, // floating point with bit unit works with space
-		{`10.5Kib`, 10.5 * (1 << 10), ""},  // floating point with bit unit works without space
-
-		// unit checks
-		{`1 bit`, 1, ""},
-		{`1 B`, 1 * 8, ""},
-		{`1 Kb`, 1e3, ""},
-		{`1 Kib`, 1 << 10, ""},
-		{`1 Mb`, 1e6, ""},
-		{`1 Mib`, 1 << 20, ""},
-		{`1 Gb`, 1e9, ""},
-		{`1 Gib`, 1 << 30, ""},
-		{`1 Tb`, 1e12, ""},
-		{`1 Tib`, 1 << 40, ""},
-	}
-
-	for _, tc := range tcs {
-		t.Run(tc.input, func(t *testing.T) {
-
-			var bits Bits
-			err := yaml.Unmarshal([]byte(tc.input), &bits)
-			if tc.expectErr != "" {
-				assert.Error(t, err)
-				assert.Regexp(t, tc.expectErr, err.Error())
-				assert.Zero(t, bits.bits)
-			} else {
-				require.NoError(t, err)
-				assert.Equal(t, tc.expectRate, bits.bits)
-			}
-		})
-
-	}
-
-}

util/envconst/envconst.go

@@ -73,24 +73,6 @@ func Int64(varname string, def int64) (d int64) {
 	return d
 }
 
-func Uint64(varname string, def uint64) (d uint64) {
-	var err error
-	if v, ok := cache.Load(varname); ok {
-		return v.(uint64)
-	}
-	e := os.Getenv(varname)
-	if e == "" {
-		d = def
-	} else {
-		d, err = strconv.ParseUint(e, 10, 64)
-		if err != nil {
-			panic(err)
-		}
-	}
-	cache.Store(varname, d)
-	return d
-}
-
 func Bool(varname string, def bool) (d bool) {
 	var err error
 	if v, ok := cache.Load(varname); ok {

util/tcpsock/tcpsock_freebind_freebsd.go

@@ -1,4 +1,3 @@
-//go:build freebsd
 // +build freebsd
 
 package tcpsock

util/tcpsock/tcpsock_freebind_linux.go

@@ -1,4 +1,3 @@
-//go:build linux
 // +build linux
 
 package tcpsock

util/tcpsock/tcpsock_freebind_unsupported.go

@@ -1,4 +1,3 @@
-//go:build !linux && !freebsd
 // +build !linux,!freebsd
 
 package tcpsock

zfs/zfs.go

@@ -7,7 +7,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"math"
 	"os"
 	"os/exec"
 	"regexp"
@@ -15,6 +14,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
@@ -26,6 +26,11 @@ import (
 	"github.com/zrepl/zrepl/zfs/zfscmd"
 )
 
+var (
+	ZFSSendPipeCapacityHint = int(envconst.Int64("ZFS_SEND_PIPE_CAPACITY_HINT", 1<<25))
+	ZFSRecvPipeCapacityHint = int(envconst.Int64("ZFS_RECV_PIPE_CAPACITY_HINT", 1<<25))
+)
+
 type DatasetPath struct {
 	comps []string
 }
@@ -323,8 +328,8 @@ func absVersion(fs string, v *ZFSSendArgVersion) (full string, err error) {
 }
 
 func pipeWithCapacityHint(capacity int) (r, w *os.File, err error) {
-	if capacity < 0 {
-		panic(fmt.Sprintf("capacity must be non-negative, got %v", capacity))
+	if capacity <= 0 {
+		panic(fmt.Sprintf("capacity must be positive %v", capacity))
 	}
 	stdoutReader, stdoutWriter, err := os.Pipe()
 	if err != nil {
@@ -334,122 +339,60 @@ func pipeWithCapacityHint(capacity int) (r, w *os.File, err error) {
 	return stdoutReader, stdoutWriter, nil
 }
 
-type sendStreamState int
-
-const (
-	sendStreamOpen sendStreamState = iota
-	sendStreamClosed
-)
-
 type SendStream struct {
-	cmd          *zfscmd.Cmd
-	kill         context.CancelFunc
-	stdoutReader io.ReadCloser // not *os.File for mocking during platformtest
-	stderrBuf    *circlog.CircularLog
+	cmd  *zfscmd.Cmd
+	kill context.CancelFunc
 
-	mtx     sync.Mutex
-	state   sendStreamState
-	exitErr *ZFSError
+	closeMtx     sync.Mutex
+	stdoutReader *os.File
+	stderrBuf    *circlog.CircularLog
+	opErr        error
 }
 
-func (s *SendStream) Read(p []byte) (n int, _ error) {
-	s.mtx.Lock()
-	defer s.mtx.Unlock()
-
-	switch s.state {
-	case sendStreamClosed:
-		return 0, os.ErrClosed
-
-	case sendStreamOpen:
-		n, readErr := s.stdoutReader.Read(p)
-		if readErr != nil {
-			debug("sendStream: read: readErr=%T %s", readErr, readErr)
-			if readErr == io.EOF {
-				// io.EOF must be bubbled up as is so that consumers can handle it properly.
-				return n, readErr
-			}
-			// Assume that the error is not retryable.
-			// Try to kill now so that we can return a nice *ZFSError with captured stderr.
-			// If the kill doesn't work, it doesn't matter because the caller must by contract call Close() anyways.
-			killErr := s.killAndWait()
-			debug("sendStream: read: killErr=%T %s", killErr, killErr)
-			if killErr == nil {
-				s.state = sendStreamClosed
-				return n, s.exitErr // return the nice error
-			} else {
-				// we remain open so that we retry
-				return n, readErr // return the normal error
-			}
-		}
-		return n, readErr
-
-	default:
-		panic("unreachable")
+func (s *SendStream) Read(p []byte) (n int, err error) {
+	s.closeMtx.Lock()
+	opErr := s.opErr
+	s.closeMtx.Unlock()
+	if opErr != nil {
+		return 0, opErr
 	}
+
+	n, err = s.stdoutReader.Read(p)
+	if err != nil {
+		debug("sendStream: read err: %T %s", err, err)
+		// TODO we assume here that any read error is permanent
+		// which is most likely the case for a local zfs send
+		kwerr := s.killAndWait(err)
+		debug("sendStream: killAndWait n=%v err= %T %s", n, kwerr, kwerr)
+		// TODO we assume here that any read error is permanent
+		return n, kwerr
+	}
+	return n, err
 }
 
 func (s *SendStream) Close() error {
 	debug("sendStream: close called")
-	s.mtx.Lock()
-	defer s.mtx.Unlock()
-
-	switch s.state {
-	case sendStreamOpen:
-		err := s.killAndWait()
-		if err != nil {
-			return err
-		} else {
-			s.state = sendStreamClosed
-			return nil
-		}
-	case sendStreamClosed:
-		return os.ErrClosed
-	default:
-		panic("unreachable")
-	}
+	return s.killAndWait(nil)
 }
 
-// returns nil iff the child process is gone (has been successfully waited upon)
-// in that case, s.exitErr is set
-func (s *SendStream) killAndWait() error {
+func (s *SendStream) killAndWait(precedingReadErr error) error {
 
 	debug("sendStream: killAndWait enter")
 	defer debug("sendStream: killAndWait leave")
+	if precedingReadErr == io.EOF {
+		// give the zfs process a little bit of time to terminate itself
+		// if it holds this deadline, exitErr will be nil
+		time.AfterFunc(200*time.Millisecond, s.kill)
+	} else {
+		s.kill()
+	}
 
-	// send SIGKILL
-	s.kill()
+	// allow async kills from Close(), that's why we only take the mutex here
+	s.closeMtx.Lock()
+	defer s.closeMtx.Unlock()
 
-	// Close our read-end of the pipe.
-	//
-	// We must do this before .Wait() because in some (not all) versions/build configs of ZFS,
-	// `zfs send` uses a separate kernel thread (taskq) to write the send stream (function `dump_bytes`).
-	// The `zfs send` thread then waits uinterruptably for the taskq thread to finish the write.
-	// And signalling the `zfs send` thread doesn't propagate to the taskq thread.
-	// So we end up in a state where we .Wait() forever.
-	// (See https://github.com/openzfs/zfs/issues/12500 and
-	//  https://github.com/zrepl/zrepl/issues/495#issuecomment-902530043)
-	//
-	// By closing our read end of the pipe before .Wait(), we unblock the taskq thread if there is any.
-	// If there is no separate taskq thread, the SIGKILL to `zfs end` would suffice and be most precise,
-	// but due to the circumstances above, there is no other portable & robust way.
-	//
-	// However, the fallout from closing the pipe is that (in non-taskq builds) `zfs sends` will get a SIGPIPE.
-	// And on Linux, that SIGPIPE appears to win over the previously issued SIGKILL.
-	// And thus, on Linux, the `zfs send` will be killed by the default SIGPIPE handler.
-	// We can observe this in the WaitStatus below.
-	// This behavior is slightly annoying because the *exec.ExitError's message ("signal: broken pipe")
-	// isn't as clear as ("signal: killed").
-	// However, it seems like we just have to live with that. (covered by platformtest)
-	var closePipeErr error
-	if s.stdoutReader != nil {
-		closePipeErr = s.stdoutReader.Close()
-		if closePipeErr == nil {
-			// avoid double-closes in case waiting below doesn't work
-			// and someone attempts Close again
-			s.stdoutReader = nil
-		} else {
-			return closePipeErr
-		}
+	if s.opErr != nil {
+		return s.opErr
 	}
 
 	waitErr := s.cmd.Wait()
@@ -464,30 +407,39 @@ func (s *SendStream) killAndWait() error {
 		}
 	}
 
-	// invariant: at this point, the child is gone and we cleaned up everything related to the SendStream
+	// now, after we know the program exited do we close the pipe
+	var closePipeErr error
+	if s.stdoutReader != nil {
+		closePipeErr = s.stdoutReader.Close()
+		if closePipeErr == nil {
+			// avoid double-closes in case anything below doesn't work
+			// and someone calls Close again
+			s.stdoutReader = nil
+		} else {
+			return closePipeErr
+		}
+	}
 
+	// we managed to tear things down, no let's give the user some pretty *ZFSError
 	if exitErr != nil {
-		// zfs send exited with an error or was killed by a signal.
-		s.exitErr = &ZFSError{
+		s.opErr = &ZFSError{
 			Stderr:  []byte(s.stderrBuf.String()),
 			WaitErr: exitErr,
 		}
 	} else {
-		// zfs send exited successfully (we know that since waitErr was either nil or wasn't an *exec.ExitError)
-		s.exitErr = nil
+		s.opErr = precedingReadErr
 	}
 
-	return nil
-}
+	// detect the edge where we're called from s.Read
+	// after the pipe EOFed and zfs send exited without errors
+	// this is actually the "hot" / nice path
+	if exitErr == nil && precedingReadErr == io.EOF {
+		return precedingReadErr
+	}
 
-func (s *SendStream) TestOnly_ReplaceStdoutReader(f io.ReadCloser) (prev io.ReadCloser) {
-	prev = s.stdoutReader
-	s.stdoutReader = f
-	return prev
+	return s.opErr
 }
 
-func (s *SendStream) TestOnly_ExitErr() *ZFSError { return s.exitErr }
-
 // NOTE: When updating this struct, make sure to update funcs Validate ValidateCorrespondsToResumeToken
 type ZFSSendArgVersion struct {
 	RelName string
@@ -906,7 +858,7 @@ func ZFSSend(ctx context.Context, sendArgs ZFSSendArgsValidated) (*SendStream, e
 	ctx, cancel := context.WithCancel(ctx)
 
 	// setup stdout with an os.Pipe to control pipe buffer size
-	stdoutReader, stdoutWriter, err := pipeWithCapacityHint(getPipeCapacityHint("ZFS_SEND_PIPE_CAPACITY_HINT"))
+	stdoutReader, stdoutWriter, err := pipeWithCapacityHint(ZFSSendPipeCapacityHint)
 	if err != nil {
 		cancel()
 		return nil, err
@@ -961,7 +913,7 @@ type DrySendInfo struct {
 	Type         DrySendType
 	Filesystem   string // parsed from To field
 	From, To     string // direct copy from ZFS output
-	SizeEstimate uint64 // 0 if size estimate is not possible
+	SizeEstimate int64  // -1 if size estimate is not possible
 }
 
 var (
@@ -1034,10 +986,11 @@ func (s *DrySendInfo) unmarshalInfoLine(l string) (regexMatched bool, err error)
 		// see https://github.com/zrepl/zrepl/issues/289
 		fields["size"] = "0"
 	}
-	s.SizeEstimate, err = strconv.ParseUint(fields["size"], 10, 64)
+	s.SizeEstimate, err = strconv.ParseInt(fields["size"], 10, 64)
 	if err != nil {
 		return true, fmt.Errorf("cannot not parse size: %s", err)
 	}
+
 	return true, nil
 }
 
@@ -1047,7 +1000,6 @@ func ZFSSendDry(ctx context.Context, sendArgs ZFSSendArgsValidated) (_ *DrySendI
 
 	if sendArgs.From != nil && strings.Contains(sendArgs.From.RelName, "#") {
 		/* TODO:
-		 * XXX feature check & support this as well
 		 * ZFS at the time of writing does not support dry-run send because size-estimation
 		 * uses fromSnap's deadlist. However, for a bookmark, that deadlist no longer exists.
 		 * Redacted send & recv will bring this functionality, see
@@ -1066,7 +1018,7 @@ func ZFSSendDry(ctx context.Context, sendArgs ZFSSendArgsValidated) (_ *DrySendI
 			Filesystem:   sendArgs.FS,
 			From:         fromAbs,
 			To:           toAbs,
-			SizeEstimate: 0}, nil
+			SizeEstimate: -1}, nil
 	}
 
 	args := make([]string, 0)
@@ -1086,19 +1038,6 @@ func ZFSSendDry(ctx context.Context, sendArgs ZFSSendArgsValidated) (_ *DrySendI
 	if err := si.unmarshalZFSOutput(output); err != nil {
 		return nil, fmt.Errorf("could not parse zfs send -n output: %s", err)
 	}
-
-	// There is a bug in OpenZFS where it estimates the size incorrectly.
-	// - zrepl: https://github.com/zrepl/zrepl/issues/463
-	// - resulting upstream bug: https://github.com/openzfs/zfs/issues/12265
-	//
-	// The wrong estimates are easy to detect because they are absurdly large.
-	// NB: we're doing the workaround for this late so that the test cases are not affected.
-	sizeEstimateThreshold := envconst.Uint64("ZREPL_ZFS_SEND_SIZE_ESTIMATE_INCORRECT_THRESHOLD", math.MaxInt64)
-	if sizeEstimateThreshold != 0 && si.SizeEstimate >= sizeEstimateThreshold {
-		debug("size estimate exceeds threshold %v, working around it: %#v %q", sizeEstimateThreshold, si, args)
-		si.SizeEstimate = 0
-	}
-
 	return &si, nil
 }
 
@@ -1221,7 +1160,7 @@ func ZFSRecv(ctx context.Context, fs string, v *ZFSSendArgVersion, stream io.Rea
 
 	stderr := bytes.NewBuffer(make([]byte, 0, RecvStderrBufSiz))
 
-	stdin, stdinWriter, err := pipeWithCapacityHint(getPipeCapacityHint("ZFS_RECV_PIPE_CAPACITY_HINT"))
+	stdin, stdinWriter, err := pipeWithCapacityHint(ZFSRecvPipeCapacityHint)
 	if err != nil {
 		return err
 	}

zfs/zfs_pipe.go

@@ -1,4 +1,3 @@
-//go:build !linux
 // +build !linux
 
 package zfs
@@ -8,14 +7,10 @@ import (
 	"sync"
 )
 
-func getPipeCapacityHint(envvar string) int {
-	return 0 // not supported
-}
-
 var zfsPipeCapacityNotSupported sync.Once
 
 func trySetPipeCapacity(p *os.File, capacity int) {
-	if debugEnabled && capacity != 0 {
+	if debugEnabled {
 		zfsPipeCapacityNotSupported.Do(func() {
 			debug("trySetPipeCapacity error: OS does not support setting pipe capacity")
 		})

zfs/zfs_pipe_linux.go

@@ -3,33 +3,11 @@ package zfs
 import (
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
-	"strconv"
-	"strings"
 
 	"golang.org/x/sys/unix"
-
-	"github.com/zrepl/zrepl/util/envconst"
 )
 
-func getPipeCapacityHint(envvar string) int {
-	var capacity int64 = 1 << 25
-
-	// Work around a race condition in Linux >= 5.8 related to pipe resizing.
-	// https://github.com/zrepl/zrepl/issues/424#issuecomment-800370928
-	// https://bugzilla.kernel.org/show_bug.cgi?id=212295
-	if _, err := os.Stat("/proc/sys/fs/pipe-max-size"); err == nil {
-		if dat, err := ioutil.ReadFile("/proc/sys/fs/pipe-max-size"); err == nil {
-			if capacity, err = strconv.ParseInt(strings.TrimSpace(string(dat)), 10, 64); err != nil {
-				capacity = 1 << 25
-			}
-		}
-	}
-
-	return int(envconst.Int64(envvar, capacity))
-}
-
 func trySetPipeCapacity(p *os.File, capacity int) {
 	res, err := unix.FcntlInt(p.Fd(), unix.F_SETPIPE_SZ, capacity)
 	if err != nil {

zfs/zfscmd/zfscmd.go

@@ -209,7 +209,3 @@ func (c *Cmd) Runtime() time.Duration {
 	}
 	return c.waitReturnedAt.Sub(c.startedAt)
 }
-
-func (c *Cmd) TestOnly_ExecCmd() *exec.Cmd {
-	return c.cmd
-}