WIP state-machine based replication

cmd/replication.v2/diff.go

@@ -0,0 +1,115 @@
+package replication
+
+import (
+	"sort"
+)
+
+type ConflictNoCommonAncestor struct {
+	SortedSenderVersions, SortedReceiverVersions []*FilesystemVersion
+}
+
+func (c *ConflictNoCommonAncestor) Error() string {
+	return "no common snapshot or suitable bookmark between sender and receiver"
+}
+
+type ConflictDiverged struct {
+	SortedSenderVersions, SortedReceiverVersions []*FilesystemVersion
+	CommonAncestor                               *FilesystemVersion
+	SenderOnly, ReceiverOnly                     []*FilesystemVersion
+}
+
+func (c *ConflictDiverged) Error() string {
+	return "the receiver's latest snapshot is not present on sender"
+}
+
+func SortVersionListByCreateTXGThenBookmarkLTSnapshot(fsvslice []*FilesystemVersion) []*FilesystemVersion {
+	lesser := func(s []*FilesystemVersion) func(i, j int) bool {
+		return func(i, j int) bool {
+			if s[i].CreateTXG < s[j].CreateTXG {
+				return true
+			}
+			if s[i].CreateTXG == s[j].CreateTXG {
+				//  Bookmark < Snapshot
+				return s[i].Type == FilesystemVersion_Bookmark && s[j].Type == FilesystemVersion_Snapshot
+			}
+			return false
+		}
+	}
+	if sort.SliceIsSorted(fsvslice, lesser(fsvslice)) {
+		return fsvslice
+	}
+	sorted := make([]*FilesystemVersion, len(fsvslice))
+	copy(sorted, fsvslice)
+	sort.Slice(sorted, lesser(sorted))
+	return sorted
+}
+
+// conflict may be a *ConflictDiverged or a *ConflictNoCommonAncestor
+func IncrementalPath(receiver, sender []*FilesystemVersion) (incPath []*FilesystemVersion, conflict error) {
+
+	if receiver == nil {
+		panic("receiver must not be nil")
+	}
+	if sender == nil {
+		panic("sender must not be nil")
+	}
+
+	receiver = SortVersionListByCreateTXGThenBookmarkLTSnapshot(receiver)
+	sender = SortVersionListByCreateTXGThenBookmarkLTSnapshot(sender)
+
+	if len(sender) == 0 {
+		return []*FilesystemVersion{}, nil
+	}
+
+	// Find most recent common ancestor by name, preferring snapshots over bookmarks
+
+	mrcaRcv := len(receiver) - 1
+	mrcaSnd := len(sender) - 1
+
+	for mrcaRcv >= 0 && mrcaSnd >= 0 {
+		if receiver[mrcaRcv].Guid == sender[mrcaSnd].Guid {
+			if mrcaSnd-1 >= 0 && sender[mrcaSnd-1].Guid == sender[mrcaSnd].Guid && sender[mrcaSnd-1].Type == FilesystemVersion_Bookmark {
+				// prefer bookmarks over snapshots as the snapshot might go away sooner
+				mrcaSnd -= 1
+			}
+			break
+		}
+		if receiver[mrcaRcv].CreateTXG < sender[mrcaSnd].CreateTXG {
+			mrcaSnd--
+		} else {
+			mrcaRcv--
+		}
+	}
+
+	if mrcaRcv == -1 || mrcaSnd == -1 {
+		return nil, &ConflictNoCommonAncestor{
+			SortedSenderVersions:   sender,
+			SortedReceiverVersions: receiver,
+		}
+	}
+
+	if mrcaRcv != len(receiver)-1 {
+		return nil, &ConflictDiverged{
+			SortedSenderVersions:   sender,
+			SortedReceiverVersions: receiver,
+			CommonAncestor:         sender[mrcaSnd],
+			SenderOnly:             sender[mrcaSnd+1:],
+			ReceiverOnly:           receiver[mrcaRcv+1:],
+		}
+	}
+
+	// incPath must not contain bookmarks except initial one,
+	incPath = make([]*FilesystemVersion, 0, len(sender))
+	incPath = append(incPath, sender[mrcaSnd])
+	// it's ok if incPath[0] is a bookmark, but not the subsequent ones in the incPath
+	for i := mrcaSnd + 1; i < len(sender); i++ {
+		if sender[i].Type == FilesystemVersion_Snapshot && incPath[len(incPath)-1].Guid != sender[i].Guid {
+			incPath = append(incPath, sender[i])
+		}
+	}
+	if len(incPath) == 1 {
+		// nothing to do
+		incPath = incPath[1:]
+	}
+	return incPath, nil
+}

cmd/replication.v2/diff_test.go

@@ -0,0 +1,268 @@
+package replication_test
+
+import (
+	"github.com/stretchr/testify/assert"
+	"github.com/zrepl/zrepl/cmd/replication"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+)
+
+func fsvlist(fsv ...string) (r []*replication.FilesystemVersion) {
+
+	r = make([]*replication.FilesystemVersion, len(fsv))
+	for i, f := range fsv {
+
+		// parse the id from fsvlist. it is used to derivce Guid,CreateTXG and Creation attrs
+		split := strings.Split(f, ",")
+		if len(split) != 2 {
+			panic("invalid fsv spec")
+		}
+		id, err := strconv.Atoi(split[1])
+		if err != nil {
+			panic(err)
+		}
+
+		if strings.HasPrefix(f, "#") {
+			r[i] = &replication.FilesystemVersion{
+				Name:      strings.TrimPrefix(f, "#"),
+				Type:      replication.FilesystemVersion_Bookmark,
+				Guid:      uint64(id),
+				CreateTXG: uint64(id),
+				Creation:  time.Unix(0, 0).Add(time.Duration(id) * time.Second).Format(time.RFC3339),
+			}
+		} else if strings.HasPrefix(f, "@") {
+			r[i] = &replication.FilesystemVersion{
+				Name:      strings.TrimPrefix(f, "@"),
+				Type:      replication.FilesystemVersion_Snapshot,
+				Guid:      uint64(id),
+				CreateTXG: uint64(id),
+				Creation:  time.Unix(0, 0).Add(time.Duration(id) * time.Second).Format(time.RFC3339),
+			}
+		} else {
+			panic("invalid character")
+		}
+	}
+	return
+}
+
+type incPathResult struct {
+	incPath  []*replication.FilesystemVersion
+	conflict error
+}
+
+type IncrementalPathTest struct {
+	Msg                    string
+	Receiver, Sender       []*replication.FilesystemVersion
+	ExpectIncPath          []*replication.FilesystemVersion
+	ExpectNoCommonAncestor bool
+	ExpectDiverged         *replication.ConflictDiverged
+	ExpectPanic            bool
+}
+
+func (tt *IncrementalPathTest) Test(t *testing.T) {
+
+	t.Logf("test: %s", tt.Msg)
+
+	if tt.ExpectPanic {
+		assert.Panics(t, func() {
+			replication.IncrementalPath(tt.Receiver, tt.Sender)
+		})
+		return
+	}
+
+	incPath, conflict := replication.IncrementalPath(tt.Receiver, tt.Sender)
+
+	if tt.ExpectIncPath != nil {
+		assert.Nil(t, conflict)
+		assert.True(t, len(incPath) == 0 || len(incPath) >= 2)
+		assert.Equal(t, tt.ExpectIncPath, incPath)
+		return
+	}
+	if conflict == nil {
+		t.Logf("conflict is (unexpectly) <nil>\nincPath: %#v", incPath)
+	}
+	if tt.ExpectNoCommonAncestor {
+		assert.IsType(t, &replication.ConflictNoCommonAncestor{}, conflict)
+		// TODO check sorting
+		return
+	}
+	if tt.ExpectDiverged != nil {
+		if !assert.IsType(t, &replication.ConflictDiverged{}, conflict) {
+			return
+		}
+		c := conflict.(*replication.ConflictDiverged)
+		// TODO check sorting
+		assert.NotZero(t, c.CommonAncestor)
+		assert.NotEmpty(t, c.ReceiverOnly)
+		assert.Equal(t, tt.ExpectDiverged.ReceiverOnly, c.ReceiverOnly)
+		assert.Equal(t, tt.ExpectDiverged.SenderOnly, c.SenderOnly)
+		return
+	}
+
+}
+
+func TestIncrementalPlan_IncrementalSnapshots(t *testing.T) {
+	l := fsvlist
+
+	tbl := []IncrementalPathTest{
+		{
+			Msg:           "basic functionality",
+			Receiver:      l("@a,1", "@b,2"),
+			Sender:        l("@a,1", "@b,2", "@c,3", "@d,4"),
+			ExpectIncPath: l("@b,2", "@c,3", "@d,4"),
+		},
+		{
+			Msg:                    "no snaps on receiver yields no common ancestor",
+			Receiver:               l(),
+			Sender:                 l("@a,1"),
+			ExpectNoCommonAncestor: true,
+		},
+		{
+			Msg:           "no snapshots on sender yields empty incremental path",
+			Receiver:      l(),
+			Sender:        l(),
+			ExpectIncPath: l(),
+		},
+		{
+			Msg:           "nothing to do yields empty incremental path",
+			Receiver:      l("@a,1"),
+			Sender:        l("@a,1"),
+			ExpectIncPath: l(),
+		},
+		{
+			Msg:                    "drifting apart",
+			Receiver:               l("@a,1", "@b,2"),
+			Sender:                 l("@c,3", "@d,4"),
+			ExpectNoCommonAncestor: true,
+		},
+		{
+			Msg:      "different snapshots on sender and receiver",
+			Receiver: l("@a,1", "@c,2"),
+			Sender:   l("@a,1", "@b,3"),
+			ExpectDiverged: &replication.ConflictDiverged{
+				CommonAncestor: l("@a,1")[0],
+				SenderOnly:     l("@b,3"),
+				ReceiverOnly:   l("@c,2"),
+			},
+		},
+		{
+			Msg:      "snapshot on receiver not present on sender",
+			Receiver: l("@a,1", "@b,2"),
+			Sender:   l("@a,1"),
+			ExpectDiverged: &replication.ConflictDiverged{
+				CommonAncestor: l("@a,1")[0],
+				SenderOnly:     l(),
+				ReceiverOnly:   l("@b,2"),
+			},
+		},
+		{
+			Msg:           "gaps before most recent common ancestor do not matter",
+			Receiver:      l("@a,1", "@b,2", "@c,3"),
+			Sender:        l("@a,1", "@c,3", "@d,4"),
+			ExpectIncPath: l("@c,3", "@d,4"),
+		},
+	}
+
+	for _, test := range tbl {
+		test.Test(t)
+	}
+
+}
+
+func TestIncrementalPlan_BookmarksSupport(t *testing.T) {
+	l := fsvlist
+
+	tbl := []IncrementalPathTest{
+		{
+			Msg:           "bookmarks are used",
+			Receiver:      l("@a,1"),
+			Sender:        l("#a,1", "@b,2"),
+			ExpectIncPath: l("#a,1", "@b,2"),
+		},
+		{
+			Msg:           "boomarks are stripped from incPath (cannot send incrementally)",
+			Receiver:      l("@a,1"),
+			Sender:        l("#a,1", "#b,2", "@c,3"),
+			ExpectIncPath: l("#a,1", "@c,3"),
+		},
+		{
+			Msg:           "bookmarks are preferred over snapshots for start of incPath",
+			Receiver:      l("@a,1"),
+			Sender:        l("#a,1", "@a,1", "@b,2"),
+			ExpectIncPath: l("#a,1", "@b,2"),
+		},
+		{
+			Msg:           "bookmarks are preferred over snapshots for start of incPath (regardless of order)",
+			Receiver:      l("@a,1"),
+			Sender:        l("@a,1", "#a,1", "@b,2"),
+			ExpectIncPath: l("#a,1", "@b,2"),
+		},
+	}
+
+	for _, test := range tbl {
+		test.Test(t)
+	}
+
+}
+
+func TestSortVersionListByCreateTXGThenBookmarkLTSnapshot(t *testing.T) {
+
+	type Test struct {
+		Msg           string
+		Input, Output []*replication.FilesystemVersion
+	}
+
+	l := fsvlist
+
+	tbl := []Test{
+		{
+			"snapshot sorting already sorted",
+			l("@a,1", "@b,2"),
+			l("@a,1", "@b,2"),
+		},
+		{
+			"bookmark sorting already sorted",
+			l("#a,1", "#b,2"),
+			l("#a,1", "#b,2"),
+		},
+		{
+			"snapshot sorting",
+			l("@b,2", "@a,1"),
+			l("@a,1", "@b,2"),
+		},
+		{
+			"bookmark sorting",
+			l("#b,2", "#a,1"),
+			l("#a,1", "#b,2"),
+		},
+	}
+
+	for _, test := range tbl {
+		t.Logf("test: %s", test.Msg)
+		inputlen := len(test.Input)
+		sorted := replication.SortVersionListByCreateTXGThenBookmarkLTSnapshot(test.Input)
+		if len(sorted) != inputlen {
+			t.Errorf("lenghts of input and output do not match: %d vs %d", inputlen, len(sorted))
+			continue
+		}
+		if !assert.Equal(t, test.Output, sorted) {
+			continue
+		}
+		last := sorted[0]
+		for _, s := range sorted[1:] {
+			if s.CreateTXG < last.CreateTXG {
+				t.Errorf("must be sorted ascending, got:\n\t%#v", sorted)
+				break
+			}
+			if s.CreateTXG == last.CreateTXG {
+				if last.Type == replication.FilesystemVersion_Bookmark && s.Type != replication.FilesystemVersion_Snapshot {
+					t.Errorf("snapshots must come after bookmarks")
+				}
+			}
+			last = s
+		}
+	}
+
+}

cmd/replication.v2/fsreplicationstate_string.go

@@ -0,0 +1,32 @@
+// Code generated by "stringer -type=FSReplicationState"; DO NOT EDIT.
+
+package replication
+
+import "strconv"
+
+const (
+	_FSReplicationState_name_0 = "FSQueuedFSActive"
+	_FSReplicationState_name_1 = "FSRetry"
+	_FSReplicationState_name_2 = "FSPermanentError"
+	_FSReplicationState_name_3 = "FSCompleted"
+)
+
+var (
+	_FSReplicationState_index_0 = [...]uint8{0, 8, 16}
+)
+
+func (i FSReplicationState) String() string {
+	switch {
+	case 1 <= i && i <= 2:
+		i -= 1
+		return _FSReplicationState_name_0[_FSReplicationState_index_0[i]:_FSReplicationState_index_0[i+1]]
+	case i == 4:
+		return _FSReplicationState_name_1
+	case i == 8:
+		return _FSReplicationState_name_2
+	case i == 16:
+		return _FSReplicationState_name_3
+	default:
+		return "FSReplicationState(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+}

cmd/replication.v2/fsreplicationstepstate_string.go

@@ -0,0 +1,16 @@
+// Code generated by "stringer -type=FSReplicationStepState"; DO NOT EDIT.
+
+package replication
+
+import "strconv"
+
+const _FSReplicationStepState_name = "StepPendingStepActiveStepRetryStepPermanentErrorStepCompleted"
+
+var _FSReplicationStepState_index = [...]uint8{0, 11, 21, 30, 48, 61}
+
+func (i FSReplicationStepState) String() string {
+	if i < 0 || i >= FSReplicationStepState(len(_FSReplicationStepState_index)-1) {
+		return "FSReplicationStepState(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _FSReplicationStepState_name[_FSReplicationStepState_index[i]:_FSReplicationStepState_index[i+1]]
+}

cmd/replication.v2/pdu.pb.go

@@ -0,0 +1,384 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: pdu.proto
+
+/*
+Package replication is a generated protocol buffer package.
+
+It is generated from these files:
+	pdu.proto
+
+It has these top-level messages:
+	ListFilesystemReq
+	ListFilesystemRes
+	Filesystem
+	ListFilesystemVersionsReq
+	ListFilesystemVersionsRes
+	FilesystemVersion
+	SendReq
+	Property
+	SendRes
+	ReceiveReq
+	ReceiveRes
+*/
+package replication
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+
+type FilesystemVersion_VersionType int32
+
+const (
+	FilesystemVersion_Snapshot FilesystemVersion_VersionType = 0
+	FilesystemVersion_Bookmark FilesystemVersion_VersionType = 1
+)
+
+var FilesystemVersion_VersionType_name = map[int32]string{
+	0: "Snapshot",
+	1: "Bookmark",
+}
+var FilesystemVersion_VersionType_value = map[string]int32{
+	"Snapshot": 0,
+	"Bookmark": 1,
+}
+
+func (x FilesystemVersion_VersionType) String() string {
+	return proto.EnumName(FilesystemVersion_VersionType_name, int32(x))
+}
+func (FilesystemVersion_VersionType) EnumDescriptor() ([]byte, []int) {
+	return fileDescriptor0, []int{5, 0}
+}
+
+type ListFilesystemReq struct {
+}
+
+func (m *ListFilesystemReq) Reset()                    { *m = ListFilesystemReq{} }
+func (m *ListFilesystemReq) String() string            { return proto.CompactTextString(m) }
+func (*ListFilesystemReq) ProtoMessage()               {}
+func (*ListFilesystemReq) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
+
+type ListFilesystemRes struct {
+	Filesystems []*Filesystem `protobuf:"bytes,1,rep,name=Filesystems" json:"Filesystems,omitempty"`
+}
+
+func (m *ListFilesystemRes) Reset()                    { *m = ListFilesystemRes{} }
+func (m *ListFilesystemRes) String() string            { return proto.CompactTextString(m) }
+func (*ListFilesystemRes) ProtoMessage()               {}
+func (*ListFilesystemRes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
+
+func (m *ListFilesystemRes) GetFilesystems() []*Filesystem {
+	if m != nil {
+		return m.Filesystems
+	}
+	return nil
+}
+
+type Filesystem struct {
+	Path        string `protobuf:"bytes,1,opt,name=Path" json:"Path,omitempty"`
+	ResumeToken string `protobuf:"bytes,2,opt,name=ResumeToken" json:"ResumeToken,omitempty"`
+}
+
+func (m *Filesystem) Reset()                    { *m = Filesystem{} }
+func (m *Filesystem) String() string            { return proto.CompactTextString(m) }
+func (*Filesystem) ProtoMessage()               {}
+func (*Filesystem) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
+
+func (m *Filesystem) GetPath() string {
+	if m != nil {
+		return m.Path
+	}
+	return ""
+}
+
+func (m *Filesystem) GetResumeToken() string {
+	if m != nil {
+		return m.ResumeToken
+	}
+	return ""
+}
+
+type ListFilesystemVersionsReq struct {
+	Filesystem string `protobuf:"bytes,1,opt,name=Filesystem" json:"Filesystem,omitempty"`
+}
+
+func (m *ListFilesystemVersionsReq) Reset()                    { *m = ListFilesystemVersionsReq{} }
+func (m *ListFilesystemVersionsReq) String() string            { return proto.CompactTextString(m) }
+func (*ListFilesystemVersionsReq) ProtoMessage()               {}
+func (*ListFilesystemVersionsReq) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
+
+func (m *ListFilesystemVersionsReq) GetFilesystem() string {
+	if m != nil {
+		return m.Filesystem
+	}
+	return ""
+}
+
+type ListFilesystemVersionsRes struct {
+	Versions []*FilesystemVersion `protobuf:"bytes,1,rep,name=Versions" json:"Versions,omitempty"`
+}
+
+func (m *ListFilesystemVersionsRes) Reset()                    { *m = ListFilesystemVersionsRes{} }
+func (m *ListFilesystemVersionsRes) String() string            { return proto.CompactTextString(m) }
+func (*ListFilesystemVersionsRes) ProtoMessage()               {}
+func (*ListFilesystemVersionsRes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
+
+func (m *ListFilesystemVersionsRes) GetVersions() []*FilesystemVersion {
+	if m != nil {
+		return m.Versions
+	}
+	return nil
+}
+
+type FilesystemVersion struct {
+	Type      FilesystemVersion_VersionType `protobuf:"varint,1,opt,name=Type,enum=replication.FilesystemVersion_VersionType" json:"Type,omitempty"`
+	Name      string                        `protobuf:"bytes,2,opt,name=Name" json:"Name,omitempty"`
+	Guid      uint64                        `protobuf:"varint,3,opt,name=Guid" json:"Guid,omitempty"`
+	CreateTXG uint64                        `protobuf:"varint,4,opt,name=CreateTXG" json:"CreateTXG,omitempty"`
+	Creation  string                        `protobuf:"bytes,5,opt,name=Creation" json:"Creation,omitempty"`
+}
+
+func (m *FilesystemVersion) Reset()                    { *m = FilesystemVersion{} }
+func (m *FilesystemVersion) String() string            { return proto.CompactTextString(m) }
+func (*FilesystemVersion) ProtoMessage()               {}
+func (*FilesystemVersion) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
+
+func (m *FilesystemVersion) GetType() FilesystemVersion_VersionType {
+	if m != nil {
+		return m.Type
+	}
+	return FilesystemVersion_Snapshot
+}
+
+func (m *FilesystemVersion) GetName() string {
+	if m != nil {
+		return m.Name
+	}
+	return ""
+}
+
+func (m *FilesystemVersion) GetGuid() uint64 {
+	if m != nil {
+		return m.Guid
+	}
+	return 0
+}
+
+func (m *FilesystemVersion) GetCreateTXG() uint64 {
+	if m != nil {
+		return m.CreateTXG
+	}
+	return 0
+}
+
+func (m *FilesystemVersion) GetCreation() string {
+	if m != nil {
+		return m.Creation
+	}
+	return ""
+}
+
+type SendReq struct {
+	Filesystem string `protobuf:"bytes,1,opt,name=Filesystem" json:"Filesystem,omitempty"`
+	From       string `protobuf:"bytes,2,opt,name=From" json:"From,omitempty"`
+	To         string `protobuf:"bytes,3,opt,name=To" json:"To,omitempty"`
+	// If ResumeToken is not empty, the resume token that CAN be tried for 'zfs send' by the sender.
+	// The sender MUST indicate in SendRes.UsedResumeToken
+	// If it does not work, the sender SHOULD clear the resume token on their side
+	// and use From and To instead
+	// If ResumeToken is not empty, the GUIDs of From and To
+	// MUST correspond to those encoded in the ResumeToken.
+	// Otherwise, the Sender MUST return an error.
+	ResumeToken string `protobuf:"bytes,4,opt,name=ResumeToken" json:"ResumeToken,omitempty"`
+	Compress    bool   `protobuf:"varint,5,opt,name=Compress" json:"Compress,omitempty"`
+	Dedup       bool   `protobuf:"varint,6,opt,name=Dedup" json:"Dedup,omitempty"`
+}
+
+func (m *SendReq) Reset()                    { *m = SendReq{} }
+func (m *SendReq) String() string            { return proto.CompactTextString(m) }
+func (*SendReq) ProtoMessage()               {}
+func (*SendReq) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
+
+func (m *SendReq) GetFilesystem() string {
+	if m != nil {
+		return m.Filesystem
+	}
+	return ""
+}
+
+func (m *SendReq) GetFrom() string {
+	if m != nil {
+		return m.From
+	}
+	return ""
+}
+
+func (m *SendReq) GetTo() string {
+	if m != nil {
+		return m.To
+	}
+	return ""
+}
+
+func (m *SendReq) GetResumeToken() string {
+	if m != nil {
+		return m.ResumeToken
+	}
+	return ""
+}
+
+func (m *SendReq) GetCompress() bool {
+	if m != nil {
+		return m.Compress
+	}
+	return false
+}
+
+func (m *SendReq) GetDedup() bool {
+	if m != nil {
+		return m.Dedup
+	}
+	return false
+}
+
+type Property struct {
+	Name  string `protobuf:"bytes,1,opt,name=Name" json:"Name,omitempty"`
+	Value string `protobuf:"bytes,2,opt,name=Value" json:"Value,omitempty"`
+}
+
+func (m *Property) Reset()                    { *m = Property{} }
+func (m *Property) String() string            { return proto.CompactTextString(m) }
+func (*Property) ProtoMessage()               {}
+func (*Property) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
+
+func (m *Property) GetName() string {
+	if m != nil {
+		return m.Name
+	}
+	return ""
+}
+
+func (m *Property) GetValue() string {
+	if m != nil {
+		return m.Value
+	}
+	return ""
+}
+
+type SendRes struct {
+	// Whether the resume token provided in the request has been used or not.
+	UsedResumeToken bool        `protobuf:"varint,1,opt,name=UsedResumeToken" json:"UsedResumeToken,omitempty"`
+	Properties      []*Property `protobuf:"bytes,2,rep,name=Properties" json:"Properties,omitempty"`
+}
+
+func (m *SendRes) Reset()                    { *m = SendRes{} }
+func (m *SendRes) String() string            { return proto.CompactTextString(m) }
+func (*SendRes) ProtoMessage()               {}
+func (*SendRes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
+
+func (m *SendRes) GetUsedResumeToken() bool {
+	if m != nil {
+		return m.UsedResumeToken
+	}
+	return false
+}
+
+func (m *SendRes) GetProperties() []*Property {
+	if m != nil {
+		return m.Properties
+	}
+	return nil
+}
+
+type ReceiveReq struct {
+	Filesystem string `protobuf:"bytes,1,opt,name=Filesystem" json:"Filesystem,omitempty"`
+	// If true, the receiver should clear the resume token before perfoming the zfs recv of the stream in the request
+	ClearResumeToken bool `protobuf:"varint,2,opt,name=ClearResumeToken" json:"ClearResumeToken,omitempty"`
+}
+
+func (m *ReceiveReq) Reset()                    { *m = ReceiveReq{} }
+func (m *ReceiveReq) String() string            { return proto.CompactTextString(m) }
+func (*ReceiveReq) ProtoMessage()               {}
+func (*ReceiveReq) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
+
+func (m *ReceiveReq) GetFilesystem() string {
+	if m != nil {
+		return m.Filesystem
+	}
+	return ""
+}
+
+func (m *ReceiveReq) GetClearResumeToken() bool {
+	if m != nil {
+		return m.ClearResumeToken
+	}
+	return false
+}
+
+type ReceiveRes struct {
+}
+
+func (m *ReceiveRes) Reset()                    { *m = ReceiveRes{} }
+func (m *ReceiveRes) String() string            { return proto.CompactTextString(m) }
+func (*ReceiveRes) ProtoMessage()               {}
+func (*ReceiveRes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
+
+func init() {
+	proto.RegisterType((*ListFilesystemReq)(nil), "replication.ListFilesystemReq")
+	proto.RegisterType((*ListFilesystemRes)(nil), "replication.ListFilesystemRes")
+	proto.RegisterType((*Filesystem)(nil), "replication.Filesystem")
+	proto.RegisterType((*ListFilesystemVersionsReq)(nil), "replication.ListFilesystemVersionsReq")
+	proto.RegisterType((*ListFilesystemVersionsRes)(nil), "replication.ListFilesystemVersionsRes")
+	proto.RegisterType((*FilesystemVersion)(nil), "replication.FilesystemVersion")
+	proto.RegisterType((*SendReq)(nil), "replication.SendReq")
+	proto.RegisterType((*Property)(nil), "replication.Property")
+	proto.RegisterType((*SendRes)(nil), "replication.SendRes")
+	proto.RegisterType((*ReceiveReq)(nil), "replication.ReceiveReq")
+	proto.RegisterType((*ReceiveRes)(nil), "replication.ReceiveRes")
+	proto.RegisterEnum("replication.FilesystemVersion_VersionType", FilesystemVersion_VersionType_name, FilesystemVersion_VersionType_value)
+}
+
+func init() { proto.RegisterFile("pdu.proto", fileDescriptor0) }
+
+var fileDescriptor0 = []byte{
+	// 454 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0x4d, 0x6f, 0xd3, 0x40,
+	0x10, 0x65, 0x53, 0xa7, 0x38, 0xe3, 0xaa, 0xa4, 0x4b, 0x11, 0x06, 0xa1, 0x2a, 0xda, 0x53, 0xe8,
+	0x21, 0x87, 0x02, 0x07, 0x40, 0xe2, 0xd0, 0xa2, 0xf6, 0x82, 0xaa, 0x6a, 0x6b, 0x4a, 0xaf, 0xa6,
+	0x1e, 0xa9, 0x4b, 0x62, 0xaf, 0xbb, 0x63, 0x23, 0xe5, 0xe7, 0xf0, 0xcf, 0xf8, 0x29, 0xc8, 0x53,
+	0x3b, 0xd9, 0x26, 0x2a, 0xca, 0xc9, 0xf3, 0xde, 0x7c, 0xbd, 0x7d, 0xeb, 0x85, 0x41, 0x99, 0xd5,
+	0x93, 0xd2, 0xd9, 0xca, 0xca, 0xc8, 0x61, 0x39, 0x33, 0x37, 0x69, 0x65, 0x6c, 0xa1, 0x9e, 0xc3,
+	0xde, 0x37, 0x43, 0xd5, 0xa9, 0x99, 0x21, 0xcd, 0xa9, 0xc2, 0x5c, 0xe3, 0x9d, 0x3a, 0x5f, 0x27,
+	0x49, 0x7e, 0x84, 0x68, 0x49, 0x50, 0x2c, 0x46, 0x5b, 0xe3, 0xe8, 0xe8, 0xe5, 0xc4, 0x1b, 0x36,
+	0xf1, 0x1a, 0xfc, 0x5a, 0x75, 0x0c, 0xb0, 0x84, 0x52, 0x42, 0x70, 0x91, 0x56, 0xb7, 0xb1, 0x18,
+	0x89, 0xf1, 0x40, 0x73, 0x2c, 0x47, 0x10, 0x69, 0xa4, 0x3a, 0xc7, 0xc4, 0x4e, 0xb1, 0x88, 0x7b,
+	0x9c, 0xf2, 0x29, 0xf5, 0x19, 0x5e, 0x3d, 0xd4, 0x74, 0x85, 0x8e, 0x8c, 0x2d, 0x48, 0xe3, 0x9d,
+	0x3c, 0xf0, 0x17, 0xb4, 0x83, 0x3d, 0x46, 0xfd, 0x78, 0xbc, 0x99, 0xe4, 0x27, 0x08, 0x3b, 0xd8,
+	0x9e, 0xea, 0xe0, 0x91, 0x53, 0xb5, 0x65, 0x7a, 0x51, 0xaf, 0xfe, 0x0a, 0xd8, 0x5b, 0xcb, 0xcb,
+	0x2f, 0x10, 0x24, 0xf3, 0x12, 0x59, 0xc8, 0xee, 0xd1, 0xe1, 0xff, 0xa7, 0x4d, 0xda, 0x6f, 0xd3,
+	0xa1, 0xb9, 0xaf, 0x71, 0xe8, 0x3c, 0xcd, 0xb1, 0xb5, 0x81, 0xe3, 0x86, 0x3b, 0xab, 0x4d, 0x16,
+	0x6f, 0x8d, 0xc4, 0x38, 0xd0, 0x1c, 0xcb, 0x37, 0x30, 0x38, 0x71, 0x98, 0x56, 0x98, 0x5c, 0x9f,
+	0xc5, 0x01, 0x27, 0x96, 0x84, 0x7c, 0x0d, 0x21, 0x03, 0x63, 0x8b, 0xb8, 0xcf, 0x93, 0x16, 0x58,
+	0xbd, 0x85, 0xc8, 0x5b, 0x2b, 0x77, 0x20, 0xbc, 0x2c, 0xd2, 0x92, 0x6e, 0x6d, 0x35, 0x7c, 0xd2,
+	0xa0, 0x63, 0x6b, 0xa7, 0x79, 0xea, 0xa6, 0x43, 0xa1, 0xfe, 0x08, 0x78, 0x7a, 0x89, 0x45, 0xb6,
+	0x81, 0xcf, 0x8d, 0xc8, 0x53, 0x67, 0xf3, 0x4e, 0x78, 0x13, 0xcb, 0x5d, 0xe8, 0x25, 0x96, 0x65,
+	0x0f, 0x74, 0x2f, 0xb1, 0xab, 0x57, 0x1d, 0xac, 0x5d, 0x35, 0x0b, 0xb7, 0x79, 0xe9, 0x90, 0x88,
+	0x85, 0x87, 0x7a, 0x81, 0xe5, 0x3e, 0xf4, 0xbf, 0x62, 0x56, 0x97, 0xf1, 0x36, 0x27, 0xee, 0x81,
+	0x7a, 0x0f, 0xe1, 0x85, 0xb3, 0x25, 0xba, 0x6a, 0xbe, 0x30, 0x4f, 0x78, 0xe6, 0xed, 0x43, 0xff,
+	0x2a, 0x9d, 0xd5, 0x9d, 0xa3, 0xf7, 0x40, 0xfd, 0xea, 0x0e, 0x46, 0x72, 0x0c, 0xcf, 0xbe, 0x13,
+	0x66, 0xbe, 0x30, 0xc1, 0x0b, 0x56, 0x69, 0xf9, 0x01, 0xa0, 0x5d, 0x65, 0x90, 0xe2, 0x1e, 0xff,
+	0x2f, 0x2f, 0x1e, 0xdc, 0x70, 0xa7, 0x44, 0x7b, 0x85, 0xea, 0x1a, 0x40, 0xe3, 0x0d, 0x9a, 0xdf,
+	0xb8, 0x89, 0x8f, 0x87, 0x30, 0x3c, 0x99, 0x61, 0xea, 0x56, 0xdf, 0x44, 0xa8, 0xd7, 0x78, 0xb5,
+	0xe3, 0x4d, 0xa6, 0x9f, 0xdb, 0xfc, 0xc6, 0xdf, 0xfd, 0x0b, 0x00, 0x00, 0xff, 0xff, 0xa4, 0x5a,
+	0xf6, 0xa7, 0xf0, 0x03, 0x00, 0x00,
+}

cmd/replication.v2/pdu.proto

@@ -0,0 +1,78 @@
+syntax = "proto3";
+
+package replication;
+
+message ListFilesystemReq {}
+
+message ListFilesystemRes {
+    repeated Filesystem Filesystems = 1;
+}
+
+message Filesystem {
+    string Path = 1;
+    string ResumeToken = 2;
+}
+
+message ListFilesystemVersionsReq {
+    string Filesystem = 1;
+}
+
+message ListFilesystemVersionsRes {
+    repeated FilesystemVersion Versions = 1;
+}
+
+message FilesystemVersion {
+    enum VersionType {
+        Snapshot = 0;
+        Bookmark = 1;
+    }
+    VersionType Type = 1;
+    string Name = 2;
+    uint64 Guid = 3;
+    uint64 CreateTXG = 4;
+    string Creation = 5; // RFC 3339
+}
+
+
+message SendReq {
+    string Filesystem  = 1;
+    string From = 2;
+    // May be empty / null to request a full transfer of From
+    string To = 3;
+
+    // If ResumeToken is not empty, the resume token that CAN be tried for 'zfs send' by the sender.
+    // The sender MUST indicate in SendRes.UsedResumeToken
+    // If it does not work, the sender SHOULD clear the resume token on their side
+    // and use From and To instead
+    // If ResumeToken is not empty, the GUIDs of From and To
+    // MUST correspond to those encoded in the ResumeToken.
+    // Otherwise, the Sender MUST return an error.
+    string ResumeToken  = 4;
+    bool Compress = 5;
+    bool Dedup = 6;
+}
+
+message Property {
+    string Name = 1;
+    string Value = 2;
+}
+
+message SendRes {
+    // The actual stream is in the stream part of the streamrpc response
+
+    // Whether the resume token provided in the request has been used or not.
+    bool UsedResumeToken = 1;
+
+    repeated Property Properties  = 2;
+}
+
+message ReceiveReq {
+    // The stream part of the streamrpc request contains the zfs send stream
+
+    string Filesystem = 1;
+
+    // If true, the receiver should clear the resume token before perfoming the zfs recv of the stream in the request
+    bool ClearResumeToken = 2;
+}
+
+message ReceiveRes {}

cmd/replication.v2/pdu_extras.go

@@ -0,0 +1,64 @@
+package replication
+
+import (
+	"fmt"
+	"github.com/zrepl/zrepl/zfs"
+	"time"
+)
+
+func (v *FilesystemVersion) RelName() string {
+	zv := v.ZFSFilesystemVersion()
+	return zv.String()
+}
+
+func (v FilesystemVersion_VersionType) ZFSVersionType() zfs.VersionType {
+	switch v {
+	case FilesystemVersion_Snapshot:
+		return zfs.Snapshot
+	case FilesystemVersion_Bookmark:
+		return zfs.Bookmark
+	default:
+		panic(fmt.Sprintf("unexpected v.Type %#v", v))
+	}
+}
+
+func FilesystemVersionFromZFS(fsv zfs.FilesystemVersion) *FilesystemVersion {
+	var t FilesystemVersion_VersionType
+	switch fsv.Type {
+	case zfs.Bookmark:
+		t = FilesystemVersion_Bookmark
+	case zfs.Snapshot:
+		t = FilesystemVersion_Snapshot
+	default:
+		panic("unknown fsv.Type: " + fsv.Type)
+	}
+	return &FilesystemVersion{
+		Type: t,
+		Name: fsv.Name,
+		Guid: fsv.Guid,
+		CreateTXG: fsv.CreateTXG,
+		Creation: fsv.Creation.Format(time.RFC3339),
+	}
+}
+
+func (v *FilesystemVersion) CreationAsTime() (time.Time, error) {
+	return time.Parse(time.RFC3339, v.Creation)
+}
+
+func (v *FilesystemVersion) ZFSFilesystemVersion() *zfs.FilesystemVersion {
+	ct := time.Time{}
+	if v.Creation != "" {
+		var err error
+		ct, err = time.Parse(time.RFC3339, v.Creation)
+		if err != nil {
+			panic(err)
+		}
+	}
+	return &zfs.FilesystemVersion{
+		Type: v.Type.ZFSVersionType(),
+		Name: v.Name,
+		Guid: v.Guid,
+		CreateTXG: v.CreateTXG,
+		Creation: ct,
+	}
+}

cmd/replication.v2/pdu_test.go

@@ -0,0 +1,64 @@
+package replication
+
+import (
+	"testing"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFilesystemVersion_RelName(t *testing.T) {
+
+	type TestCase struct {
+		In FilesystemVersion
+		Out string
+		Panic bool
+	}
+
+	tcs := []TestCase{
+		{
+			In: FilesystemVersion{
+				Type: FilesystemVersion_Snapshot,
+				Name: "foobar",
+			},
+			Out: "@foobar",
+		},
+		{
+			In: FilesystemVersion{
+				Type: FilesystemVersion_Bookmark,
+				Name: "foobar",
+			},
+			Out: "#foobar",
+		},
+		{
+			In: FilesystemVersion{
+				Type: 2342,
+				Name: "foobar",
+			},
+			Panic: true,
+		},
+	}
+
+	for _, tc := range tcs {
+		if tc.Panic {
+			assert.Panics(t, func() {
+				tc.In.RelName()
+			})
+		} else {
+			o := tc.In.RelName()
+			assert.Equal(t, tc.Out, o)
+		}
+	}
+
+}
+
+func TestFilesystemVersion_ZFSFilesystemVersion(t *testing.T) {
+
+	empty := &FilesystemVersion{}
+	emptyZFS := empty.ZFSFilesystemVersion()
+	assert.Zero(t, emptyZFS.Creation)
+
+	dateInvalid := &FilesystemVersion{Creation:"foobar"}
+	assert.Panics(t, func() {
+		dateInvalid.ZFSFilesystemVersion()
+	})
+
+}

cmd/replication.v2/plan.go

@@ -0,0 +1,474 @@
+package replication
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"sort"
+	"time"
+)
+
+//go:generate stringer -type=ReplicationState
+type ReplicationState int
+
+const (
+	Planning ReplicationState = iota
+	PlanningError
+	Working
+	WorkingWait
+	Completed
+	ContextDone
+)
+
+type Replication struct {
+	state ReplicationState
+
+	// Working / WorkingWait
+
+	pending, completed []*FSReplication
+
+	// PlanningError
+	planningError error
+
+	// ContextDone
+	contextError error
+}
+
+type FSReplicationState int
+
+//go:generate stringer -type=FSReplicationState
+const (
+	FSQueued FSReplicationState = 1 << iota
+	FSActive
+	FSRetry
+	FSPermanentError
+	FSCompleted
+)
+
+type FSReplication struct {
+	state              FSReplicationState
+	fs                 *Filesystem
+	permanentError	   error
+	retryAt            time.Time
+	completed, pending []*FSReplicationStep
+}
+
+func newFSReplicationPermanentError(fs *Filesystem, err error) *FSReplication {
+	return &FSReplication{
+		state: FSPermanentError,
+		fs: fs,
+		permanentError: err,
+	}
+}
+
+type FSReplicationBuilder struct {
+	r *FSReplication
+	steps []*FSReplicationStep
+}
+
+func buildNewFSReplication(fs *Filesystem) *FSReplicationBuilder {
+	return &FSReplicationBuilder{
+		r: &FSReplication{
+			fs: fs,
+			pending: make([]*FSReplicationStep, 0),
+		},
+	}
+}
+
+func (b *FSReplicationBuilder) AddStep(from, to *FilesystemVersion) *FSReplication {
+	step := &FSReplicationStep{
+		state: StepPending,
+		fsrep: b.r,
+		from: from,
+		to: to,
+	}
+	b.r.pending = append(b.r.pending, step)
+	return b.r
+}
+
+func (b *FSReplicationBuilder) Complete() *FSReplication {
+	if len(b.r.pending) > 0 {
+		b.r.state = FSQueued
+	} else {
+		b.r.state = FSCompleted
+	}
+	r := b.r
+	return r
+}
+
+//go:generate stringer -type=FSReplicationStepState
+type FSReplicationStepState int
+
+const (
+	StepPending FSReplicationStepState = iota
+	StepActive
+	StepRetry
+	StepPermanentError
+	StepCompleted
+)
+
+type FSReplicationStep struct {
+	state    FSReplicationStepState
+	from, to *FilesystemVersion
+	fsrep    *FSReplication
+
+	// both retry and permanent error
+	err error
+}
+
+func (r *Replication) Drive(ctx context.Context, ep EndpointPair, retryNow chan struct{}) {
+	for !(r.state == Completed || r.state == ContextDone) {
+		pre := r.state
+		preTime := time.Now()
+		r.doDrive(ctx, ep, retryNow)
+		delta := time.Now().Sub(preTime)
+		post := r.state
+		getLogger(ctx).
+			WithField("transition", fmt.Sprintf("%s => %s", pre, post)).
+			WithField("duration", delta).
+			Debug("state transition")
+	}
+}
+
+func (r *Replication) doDrive(ctx context.Context, ep EndpointPair, retryNow chan struct{}) {
+
+	switch r.state {
+
+	case Planning:
+		r.tryBuildPlan(ctx, ep)
+
+	case PlanningError:
+		w := time.NewTimer(10 * time.Second) // FIXME constant make configurable
+		defer w.Stop()
+		select {
+		case <-ctx.Done():
+			r.state = ContextDone
+			r.contextError = ctx.Err()
+		case <-retryNow:
+			r.state = Planning
+			r.planningError = nil
+		case <-w.C:
+			r.state = Planning
+			r.planningError = nil
+		}
+
+	case Working:
+
+		if len(r.pending) == 0 {
+			r.state = Completed
+			return
+		}
+
+		sort.Slice(r.pending, func(i, j int) bool {
+			a, b := r.pending[i], r.pending[j]
+			statePrio := func(x *FSReplication) int {
+				if !(x.state == FSQueued || x.state == FSRetry) {
+					panic(x)
+				}
+				if x.state == FSQueued {
+					return 0
+				} else {
+					return 1
+				}
+			}
+			aprio, bprio := statePrio(a), statePrio(b)
+			if aprio != bprio {
+				return aprio < bprio
+			}
+			// now we know they are the same state
+			if a.state == FSQueued {
+				return a.nextStepDate().Before(b.nextStepDate())
+			}
+			if a.state == FSRetry {
+				return a.retryAt.Before(b.retryAt)
+			}
+			panic("should not be reached")
+		})
+
+		fsrep := r.pending[0]
+
+		if fsrep.state == FSRetry {
+			r.state = WorkingWait
+			return
+		}
+		if fsrep.state != FSQueued {
+			panic(fsrep)
+		}
+
+		fsState := fsrep.takeStep(ctx, ep)
+		if fsState&(FSPermanentError|FSCompleted) != 0 {
+			r.pending = r.pending[1:]
+			r.completed = append(r.completed, fsrep)
+		}
+
+	case WorkingWait:
+		fsrep := r.pending[0]
+		w := time.NewTimer(fsrep.retryAt.Sub(time.Now()))
+		defer w.Stop()
+		select {
+		case <-ctx.Done():
+			r.state = ContextDone
+			r.contextError = ctx.Err()
+		case <-retryNow:
+			for _, fsr := range r.pending {
+				fsr.retryNow()
+			}
+			r.state = Working
+		case <-w.C:
+			fsrep.retryNow() // avoid timer jitter
+			r.state = Working
+		}
+	}
+}
+
+func (r *Replication) tryBuildPlan(ctx context.Context, ep EndpointPair) ReplicationState {
+
+	log := getLogger(ctx)
+
+	planningError := func(err error) ReplicationState {
+		r.state = PlanningError
+		r.planningError = err
+		return r.state
+	}
+	done := func() ReplicationState {
+		r.state = Working
+		r.planningError = nil
+		return r.state
+	}
+
+	sfss, err := ep.Sender().ListFilesystems(ctx)
+	if err != nil {
+		log.WithError(err).Error("error listing sender filesystems")
+		return planningError(err)
+	}
+
+	rfss, err := ep.Receiver().ListFilesystems(ctx)
+	if err != nil {
+		log.WithError(err).Error("error listing receiver filesystems")
+		return planningError(err)
+	}
+
+	r.pending = make([]*FSReplication, 0, len(sfss))
+	r.completed = make([]*FSReplication, 0, len(sfss))
+	mainlog := log
+	for _, fs := range sfss {
+
+		log := mainlog.WithField("filesystem", fs.Path)
+
+		log.Info("assessing filesystem")
+
+		sfsvs, err := ep.Sender().ListFilesystemVersions(ctx, fs.Path)
+		if err != nil {
+			log.WithError(err).Error("cannot get remote filesystem versions")
+			return planningError(err)
+		}
+
+		if len(sfsvs) <= 1 {
+			err := errors.New("sender does not have any versions")
+			log.Error(err.Error())
+			r.completed = append(r.completed, newFSReplicationPermanentError(fs, err))
+			continue
+		}
+
+		receiverFSExists := false
+		for _, rfs := range rfss {
+			if rfs.Path == fs.Path {
+				receiverFSExists = true
+			}
+		}
+
+		var rfsvs []*FilesystemVersion
+		if receiverFSExists {
+			rfsvs, err = ep.Receiver().ListFilesystemVersions(ctx, fs.Path)
+			if err != nil {
+				if _, ok := err.(FilteredError); ok {
+					log.Info("receiver ignores filesystem")
+					continue
+				}
+				log.WithError(err).Error("receiver error")
+				return planningError(err)
+			}
+		} else {
+			rfsvs = []*FilesystemVersion{}
+		}
+
+		path, conflict := IncrementalPath(rfsvs, sfsvs)
+		if conflict != nil {
+			var msg string
+			path, msg = resolveConflict(conflict) // no shadowing allowed!
+			if path != nil {
+				log.WithField("conflict", conflict).Info("conflict")
+				log.WithField("resolution", msg).Info("automatically resolved")
+			} else {
+				log.WithField("conflict", conflict).Error("conflict")
+				log.WithField("problem", msg).Error("cannot resolve conflict")
+			}
+		}
+		if path == nil {
+			r.completed = append(r.completed, newFSReplicationPermanentError(fs, conflict))
+			continue
+		}
+
+		fsreplbuilder := buildNewFSReplication(fs)
+		if len(path) == 1 {
+			fsreplbuilder.AddStep(nil, path[0])
+		} else {
+			for i := 0; i < len(path)-1; i++ {
+				fsreplbuilder.AddStep(path[i], path[i+1])
+			}
+		}
+		fsrepl := fsreplbuilder.Complete()
+		switch fsrepl.state {
+		case FSCompleted:
+			r.completed = append(r.completed, fsreplbuilder.Complete())
+		case FSQueued:
+			r.pending = append(r.pending, fsreplbuilder.Complete())
+		default:
+			panic(fsrepl)
+		}
+
+	}
+
+	return done()
+}
+
+func (f *FSReplication) nextStepDate() time.Time {
+	if f.state != FSQueued {
+		panic(f)
+	}
+	ct, err := f.pending[0].to.CreationAsTime()
+	if err != nil {
+		panic(err) // FIXME
+	}
+	return ct
+}
+
+func (f *FSReplication) takeStep(ctx context.Context, ep EndpointPair) FSReplicationState {
+	if f.state != FSQueued {
+		panic(f)
+	}
+
+	f.state = FSActive
+	step := f.pending[0]
+	stepState := step.do(ctx, ep)
+
+	switch stepState {
+	case StepCompleted:
+		f.pending = f.pending[1:]
+		f.completed = append(f.completed, step)
+		if len(f.pending) > 0 {
+			f.state = FSQueued
+		} else {
+			f.state = FSCompleted
+		}
+
+	case StepRetry:
+		f.state = FSRetry
+		f.retryAt = time.Now().Add(10 * time.Second) // FIXME hardcoded constant
+
+	case StepPermanentError:
+		f.state = FSPermanentError
+
+	}
+	return f.state
+}
+
+func (f *FSReplication) retryNow() {
+	if f.state != FSRetry {
+		panic(f)
+	}
+	f.retryAt = time.Time{}
+	f.state = FSQueued
+}
+
+func (s *FSReplicationStep) do(ctx context.Context, ep EndpointPair) FSReplicationStepState {
+
+	fs := s.fsrep.fs
+
+	log := getLogger(ctx).
+		WithField("filesystem", fs.Path).
+		WithField("step", s.String())
+
+	updateStateError := func(err error) FSReplicationStepState {
+		s.err = err
+		switch err {
+			case io.EOF: fallthrough
+			case io.ErrUnexpectedEOF: fallthrough
+			case io.ErrClosedPipe:
+				return StepRetry
+		}
+		if _, ok := err.(net.Error); ok {
+			return StepRetry
+		}
+		return StepPermanentError
+	}
+
+	updateStateCompleted := func() FSReplicationStepState {
+		s.err = nil
+		s.state = StepCompleted
+		return s.state
+	}
+
+	// FIXME refresh fs resume token
+	fs.ResumeToken = ""
+
+	var sr *SendReq
+	if fs.ResumeToken != "" {
+		sr = &SendReq{
+			Filesystem:  fs.Path,
+			ResumeToken: fs.ResumeToken,
+		}
+	} else if s.from == nil {
+		sr = &SendReq{
+			Filesystem: fs.Path,
+			From:       s.to.RelName(), // FIXME fix protocol to use To, like zfs does internally
+		}
+	} else {
+		sr = &SendReq{
+			Filesystem: fs.Path,
+			From:       s.from.RelName(),
+			To:         s.to.RelName(),
+		}
+	}
+
+	log.WithField("request", sr).Debug("initiate send request")
+	sres, sstream, err := ep.Sender().Send(ctx, sr)
+	if err != nil {
+		log.WithError(err).Error("send request failed")
+		return updateStateError(err)
+	}
+	if sstream == nil {
+		err := errors.New("send request did not return a stream, broken endpoint implementation")
+		return updateStateError(err)
+	}
+
+	rr := &ReceiveReq{
+		Filesystem:       fs.Path,
+		ClearResumeToken: !sres.UsedResumeToken,
+	}
+	log.WithField("request", rr).Debug("initiate receive request")
+	err = ep.Receiver().Receive(ctx, rr, sstream)
+	if err != nil {
+		log.WithError(err).Error("receive request failed (might also be error on sender)")
+		sstream.Close()
+		// This failure could be due to
+		// 	- an unexpected exit of ZFS on the sending side
+		//  - an unexpected exit of ZFS on the receiving side
+		//  - a connectivity issue
+		return updateStateError(err)
+	}
+	log.Info("receive finished")
+	return updateStateCompleted()
+
+}
+
+func (s *FSReplicationStep) String() string {
+	if s.from == nil { // FIXME: ZFS semantics are that to is nil on non-incremental send
+		return fmt.Sprintf("%s%s (full)", s.fsrep.fs.Path, s.to.RelName())
+	} else {
+		return fmt.Sprintf("%s(%s => %s)", s.fsrep.fs.Path, s.from.RelName(), s.to.RelName())
+	}
+}
+

cmd/replication.v2/replication.go

@@ -0,0 +1,137 @@
+package replication
+
+import (
+	"context"
+	"fmt"
+	"github.com/zrepl/zrepl/logger"
+	"io"
+)
+
+type ReplicationEndpoint interface {
+	// Does not include placeholder filesystems
+	ListFilesystems(ctx context.Context) ([]*Filesystem, error)
+	ListFilesystemVersions(ctx context.Context, fs string) ([]*FilesystemVersion, error) // fix depS
+	Send(ctx context.Context, r *SendReq) (*SendRes, io.ReadCloser, error)
+	Receive(ctx context.Context, r *ReceiveReq, sendStream io.ReadCloser) error
+}
+
+type FilteredError struct{ fs string }
+
+func NewFilteredError(fs string) FilteredError {
+	return FilteredError{fs}
+}
+
+func (f FilteredError) Error() string { return "endpoint does not allow access to filesystem " + f.fs }
+
+type ReplicationMode int
+
+const (
+	ReplicationModePull ReplicationMode = iota
+	ReplicationModePush
+)
+
+type EndpointPair struct {
+	a, b ReplicationEndpoint
+	m    ReplicationMode
+}
+
+func NewEndpointPairPull(sender, receiver ReplicationEndpoint) EndpointPair {
+	return EndpointPair{sender, receiver, ReplicationModePull}
+}
+
+func NewEndpointPairPush(sender, receiver ReplicationEndpoint) EndpointPair {
+	return EndpointPair{receiver, sender, ReplicationModePush}
+}
+
+func (p EndpointPair) Sender() ReplicationEndpoint {
+	switch p.m {
+	case ReplicationModePull:
+		return p.a
+	case ReplicationModePush:
+		return p.b
+	}
+	panic("should not be reached")
+	return nil
+}
+
+func (p EndpointPair) Receiver() ReplicationEndpoint {
+	switch p.m {
+	case ReplicationModePull:
+		return p.b
+	case ReplicationModePush:
+		return p.a
+	}
+	panic("should not be reached")
+	return nil
+}
+
+func (p EndpointPair) Mode() ReplicationMode {
+	return p.m
+}
+
+type contextKey int
+
+const (
+	contextKeyLog contextKey = iota
+)
+
+//type Logger interface {
+//	Infof(fmt string, args ...interface{})
+//	Errorf(fmt string, args ...interface{})
+//}
+
+//var _ Logger = nullLogger{}
+
+//type nullLogger struct{}
+//
+//func (nullLogger) Infof(fmt string, args ...interface{}) {}
+//func (nullLogger) Errorf(fmt string, args ...interface{}) {}
+
+type Logger = logger.Logger
+
+func ContextWithLogger(ctx context.Context, l Logger) context.Context {
+	return context.WithValue(ctx, contextKeyLog, l)
+}
+
+func getLogger(ctx context.Context) Logger {
+	l, ok := ctx.Value(contextKeyLog).(Logger)
+	if !ok {
+		l = logger.NewNullLogger()
+	}
+	return l
+}
+
+func resolveConflict(conflict error) (path []*FilesystemVersion, msg string) {
+	if noCommonAncestor, ok := conflict.(*ConflictNoCommonAncestor); ok {
+		if len(noCommonAncestor.SortedReceiverVersions) == 0 {
+			// FIXME hard-coded replication policy: most recent
+			// snapshot as source
+			var mostRecentSnap *FilesystemVersion
+			for n := len(noCommonAncestor.SortedSenderVersions) - 1; n >= 0; n-- {
+				if noCommonAncestor.SortedSenderVersions[n].Type == FilesystemVersion_Snapshot {
+					mostRecentSnap = noCommonAncestor.SortedSenderVersions[n]
+					break
+				}
+			}
+			if mostRecentSnap == nil {
+				return nil, "no snapshots available on sender side"
+			}
+			return []*FilesystemVersion{mostRecentSnap}, fmt.Sprintf("start replication at most recent snapshot %s", mostRecentSnap.RelName())
+		}
+	}
+	return nil, "no automated way to handle conflict type"
+}
+
+// Replicate replicates filesystems from ep.Sender() to ep.Receiver().
+//
+// All filesystems presented by the sending side are replicated,
+// unless the receiver rejects a Receive request with a *FilteredError.
+//
+// If an error occurs when replicating a filesystem, that error is logged to the logger in ctx.
+// Replicate continues with the replication of the remaining file systems.
+// Depending on the type of error, failed replications are retried in an unspecified order (currently FIFO).
+func Replicate(ctx context.Context, ep EndpointPair, retryNow chan struct{}) {
+	r := Replication{}
+	r.Drive(ctx, ep, retryNow)
+}
+

cmd/replication.v2/replication_test.go

@@ -0,0 +1,181 @@
+package replication_test
+
+import (
+	"context"
+	"github.com/stretchr/testify/assert"
+	"github.com/zrepl/zrepl/cmd/replication"
+	"io"
+	"testing"
+)
+
+type IncrementalPathSequenceStep struct {
+	SendRequest    *replication.SendReq
+	SendResponse   *replication.SendRes
+	SendReader io.ReadCloser
+	SendError      error
+	ReceiveRequest *replication.ReceiveReq
+	ReceiveError   error
+}
+
+type MockIncrementalPathRecorder struct {
+	T        *testing.T
+	Sequence []IncrementalPathSequenceStep
+	Pos      int
+}
+
+func (m *MockIncrementalPathRecorder) Receive(ctx context.Context, r *replication.ReceiveReq, rs io.ReadCloser) (error) {
+	if m.Pos >= len(m.Sequence) {
+		m.T.Fatal("unexpected Receive")
+	}
+	i := m.Sequence[m.Pos]
+	m.Pos++
+	if !assert.Equal(m.T, i.ReceiveRequest, r) {
+		m.T.FailNow()
+	}
+	return i.ReceiveError
+}
+
+func (m *MockIncrementalPathRecorder) Send(ctx context.Context, r *replication.SendReq) (*replication.SendRes, io.ReadCloser, error) {
+	if m.Pos >= len(m.Sequence) {
+		m.T.Fatal("unexpected Send")
+	}
+	i := m.Sequence[m.Pos]
+	m.Pos++
+	if !assert.Equal(m.T, i.SendRequest, r) {
+		m.T.FailNow()
+	}
+	return i.SendResponse, i.SendReader, i.SendError
+}
+
+func (m *MockIncrementalPathRecorder) Finished() bool {
+	return m.Pos == len(m.Sequence)
+}
+
+//type IncrementalPathReplicatorTest struct {
+//	Msg        string
+//	Filesystem *replication.Filesystem
+//	Path       []*replication.FilesystemVersion
+//	Steps      []IncrementalPathSequenceStep
+//}
+//
+//func (test *IncrementalPathReplicatorTest) Test(t *testing.T) {
+//
+//	t.Log(test.Msg)
+//
+//	rec := &MockIncrementalPathRecorder{
+//		T:        t,
+//		Sequence: test.Steps,
+//	}
+//
+//	ctx := replication.ContextWithLogger(context.Background(), testLog{t})
+//
+//	ipr := replication.NewIncrementalPathReplicator()
+//	ipr.Replicate(
+//		ctx,
+//		rec,
+//		rec,
+//		DiscardCopier{},
+//		test.Filesystem,
+//		test.Path,
+//	)
+//
+//	assert.True(t, rec.Finished())
+//
+//}
+
+//type testLog struct {
+//	t *testing.T
+//}
+//
+//var _ replication.Logger = testLog{}
+//
+//func (t testLog) Infof(fmt string, args ...interface{}) {
+//	t.t.Logf(fmt, args)
+//}
+//func (t testLog) Errorf(fmt string, args ...interface{}) {
+//	t.t.Logf(fmt, args)
+//}
+
+
+//func TestIncrementalPathReplicator_Replicate(t *testing.T) {
+//
+//	tbl := []IncrementalPathReplicatorTest{
+//		{
+//			Msg: "generic happy place with resume token",
+//			Filesystem: &replication.Filesystem{
+//				Path:        "foo/bar",
+//				ResumeToken: "blafoo",
+//			},
+//			Path: fsvlist("@a,1", "@b,2", "@c,3"),
+//			Steps: []IncrementalPathSequenceStep{
+//				{
+//					SendRequest: &replication.SendReq{
+//						Filesystem:  "foo/bar",
+//						From:        "@a,1",
+//						To:          "@b,2",
+//						ResumeToken: "blafoo",
+//					},
+//					SendResponse: &replication.SendRes{
+//						UsedResumeToken: true,
+//					},
+//				},
+//				{
+//					ReceiveRequest: &replication.ReceiveReq{
+//						Filesystem:  "foo/bar",
+//						ClearResumeToken: false,
+//					},
+//				},
+//				{
+//					SendRequest: &replication.SendReq{
+//						Filesystem: "foo/bar",
+//						From:       "@b,2",
+//						To:         "@c,3",
+//					},
+//				},
+//				{
+//					ReceiveRequest: &replication.ReceiveReq{
+//						Filesystem: "foo/bar",
+//					},
+//				},
+//			},
+//		},
+//		{
+//			Msg: "no action on empty sequence",
+//			Filesystem: &replication.Filesystem{
+//				Path: "foo/bar",
+//			},
+//			Path:  fsvlist(),
+//			Steps: []IncrementalPathSequenceStep{},
+//		},
+//		{
+//			Msg: "full send on single entry path",
+//			Filesystem: &replication.Filesystem{
+//				Path: "foo/bar",
+//			},
+//			Path:  fsvlist("@justone,1"),
+//			Steps: []IncrementalPathSequenceStep{
+//				{
+//					SendRequest: &replication.SendReq{
+//						Filesystem: "foo/bar",
+//						From: "@justone,1",
+//						To: "", // empty means full send
+//					},
+//					SendResponse: &replication.SendRes{
+//						UsedResumeToken: false,
+//					},
+//				},
+//				{
+//					ReceiveRequest: &replication.ReceiveReq{
+//						Filesystem: "foo/bar",
+//						ClearResumeToken: false,
+//					},
+//				},
+//			},
+//		},
+//	}
+//
+//	for _, test := range tbl {
+//		test.Test(t)
+//	}
+//
+//}

cmd/replication.v2/replicationstate_string.go

@@ -0,0 +1,16 @@
+// Code generated by "stringer -type=ReplicationState"; DO NOT EDIT.
+
+package replication
+
+import "strconv"
+
+const _ReplicationState_name = "PlanningPlanningErrorWorkingWorkingWaitCompletedContextDone"
+
+var _ReplicationState_index = [...]uint8{0, 8, 21, 28, 39, 48, 59}
+
+func (i ReplicationState) String() string {
+	if i < 0 || i >= ReplicationState(len(_ReplicationState_index)-1) {
+		return "ReplicationState(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _ReplicationState_name[_ReplicationState_index[i]:_ReplicationState_index[i+1]]
+}