enforce encapsulation by breaking up replication package into packages

not perfect yet, public shouldn't be required to use 'common' package to
use replication package
This commit is contained in:
Christian Schwarz
2018-08-16 21:05:21 +02:00
parent c7d28fee8f
commit 38532abf45
22 changed files with 1057 additions and 1491 deletions
+91
View File
@@ -0,0 +1,91 @@
package common
import (
"context"
"io"
"github.com/zrepl/zrepl/cmd/replication/pdu"
"github.com/zrepl/zrepl/logger"
)
type contextKey int
const (
contextKeyLog contextKey = iota
)
type Logger = logger.Logger
func ContextWithLogger(ctx context.Context, l Logger) context.Context {
return context.WithValue(ctx, contextKeyLog, l)
}
func GetLogger(ctx context.Context) Logger {
l, ok := ctx.Value(contextKeyLog).(Logger)
if !ok {
l = logger.NewNullLogger()
}
return l
}
type ReplicationEndpoint interface {
// Does not include placeholder filesystems
ListFilesystems(ctx context.Context) ([]*pdu.Filesystem, error)
ListFilesystemVersions(ctx context.Context, fs string) ([]*pdu.FilesystemVersion, error) // fix depS
Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.ReadCloser, error)
Receive(ctx context.Context, r *pdu.ReceiveReq, sendStream io.ReadCloser) error
}
type FilteredError struct{ fs string }
func NewFilteredError(fs string) *FilteredError {
return &FilteredError{fs}
}
func (f FilteredError) Error() string { return "endpoint does not allow access to filesystem " + f.fs }
type ReplicationMode int
const (
ReplicationModePull ReplicationMode = iota
ReplicationModePush
)
type EndpointPair struct {
a, b ReplicationEndpoint
m ReplicationMode
}
func NewEndpointPairPull(sender, receiver ReplicationEndpoint) EndpointPair {
return EndpointPair{sender, receiver, ReplicationModePull}
}
func NewEndpointPairPush(sender, receiver ReplicationEndpoint) EndpointPair {
return EndpointPair{receiver, sender, ReplicationModePush}
}
func (p EndpointPair) Sender() ReplicationEndpoint {
switch p.m {
case ReplicationModePull:
return p.a
case ReplicationModePush:
return p.b
}
panic("should not be reached")
return nil
}
func (p EndpointPair) Receiver() ReplicationEndpoint {
switch p.m {
case ReplicationModePull:
return p.b
case ReplicationModePush:
return p.a
}
panic("should not be reached")
return nil
}
func (p EndpointPair) Mode() ReplicationMode {
return p.m
}
+361
View File
@@ -0,0 +1,361 @@
package fsfsm
import (
"context"
"errors"
"fmt"
"io"
"math/bits"
"net"
"sync"
"time"
"github.com/zrepl/zrepl/cmd/replication/pdu"
. "github.com/zrepl/zrepl/cmd/replication/common"
)
type StepReport struct {
From, To string
Status string
Problem string
}
type FilesystemReplicationReport struct {
Filesystem string
Status string
Problem string
Completed,Pending []*StepReport
}
//go:generate stringer -type=FSReplicationState
type FSReplicationState uint
const (
FSReady FSReplicationState = 1 << iota
FSRetryWait
FSPermanentError
FSCompleted
)
func (s FSReplicationState) fsrsf() fsrsf {
idx := bits.TrailingZeros(uint(s))
if idx == bits.UintSize {
panic(s)
}
m := []fsrsf{
fsrsfReady,
fsrsfRetryWait,
nil,
nil,
}
return m[idx]
}
type FSReplication struct {
// lock protects all fields in this struct, but not the data behind pointers
lock sync.Mutex
state FSReplicationState
fs string
err error
retryWaitUntil time.Time
completed, pending []*FSReplicationStep
}
func (f *FSReplication) State() FSReplicationState {
f.lock.Lock()
defer f.lock.Unlock()
return f.state
}
type FSReplicationBuilder struct {
r *FSReplication
}
func BuildFSReplication(fs string) *FSReplicationBuilder {
return &FSReplicationBuilder{&FSReplication{fs: fs}}
}
func (b *FSReplicationBuilder) AddStep(from, to FilesystemVersion) *FSReplicationBuilder {
step := &FSReplicationStep{
state: StepReady,
fsrep: b.r,
from: from,
to: to,
}
b.r.pending = append(b.r.pending, step)
return b
}
func (b *FSReplicationBuilder) Done() (r *FSReplication) {
if len(b.r.pending) > 0 {
b.r.state = FSReady
} else {
b.r.state = FSCompleted
}
r = b.r
b.r = nil
return r
}
func NewFSReplicationWithPermanentError(fs string, err error) *FSReplication {
return &FSReplication{
state: FSPermanentError,
fs: fs,
err: err,
}
}
//go:generate stringer -type=FSReplicationStepState
type FSReplicationStepState uint
const (
StepReady FSReplicationStepState = 1 << iota
StepRetry
StepPermanentError
StepCompleted
)
type FilesystemVersion interface {
SnapshotTime() time.Time
RelName() string
}
type FSReplicationStep struct {
// only protects state, err
// from, to and fsrep are assumed to be immutable
lock sync.Mutex
state FSReplicationStepState
from, to FilesystemVersion
fsrep *FSReplication
// both retry and permanent error
err error
}
func (f *FSReplication) TakeStep(ctx context.Context, ep EndpointPair) (post FSReplicationState, nextStepDate time.Time) {
var u fsrUpdater = func(fu func(*FSReplication)) FSReplicationState {
f.lock.Lock()
defer f.lock.Unlock()
if fu != nil {
fu(f)
}
return f.state
}
var s fsrsf = u(nil).fsrsf()
pre := u(nil)
preTime := time.Now()
s = s(ctx, ep, u)
delta := time.Now().Sub(preTime)
post = u(func(f *FSReplication) {
if len(f.pending) == 0 {
return
}
nextStepDate = f.pending[0].to.SnapshotTime()
})
GetLogger(ctx).
WithField("fs", f.fs).
WithField("transition", fmt.Sprintf("%s => %s", pre, post)).
WithField("duration", delta).
Debug("fsr step taken")
return post, nextStepDate
}
type fsrUpdater func(func(fsr *FSReplication)) FSReplicationState
type fsrsf func(ctx context.Context, ep EndpointPair, u fsrUpdater) fsrsf
func fsrsfReady(ctx context.Context, ep EndpointPair, u fsrUpdater) fsrsf {
var current *FSReplicationStep
s := u(func(f *FSReplication) {
if len(f.pending) == 0 {
f.state = FSCompleted
return
}
current = f.pending[0]
})
if s != FSReady {
return s.fsrsf()
}
stepState := current.do(ctx, ep)
return u(func(f *FSReplication) {
switch stepState {
case StepCompleted:
f.completed = append(f.completed, current)
f.pending = f.pending[1:]
if len(f.pending) > 0 {
f.state = FSReady
} else {
f.state = FSCompleted
}
case StepRetry:
f.retryWaitUntil = time.Now().Add(10 * time.Second) // FIXME make configurable
f.state = FSRetryWait
case StepPermanentError:
f.state = FSPermanentError
f.err = errors.New("a replication step failed with a permanent error")
default:
panic(f)
}
}).fsrsf()
}
func fsrsfRetryWait(ctx context.Context, ep EndpointPair, u fsrUpdater) fsrsf {
var sleepUntil time.Time
u(func(f *FSReplication) {
sleepUntil = f.retryWaitUntil
})
t := time.NewTimer(sleepUntil.Sub(time.Now()))
defer t.Stop()
select {
case <-ctx.Done():
return u(func(f *FSReplication) {
f.state = FSPermanentError
f.err = ctx.Err()
}).fsrsf()
case <-t.C:
}
return u(func(f *FSReplication) {
f.state = FSReady
}).fsrsf()
}
// access to fsr's members must be exclusive
func (fsr *FSReplication) Report() *FilesystemReplicationReport {
fsr.lock.Lock()
defer fsr.lock.Unlock()
rep := FilesystemReplicationReport{
Filesystem: fsr.fs,
Status: fsr.state.String(),
}
if fsr.state&FSPermanentError != 0 {
rep.Problem = fsr.err.Error()
return &rep
}
rep.Completed = make([]*StepReport, len(fsr.completed))
for i := range fsr.completed {
rep.Completed[i] = fsr.completed[i].Report()
}
rep.Pending = make([]*StepReport, len(fsr.pending))
for i := range fsr.pending {
rep.Pending[i] = fsr.pending[i].Report()
}
return &rep
}
func (s *FSReplicationStep) do(ctx context.Context, ep EndpointPair) FSReplicationStepState {
fs := s.fsrep.fs
log := GetLogger(ctx).
WithField("filesystem", fs).
WithField("step", s.String())
updateStateError := func(err error) FSReplicationStepState {
s.lock.Lock()
defer s.lock.Unlock()
s.err = err
switch err {
case io.EOF:
fallthrough
case io.ErrUnexpectedEOF:
fallthrough
case io.ErrClosedPipe:
s.state = StepRetry
return s.state
}
if _, ok := err.(net.Error); ok {
s.state = StepRetry
return s.state
}
s.state = StepPermanentError
return s.state
}
updateStateCompleted := func() FSReplicationStepState {
s.lock.Lock()
defer s.lock.Unlock()
s.err = nil
s.state = StepCompleted
return s.state
}
var sr *pdu.SendReq
if s.from == nil {
sr = &pdu.SendReq{
Filesystem: fs,
From: s.to.RelName(), // FIXME fix protocol to use To, like zfs does internally
}
} else {
sr = &pdu.SendReq{
Filesystem: fs,
From: s.from.RelName(),
To: s.to.RelName(),
}
}
log.WithField("request", sr).Debug("initiate send request")
sres, sstream, err := ep.Sender().Send(ctx, sr)
if err != nil {
log.WithError(err).Error("send request failed")
return updateStateError(err)
}
if sstream == nil {
err := errors.New("send request did not return a stream, broken endpoint implementation")
return updateStateError(err)
}
rr := &pdu.ReceiveReq{
Filesystem: fs,
ClearResumeToken: !sres.UsedResumeToken,
}
log.WithField("request", rr).Debug("initiate receive request")
err = ep.Receiver().Receive(ctx, rr, sstream)
if err != nil {
log.WithError(err).Error("receive request failed (might also be error on sender)")
sstream.Close()
// This failure could be due to
// - an unexpected exit of ZFS on the sending side
// - an unexpected exit of ZFS on the receiving side
// - a connectivity issue
return updateStateError(err)
}
log.Info("receive finished")
return updateStateCompleted()
}
func (s *FSReplicationStep) String() string {
if s.from == nil { // FIXME: ZFS semantics are that to is nil on non-incremental send
return fmt.Sprintf("%s%s (full)", s.fsrep.fs, s.to.RelName())
} else {
return fmt.Sprintf("%s(%s => %s)", s.fsrep.fs, s.from, s.to.RelName())
}
}
func (step *FSReplicationStep) Report() *StepReport {
var from string // FIXME follow same convention as ZFS: to should be nil on full send
if step.from != nil {
from = step.from.RelName()
}
rep := StepReport{
From: from,
To: step.to.RelName(),
Status: step.state.String(),
}
return &rep
}
@@ -0,0 +1,29 @@
// Code generated by "stringer -type=FSReplicationState"; DO NOT EDIT.
package fsfsm
import "strconv"
const (
_FSReplicationState_name_0 = "FSReadyFSRetryWait"
_FSReplicationState_name_1 = "FSPermanentError"
_FSReplicationState_name_2 = "FSCompleted"
)
var (
_FSReplicationState_index_0 = [...]uint8{0, 7, 18}
)
func (i FSReplicationState) String() string {
switch {
case 1 <= i && i <= 2:
i -= 1
return _FSReplicationState_name_0[_FSReplicationState_index_0[i]:_FSReplicationState_index_0[i+1]]
case i == 4:
return _FSReplicationState_name_1
case i == 8:
return _FSReplicationState_name_2
default:
return "FSReplicationState(" + strconv.FormatInt(int64(i), 10) + ")"
}
}
@@ -0,0 +1,29 @@
// Code generated by "stringer -type=FSReplicationStepState"; DO NOT EDIT.
package fsfsm
import "strconv"
const (
_FSReplicationStepState_name_0 = "StepReadyStepRetry"
_FSReplicationStepState_name_1 = "StepPermanentError"
_FSReplicationStepState_name_2 = "StepCompleted"
)
var (
_FSReplicationStepState_index_0 = [...]uint8{0, 9, 18}
)
func (i FSReplicationStepState) String() string {
switch {
case 1 <= i && i <= 2:
i -= 1
return _FSReplicationStepState_name_0[_FSReplicationStepState_index_0[i]:_FSReplicationStepState_index_0[i+1]]
case i == 4:
return _FSReplicationStepState_name_1
case i == 8:
return _FSReplicationStepState_name_2
default:
return "FSReplicationStepState(" + strconv.FormatInt(int64(i), 10) + ")"
}
}
+117
View File
@@ -0,0 +1,117 @@
package mainfsm
import (
"sort"
. "github.com/zrepl/zrepl/cmd/replication/pdu"
)
type ConflictNoCommonAncestor struct {
SortedSenderVersions, SortedReceiverVersions []*FilesystemVersion
}
func (c *ConflictNoCommonAncestor) Error() string {
return "no common snapshot or suitable bookmark between sender and receiver"
}
type ConflictDiverged struct {
SortedSenderVersions, SortedReceiverVersions []*FilesystemVersion
CommonAncestor *FilesystemVersion
SenderOnly, ReceiverOnly []*FilesystemVersion
}
func (c *ConflictDiverged) Error() string {
return "the receiver's latest snapshot is not present on sender"
}
func SortVersionListByCreateTXGThenBookmarkLTSnapshot(fsvslice []*FilesystemVersion) []*FilesystemVersion {
lesser := func(s []*FilesystemVersion) func(i, j int) bool {
return func(i, j int) bool {
if s[i].CreateTXG < s[j].CreateTXG {
return true
}
if s[i].CreateTXG == s[j].CreateTXG {
// Bookmark < Snapshot
return s[i].Type == FilesystemVersion_Bookmark && s[j].Type == FilesystemVersion_Snapshot
}
return false
}
}
if sort.SliceIsSorted(fsvslice, lesser(fsvslice)) {
return fsvslice
}
sorted := make([]*FilesystemVersion, len(fsvslice))
copy(sorted, fsvslice)
sort.Slice(sorted, lesser(sorted))
return sorted
}
// conflict may be a *ConflictDiverged or a *ConflictNoCommonAncestor
func IncrementalPath(receiver, sender []*FilesystemVersion) (incPath []*FilesystemVersion, conflict error) {
if receiver == nil {
panic("receiver must not be nil")
}
if sender == nil {
panic("sender must not be nil")
}
receiver = SortVersionListByCreateTXGThenBookmarkLTSnapshot(receiver)
sender = SortVersionListByCreateTXGThenBookmarkLTSnapshot(sender)
if len(sender) == 0 {
return []*FilesystemVersion{}, nil
}
// Find most recent common ancestor by name, preferring snapshots over bookmarks
mrcaRcv := len(receiver) - 1
mrcaSnd := len(sender) - 1
for mrcaRcv >= 0 && mrcaSnd >= 0 {
if receiver[mrcaRcv].Guid == sender[mrcaSnd].Guid {
if mrcaSnd-1 >= 0 && sender[mrcaSnd-1].Guid == sender[mrcaSnd].Guid && sender[mrcaSnd-1].Type == FilesystemVersion_Bookmark {
// prefer bookmarks over snapshots as the snapshot might go away sooner
mrcaSnd -= 1
}
break
}
if receiver[mrcaRcv].CreateTXG < sender[mrcaSnd].CreateTXG {
mrcaSnd--
} else {
mrcaRcv--
}
}
if mrcaRcv == -1 || mrcaSnd == -1 {
return nil, &ConflictNoCommonAncestor{
SortedSenderVersions: sender,
SortedReceiverVersions: receiver,
}
}
if mrcaRcv != len(receiver)-1 {
return nil, &ConflictDiverged{
SortedSenderVersions: sender,
SortedReceiverVersions: receiver,
CommonAncestor: sender[mrcaSnd],
SenderOnly: sender[mrcaSnd+1:],
ReceiverOnly: receiver[mrcaRcv+1:],
}
}
// incPath must not contain bookmarks except initial one,
incPath = make([]*FilesystemVersion, 0, len(sender))
incPath = append(incPath, sender[mrcaSnd])
// it's ok if incPath[0] is a bookmark, but not the subsequent ones in the incPath
for i := mrcaSnd + 1; i < len(sender); i++ {
if sender[i].Type == FilesystemVersion_Snapshot && incPath[len(incPath)-1].Guid != sender[i].Guid {
incPath = append(incPath, sender[i])
}
}
if len(incPath) == 1 {
// nothing to do
incPath = incPath[1:]
}
return incPath, nil
}
+342
View File
@@ -0,0 +1,342 @@
package mainfsm
import (
"context"
"errors"
"fmt"
"math/bits"
"sync"
"time"
. "github.com/zrepl/zrepl/cmd/replication/common"
"github.com/zrepl/zrepl/cmd/replication/pdu"
"github.com/zrepl/zrepl/cmd/replication/internal/fsfsm"
. "github.com/zrepl/zrepl/cmd/replication/internal/mainfsm/queue"
)
//go:generate stringer -type=ReplicationState
type ReplicationState uint
const (
Planning ReplicationState = 1 << iota
PlanningError
Working
WorkingWait
Completed
ContextDone
)
func (s ReplicationState) rsf() replicationStateFunc {
idx := bits.TrailingZeros(uint(s))
if idx == bits.UintSize {
panic(s) // invalid value
}
m := []replicationStateFunc{
rsfPlanning,
rsfPlanningError,
rsfWorking,
rsfWorkingWait,
nil,
nil,
}
return m[idx]
}
type Replication struct {
// lock protects all fields of this struct (but not the fields behind pointers!)
lock sync.Mutex
state ReplicationState
// Working, WorkingWait, Completed, ContextDone
queue *ReplicationQueue
completed []*fsfsm.FSReplication
active *ReplicationQueueItemHandle
// PlanningError
planningError error
// ContextDone
contextError error
// PlanningError, WorkingWait
sleepUntil time.Time
}
type Report struct {
Status string
Problem string
Completed []*fsfsm.FilesystemReplicationReport
Pending []*fsfsm.FilesystemReplicationReport
Active *fsfsm.FilesystemReplicationReport
}
func NewReplication() *Replication {
r := Replication{
state: Planning,
}
return &r
}
type replicationUpdater func(func(*Replication)) (newState ReplicationState)
type replicationStateFunc func(context.Context, EndpointPair, replicationUpdater) replicationStateFunc
func (r *Replication) Drive(ctx context.Context, ep EndpointPair) {
var u replicationUpdater = func(f func(*Replication)) ReplicationState {
r.lock.Lock()
defer r.lock.Unlock()
if f != nil {
f(r)
}
return r.state
}
var s replicationStateFunc = rsfPlanning
var pre, post ReplicationState
for s != nil {
preTime := time.Now()
pre = u(nil)
s = s(ctx, ep, u)
delta := time.Now().Sub(preTime)
post = u(nil)
GetLogger(ctx).
WithField("transition", fmt.Sprintf("%s => %s", pre, post)).
WithField("duration", delta).
Debug("main state transition")
}
GetLogger(ctx).
WithField("final_state", post).
Debug("main final state")
}
func resolveConflict(conflict error) (path []*pdu.FilesystemVersion, msg string) {
if noCommonAncestor, ok := conflict.(*ConflictNoCommonAncestor); ok {
if len(noCommonAncestor.SortedReceiverVersions) == 0 {
// FIXME hard-coded replication policy: most recent
// snapshot as source
var mostRecentSnap *pdu.FilesystemVersion
for n := len(noCommonAncestor.SortedSenderVersions) - 1; n >= 0; n-- {
if noCommonAncestor.SortedSenderVersions[n].Type == pdu.FilesystemVersion_Snapshot {
mostRecentSnap = noCommonAncestor.SortedSenderVersions[n]
break
}
}
if mostRecentSnap == nil {
return nil, "no snapshots available on sender side"
}
return []*pdu.FilesystemVersion{mostRecentSnap}, fmt.Sprintf("start replication at most recent snapshot %s", mostRecentSnap.RelName())
}
}
return nil, "no automated way to handle conflict type"
}
func rsfPlanning(ctx context.Context, ep EndpointPair, u replicationUpdater) replicationStateFunc {
log := GetLogger(ctx)
handlePlanningError := func(err error) replicationStateFunc {
return u(func(r *Replication) {
r.planningError = err
r.state = PlanningError
}).rsf()
}
sfss, err := ep.Sender().ListFilesystems(ctx)
if err != nil {
log.WithError(err).Error("error listing sender filesystems")
return handlePlanningError(err)
}
rfss, err := ep.Receiver().ListFilesystems(ctx)
if err != nil {
log.WithError(err).Error("error listing receiver filesystems")
return handlePlanningError(err)
}
q := NewReplicationQueue()
mainlog := log
for _, fs := range sfss {
log := mainlog.WithField("filesystem", fs.Path)
log.Info("assessing filesystem")
sfsvs, err := ep.Sender().ListFilesystemVersions(ctx, fs.Path)
if err != nil {
log.WithError(err).Error("cannot get remote filesystem versions")
return handlePlanningError(err)
}
if len(sfsvs) <= 1 {
err := errors.New("sender does not have any versions")
log.Error(err.Error())
q.Add(fsfsm.NewFSReplicationWithPermanentError(fs.Path, err))
continue
}
receiverFSExists := false
for _, rfs := range rfss {
if rfs.Path == fs.Path {
receiverFSExists = true
}
}
var rfsvs []*pdu.FilesystemVersion
if receiverFSExists {
rfsvs, err = ep.Receiver().ListFilesystemVersions(ctx, fs.Path)
if err != nil {
if _, ok := err.(*FilteredError); ok {
log.Info("receiver ignores filesystem")
continue
}
log.WithError(err).Error("receiver error")
return handlePlanningError(err)
}
} else {
rfsvs = []*pdu.FilesystemVersion{}
}
path, conflict := IncrementalPath(rfsvs, sfsvs)
if conflict != nil {
var msg string
path, msg = resolveConflict(conflict) // no shadowing allowed!
if path != nil {
log.WithField("conflict", conflict).Info("conflict")
log.WithField("resolution", msg).Info("automatically resolved")
} else {
log.WithField("conflict", conflict).Error("conflict")
log.WithField("problem", msg).Error("cannot resolve conflict")
}
}
if path == nil {
q.Add(fsfsm.NewFSReplicationWithPermanentError(fs.Path, conflict))
continue
}
fsrfsm := fsfsm.BuildFSReplication(fs.Path)
if len(path) == 1 {
fsrfsm.AddStep(nil, path[0])
} else {
for i := 0; i < len(path)-1; i++ {
fsrfsm.AddStep(path[i], path[i+1])
}
}
qitem := fsrfsm.Done()
q.Add(qitem)
}
return u(func(r *Replication) {
r.completed = nil
r.queue = q
r.planningError = nil
r.state = Working
}).rsf()
}
func rsfPlanningError(ctx context.Context, ep EndpointPair, u replicationUpdater) replicationStateFunc {
sleepTime := 10 * time.Second
u(func(r *Replication) {
r.sleepUntil = time.Now().Add(sleepTime)
})
t := time.NewTimer(sleepTime) // FIXME make constant onfigurable
defer t.Stop()
select {
case <-ctx.Done():
return u(func(r *Replication) {
r.state = ContextDone
r.contextError = ctx.Err()
}).rsf()
case <-t.C:
return u(func(r *Replication) {
r.state = Planning
}).rsf()
}
}
func rsfWorking(ctx context.Context, ep EndpointPair, u replicationUpdater) replicationStateFunc {
var active *ReplicationQueueItemHandle
rsfNext := u(func(r *Replication) {
done, next := r.queue.GetNext()
r.completed = append(r.completed, done...)
if next == nil {
r.state = Completed
}
r.active = next
active = next
}).rsf()
if active == nil {
return rsfNext
}
state, nextStepDate := active.GetFSReplication().TakeStep(ctx, ep)
return u(func(r *Replication) {
active.Update(state, nextStepDate)
r.active = nil
}).rsf()
}
func rsfWorkingWait(ctx context.Context, ep EndpointPair, u replicationUpdater) replicationStateFunc {
sleepTime := 10 * time.Second
u(func(r *Replication) {
r.sleepUntil = time.Now().Add(sleepTime)
})
t := time.NewTimer(sleepTime)
defer t.Stop()
select {
case <-ctx.Done():
return u(func(r *Replication) {
r.state = ContextDone
r.contextError = ctx.Err()
}).rsf()
case <-t.C:
return u(func(r *Replication) {
r.state = Working
}).rsf()
}
}
func (r *Replication) Report() *Report {
r.lock.Lock()
defer r.lock.Unlock()
rep := Report{
Status: r.state.String(),
}
if r.state&(Planning|PlanningError|ContextDone) != 0 {
switch r.state {
case PlanningError:
rep.Problem = r.planningError.Error()
case ContextDone:
rep.Problem = r.contextError.Error()
}
return &rep
}
rep.Pending = make([]*fsfsm.FilesystemReplicationReport, 0, r.queue.Len())
rep.Completed = make([]*fsfsm.FilesystemReplicationReport, 0, len(r.completed)) // room for active (potentially)
var active *fsfsm.FSReplication
if r.active != nil {
active = r.active.GetFSReplication()
rep.Active = active.Report()
}
r.queue.Foreach(func (h *ReplicationQueueItemHandle){
fsr := h.GetFSReplication()
if active != fsr {
rep.Pending = append(rep.Pending, fsr.Report())
}
})
for _, fsr := range r.completed {
rep.Completed = append(rep.Completed, fsr.Report())
}
return &rep
}
@@ -0,0 +1,124 @@
package queue
import (
"time"
"sort"
. "github.com/zrepl/zrepl/cmd/replication/internal/fsfsm"
)
type replicationQueueItem struct {
retriesSinceLastError int
// duplicates fsr.state to avoid accessing and locking fsr
state FSReplicationState
// duplicates fsr.current.nextStepDate to avoid accessing & locking fsr
nextStepDate time.Time
fsr *FSReplication
}
type ReplicationQueue []*replicationQueueItem
func NewReplicationQueue() *ReplicationQueue {
q := make(ReplicationQueue, 0)
return &q
}
func (q ReplicationQueue) Len() int { return len(q) }
func (q ReplicationQueue) Swap(i, j int) { q[i], q[j] = q[j], q[i] }
type lessmapEntry struct{
prio int
less func(a,b *replicationQueueItem) bool
}
var lessmap = map[FSReplicationState]lessmapEntry {
FSReady: {
prio: 0,
less: func(a, b *replicationQueueItem) bool {
return a.nextStepDate.Before(b.nextStepDate)
},
},
FSRetryWait: {
prio: 1,
less: func(a, b *replicationQueueItem) bool {
return a.retriesSinceLastError < b.retriesSinceLastError
},
},
}
func (q ReplicationQueue) Less(i, j int) bool {
a, b := q[i], q[j]
al, aok := lessmap[a.state]
if !aok {
panic(a)
}
bl, bok := lessmap[b.state]
if !bok {
panic(b)
}
if al.prio != bl.prio {
return al.prio < bl.prio
}
return al.less(a, b)
}
func (q *ReplicationQueue) sort() (done []*FSReplication) {
// pre-scan for everything that is not ready
newq := make(ReplicationQueue, 0, len(*q))
done = make([]*FSReplication, 0, len(*q))
for _, qitem := range *q {
if _, ok := lessmap[qitem.state]; !ok {
done = append(done, qitem.fsr)
} else {
newq = append(newq, qitem)
}
}
sort.Stable(newq) // stable to avoid flickering in reports
*q = newq
return done
}
// next remains valid until the next call to GetNext()
func (q *ReplicationQueue) GetNext() (done []*FSReplication, next *ReplicationQueueItemHandle) {
done = q.sort()
if len(*q) == 0 {
return done, nil
}
next = &ReplicationQueueItemHandle{(*q)[0]}
return done, next
}
func (q *ReplicationQueue) Add(fsr *FSReplication) {
*q = append(*q, &replicationQueueItem{
fsr: fsr,
state: fsr.State(),
})
}
func (q *ReplicationQueue) Foreach(fu func(*ReplicationQueueItemHandle)) {
for _, qitem := range *q {
fu(&ReplicationQueueItemHandle{qitem})
}
}
type ReplicationQueueItemHandle struct {
i *replicationQueueItem
}
func (h ReplicationQueueItemHandle) GetFSReplication() *FSReplication {
return h.i.fsr
}
func (h ReplicationQueueItemHandle) Update(newState FSReplicationState, nextStepDate time.Time) {
h.i.state = newState
h.i.nextStepDate = nextStepDate
if h.i.state&FSReady != 0 {
h.i.retriesSinceLastError = 0
} else if h.i.state&FSRetryWait != 0 {
h.i.retriesSinceLastError++
}
}
@@ -0,0 +1,35 @@
// Code generated by "stringer -type=ReplicationState"; DO NOT EDIT.
package mainfsm
import "strconv"
const (
_ReplicationState_name_0 = "PlanningPlanningError"
_ReplicationState_name_1 = "Working"
_ReplicationState_name_2 = "WorkingWait"
_ReplicationState_name_3 = "Completed"
_ReplicationState_name_4 = "ContextDone"
)
var (
_ReplicationState_index_0 = [...]uint8{0, 8, 21}
)
func (i ReplicationState) String() string {
switch {
case 1 <= i && i <= 2:
i -= 1
return _ReplicationState_name_0[_ReplicationState_index_0[i]:_ReplicationState_index_0[i+1]]
case i == 4:
return _ReplicationState_name_1
case i == 8:
return _ReplicationState_name_2
case i == 16:
return _ReplicationState_name_3
case i == 32:
return _ReplicationState_name_4
default:
return "ReplicationState(" + strconv.FormatInt(int64(i), 10) + ")"
}
}
+384
View File
@@ -0,0 +1,384 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// source: pdu.proto
/*
Package pdu is a generated protocol buffer package.
It is generated from these files:
pdu.proto
It has these top-level messages:
ListFilesystemReq
ListFilesystemRes
Filesystem
ListFilesystemVersionsReq
ListFilesystemVersionsRes
FilesystemVersion
SendReq
Property
SendRes
ReceiveReq
ReceiveRes
*/
package pdu
import proto "github.com/golang/protobuf/proto"
import fmt "fmt"
import math "math"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
type FilesystemVersion_VersionType int32
const (
FilesystemVersion_Snapshot FilesystemVersion_VersionType = 0
FilesystemVersion_Bookmark FilesystemVersion_VersionType = 1
)
var FilesystemVersion_VersionType_name = map[int32]string{
0: "Snapshot",
1: "Bookmark",
}
var FilesystemVersion_VersionType_value = map[string]int32{
"Snapshot": 0,
"Bookmark": 1,
}
func (x FilesystemVersion_VersionType) String() string {
return proto.EnumName(FilesystemVersion_VersionType_name, int32(x))
}
func (FilesystemVersion_VersionType) EnumDescriptor() ([]byte, []int) {
return fileDescriptor0, []int{5, 0}
}
type ListFilesystemReq struct {
}
func (m *ListFilesystemReq) Reset() { *m = ListFilesystemReq{} }
func (m *ListFilesystemReq) String() string { return proto.CompactTextString(m) }
func (*ListFilesystemReq) ProtoMessage() {}
func (*ListFilesystemReq) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
type ListFilesystemRes struct {
Filesystems []*Filesystem `protobuf:"bytes,1,rep,name=Filesystems" json:"Filesystems,omitempty"`
}
func (m *ListFilesystemRes) Reset() { *m = ListFilesystemRes{} }
func (m *ListFilesystemRes) String() string { return proto.CompactTextString(m) }
func (*ListFilesystemRes) ProtoMessage() {}
func (*ListFilesystemRes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
func (m *ListFilesystemRes) GetFilesystems() []*Filesystem {
if m != nil {
return m.Filesystems
}
return nil
}
type Filesystem struct {
Path string `protobuf:"bytes,1,opt,name=Path" json:"Path,omitempty"`
ResumeToken string `protobuf:"bytes,2,opt,name=ResumeToken" json:"ResumeToken,omitempty"`
}
func (m *Filesystem) Reset() { *m = Filesystem{} }
func (m *Filesystem) String() string { return proto.CompactTextString(m) }
func (*Filesystem) ProtoMessage() {}
func (*Filesystem) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
func (m *Filesystem) GetPath() string {
if m != nil {
return m.Path
}
return ""
}
func (m *Filesystem) GetResumeToken() string {
if m != nil {
return m.ResumeToken
}
return ""
}
type ListFilesystemVersionsReq struct {
Filesystem string `protobuf:"bytes,1,opt,name=Filesystem" json:"Filesystem,omitempty"`
}
func (m *ListFilesystemVersionsReq) Reset() { *m = ListFilesystemVersionsReq{} }
func (m *ListFilesystemVersionsReq) String() string { return proto.CompactTextString(m) }
func (*ListFilesystemVersionsReq) ProtoMessage() {}
func (*ListFilesystemVersionsReq) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
func (m *ListFilesystemVersionsReq) GetFilesystem() string {
if m != nil {
return m.Filesystem
}
return ""
}
type ListFilesystemVersionsRes struct {
Versions []*FilesystemVersion `protobuf:"bytes,1,rep,name=Versions" json:"Versions,omitempty"`
}
func (m *ListFilesystemVersionsRes) Reset() { *m = ListFilesystemVersionsRes{} }
func (m *ListFilesystemVersionsRes) String() string { return proto.CompactTextString(m) }
func (*ListFilesystemVersionsRes) ProtoMessage() {}
func (*ListFilesystemVersionsRes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
func (m *ListFilesystemVersionsRes) GetVersions() []*FilesystemVersion {
if m != nil {
return m.Versions
}
return nil
}
type FilesystemVersion struct {
Type FilesystemVersion_VersionType `protobuf:"varint,1,opt,name=Type,enum=pdu.FilesystemVersion_VersionType" json:"Type,omitempty"`
Name string `protobuf:"bytes,2,opt,name=Name" json:"Name,omitempty"`
Guid uint64 `protobuf:"varint,3,opt,name=Guid" json:"Guid,omitempty"`
CreateTXG uint64 `protobuf:"varint,4,opt,name=CreateTXG" json:"CreateTXG,omitempty"`
Creation string `protobuf:"bytes,5,opt,name=Creation" json:"Creation,omitempty"`
}
func (m *FilesystemVersion) Reset() { *m = FilesystemVersion{} }
func (m *FilesystemVersion) String() string { return proto.CompactTextString(m) }
func (*FilesystemVersion) ProtoMessage() {}
func (*FilesystemVersion) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
func (m *FilesystemVersion) GetType() FilesystemVersion_VersionType {
if m != nil {
return m.Type
}
return FilesystemVersion_Snapshot
}
func (m *FilesystemVersion) GetName() string {
if m != nil {
return m.Name
}
return ""
}
func (m *FilesystemVersion) GetGuid() uint64 {
if m != nil {
return m.Guid
}
return 0
}
func (m *FilesystemVersion) GetCreateTXG() uint64 {
if m != nil {
return m.CreateTXG
}
return 0
}
func (m *FilesystemVersion) GetCreation() string {
if m != nil {
return m.Creation
}
return ""
}
type SendReq struct {
Filesystem string `protobuf:"bytes,1,opt,name=Filesystem" json:"Filesystem,omitempty"`
From string `protobuf:"bytes,2,opt,name=From" json:"From,omitempty"`
// May be empty / null to request a full transfer of From
To string `protobuf:"bytes,3,opt,name=To" json:"To,omitempty"`
// If ResumeToken is not empty, the resume token that CAN be tried for 'zfs send' by the sender.
// The sender MUST indicate in SendRes.UsedResumeToken
// If it does not work, the sender SHOULD clear the resume token on their side
// and use From and To instead
// If ResumeToken is not empty, the GUIDs of From and To
// MUST correspond to those encoded in the ResumeToken.
// Otherwise, the Sender MUST return an error.
ResumeToken string `protobuf:"bytes,4,opt,name=ResumeToken" json:"ResumeToken,omitempty"`
Compress bool `protobuf:"varint,5,opt,name=Compress" json:"Compress,omitempty"`
Dedup bool `protobuf:"varint,6,opt,name=Dedup" json:"Dedup,omitempty"`
}
func (m *SendReq) Reset() { *m = SendReq{} }
func (m *SendReq) String() string { return proto.CompactTextString(m) }
func (*SendReq) ProtoMessage() {}
func (*SendReq) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
func (m *SendReq) GetFilesystem() string {
if m != nil {
return m.Filesystem
}
return ""
}
func (m *SendReq) GetFrom() string {
if m != nil {
return m.From
}
return ""
}
func (m *SendReq) GetTo() string {
if m != nil {
return m.To
}
return ""
}
func (m *SendReq) GetResumeToken() string {
if m != nil {
return m.ResumeToken
}
return ""
}
func (m *SendReq) GetCompress() bool {
if m != nil {
return m.Compress
}
return false
}
func (m *SendReq) GetDedup() bool {
if m != nil {
return m.Dedup
}
return false
}
type Property struct {
Name string `protobuf:"bytes,1,opt,name=Name" json:"Name,omitempty"`
Value string `protobuf:"bytes,2,opt,name=Value" json:"Value,omitempty"`
}
func (m *Property) Reset() { *m = Property{} }
func (m *Property) String() string { return proto.CompactTextString(m) }
func (*Property) ProtoMessage() {}
func (*Property) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
func (m *Property) GetName() string {
if m != nil {
return m.Name
}
return ""
}
func (m *Property) GetValue() string {
if m != nil {
return m.Value
}
return ""
}
type SendRes struct {
// Whether the resume token provided in the request has been used or not.
UsedResumeToken bool `protobuf:"varint,1,opt,name=UsedResumeToken" json:"UsedResumeToken,omitempty"`
Properties []*Property `protobuf:"bytes,2,rep,name=Properties" json:"Properties,omitempty"`
}
func (m *SendRes) Reset() { *m = SendRes{} }
func (m *SendRes) String() string { return proto.CompactTextString(m) }
func (*SendRes) ProtoMessage() {}
func (*SendRes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
func (m *SendRes) GetUsedResumeToken() bool {
if m != nil {
return m.UsedResumeToken
}
return false
}
func (m *SendRes) GetProperties() []*Property {
if m != nil {
return m.Properties
}
return nil
}
type ReceiveReq struct {
Filesystem string `protobuf:"bytes,1,opt,name=Filesystem" json:"Filesystem,omitempty"`
// If true, the receiver should clear the resume token before perfoming the zfs recv of the stream in the request
ClearResumeToken bool `protobuf:"varint,2,opt,name=ClearResumeToken" json:"ClearResumeToken,omitempty"`
}
func (m *ReceiveReq) Reset() { *m = ReceiveReq{} }
func (m *ReceiveReq) String() string { return proto.CompactTextString(m) }
func (*ReceiveReq) ProtoMessage() {}
func (*ReceiveReq) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
func (m *ReceiveReq) GetFilesystem() string {
if m != nil {
return m.Filesystem
}
return ""
}
func (m *ReceiveReq) GetClearResumeToken() bool {
if m != nil {
return m.ClearResumeToken
}
return false
}
type ReceiveRes struct {
}
func (m *ReceiveRes) Reset() { *m = ReceiveRes{} }
func (m *ReceiveRes) String() string { return proto.CompactTextString(m) }
func (*ReceiveRes) ProtoMessage() {}
func (*ReceiveRes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
func init() {
proto.RegisterType((*ListFilesystemReq)(nil), "pdu.ListFilesystemReq")
proto.RegisterType((*ListFilesystemRes)(nil), "pdu.ListFilesystemRes")
proto.RegisterType((*Filesystem)(nil), "pdu.Filesystem")
proto.RegisterType((*ListFilesystemVersionsReq)(nil), "pdu.ListFilesystemVersionsReq")
proto.RegisterType((*ListFilesystemVersionsRes)(nil), "pdu.ListFilesystemVersionsRes")
proto.RegisterType((*FilesystemVersion)(nil), "pdu.FilesystemVersion")
proto.RegisterType((*SendReq)(nil), "pdu.SendReq")
proto.RegisterType((*Property)(nil), "pdu.Property")
proto.RegisterType((*SendRes)(nil), "pdu.SendRes")
proto.RegisterType((*ReceiveReq)(nil), "pdu.ReceiveReq")
proto.RegisterType((*ReceiveRes)(nil), "pdu.ReceiveRes")
proto.RegisterEnum("pdu.FilesystemVersion_VersionType", FilesystemVersion_VersionType_name, FilesystemVersion_VersionType_value)
}
func init() { proto.RegisterFile("pdu.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
// 445 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0xc1, 0x6e, 0xd3, 0x40,
0x10, 0x65, 0x13, 0xa7, 0x38, 0x93, 0xd2, 0xa6, 0x4b, 0x85, 0x0c, 0x42, 0x28, 0xda, 0x53, 0x40,
0x22, 0x12, 0x01, 0x71, 0xe1, 0xd6, 0xa2, 0xf4, 0x82, 0xa0, 0xda, 0x9a, 0xaa, 0x57, 0x17, 0x8f,
0x54, 0x2b, 0xb1, 0x77, 0xbb, 0x63, 0x23, 0xe5, 0x73, 0xf8, 0x2b, 0x3e, 0x07, 0x79, 0x6a, 0x27,
0x4b, 0x0c, 0x52, 0x4e, 0x99, 0xf7, 0x66, 0x32, 0xf3, 0xe6, 0xcd, 0x1a, 0x86, 0x36, 0xad, 0x66,
0xd6, 0x99, 0xd2, 0xc8, 0xbe, 0x4d, 0x2b, 0xf5, 0x14, 0x4e, 0xbe, 0x64, 0x54, 0x2e, 0xb2, 0x15,
0xd2, 0x9a, 0x4a, 0xcc, 0x35, 0xde, 0xab, 0x45, 0x97, 0x24, 0xf9, 0x0e, 0x46, 0x5b, 0x82, 0x22,
0x31, 0xe9, 0x4f, 0x47, 0xf3, 0xe3, 0x59, 0xdd, 0xcf, 0x2b, 0xf4, 0x6b, 0xd4, 0x19, 0xc0, 0x16,
0x4a, 0x09, 0xc1, 0x65, 0x52, 0xde, 0x45, 0x62, 0x22, 0xa6, 0x43, 0xcd, 0xb1, 0x9c, 0xc0, 0x48,
0x23, 0x55, 0x39, 0xc6, 0x66, 0x89, 0x45, 0xd4, 0xe3, 0x94, 0x4f, 0xa9, 0x4f, 0xf0, 0xfc, 0x6f,
0x2d, 0xd7, 0xe8, 0x28, 0x33, 0x05, 0x69, 0xbc, 0x97, 0xaf, 0xfc, 0x01, 0x4d, 0x63, 0x8f, 0x51,
0xdf, 0xfe, 0xff, 0x67, 0x92, 0x73, 0x08, 0x5b, 0xd8, 0x6c, 0xf3, 0x6c, 0x67, 0x9b, 0x26, 0xad,
0x37, 0x75, 0xea, 0xb7, 0x80, 0x93, 0x4e, 0x5e, 0x7e, 0x84, 0x20, 0x5e, 0x5b, 0x64, 0x01, 0x47,
0x73, 0xf5, 0xef, 0x2e, 0xb3, 0xe6, 0xb7, 0xae, 0xd4, 0x5c, 0x5f, 0x3b, 0xf2, 0x35, 0xc9, 0xb1,
0x59, 0x9b, 0xe3, 0x9a, 0xbb, 0xa8, 0xb2, 0x34, 0xea, 0x4f, 0xc4, 0x34, 0xd0, 0x1c, 0xcb, 0x97,
0x30, 0x3c, 0x77, 0x98, 0x94, 0x18, 0xdf, 0x5c, 0x44, 0x01, 0x27, 0xb6, 0x84, 0x7c, 0x01, 0x21,
0x83, 0xcc, 0x14, 0xd1, 0x80, 0x3b, 0x6d, 0xb0, 0x7a, 0x0d, 0x23, 0x6f, 0xac, 0x3c, 0x84, 0xf0,
0xaa, 0x48, 0x2c, 0xdd, 0x99, 0x72, 0xfc, 0xa8, 0x46, 0x67, 0xc6, 0x2c, 0xf3, 0xc4, 0x2d, 0xc7,
0x42, 0xfd, 0x12, 0xf0, 0xf8, 0x0a, 0x8b, 0x74, 0x0f, 0x5f, 0x6b, 0x91, 0x0b, 0x67, 0xf2, 0x56,
0x78, 0x1d, 0xcb, 0x23, 0xe8, 0xc5, 0x86, 0x65, 0x0f, 0x75, 0x2f, 0x36, 0xbb, 0xa7, 0x0d, 0x3a,
0xa7, 0x65, 0xe1, 0x26, 0xb7, 0x0e, 0x89, 0x58, 0x78, 0xa8, 0x37, 0x58, 0x9e, 0xc2, 0xe0, 0x33,
0xa6, 0x95, 0x8d, 0x0e, 0x38, 0xf1, 0x00, 0xd4, 0x07, 0x08, 0x2f, 0x9d, 0xb1, 0xe8, 0xca, 0xf5,
0xc6, 0x3c, 0xe1, 0x99, 0x77, 0x0a, 0x83, 0xeb, 0x64, 0x55, 0xb5, 0x8e, 0x3e, 0x00, 0x75, 0xdb,
0x2e, 0x46, 0x72, 0x0a, 0xc7, 0xdf, 0x09, 0x53, 0x5f, 0x98, 0xe0, 0x01, 0xbb, 0xb4, 0x7c, 0x0b,
0xd0, 0x8c, 0xca, 0x90, 0xa2, 0x1e, 0xbf, 0x8f, 0x27, 0x7c, 0xd9, 0x56, 0x81, 0xf6, 0x0a, 0xd4,
0x0d, 0x80, 0xc6, 0x1f, 0x98, 0xfd, 0xc4, 0x7d, 0xfc, 0x7b, 0x03, 0xe3, 0xf3, 0x15, 0x26, 0x6e,
0xf7, 0xed, 0x87, 0xba, 0xc3, 0xab, 0x43, 0xaf, 0x33, 0xdd, 0x1e, 0xf0, 0xb7, 0xfb, 0xfe, 0x4f,
0x00, 0x00, 0x00, 0xff, 0xff, 0xfd, 0xc4, 0x8d, 0xb7, 0xc8, 0x03, 0x00, 0x00,
}
+78
View File
@@ -0,0 +1,78 @@
syntax = "proto3";
package pdu;
message ListFilesystemReq {}
message ListFilesystemRes {
repeated Filesystem Filesystems = 1;
}
message Filesystem {
string Path = 1;
string ResumeToken = 2;
}
message ListFilesystemVersionsReq {
string Filesystem = 1;
}
message ListFilesystemVersionsRes {
repeated FilesystemVersion Versions = 1;
}
message FilesystemVersion {
enum VersionType {
Snapshot = 0;
Bookmark = 1;
}
VersionType Type = 1;
string Name = 2;
uint64 Guid = 3;
uint64 CreateTXG = 4;
string Creation = 5; // RFC 3339
}
message SendReq {
string Filesystem = 1;
string From = 2;
// May be empty / null to request a full transfer of From
string To = 3;
// If ResumeToken is not empty, the resume token that CAN be tried for 'zfs send' by the sender.
// The sender MUST indicate in SendRes.UsedResumeToken
// If it does not work, the sender SHOULD clear the resume token on their side
// and use From and To instead
// If ResumeToken is not empty, the GUIDs of From and To
// MUST correspond to those encoded in the ResumeToken.
// Otherwise, the Sender MUST return an error.
string ResumeToken = 4;
bool Compress = 5;
bool Dedup = 6;
}
message Property {
string Name = 1;
string Value = 2;
}
message SendRes {
// The actual stream is in the stream part of the streamrpc response
// Whether the resume token provided in the request has been used or not.
bool UsedResumeToken = 1;
repeated Property Properties = 2;
}
message ReceiveReq {
// The stream part of the streamrpc request contains the zfs send stream
string Filesystem = 1;
// If true, the receiver should clear the resume token before perfoming the zfs recv of the stream in the request
bool ClearResumeToken = 2;
}
message ReceiveRes {}
+73
View File
@@ -0,0 +1,73 @@
package pdu
import (
"fmt"
"github.com/zrepl/zrepl/zfs"
"time"
)
func (v *FilesystemVersion) RelName() string {
zv := v.ZFSFilesystemVersion()
return zv.String()
}
func (v FilesystemVersion_VersionType) ZFSVersionType() zfs.VersionType {
switch v {
case FilesystemVersion_Snapshot:
return zfs.Snapshot
case FilesystemVersion_Bookmark:
return zfs.Bookmark
default:
panic(fmt.Sprintf("unexpected v.Type %#v", v))
}
}
func FilesystemVersionFromZFS(fsv zfs.FilesystemVersion) *FilesystemVersion {
var t FilesystemVersion_VersionType
switch fsv.Type {
case zfs.Bookmark:
t = FilesystemVersion_Bookmark
case zfs.Snapshot:
t = FilesystemVersion_Snapshot
default:
panic("unknown fsv.Type: " + fsv.Type)
}
return &FilesystemVersion{
Type: t,
Name: fsv.Name,
Guid: fsv.Guid,
CreateTXG: fsv.CreateTXG,
Creation: fsv.Creation.Format(time.RFC3339),
}
}
func (v *FilesystemVersion) CreationAsTime() (time.Time, error) {
return time.Parse(time.RFC3339, v.Creation)
}
// implement fsfsm.FilesystemVersion
func (v *FilesystemVersion) SnapshotTime() time.Time {
t, err := v.CreationAsTime()
if err != nil {
panic(err) // FIXME
}
return t
}
func (v *FilesystemVersion) ZFSFilesystemVersion() *zfs.FilesystemVersion {
ct := time.Time{}
if v.Creation != "" {
var err error
ct, err = time.Parse(time.RFC3339, v.Creation)
if err != nil {
panic(err)
}
}
return &zfs.FilesystemVersion{
Type: v.Type.ZFSVersionType(),
Name: v.Name,
Guid: v.Guid,
CreateTXG: v.CreateTXG,
Creation: ct,
}
}
+64
View File
@@ -0,0 +1,64 @@
package pdu
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestFilesystemVersion_RelName(t *testing.T) {
type TestCase struct {
In FilesystemVersion
Out string
Panic bool
}
tcs := []TestCase{
{
In: FilesystemVersion{
Type: FilesystemVersion_Snapshot,
Name: "foobar",
},
Out: "@foobar",
},
{
In: FilesystemVersion{
Type: FilesystemVersion_Bookmark,
Name: "foobar",
},
Out: "#foobar",
},
{
In: FilesystemVersion{
Type: 2342,
Name: "foobar",
},
Panic: true,
},
}
for _, tc := range tcs {
if tc.Panic {
assert.Panics(t, func() {
tc.In.RelName()
})
} else {
o := tc.In.RelName()
assert.Equal(t, tc.Out, o)
}
}
}
func TestFilesystemVersion_ZFSFilesystemVersion(t *testing.T) {
empty := &FilesystemVersion{}
emptyZFS := empty.ZFSFilesystemVersion()
assert.Zero(t, emptyZFS.Creation)
dateInvalid := &FilesystemVersion{Creation:"foobar"}
assert.Panics(t, func() {
dateInvalid.ZFSFilesystemVersion()
})
}
+19
View File
@@ -0,0 +1,19 @@
package replication
import (
"context"
"github.com/zrepl/zrepl/cmd/replication/common"
"github.com/zrepl/zrepl/cmd/replication/internal/mainfsm"
)
type Report = mainfsm.Report
type Replication interface {
Drive(ctx context.Context, ep common.EndpointPair)
Report() *Report
}
func NewReplication() Replication {
return mainfsm.NewReplication()
}