adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

replication + endpoint: replication guarantees: guarantee_{resumability,incremental,nothing}

Browse Source 
This commit

- adds a configuration in which no step holds, replication cursors, etc. are created
- removes the send.step_holds.disable_incremental setting
- creates a new config option `replication` for active-side jobs
- adds the replication.protection.{initial,incremental} settings, each
  of which can have values
    - `guarantee_resumability`
    - `guarantee_incremental`
    - `guarantee_nothing`
  (refer to docs/configuration/replication.rst for semantics)

The `replication` config from an active side is sent to both endpoint.Sender and endpoint.Receiver
for each replication step. Sender and Receiver then act accordingly.

For `guarantee_incremental`, we add the new `tentative-replication-cursor` abstraction.
The necessity for that abstraction is outlined in https://github.com/zrepl/zrepl/issues/340.

fixes https://github.com/zrepl/zrepl/issues/340
This commit is contained in:
Christian Schwarz
2020-06-27 23:53:33 +02:00
parent 27673a23e9
commit 30cdc1430e
36 changed files with 1503 additions and 757 deletions
Download Patch File Download Diff File Expand all files Collapse all files
daemon/job/active.go
+18 -26
View File
@@ -12,7 +12,6 @@ import (
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/daemon/job/reset"
"github.com/zrepl/zrepl/daemon/job/wakeup"
"github.com/zrepl/zrepl/daemon/pruner"
@@ -145,23 +144,24 @@ func (m *modePush) ResetConnectBackoff() {
func modePushFromConfig(g *config.Global, in *config.PushJob, jobID endpoint.JobID) (*modePush, error) {
m := &modePush{}
var err error
fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
m.senderConfig, err = buildSenderConfig(in, jobID)
if err != nil {
return nil, errors.Wrap(err, "cannot build filesystem filter")
return nil, errors.Wrap(err, "sender config")
}
m.senderConfig = &endpoint.SenderConfig{
FSF: fsf,
Encrypt: &zfs.NilBool{B: in.Send.Encrypted},
DisableIncrementalStepHolds: in.Send.StepHolds.DisableIncremental,
JobID: jobID,
replicationConfig, err := logic.ReplicationConfigFromConfig(in.Replication)
if err != nil {
return nil, errors.Wrap(err, "field `replication`")
}
m.plannerPolicy = &logic.PlannerPolicy{
EncryptedSend: logic.TriFromBool(in.Send.Encrypted),
EncryptedSend: logic.TriFromBool(in.Send.Encrypted),
ReplicationConfig: *replicationConfig,
}
if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
if m.snapper, err = snapper.FromConfig(g, m.senderConfig.FSF, in.Snapshotting); err != nil {
return nil, errors.Wrap(err, "cannot build snapper")
}
@@ -173,7 +173,6 @@ type modePull struct {
receiver *endpoint.Receiver
receiverConfig endpoint.ReceiverConfig
sender *rpc.Client
rootFS *zfs.DatasetPath
plannerPolicy *logic.PlannerPolicy
interval config.PositiveDurationOrManual
}
@@ -247,26 +246,19 @@ func modePullFromConfig(g *config.Global, in *config.PullJob, jobID endpoint.Job
m = &modePull{}
m.interval = in.Interval
m.rootFS, err = zfs.NewDatasetPath(in.RootFS)
replicationConfig, err := logic.ReplicationConfigFromConfig(in.Replication)
if err != nil {
return nil, errors.New("RootFS is not a valid zfs filesystem path")
}
if m.rootFS.Length() <= 0 {
return nil, errors.New("RootFS must not be empty") // duplicates error check of receiver
return nil, errors.Wrap(err, "field `replication`")
}
m.plannerPolicy = &logic.PlannerPolicy{
EncryptedSend: logic.DontCare,
EncryptedSend: logic.DontCare,
ReplicationConfig: *replicationConfig,
}
m.receiverConfig = endpoint.ReceiverConfig{
JobID: jobID,
RootWithoutClientComponent: m.rootFS,
AppendClientIdentity: false, // !
UpdateLastReceivedHold: true,
}
if err := m.receiverConfig.Validate(); err != nil {
return nil, errors.Wrap(err, "cannot build receiver config")
m.receiverConfig, err = buildReceiverConfig(in, jobID)
if err != nil {
return nil, err
}
return m, nil
@@ -365,7 +357,7 @@ func (j *ActiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
_ = j.mode.(*modePush) // make sure we didn't introduce a new job type
return nil, false
}
return pull.rootFS.Copy(), true
return pull.receiverConfig.RootWithoutClientComponent.Copy(), true
}
func (j *ActiveSide) SenderConfig() *endpoint.SenderConfig {
daemon/job/build_jobs_sendrecvoptions.go
+55
View File
@@ -0,0 +1,55 @@
package job
import (
"github.com/pkg/errors"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/endpoint"
"github.com/zrepl/zrepl/zfs"
)
type SendingJobConfig interface {
GetFilesystems() config.FilesystemsFilter
GetSendOptions() *config.SendOptions // must not be nil
}
func buildSenderConfig(in SendingJobConfig, jobID endpoint.JobID) (*endpoint.SenderConfig, error) {
fsf, err := filters.DatasetMapFilterFromConfig(in.GetFilesystems())
if err != nil {
return nil, errors.Wrap(err, "cannot build filesystem filter")
}
return &endpoint.SenderConfig{
FSF: fsf,
Encrypt: &zfs.NilBool{B: in.GetSendOptions().Encrypted},
JobID: jobID,
}, nil
}
type ReceivingJobConfig interface {
GetRootFS() string
GetAppendClientIdentity() bool
GetRecvOptions() *config.RecvOptions
}
func buildReceiverConfig(in ReceivingJobConfig, jobID endpoint.JobID) (rc endpoint.ReceiverConfig, err error) {
rootFs, err := zfs.NewDatasetPath(in.GetRootFS())
if err != nil {
return rc, errors.New("root_fs is not a valid zfs filesystem path")
}
if rootFs.Length() <= 0 {
return rc, errors.New("root_fs must not be empty") // duplicates error check of receiver
}
rc = endpoint.ReceiverConfig{
JobID: jobID,
RootWithoutClientComponent: rootFs,
AppendClientIdentity: in.GetAppendClientIdentity(),
}
if err := rc.Validate(); err != nil {
return rc, errors.Wrap(err, "cannot build receiver config")
}
return rc, nil
}
daemon/job/passive.go
+6 -22
View File
@@ -9,7 +9,6 @@ import (
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/daemon/snapper"
"github.com/zrepl/zrepl/endpoint"
@@ -48,19 +47,9 @@ func (m *modeSink) SnapperReport() *snapper.Report { return nil }
func modeSinkFromConfig(g *config.Global, in *config.SinkJob, jobID endpoint.JobID) (m *modeSink, err error) {
m = &modeSink{}
rootDataset, err := zfs.NewDatasetPath(in.RootFS)
m.receiverConfig, err = buildReceiverConfig(in, jobID)
if err != nil {
return nil, errors.New("root dataset is not a valid zfs filesystem path")
}
m.receiverConfig = endpoint.ReceiverConfig{
JobID: jobID,
RootWithoutClientComponent: rootDataset,
AppendClientIdentity: true, // !
UpdateLastReceivedHold: true,
}
if err := m.receiverConfig.Validate(); err != nil {
return nil, errors.Wrap(err, "cannot build receiver config")
return nil, err
}
return m, nil
@@ -74,18 +63,13 @@ type modeSource struct {
func modeSourceFromConfig(g *config.Global, in *config.SourceJob, jobID endpoint.JobID) (m *modeSource, err error) {
// FIXME exact dedup of modePush
m = &modeSource{}
fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
m.senderConfig, err = buildSenderConfig(in, jobID)
if err != nil {
return nil, errors.Wrap(err, "cannot build filesystem filter")
}
m.senderConfig = &endpoint.SenderConfig{
FSF: fsf,
Encrypt: &zfs.NilBool{B: in.Send.Encrypted},
DisableIncrementalStepHolds: in.Send.StepHolds.DisableIncremental,
JobID: jobID,
return nil, errors.Wrap(err, "send options")
}
if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
if m.snapper, err = snapper.FromConfig(g, m.senderConfig.FSF, in.Snapshotting); err != nil {
return nil, errors.Wrap(err, "cannot build snapper")
}
daemon/job/snapjob.go
-2
View File
@@ -175,8 +175,6 @@ func (j *SnapJob) doPrune(ctx context.Context) {
FSF: j.fsfilter,
// FIXME encryption setting is irrelevant for SnapJob because the endpoint is only used as pruner.Target
Encrypt: &zfs.NilBool{B: true},
// FIXME DisableIncrementalStepHolds setting is irrelevant for SnapJob because the endpoint is only used as pruner.Target
DisableIncrementalStepHolds: false,
})
j.pruner = j.prunerFactory.BuildLocalPruner(ctx, sender, alwaysUpToDateReplicationCursorHistory{sender})
log.Info("start pruning")
daemon/snapper/snapper.go
+3 -4
View File
@@ -11,7 +11,6 @@ import (
"github.com/zrepl/zrepl/daemon/logging/trace"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/daemon/hooks"
"github.com/zrepl/zrepl/daemon/logging"
"github.com/zrepl/zrepl/logger"
@@ -49,7 +48,7 @@ type args struct {
ctx context.Context
prefix string
interval time.Duration
fsf *filters.DatasetMapFilter
fsf zfs.DatasetFilter
snapshotsTaken chan<- struct{}
hooks *hooks.List
dryRun bool
@@ -109,7 +108,7 @@ func getLogger(ctx context.Context) Logger {
return logging.GetLogger(ctx, logging.SubsysSnapshot)
}
func PeriodicFromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in *config.SnapshottingPeriodic) (*Snapper, error) {
func PeriodicFromConfig(g *config.Global, fsf zfs.DatasetFilter, in *config.SnapshottingPeriodic) (*Snapper, error) {
if in.Prefix == "" {
return nil, errors.New("prefix must not be empty")
}
@@ -383,7 +382,7 @@ func wait(a args, u updater) state {
}
}
func listFSes(ctx context.Context, mf *filters.DatasetMapFilter) (fss []*zfs.DatasetPath, err error) {
func listFSes(ctx context.Context, mf zfs.DatasetFilter) (fss []*zfs.DatasetPath, err error) {
return zfs.ZFSListMapping(ctx, mf)
}
daemon/snapper/snapper_all.go
+2 -2
View File
@@ -5,7 +5,7 @@ import (
"fmt"
"github.com/zrepl/zrepl/config"
"github.com/zrepl/zrepl/daemon/filters"
"github.com/zrepl/zrepl/zfs"
)
// FIXME: properly abstract snapshotting:
@@ -32,7 +32,7 @@ func (s *PeriodicOrManual) Report() *Report {
return nil
}
func FromConfig(g *config.Global, fsf *filters.DatasetMapFilter, in config.SnapshottingEnum) (*PeriodicOrManual, error) {
func FromConfig(g *config.Global, fsf zfs.DatasetFilter, in config.SnapshottingEnum) (*PeriodicOrManual, error) {
switch v := in.Ret.(type) {
case *config.SnapshottingPeriodic:
snapper, err := PeriodicFromConfig(g, fsf, v)